justinzeus/scholarr
1
0
Fork 0
Code Issues 1 Pull requests 18 Projects Releases Packages Wiki Activity Actions

ci: add ruff linting and mypy type checking

Browse source 
Add ruff and mypy to dev dependencies with configuration in pyproject.toml.
Add a lint CI job that runs ruff check, ruff format --check, and mypy.
Auto-fix import sorting and formatting across the codebase. Exclude
alembic/versions from linting (auto-generated migrations). Ignore B008
(FastAPI Depends pattern) and RUF001 (unicode in user-facing strings).

21 ruff lint errors and 50 mypy errors remain for manual review.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Justin Visser 2026-02-26 22:11:41 +01:00
parent 399276ea69
commit bf04c77aa9
137 changed files with 3066 additions and 1900 deletions
Download patch file Download diff file Expand all files Collapse all files

12
tests/conftest.py
View file

@ -6,12 +6,12 @@ import re
from collections.abc import AsyncIterator, Iterator
import pytest
from alembic import command
from alembic.config import Config
from sqlalchemy.engine import make_url
from sqlalchemy import text
from sqlalchemy.engine import make_url
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
from alembic import command
from app.auth.deps import get_login_rate_limiter
from app.db.session import close_engine
from app.settings import settings
@ -52,9 +52,7 @@ def _resolve_test_database_url() -> str | None:
parsed = make_url(base)
if not parsed.database:
return None
derived_database = (
parsed.database if parsed.database.endswith("_test") else f"{parsed.database}_test"
)
derived_database = parsed.database if parsed.database.endswith("_test") else f"{parsed.database}_test"
return parsed.set(database=derived_database).render_as_string(hide_password=False)
@ -85,9 +83,7 @@ def ensure_test_database_exists(database_url: str) -> Iterator[None]:
if not database_name:
raise RuntimeError("TEST_DATABASE_URL must include a database name.")
if not DB_NAME_SAFE_RE.fullmatch(database_name):
raise RuntimeError(
"TEST_DATABASE_URL database name must match [A-Za-z0-9_]+ for safe auto-provisioning."
)
raise RuntimeError("TEST_DATABASE_URL database name must match [A-Za-z0-9_]+ for safe auto-provisioning.")
admin_name = "postgres" if database_name != "postgres" else "template1"
admin_url = parsed.set(database=admin_name)

1
tests/integration/helpers.py
View file

@ -6,6 +6,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
from app.auth.security import PasswordService
def login_user(client: TestClient, *, email: str, password: str) -> None:
bootstrap_response = client.get("/api/v1/auth/csrf")
assert bootstrap_response.status_code == 200

27
tests/integration/test_api_v1.py
View file

@ -1,7 +1,7 @@
from __future__ import annotations
import asyncio
from datetime import datetime, timedelta, timezone
from datetime import UTC, datetime, timedelta
from pathlib import Path
import pytest
@ -12,8 +12,8 @@ from sqlalchemy.ext.asyncio import AsyncSession
from app.api.runtime_deps import get_scholar_source
from app.main import app
from app.services.domains.publications.types import PublicationListItem
from app.services.domains.settings import application as user_settings_service
from app.services.domains.scholar.source import FetchResult
from app.services.domains.settings import application as user_settings_service
from app.settings import settings
from tests.integration.helpers import insert_user, login_user
@ -412,10 +412,7 @@ async def test_api_admin_dbops_integrity_and_repair_flow(db_session: AsyncSessio
assert integrity_response.status_code == 200
integrity_payload = integrity_response.json()["data"]
assert integrity_payload["status"] in {"ok", "warning", "failed"}
assert any(
check["name"] == "missing_pdf_url"
for check in integrity_payload["checks"]
)
assert any(check["name"] == "missing_pdf_url" for check in integrity_payload["checks"])
repair_response = client.post(
"/api/v1/admin/db/repairs/publication-links",
@ -450,10 +447,7 @@ async def test_api_admin_dbops_integrity_and_repair_flow(db_session: AsyncSessio
assert page_two_payload["has_prev"] is True
untracked_response = client.get("/api/v1/admin/db/pdf-queue?limit=20&status=untracked")
assert untracked_response.status_code == 200
assert any(
item["status"] == "untracked"
for item in untracked_response.json()["data"]["items"]
)
assert any(item["status"] == "untracked" for item in untracked_response.json()["data"]["items"])
link_count_result = await db_session.execute(
text("SELECT count(*) FROM scholar_publications WHERE publication_id = :publication_id"),
@ -1272,9 +1266,9 @@ async def test_api_settings_get_and_update(db_session: AsyncSession) -> None:
assert settings_payload["policy"]["min_run_interval_minutes"] == user_settings_service.resolve_run_interval_minimum(
settings.ingestion_min_run_interval_minutes
)
assert settings_payload["policy"]["min_request_delay_seconds"] == user_settings_service.resolve_request_delay_minimum(
settings.ingestion_min_request_delay_seconds
)
assert settings_payload["policy"][
"min_request_delay_seconds"
] == user_settings_service.resolve_request_delay_minimum(settings.ingestion_min_request_delay_seconds)
assert settings_payload["policy"]["automation_allowed"] is settings.ingestion_automation_allowed
assert settings_payload["policy"]["manual_run_allowed"] is settings.ingestion_manual_run_allowed
assert settings_payload["policy"]["blocked_failure_threshold"] == max(
@ -1482,10 +1476,7 @@ async def test_api_runs_manual_and_queue_actions(db_session: AsyncSession) -> No
queue_list_response = client.get("/api/v1/runs/queue/items")
assert queue_list_response.status_code == 200
assert any(
int(item["id"]) == queue_item_id
for item in queue_list_response.json()["data"]["queue_items"]
)
assert any(int(item["id"]) == queue_item_id for item in queue_list_response.json()["data"]["queue_items"])
retry_response = client.post(f"/api/v1/runs/queue/{queue_item_id}/retry", headers=headers)
assert retry_response.status_code == 200
@ -1766,7 +1757,7 @@ async def test_api_settings_clears_expired_scrape_safety_cooldown(
)
user_settings.scrape_safety_state = {"blocked_start_count": 2}
user_settings.scrape_cooldown_reason = "blocked_failure_threshold_exceeded"
user_settings.scrape_cooldown_until = datetime.now(timezone.utc) - timedelta(seconds=15)
user_settings.scrape_cooldown_until = datetime.now(UTC) - timedelta(seconds=15)
await db_session.commit()
client = TestClient(app)

5
tests/integration/test_data_repair_jobs.py
View file

@ -160,10 +160,7 @@ async def _assert_apply_effects(
)
baseline_completed = await _count_rows(
db_session,
sql=(
"SELECT count(*) FROM scholar_profiles "
"WHERE id = :scholar_profile_id AND baseline_completed = false"
),
sql=("SELECT count(*) FROM scholar_profiles WHERE id = :scholar_profile_id AND baseline_completed = false"),
params={"scholar_profile_id": scholar_profile_id},
)
publication_count = await _count_rows(

40
tests/integration/test_deferred_enrichment.py
View file

@ -1,22 +1,24 @@
from __future__ import annotations
from datetime import UTC, datetime
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from unittest.mock import patch, AsyncMock, MagicMock
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from datetime import datetime, timezone
from app.db.models import CrawlRun, Publication, ScholarProfile, ScholarPublication, RunStatus, RunTriggerType
from app.db.models import CrawlRun, Publication, RunStatus, RunTriggerType, ScholarProfile, ScholarPublication
from app.services.domains.ingestion.application import ScholarIngestionService
from app.services.domains.openalex.types import OpenAlexWork
from tests.integration.helpers import insert_user
@pytest.mark.integration
@pytest.mark.asyncio
async def test_deferred_enrichment_sweeps_previous_runs(db_session: AsyncSession) -> None:
# 1. Setup: Create user and scholar
user_id = await insert_user(db_session, email="test@example.com", password="password123")
scholar = ScholarProfile(
user_id=user_id,
scholar_id="SaiiI5MAAAAJ",
@ -25,17 +27,17 @@ async def test_deferred_enrichment_sweeps_previous_runs(db_session: AsyncSession
)
db_session.add(scholar)
await db_session.flush()
# 2. Simulate a previous FAILED run that left an un-enriched publication
failed_run = CrawlRun(
user_id=user_id,
status=RunStatus.FAILED,
trigger_type=RunTriggerType.MANUAL,
start_dt=datetime.now(timezone.utc),
start_dt=datetime.now(UTC),
)
db_session.add(failed_run)
await db_session.flush()
pub = Publication(
title_raw="A fast quantum mechanical algorithm for database search",
title_normalized="a fast quantum mechanical algorithm for database search",
@ -45,7 +47,7 @@ async def test_deferred_enrichment_sweeps_previous_runs(db_session: AsyncSession
)
db_session.add(pub)
await db_session.flush()
link = ScholarPublication(
scholar_profile_id=scholar.id,
publication_id=pub.id,
@ -53,17 +55,17 @@ async def test_deferred_enrichment_sweeps_previous_runs(db_session: AsyncSession
)
db_session.add(link)
await db_session.commit()
# 3. Create a NEW run
new_run = CrawlRun(
user_id=user_id,
status=RunStatus.RUNNING,
trigger_type=RunTriggerType.MANUAL,
start_dt=datetime.now(timezone.utc),
start_dt=datetime.now(UTC),
)
db_session.add(new_run)
await db_session.commit()
# 4. Mock OpenAlex client to return enrichment data
mock_work = OpenAlexWork(
openalex_id="W1234567",
@ -76,26 +78,30 @@ async def test_deferred_enrichment_sweeps_previous_runs(db_session: AsyncSession
is_oa=True,
oa_url="http://example.com/grover.pdf",
)
mock_source = MagicMock()
service = ScholarIngestionService(source=mock_source)
# We patch the client at its source, and also mock arXiv to avoid real HTTP calls
with patch("app.services.domains.openalex.client.OpenAlexClient") as MockClient, \
patch("app.services.domains.arxiv.application.discover_arxiv_id_for_publication", new=AsyncMock(return_value=None)):
with (
patch("app.services.domains.openalex.client.OpenAlexClient") as MockClient,
patch(
"app.services.domains.arxiv.application.discover_arxiv_id_for_publication", new=AsyncMock(return_value=None)
),
):
mock_instance = MockClient.return_value
mock_instance.get_works_by_filter = AsyncMock(return_value=[mock_work])
# 5. Execute the enrichment pass for the NEW run
await service._enrich_pending_publications(db_session, run_id=new_run.id)
await db_session.commit()
# 6. Verification: The publication from the FAILED run should now be enriched
await db_session.refresh(pub)
assert pub.openalex_enriched is True
assert pub.citation_count == 1000
assert pub.pdf_url == "http://example.com/grover.pdf"
# Double check it was indeed processed
stmt = select(Publication).where(Publication.id == pub.id)
result = await db_session.execute(stmt)

3
tests/integration/test_fixture_probe_runs.py
View file

@ -34,8 +34,7 @@ def _blocked_fetch(*, scholar_id: str, body: str) -> FetchResult:
requested_url=f"https://scholar.google.com/citations?hl=en&user={scholar_id}",
status_code=200,
final_url=(
"https://accounts.google.com/v3/signin/identifier"
"?continue=https%3A%2F%2Fscholar.google.com%2Fcitations"
"https://accounts.google.com/v3/signin/identifier?continue=https%3A%2F%2Fscholar.google.com%2Fcitations"
),
body=body,
error=None,

4
tests/integration/test_migrations.py
View file

@ -29,9 +29,7 @@ EXPECTED_REVISION = "20260226_0024"
@pytest.mark.migrations
@pytest.mark.asyncio
async def test_migration_creates_expected_tables(db_session: AsyncSession) -> None:
result = await db_session.execute(
text("SELECT tablename FROM pg_tables WHERE schemaname = 'public'")
)
result = await db_session.execute(text("SELECT tablename FROM pg_tables WHERE schemaname = 'public'"))
table_names = {row[0] for row in result}
assert EXPECTED_TABLES.issubset(table_names)

6
tests/integration/test_run_lifecycle_consistency.py
View file

@ -338,8 +338,8 @@ async def test_partial_discovery_exception_keeps_new_pub_count_consistent(
monkeypatch.setattr(service, "_resolve_publication", _resolve_publication_stub)
from app.services.domains.scholar.parser_types import PublicationCandidate
from app.db.models import ScholarProfile
from app.services.domains.scholar.parser_types import PublicationCandidate
scholar = await db_session.get(ScholarProfile, scholar_profile_id)
assert scholar is not None
@ -505,9 +505,7 @@ async def test_publications_pagination_snapshot_stays_stable_across_inserts(
)
assert first_page_again_response.status_code == 200
first_page_again = first_page_again_response.json()["data"]
first_page_again_ids = [
int(item["publication_id"]) for item in first_page_again["publications"]
]
first_page_again_ids = [int(item["publication_id"]) for item in first_page_again["publications"]]
assert first_page_again_ids == first_page_ids
assert not (set(first_page_ids) & set(second_page_ids))

1
tests/smoke/test_db_connectivity.py
View file

@ -20,4 +20,3 @@ async def test_database_connectivity_from_database_url() -> None:
assert result.scalar_one() == 1
finally:
await engine.dispose()

2
tests/smoke/test_migration_schema.py
View file

@ -1,6 +1,6 @@
import pytest
from alembic.config import Config
from alembic.script import ScriptDirectory
import pytest
from sqlalchemy import text
from sqlalchemy.ext.asyncio import AsyncSession

9
tests/unit/services/domains/arxiv/test_cache.py
View file

@ -2,7 +2,7 @@ from __future__ import annotations
import asyncio
import gc
from datetime import datetime, timedelta, timezone
from datetime import UTC, datetime, timedelta
import pytest
from sqlalchemy import select
@ -59,7 +59,7 @@ def test_build_query_fingerprint_normalizes_search_and_id_params() -> None:
@pytest.mark.asyncio
async def test_cache_entry_expires_and_is_deleted(db_session: AsyncSession) -> None:
query_fingerprint = build_query_fingerprint(params={"search_query": "ti:test", "start": 0})
now_utc = datetime(2026, 2, 26, 12, 0, tzinfo=timezone.utc)
now_utc = datetime(2026, 2, 26, 12, 0, tzinfo=UTC)
await set_cached_feed(
query_fingerprint=query_fingerprint,
@ -79,9 +79,7 @@ async def test_cache_entry_expires_and_is_deleted(db_session: AsyncSession) -> N
)
result = await db_session.execute(
select(ArxivQueryCacheEntry).where(
ArxivQueryCacheEntry.query_fingerprint == query_fingerprint
)
select(ArxivQueryCacheEntry).where(ArxivQueryCacheEntry.query_fingerprint == query_fingerprint)
)
assert hit is not None
assert miss is None
@ -118,6 +116,7 @@ async def test_inflight_owner_failure_without_joiner_has_no_unretrieved_exceptio
loop.set_exception_handler(_capture_exception)
try:
async def _failing_fetch() -> ArxivFeed:
raise RuntimeError("owner_failed")

25
tests/unit/services/domains/arxiv/test_client.py
View file

@ -1,7 +1,8 @@
from __future__ import annotations
import asyncio
from datetime import datetime, timezone
from datetime import UTC, datetime
import httpx
import pytest
from sqlalchemy.ext.asyncio import AsyncSession
@ -35,7 +36,9 @@ async def test_client_search_builds_query_and_sort_params() -> None:
captured["params"] = params
captured["request_email"] = request_email
captured["timeout_seconds"] = timeout_seconds
return httpx.Response(200, text=_CLIENT_FEED_XML, request=httpx.Request("GET", "https://export.arxiv.org/api/query"))
return httpx.Response(
200, text=_CLIENT_FEED_XML, request=httpx.Request("GET", "https://export.arxiv.org/api/query")
)
client = ArxivClient(request_fn=_request_fn)
feed = await client.search(
@ -66,7 +69,9 @@ async def test_client_lookup_ids_builds_id_list_param() -> None:
async def _request_fn(*, params, request_email, timeout_seconds):
captured["params"] = params
return httpx.Response(200, text=_CLIENT_FEED_XML, request=httpx.Request("GET", "https://export.arxiv.org/api/query"))
return httpx.Response(
200, text=_CLIENT_FEED_XML, request=httpx.Request("GET", "https://export.arxiv.org/api/query")
)
client = ArxivClient(request_fn=_request_fn)
await client.lookup_ids(id_list=["1234.5678", " 9999.0001 "], start=0, max_results=2)
@ -76,7 +81,9 @@ async def test_client_lookup_ids_builds_id_list_param() -> None:
@pytest.mark.asyncio
async def test_client_search_rejects_invalid_sort_by() -> None:
async def _unused_request_fn(*, params, request_email, timeout_seconds):
return httpx.Response(200, text=_CLIENT_FEED_XML, request=httpx.Request("GET", "https://export.arxiv.org/api/query"))
return httpx.Response(
200, text=_CLIENT_FEED_XML, request=httpx.Request("GET", "https://export.arxiv.org/api/query")
)
client = ArxivClient(request_fn=_unused_request_fn)
with pytest.raises(ArxivClientValidationError):
@ -86,7 +93,9 @@ async def test_client_search_rejects_invalid_sort_by() -> None:
@pytest.mark.asyncio
async def test_client_lookup_ids_rejects_empty_list() -> None:
async def _unused_request_fn(*, params, request_email, timeout_seconds):
return httpx.Response(200, text=_CLIENT_FEED_XML, request=httpx.Request("GET", "https://export.arxiv.org/api/query"))
return httpx.Response(
200, text=_CLIENT_FEED_XML, request=httpx.Request("GET", "https://export.arxiv.org/api/query")
)
client = ArxivClient(request_fn=_unused_request_fn)
with pytest.raises(ArxivClientValidationError):
@ -96,7 +105,9 @@ async def test_client_lookup_ids_rejects_empty_list() -> None:
@pytest.mark.asyncio
async def test_client_search_rejects_negative_start() -> None:
async def _unused_request_fn(*, params, request_email, timeout_seconds):
return httpx.Response(200, text=_CLIENT_FEED_XML, request=httpx.Request("GET", "https://export.arxiv.org/api/query"))
return httpx.Response(
200, text=_CLIENT_FEED_XML, request=httpx.Request("GET", "https://export.arxiv.org/api/query")
)
client = ArxivClient(request_fn=_unused_request_fn)
with pytest.raises(ArxivClientValidationError):
@ -172,7 +183,7 @@ async def test_request_feed_skips_live_call_when_global_cooldown_is_active(
return ArxivCooldownStatus(
is_active=True,
remaining_seconds=61.0,
cooldown_until=datetime(2026, 2, 26, 12, 1, tzinfo=timezone.utc),
cooldown_until=datetime(2026, 2, 26, 12, 1, tzinfo=UTC),
)
async def _unexpected_limit_call(*, fetch, source_path): # pragma: no cover - defensive

4
tests/unit/services/domains/arxiv/test_guards.py
View file

@ -1,6 +1,6 @@
from __future__ import annotations
from datetime import datetime, timezone
from datetime import UTC, datetime
from app.services.domains.arxiv.guards import arxiv_skip_reason_for_item
from app.services.domains.publication_identifiers.types import DisplayIdentifier
@ -25,7 +25,7 @@ def _item(
pub_url=pub_url,
pdf_url=pdf_url,
is_read=False,
first_seen_at=datetime.now(timezone.utc),
first_seen_at=datetime.now(UTC),
is_new_in_latest_run=True,
display_identifier=display_identifier,
)

4
tests/unit/services/domains/arxiv/test_parser.py
View file

@ -50,6 +50,8 @@ def test_parse_arxiv_feed_raises_on_invalid_xml() -> None:
def test_parse_arxiv_feed_raises_on_invalid_opensearch_integer() -> None:
payload = _VALID_FEED_XML.replace("<opensearch:totalResults>2</opensearch:totalResults>", "<opensearch:totalResults>x</opensearch:totalResults>")
payload = _VALID_FEED_XML.replace(
"<opensearch:totalResults>2</opensearch:totalResults>", "<opensearch:totalResults>x</opensearch:totalResults>"
)
with pytest.raises(ArxivParseError):
parse_arxiv_feed(payload)

14
tests/unit/services/domains/arxiv/test_rate_limit.py
View file

@ -1,7 +1,7 @@
from __future__ import annotations
import asyncio
from datetime import datetime, timedelta, timezone
from datetime import UTC, datetime, timedelta
import httpx
import pytest
@ -20,7 +20,7 @@ async def test_arxiv_rate_limit_respects_cooldown(db_session: AsyncSession) -> N
db_session.add(
ArxivRuntimeState(
state_key=ARXIV_RUNTIME_STATE_KEY,
cooldown_until=datetime.now(timezone.utc) + timedelta(seconds=30),
cooldown_until=datetime.now(UTC) + timedelta(seconds=30),
)
)
await db_session.commit()
@ -43,6 +43,7 @@ async def test_arxiv_rate_limit_persists_cooldown_after_429(db_session: AsyncSes
object.__setattr__(settings, "arxiv_min_interval_seconds", 0.0)
object.__setattr__(settings, "arxiv_rate_limit_cooldown_seconds", 5.0)
try:
async def _fetch() -> httpx.Response:
return httpx.Response(429, text="rate limited")
@ -57,7 +58,7 @@ async def test_arxiv_rate_limit_persists_cooldown_after_429(db_session: AsyncSes
)
state = result.scalar_one()
assert state.cooldown_until is not None
assert state.cooldown_until > datetime.now(timezone.utc)
assert state.cooldown_until > datetime.now(UTC)
@pytest.mark.asyncio
@ -120,6 +121,7 @@ async def test_arxiv_rate_limit_logs_cooldown_activation(
object.__setattr__(settings, "arxiv_min_interval_seconds", 0.0)
object.__setattr__(settings, "arxiv_rate_limit_cooldown_seconds", 5.0)
try:
async def _fetch() -> httpx.Response:
return httpx.Response(429, text="rate limited")
@ -133,9 +135,7 @@ async def test_arxiv_rate_limit_logs_cooldown_activation(
object.__setattr__(settings, "arxiv_min_interval_seconds", previous_interval)
object.__setattr__(settings, "arxiv_rate_limit_cooldown_seconds", previous_cooldown)
cooldown_events = [
entry for entry in logged_warning if entry.get("event") == "arxiv.cooldown_activated"
]
cooldown_events = [entry for entry in logged_warning if entry.get("event") == "arxiv.cooldown_activated"]
assert cooldown_events
assert cooldown_events[0]["source_path"] == "lookup_ids"
assert float(cooldown_events[0]["cooldown_remaining_seconds"]) > 0.0
@ -143,7 +143,7 @@ async def test_arxiv_rate_limit_logs_cooldown_activation(
@pytest.mark.asyncio
async def test_get_arxiv_cooldown_status_reads_active_cooldown(db_session: AsyncSession) -> None:
now_utc = datetime(2026, 2, 26, 13, 0, tzinfo=timezone.utc)
now_utc = datetime(2026, 2, 26, 13, 0, tzinfo=UTC)
existing = await db_session.get(ArxivRuntimeState, ARXIV_RUNTIME_STATE_KEY)
if existing is None:
db_session.add(

24
tests/unit/services/domains/openalex/test_client.py
View file

@ -1,6 +1,6 @@
import pytest
from app.services.domains.openalex.types import OpenAlexWork
def test_parse_openalex_work_from_api_dict() -> None:
raw_api_response = {
"id": "https://openalex.org/W2741809807",
@ -13,28 +13,19 @@ def test_parse_openalex_work_from_api_dict() -> None:
"doi": "https://doi.org/10.1038/s41586-020-0315-z",
"mag": "2741809807",
"pmid": "https://pubmed.ncbi.nlm.nih.gov/32040050",
"pmcid": "https://www.ncbi.nlm.nih.gov/pmc/articles/PMC7325852"
},
"open_access": {
"is_oa": True,
"oa_url": "https://example.com/pdf"
"pmcid": "https://www.ncbi.nlm.nih.gov/pmc/articles/PMC7325852",
},
"open_access": {"is_oa": True, "oa_url": "https://example.com/pdf"},
"authorships": [
{
"author_position": "first",
"author": {
"id": "https://openalex.org/A1969205032",
"display_name": "Giuseppe Carleo"
}
"author": {"id": "https://openalex.org/A1969205032", "display_name": "Giuseppe Carleo"},
},
{
"author_position": "middle",
"author": {
"id": "https://openalex.org/A4356881717",
"display_name": "Ignacio Cirac"
}
}
]
"author": {"id": "https://openalex.org/A4356881717", "display_name": "Ignacio Cirac"},
},
],
}
work = OpenAlexWork.from_api_dict(raw_api_response)
@ -52,6 +43,7 @@ def test_parse_openalex_work_from_api_dict() -> None:
assert work.authors[0].display_name == "Giuseppe Carleo"
assert work.authors[1].display_name == "Ignacio Cirac"
def test_parse_openalex_work_empty() -> None:
work = OpenAlexWork.from_api_dict({"id": "W123"})
assert work.openalex_id == "W123"

46
tests/unit/services/domains/openalex/test_matching.py
View file

@ -1,54 +1,62 @@
import pytest
from app.services.domains.openalex.types import OpenAlexWork
from app.services.domains.openalex.matching import find_best_match
from app.services.domains.openalex.types import OpenAlexWork
def test_find_best_match_exact_title():
cand1 = OpenAlexWork.from_api_dict({"id": "W1", "title": "Exact Title of the Paper"})
cand2 = OpenAlexWork.from_api_dict({"id": "W2", "title": "Totally Different Paper"})
match = find_best_match("Exact Title of the Paper", 2020, "Author A", [cand1, cand2])
assert match is not None
assert match.openalex_id == "W1"
def test_find_best_match_fuzzy_title():
# Only differences are punctuation or minor phrasing (e.g., matching a preprint title vs published)
cand1 = OpenAlexWork.from_api_dict({"id": "W1", "title": "Fuzzier Title: A Study on OpenAlex"})
cand2 = OpenAlexWork.from_api_dict({"id": "W2", "title": "Some completely unrelated work"})
match = find_best_match("Fuzzier Title A Study on OpenAlex", 2021, "Author B", [cand1, cand2])
assert match is not None
assert match.openalex_id == "W1"
def test_find_best_match_rejects_low_score():
cand1 = OpenAlexWork.from_api_dict({"id": "W1", "title": "Cats in hats"})
match = find_best_match("Dogs with logs", 2020, "Author A", [cand1])
assert match is None
def test_find_best_match_year_tiebreaker():
# Both titles are very similar, one has exact year.
cand1 = OpenAlexWork.from_api_dict({"id": "W1", "title": "The exact same title", "publication_year": 2018})
cand2 = OpenAlexWork.from_api_dict({"id": "W2", "title": "The exact same title", "publication_year": 2020})
match = find_best_match("The exact same title", 2020, "Author A", [cand1, cand2])
assert match is not None
assert match.openalex_id == "W2"
def test_find_best_match_author_tiebreaker():
# Titles and years match exactly. Author overlap decides it.
cand1 = OpenAlexWork.from_api_dict({
"id": "W1",
"title": "A popular title",
"publication_year": 2020,
"authorships": [{"author": {"display_name": "Smith, J"}}]
})
cand2 = OpenAlexWork.from_api_dict({
"id": "W2",
"title": "A popular title",
"publication_year": 2020,
"authorships": [{"author": {"display_name": "Doe, J"}}]
})
cand1 = OpenAlexWork.from_api_dict(
{
"id": "W1",
"title": "A popular title",
"publication_year": 2020,
"authorships": [{"author": {"display_name": "Smith, J"}}],
}
)
cand2 = OpenAlexWork.from_api_dict(
{
"id": "W2",
"title": "A popular title",
"publication_year": 2020,
"authorships": [{"author": {"display_name": "Doe, J"}}],
}
)
# Target authors contains "Doe"
match = find_best_match("A popular title", 2020, "A Einstein, J Doe", [cand1, cand2])
assert match is not None

12
tests/unit/services/domains/publications/test_dedup.py
View file

@ -110,12 +110,14 @@ async def test_merge_duplicate_publication_migrates_links_and_identifiers() -> N
async def test_merge_duplicate_publication_rejects_missing_publications() -> None:
session = AsyncMock()
with patch(
"app.services.domains.publications.dedup._load_publication",
new=AsyncMock(side_effect=[None, None]),
with (
patch(
"app.services.domains.publications.dedup._load_publication",
new=AsyncMock(side_effect=[None, None]),
),
pytest.raises(ValueError),
):
with pytest.raises(ValueError):
await merge_duplicate_publication(session, winner_id=1, dup_id=2)
await merge_duplicate_publication(session, winner_id=1, dup_id=2)
@pytest.mark.asyncio

91
tests/unit/test_fingerprints.py
View file

@ -1,7 +1,5 @@
from __future__ import annotations
import pytest
from app.services.domains.ingestion.fingerprints import (
_dedupe_publication_candidates,
canonical_title_for_dedup,
@ -36,28 +34,40 @@ class TestFuzzyTitlesMatch:
assert fuzzy_titles_match("Deep Learning for NLP", "deep learning for nlp") is True
def test_minor_word_difference(self) -> None:
assert fuzzy_titles_match(
"A Survey on Deep Learning Methods for NLP",
"Survey on Deep Learning Methods for NLP",
) is True
assert (
fuzzy_titles_match(
"A Survey on Deep Learning Methods for NLP",
"Survey on Deep Learning Methods for NLP",
)
is True
)
def test_punctuation_difference(self) -> None:
assert fuzzy_titles_match(
"Attention Is All You Need",
"Attention Is All You Need.",
) is True
assert (
fuzzy_titles_match(
"Attention Is All You Need",
"Attention Is All You Need.",
)
is True
)
def test_colon_vs_dash_subtitle(self) -> None:
assert fuzzy_titles_match(
"Deep Learning: A Comprehensive Survey",
"Deep Learning - A Comprehensive Survey",
) is True
assert (
fuzzy_titles_match(
"Deep Learning: A Comprehensive Survey",
"Deep Learning - A Comprehensive Survey",
)
is True
)
def test_completely_different_titles(self) -> None:
assert fuzzy_titles_match(
"Deep Learning for NLP",
"Climate Change Impact on Agriculture",
) is False
assert (
fuzzy_titles_match(
"Deep Learning for NLP",
"Climate Change Impact on Agriculture",
)
is False
)
def test_short_title_no_false_positive(self) -> None:
assert fuzzy_titles_match("On Trees", "On Forests") is False
@ -67,16 +77,22 @@ class TestFuzzyTitlesMatch:
def test_custom_threshold(self) -> None:
# Lower threshold catches more distant matches
assert fuzzy_titles_match(
"A Survey on Deep Learning",
"Survey on Machine Learning Approaches",
threshold=0.3,
) is True
assert (
fuzzy_titles_match(
"A Survey on Deep Learning",
"Survey on Machine Learning Approaches",
threshold=0.3,
)
is True
)
# Default threshold rejects them
assert fuzzy_titles_match(
"A Survey on Deep Learning",
"Survey on Machine Learning Approaches",
) is False
assert (
fuzzy_titles_match(
"A Survey on Deep Learning",
"Survey on Machine Learning Approaches",
)
is False
)
class TestDedupePublicationCandidates:
@ -203,27 +219,19 @@ class TestDedupePublicationCandidates:
class TestCanonicalTitleForDedup:
def test_strips_doi_suffix(self) -> None:
title = "Adam: A Method for Stochastic Optimization. doi: 10.48550/arxiv.1412.6980"
assert canonical_title_for_dedup(title) == normalize_title(
"Adam: A Method for Stochastic Optimization"
)
assert canonical_title_for_dedup(title) == normalize_title("Adam: A Method for Stochastic Optimization")
def test_strips_arxiv_metadata_suffix(self) -> None:
title = "Adam: A Method for Stochastic Optimization. arXiv, Jan 29, 2017"
assert canonical_title_for_dedup(title) == normalize_title(
"Adam: A Method for Stochastic Optimization"
)
assert canonical_title_for_dedup(title) == normalize_title("Adam: A Method for Stochastic Optimization")
def test_strips_preprint_parenthetical(self) -> None:
title = "Adam: A method for stochastic optimization, preprint (2014)"
assert canonical_title_for_dedup(title) == normalize_title(
"Adam: A method for stochastic optimization"
)
assert canonical_title_for_dedup(title) == normalize_title("Adam: A method for stochastic optimization")
def test_strips_venue_sentence_suffix(self) -> None:
title = "Adam a method for stochastic optimization. Comput. Sci"
assert canonical_title_for_dedup(title) == normalize_title(
"Adam a method for stochastic optimization"
)
assert canonical_title_for_dedup(title) == normalize_title("Adam a method for stochastic optimization")
def test_strips_trailing_year_in_parens(self) -> None:
assert canonical_title_for_dedup("Deep Learning (2018)") == normalize_title("Deep Learning")
@ -242,10 +250,7 @@ class TestCanonicalTitleForDedup:
assert len(set(canonicals)) == 1, f"Expected one canonical, got: {canonicals}"
def test_strips_mojibake_conference_suffix(self) -> None:
noisy = (
"†œAdam: A method for stochastic optimization, "
"†3rd Int. Conf. Learn. Represent. ICLR 2015-Conf"
)
noisy = "†œAdam: A method for stochastic optimization, †3rd Int. Conf. Learn. Represent. ICLR 2015-Conf"
clean = "Adam: A method for stochastic optimization"
assert canonical_title_for_dedup(noisy) == normalize_title(clean)

1
tests/unit/test_ingestion_arxiv_rate_limit.py
View file

@ -31,6 +31,7 @@ async def test_discover_identifiers_for_enrichment_disables_arxiv_on_rate_limit(
_raise_rate_limit,
)
monkeypatch.setattr(identifier_service, "sync_identifiers_for_publication_fields", _sync_fields)
async def _publish_noop(*args, **kwargs) -> None:
_ = (args, kwargs)

2
tests/unit/test_logging.py
View file

@ -7,10 +7,10 @@ from unittest.mock import AsyncMock
from fastapi.testclient import TestClient
from app.http.middleware import REQUEST_ID_HEADER, parse_skip_paths
from app.logging_config import ConsoleLogFormatter, JsonLogFormatter, parse_redact_fields
from app.logging_utils import structured_log
from app.main import app
from app.http.middleware import REQUEST_ID_HEADER, parse_skip_paths
def test_json_log_formatter_redacts_sensitive_fields() -> None:

18
tests/unit/test_publication_pdf_queue_policy.py
View file

@ -1,6 +1,6 @@
from __future__ import annotations
from datetime import datetime, timedelta, timezone
from datetime import UTC, datetime, timedelta
from types import SimpleNamespace
import pytest
@ -25,7 +25,9 @@ def _job(
)
def _row(*, pub_url: str | None = "https://scholar.google.com/citations?view_op=view_citation&citation_for_view=abc:xyz") -> SimpleNamespace:
def _row(
*, pub_url: str | None = "https://scholar.google.com/citations?view_op=view_citation&citation_for_view=abc:xyz"
) -> SimpleNamespace:
return SimpleNamespace(
publication_id=1,
scholar_profile_id=1,
@ -39,13 +41,13 @@ def _row(*, pub_url: str | None = "https://scholar.google.com/citations?view_op=
pdf_url=None,
is_read=False,
is_favorite=False,
first_seen_at=datetime(2026, 2, 22, 12, 0, tzinfo=timezone.utc),
first_seen_at=datetime(2026, 2, 22, 12, 0, tzinfo=UTC),
is_new_in_latest_run=True,
)
def test_pdf_queue_auto_enqueue_blocks_recent_attempt(monkeypatch: pytest.MonkeyPatch) -> None:
now = datetime(2026, 2, 21, 12, 0, tzinfo=timezone.utc)
now = datetime(2026, 2, 21, 12, 0, tzinfo=UTC)
monkeypatch.setattr(pdf_queue, "_utcnow", lambda: now)
monkeypatch.setattr(pdf_queue, "_auto_retry_first_interval_seconds", lambda: 3_600)
monkeypatch.setattr(pdf_queue, "_auto_retry_interval_seconds", lambda: 86_400)
@ -59,7 +61,7 @@ def test_pdf_queue_auto_enqueue_blocks_recent_attempt(monkeypatch: pytest.Monkey
def test_pdf_queue_auto_enqueue_blocks_recent_first_retry(monkeypatch: pytest.MonkeyPatch) -> None:
now = datetime(2026, 2, 21, 12, 0, tzinfo=timezone.utc)
now = datetime(2026, 2, 21, 12, 0, tzinfo=UTC)
monkeypatch.setattr(pdf_queue, "_utcnow", lambda: now)
monkeypatch.setattr(pdf_queue, "_auto_retry_first_interval_seconds", lambda: 3_600)
monkeypatch.setattr(pdf_queue, "_auto_retry_interval_seconds", lambda: 86_400)
@ -73,7 +75,7 @@ def test_pdf_queue_auto_enqueue_blocks_recent_first_retry(monkeypatch: pytest.Mo
def test_pdf_queue_auto_enqueue_blocks_after_max_attempts(monkeypatch: pytest.MonkeyPatch) -> None:
now = datetime(2026, 2, 21, 12, 0, tzinfo=timezone.utc)
now = datetime(2026, 2, 21, 12, 0, tzinfo=UTC)
monkeypatch.setattr(pdf_queue, "_utcnow", lambda: now)
monkeypatch.setattr(pdf_queue, "_auto_retry_first_interval_seconds", lambda: 3_600)
monkeypatch.setattr(pdf_queue, "_auto_retry_interval_seconds", lambda: 86_400)
@ -87,7 +89,7 @@ def test_pdf_queue_auto_enqueue_blocks_after_max_attempts(monkeypatch: pytest.Mo
def test_pdf_queue_auto_enqueue_blocks_second_retry_within_day(monkeypatch: pytest.MonkeyPatch) -> None:
now = datetime(2026, 2, 21, 12, 0, tzinfo=timezone.utc)
now = datetime(2026, 2, 21, 12, 0, tzinfo=UTC)
monkeypatch.setattr(pdf_queue, "_utcnow", lambda: now)
monkeypatch.setattr(pdf_queue, "_auto_retry_first_interval_seconds", lambda: 3_600)
monkeypatch.setattr(pdf_queue, "_auto_retry_interval_seconds", lambda: 86_400)
@ -103,7 +105,7 @@ def test_pdf_queue_auto_enqueue_blocks_second_retry_within_day(monkeypatch: pyte
def test_pdf_queue_manual_requeue_bypasses_cooldown_and_max_attempts(
monkeypatch: pytest.MonkeyPatch,
) -> None:
now = datetime(2026, 2, 21, 12, 0, tzinfo=timezone.utc)
now = datetime(2026, 2, 21, 12, 0, tzinfo=UTC)
monkeypatch.setattr(pdf_queue, "_utcnow", lambda: now)
monkeypatch.setattr(pdf_queue, "_auto_retry_first_interval_seconds", lambda: 3_600)
monkeypatch.setattr(pdf_queue, "_auto_retry_interval_seconds", lambda: 86_400)

8
tests/unit/test_publication_pdf_resolution_pipeline.py
View file

@ -1,13 +1,13 @@
from __future__ import annotations
from datetime import datetime, timezone
from datetime import UTC, datetime
from types import SimpleNamespace
import pytest
from app.services.domains.arxiv.errors import ArxivRateLimitError
from app.services.domains.publications import pdf_resolution_pipeline as pipeline
from app.services.domains.publication_identifiers.types import DisplayIdentifier
from app.services.domains.publications import pdf_resolution_pipeline as pipeline
from app.services.domains.unpaywall.application import OaResolutionOutcome
@ -25,7 +25,7 @@ def _row(*, display_identifier: DisplayIdentifier | None = None) -> SimpleNamesp
pdf_url=None,
is_read=False,
is_favorite=False,
first_seen_at=datetime(2026, 2, 22, 12, 0, tzinfo=timezone.utc),
first_seen_at=datetime(2026, 2, 22, 12, 0, tzinfo=UTC),
is_new_in_latest_run=True,
)
@ -61,7 +61,7 @@ async def test_pipeline_prefers_openalex_before_arxiv(monkeypatch: pytest.Monkey
async def _fail_arxiv(row, *, request_email: str | None = None, allow_lookup: bool = True):
_ = (row, request_email, allow_lookup)
raise AssertionError(f"arXiv should not run when OpenAlex candidate exists.")
raise AssertionError("arXiv should not run when OpenAlex candidate exists.")
monkeypatch.setattr(pipeline, "_openalex_outcome", _fake_openalex)
monkeypatch.setattr(pipeline, "_arxiv_outcome", _fail_arxiv)

6
tests/unit/test_request_delay_policy.py
View file

@ -1,9 +1,9 @@
from __future__ import annotations
from datetime import datetime, timezone
from datetime import UTC, datetime
from app.services.domains.ingestion.application import ScholarIngestionService
from app.services.domains.ingestion import scheduler as scheduler_module
from app.services.domains.ingestion.application import ScholarIngestionService
from app.settings import settings
@ -37,7 +37,7 @@ def test_scheduler_candidate_row_clamps_request_delay() -> None:
try:
candidate = scheduler_module.SchedulerService._candidate_from_row(
(1, 15, 1, None, None),
now_utc=datetime(2026, 2, 21, tzinfo=timezone.utc),
now_utc=datetime(2026, 2, 21, tzinfo=UTC),
)
assert candidate is not None
assert candidate.request_delay_seconds == 6

12
tests/unit/test_run_safety.py
View file

@ -1,6 +1,6 @@
from __future__ import annotations
from datetime import datetime, timedelta, timezone
from datetime import UTC, datetime, timedelta
from app.db.models import UserSetting
from app.services.domains.ingestion import safety as run_safety
@ -8,7 +8,7 @@ from app.services.domains.ingestion import safety as run_safety
def test_apply_run_safety_outcome_triggers_blocked_cooldown() -> None:
settings = UserSetting(user_id=1, scrape_safety_state={})
now = datetime(2026, 2, 19, 20, 0, tzinfo=timezone.utc)
now = datetime(2026, 2, 19, 20, 0, tzinfo=UTC)
safety_state, reason = run_safety.apply_run_safety_outcome(
settings,
@ -32,7 +32,7 @@ def test_apply_run_safety_outcome_triggers_blocked_cooldown() -> None:
def test_clear_expired_cooldown() -> None:
now = datetime(2026, 2, 19, 20, 0, tzinfo=timezone.utc)
now = datetime(2026, 2, 19, 20, 0, tzinfo=UTC)
settings = UserSetting(
user_id=1,
scrape_safety_state={"blocked_start_count": 3},
@ -51,7 +51,7 @@ def test_clear_expired_cooldown() -> None:
def test_apply_run_safety_outcome_triggers_network_cooldown() -> None:
settings = UserSetting(user_id=1, scrape_safety_state={})
now = datetime(2026, 2, 19, 20, 0, tzinfo=timezone.utc)
now = datetime(2026, 2, 19, 20, 0, tzinfo=UTC)
safety_state, reason = run_safety.apply_run_safety_outcome(
settings,
@ -75,7 +75,7 @@ def test_apply_run_safety_outcome_triggers_network_cooldown() -> None:
def test_register_cooldown_blocked_start_increments_counter() -> None:
now = datetime(2026, 2, 19, 20, 0, tzinfo=timezone.utc)
now = datetime(2026, 2, 19, 20, 0, tzinfo=UTC)
settings = UserSetting(
user_id=1,
scrape_safety_state={"blocked_start_count": 1},
@ -91,7 +91,7 @@ def test_register_cooldown_blocked_start_increments_counter() -> None:
def test_get_safety_event_context_contains_structured_fields() -> None:
now = datetime(2026, 2, 19, 20, 0, tzinfo=timezone.utc)
now = datetime(2026, 2, 19, 20, 0, tzinfo=UTC)
settings = UserSetting(
user_id=1,
scrape_safety_state={"cooldown_entry_count": 4},

5
tests/unit/test_scholar_search_safety.py
View file

@ -3,9 +3,9 @@ from __future__ import annotations
import pytest
from sqlalchemy.ext.asyncio import AsyncSession
from app.services.domains.scholars import application as scholar_service
from app.services.domains.scholar.parser import ParseState
from app.services.domains.scholar.source import FetchResult
from app.services.domains.scholars import application as scholar_service
pytestmark = [pytest.mark.integration, pytest.mark.db]
@ -51,8 +51,7 @@ def _blocked_author_search_fetch() -> FetchResult:
requested_url="https://scholar.google.com/citations?hl=en&view_op=search_authors&mauthors=ada",
status_code=200,
final_url=(
"https://accounts.google.com/v3/signin/identifier"
"?continue=https%3A%2F%2Fscholar.google.com%2Fcitations"
"https://accounts.google.com/v3/signin/identifier?continue=https%3A%2F%2Fscholar.google.com%2Fcitations"
),
body="<html><body>Sign in</body></html>",
error=None,

6
tests/unit/test_unpaywall_pdf_discovery.py
View file

@ -61,7 +61,7 @@ class _FakeClient:
def __init__(self, pages: dict[str, _FakeResponse]) -> None:
self._pages = pages
async def get(self, url: str, follow_redirects: bool = True): # noqa: ARG002
async def get(self, url: str, follow_redirects: bool = True):
return self._pages[url]
@ -69,7 +69,7 @@ class _FakeClient:
async def test_resolve_pdf_from_landing_page_follows_one_hop_html_candidate(
monkeypatch: pytest.MonkeyPatch,
) -> None:
async def _skip_wait(*args, **kwargs): # noqa: ARG001, ANN002
async def _skip_wait(*args, **kwargs):
return None
monkeypatch.setattr(pdf_discovery, "wait_for_unpaywall_slot", _skip_wait)
@ -81,7 +81,7 @@ async def test_resolve_pdf_from_landing_page_follows_one_hop_html_candidate(
landing_url: _FakeResponse(
status_code=200,
content_type="text/html",
text=f"<html><body><a href=\"{hop_url}\">View article</a></body></html>",
text=f'<html><body><a href="{hop_url}">View article</a></body></html>',
),
hop_url: _FakeResponse(
status_code=200,

6
tests/unit/test_unpaywall_resolution.py
View file

@ -1,12 +1,12 @@
from __future__ import annotations
from dataclasses import replace
from datetime import datetime, timezone
from datetime import UTC, datetime
import pytest
from app.services.domains.publications.types import PublicationListItem
from app.services.domains.publication_identifiers.types import DisplayIdentifier
from app.services.domains.publications.types import PublicationListItem
from app.services.domains.unpaywall import application as unpaywall_app
@ -35,7 +35,7 @@ def _item(publication_id: int) -> PublicationListItem:
display_identifier=None,
pdf_url=None,
is_read=False,
first_seen_at=datetime.now(timezone.utc),
first_seen_at=datetime.now(UTC),
is_new_in_latest_run=True,
)