justinzeus/scholarr
1
0
Fork 0
Code Issues 1 Pull requests 18 Projects Releases Packages Wiki Activity Actions

Big changes

Browse source
This commit is contained in:
Justin Visser 2026-02-21 17:33:05 +01:00
parent 4240ad38e2
commit e3f0d63fec
99 changed files with 8804 additions and 1731 deletions
Download patch file Download diff file Expand all files Collapse all files

3
tests/conftest.py
View file

@ -21,6 +21,9 @@ RESET_SQL = text(
TRUNCATE TABLE
author_search_cache_entries,
author_search_runtime_state,
data_repair_jobs,
publication_pdf_job_events,
publication_pdf_jobs,
ingestion_queue_items,
scholar_publications,
crawl_runs,

592
tests/integration/test_api_v1.py
View file

@ -10,6 +10,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
from app.api.runtime_deps import get_scholar_source
from app.main import app
from app.services.domains.publications.types import PublicationListItem
from app.services.domains.settings import application as user_settings_service
from app.services.domains.scholar.source import FetchResult
from app.settings import settings
@ -65,6 +66,49 @@ def _assert_safety_state_contract(payload: dict[str, object]) -> None:
assert set(counters.keys()) == SAFETY_COUNTER_KEYS
async def _seed_publication_link_for_user(
db_session: AsyncSession,
*,
user_id: int,
scholar_id: str,
title: str,
fingerprint: str,
) -> tuple[int, int]:
scholar_result = await db_session.execute(
text(
"""
INSERT INTO scholar_profiles (user_id, scholar_id, display_name, is_enabled)
VALUES (:user_id, :scholar_id, :display_name, true)
RETURNING id
"""
),
{"user_id": user_id, "scholar_id": scholar_id, "display_name": scholar_id},
)
scholar_profile_id = int(scholar_result.scalar_one())
publication_result = await db_session.execute(
text(
"""
INSERT INTO publications (fingerprint_sha256, title_raw, title_normalized, citation_count)
VALUES (:fingerprint, :title_raw, :title_normalized, 4)
RETURNING id
"""
),
{"fingerprint": fingerprint, "title_raw": title, "title_normalized": title.lower()},
)
publication_id = int(publication_result.scalar_one())
await db_session.execute(
text(
"""
INSERT INTO scholar_publications (scholar_profile_id, publication_id, is_read)
VALUES (:scholar_profile_id, :publication_id, false)
"""
),
{"scholar_profile_id": scholar_profile_id, "publication_id": publication_id},
)
await db_session.commit()
return scholar_profile_id, publication_id
@pytest.mark.integration
@pytest.mark.db
@pytest.mark.asyncio
@ -321,6 +365,265 @@ async def test_api_admin_user_management_endpoints(db_session: AsyncSession) ->
assert created_exists.scalar_one() == 1
@pytest.mark.integration
@pytest.mark.db
@pytest.mark.asyncio
async def test_api_admin_dbops_integrity_and_repair_flow(db_session: AsyncSession) -> None:
await insert_user(db_session, email="api-admin-dbops@example.com", password="admin-password", is_admin=True)
target_user_id = await insert_user(db_session, email="api-dbops-target@example.com", password="target-password")
_scholar_id, publication_id = await _seed_publication_link_for_user(
db_session,
user_id=target_user_id,
scholar_id="dbopsTarget01",
title="DB Ops Target Paper",
fingerprint=f"{(target_user_id + 31):064x}",
)
client = TestClient(app)
login_user(client, email="api-admin-dbops@example.com", password="admin-password")
headers = _api_csrf_headers(client)
integrity_response = client.get("/api/v1/admin/db/integrity")
assert integrity_response.status_code == 200
integrity_payload = integrity_response.json()["data"]
assert integrity_payload["status"] in {"ok", "warning", "failed"}
assert any(
check["name"] == "missing_pdf_url"
for check in integrity_payload["checks"]
)
repair_response = client.post(
"/api/v1/admin/db/repairs/publication-links",
json={"user_id": target_user_id, "dry_run": True},
headers=headers,
)
assert repair_response.status_code == 200
repair_data = repair_response.json()["data"]
assert repair_data["status"] == "completed"
assert bool(repair_data["summary"]["dry_run"]) is True
job_id = int(repair_data["job_id"])
jobs_response = client.get("/api/v1/admin/db/repair-jobs?limit=10")
assert jobs_response.status_code == 200
jobs = jobs_response.json()["data"]["jobs"]
assert any(int(job["id"]) == job_id for job in jobs)
pdf_queue_response = client.get("/api/v1/admin/db/pdf-queue?limit=20")
assert pdf_queue_response.status_code == 200
pdf_queue_payload = pdf_queue_response.json()["data"]
assert isinstance(pdf_queue_payload["items"], list)
assert int(pdf_queue_payload["page"]) == 1
assert int(pdf_queue_payload["page_size"]) == 20
assert int(pdf_queue_payload["total_count"]) >= 0
assert isinstance(pdf_queue_payload["has_next"], bool)
assert isinstance(pdf_queue_payload["has_prev"], bool)
page_two_response = client.get("/api/v1/admin/db/pdf-queue?page=2&page_size=1")
assert page_two_response.status_code == 200
page_two_payload = page_two_response.json()["data"]
assert int(page_two_payload["page"]) == 2
assert int(page_two_payload["page_size"]) == 1
assert page_two_payload["has_prev"] is True
untracked_response = client.get("/api/v1/admin/db/pdf-queue?limit=20&status=untracked")
assert untracked_response.status_code == 200
assert any(
item["status"] == "untracked"
for item in untracked_response.json()["data"]["items"]
)
link_count_result = await db_session.execute(
text("SELECT count(*) FROM scholar_publications WHERE publication_id = :publication_id"),
{"publication_id": publication_id},
)
assert int(link_count_result.scalar_one()) == 1
@pytest.mark.integration
@pytest.mark.db
@pytest.mark.asyncio
async def test_api_admin_dbops_pdf_queue_requeue_endpoint(
db_session: AsyncSession,
monkeypatch: pytest.MonkeyPatch,
) -> None:
await insert_user(db_session, email="api-admin-pdf@example.com", password="admin-password", is_admin=True)
target_user_id = await insert_user(db_session, email="api-pdf-target@example.com", password="target-password")
_scholar_id, publication_id = await _seed_publication_link_for_user(
db_session,
user_id=target_user_id,
scholar_id="dbopsPdf01",
title="PDF Queue Target",
fingerprint=f"{(target_user_id + 73):064x}",
)
monkeypatch.setattr(
"app.services.domains.publications.pdf_queue._schedule_rows",
lambda **_kwargs: None,
)
client = TestClient(app)
login_user(client, email="api-admin-pdf@example.com", password="admin-password")
headers = _api_csrf_headers(client)
first_response = client.post(
f"/api/v1/admin/db/pdf-queue/{publication_id}/requeue",
headers=headers,
)
assert first_response.status_code == 200
first_data = first_response.json()["data"]
assert first_data["publication_id"] == publication_id
assert first_data["queued"] is True
assert first_data["status"] == "queued"
second_response = client.post(
f"/api/v1/admin/db/pdf-queue/{publication_id}/requeue",
headers=headers,
)
assert second_response.status_code == 200
second_data = second_response.json()["data"]
assert second_data["queued"] is False
assert second_data["status"] == "blocked"
@pytest.mark.integration
@pytest.mark.db
@pytest.mark.asyncio
async def test_api_admin_dbops_pdf_queue_requeue_all_endpoint(
db_session: AsyncSession,
monkeypatch: pytest.MonkeyPatch,
) -> None:
await insert_user(db_session, email="api-admin-pdf-all@example.com", password="admin-password", is_admin=True)
target_user_id = await insert_user(db_session, email="api-pdf-all-target@example.com", password="target-password")
_scholar_id, _publication_id = await _seed_publication_link_for_user(
db_session,
user_id=target_user_id,
scholar_id="dbopsPdfAll01",
title="PDF Queue All Target",
fingerprint=f"{(target_user_id + 83):064x}",
)
monkeypatch.setattr(
"app.services.domains.publications.pdf_queue._schedule_rows",
lambda **_kwargs: None,
)
client = TestClient(app)
login_user(client, email="api-admin-pdf-all@example.com", password="admin-password")
headers = _api_csrf_headers(client)
response = client.post(
"/api/v1/admin/db/pdf-queue/requeue-all?limit=500",
headers=headers,
)
assert response.status_code == 200
payload = response.json()["data"]
assert int(payload["requested_count"]) >= 1
assert int(payload["queued_count"]) >= 1
@pytest.mark.integration
@pytest.mark.db
@pytest.mark.asyncio
async def test_api_admin_dbops_forbidden_for_non_admin_and_validates_scope(db_session: AsyncSession) -> None:
await insert_user(db_session, email="api-admin-dbops2@example.com", password="admin-password", is_admin=True)
member_user_id = await insert_user(db_session, email="api-dbops-member@example.com", password="member-password")
client = TestClient(app)
login_user(client, email="api-dbops-member@example.com", password="member-password")
headers = _api_csrf_headers(client)
forbidden_integrity = client.get("/api/v1/admin/db/integrity")
assert forbidden_integrity.status_code == 403
assert forbidden_integrity.json()["error"]["code"] == "forbidden"
forbidden_pdf_queue = client.get("/api/v1/admin/db/pdf-queue")
assert forbidden_pdf_queue.status_code == 403
assert forbidden_pdf_queue.json()["error"]["code"] == "forbidden"
forbidden_requeue = client.post(
"/api/v1/admin/db/pdf-queue/1/requeue",
headers=headers,
)
assert forbidden_requeue.status_code == 403
assert forbidden_requeue.json()["error"]["code"] == "forbidden"
forbidden_requeue_all = client.post(
"/api/v1/admin/db/pdf-queue/requeue-all?limit=100",
headers=headers,
)
assert forbidden_requeue_all.status_code == 403
assert forbidden_requeue_all.json()["error"]["code"] == "forbidden"
forbidden_repair = client.post(
"/api/v1/admin/db/repairs/publication-links",
json={"user_id": member_user_id, "dry_run": True},
headers=headers,
)
assert forbidden_repair.status_code == 403
assert forbidden_repair.json()["error"]["code"] == "forbidden"
admin_headers = _api_bootstrap_csrf_headers(client)
admin_login = client.post(
"/api/v1/auth/login",
json={"email": "api-admin-dbops2@example.com", "password": "admin-password"},
headers=admin_headers,
)
assert admin_login.status_code == 200
post_login_headers = {"X-CSRF-Token": admin_login.json()["data"]["csrf_token"]}
invalid_scope = client.post(
"/api/v1/admin/db/repairs/publication-links",
json={"user_id": member_user_id, "dry_run": True},
headers=post_login_headers,
)
assert invalid_scope.status_code == 400
assert invalid_scope.json()["error"]["code"] == "invalid_repair_scope"
@pytest.mark.integration
@pytest.mark.db
@pytest.mark.asyncio
async def test_api_admin_dbops_all_users_apply_requires_confirmation(db_session: AsyncSession) -> None:
await insert_user(db_session, email="api-admin-dbops3@example.com", password="admin-password", is_admin=True)
first_user_id = await insert_user(db_session, email="api-dbops-a@example.com", password="user-password")
second_user_id = await insert_user(db_session, email="api-dbops-b@example.com", password="user-password")
await _seed_publication_link_for_user(
db_session,
user_id=first_user_id,
scholar_id="dbopsAll01",
title="DB Ops All User Paper One",
fingerprint=f"{(first_user_id + 61):064x}",
)
await _seed_publication_link_for_user(
db_session,
user_id=second_user_id,
scholar_id="dbopsAll02",
title="DB Ops All User Paper Two",
fingerprint=f"{(second_user_id + 71):064x}",
)
client = TestClient(app)
login_user(client, email="api-admin-dbops3@example.com", password="admin-password")
headers = _api_csrf_headers(client)
missing_confirmation = client.post(
"/api/v1/admin/db/repairs/publication-links",
json={"scope_mode": "all_users", "dry_run": False},
headers=headers,
)
assert missing_confirmation.status_code == 422
assert "confirmation_text" in str(missing_confirmation.json())
apply_response = client.post(
"/api/v1/admin/db/repairs/publication-links",
json={
"scope_mode": "all_users",
"dry_run": False,
"confirmation_text": "REPAIR ALL USERS",
},
headers=headers,
)
assert apply_response.status_code == 200
apply_data = apply_response.json()["data"]
assert apply_data["status"] == "completed"
assert apply_data["scope"]["scope_mode"] == "all_users"
assert int(apply_data["summary"]["links_deleted"]) >= 2
remaining_links = await db_session.execute(text("SELECT count(*) FROM scholar_publications"))
assert int(remaining_links.scalar_one()) == 0
@pytest.mark.integration
@pytest.mark.db
@pytest.mark.asyncio
@ -1400,12 +1703,19 @@ async def test_api_publications_list_and_mark_read(db_session: AsyncSession) ->
assert list_response.status_code == 200
data = list_response.json()["data"]
assert data["mode"] == "all"
assert data["favorite_only"] is False
assert data["total_count"] == 2
assert data["unread_count"] == 2
assert data["favorites_count"] == 0
assert data["latest_count"] == 0
assert data["new_count"] == data["latest_count"]
assert data["page"] == 1
assert data["page_size"] == 100
assert data["has_prev"] is False
assert data["has_next"] is False
assert isinstance(data["publications"], list)
assert len(data["publications"]) == 2
assert all(bool(item["is_favorite"]) is False for item in data["publications"])
pdf_urls = {item["title"]: item["pdf_url"] for item in data["publications"]}
assert pdf_urls["Paper A"] == "https://example.org/paper-a.pdf"
assert pdf_urls["Paper B"] is None
@ -1465,6 +1775,220 @@ async def test_api_publications_list_and_mark_read(db_session: AsyncSession) ->
assert mark_response.json()["data"]["updated_count"] == 1
@pytest.mark.integration
@pytest.mark.db
@pytest.mark.asyncio
async def test_api_publications_list_supports_pagination(db_session: AsyncSession) -> None:
user_id = await insert_user(
db_session,
email="api-pubs-paging@example.com",
password="api-password",
)
scholar_result = await db_session.execute(
text(
"""
INSERT INTO scholar_profiles (user_id, scholar_id, display_name, is_enabled)
VALUES (:user_id, :scholar_id, :display_name, true)
RETURNING id
"""
),
{
"user_id": user_id,
"scholar_id": "pagingScholar01",
"display_name": "Paging Scholar",
},
)
scholar_profile_id = int(scholar_result.scalar_one())
publication_ids: list[int] = []
for index in range(3):
created = await db_session.execute(
text(
"""
INSERT INTO publications (fingerprint_sha256, title_raw, title_normalized, citation_count)
VALUES (:fingerprint, :title_raw, :title_normalized, 1)
RETURNING id
"""
),
{
"fingerprint": f"{(user_id + 500 + index):064x}",
"title_raw": f"Paged Paper {index}",
"title_normalized": f"paged paper {index}",
},
)
publication_ids.append(int(created.scalar_one()))
await db_session.execute(
text(
"""
INSERT INTO scholar_publications (scholar_profile_id, publication_id, is_read, is_favorite)
VALUES
(:scholar_profile_id, :publication_1, false, false),
(:scholar_profile_id, :publication_2, false, false),
(:scholar_profile_id, :publication_3, false, false)
"""
),
{
"scholar_profile_id": scholar_profile_id,
"publication_1": publication_ids[0],
"publication_2": publication_ids[1],
"publication_3": publication_ids[2],
},
)
await db_session.commit()
client = TestClient(app)
login_user(client, email="api-pubs-paging@example.com", password="api-password")
first_page = client.get("/api/v1/publications?mode=all&page=1&page_size=2")
assert first_page.status_code == 200
first_data = first_page.json()["data"]
assert first_data["total_count"] == 3
assert first_data["page"] == 1
assert first_data["page_size"] == 2
assert first_data["has_prev"] is False
assert first_data["has_next"] is True
assert len(first_data["publications"]) == 2
second_page = client.get("/api/v1/publications?mode=all&page=2&page_size=2")
assert second_page.status_code == 200
second_data = second_page.json()["data"]
assert second_data["total_count"] == 3
assert second_data["page"] == 2
assert second_data["page_size"] == 2
assert second_data["has_prev"] is True
assert second_data["has_next"] is False
assert len(second_data["publications"]) == 1
@pytest.mark.integration
@pytest.mark.db
@pytest.mark.asyncio
async def test_api_publications_favorite_toggle_and_filter(db_session: AsyncSession) -> None:
user_id = await insert_user(
db_session,
email="api-pubs-favorites@example.com",
password="api-password",
)
scholar_result = await db_session.execute(
text(
"""
INSERT INTO scholar_profiles (user_id, scholar_id, display_name, is_enabled)
VALUES (:user_id, :scholar_id, :display_name, true)
RETURNING id
"""
),
{
"user_id": user_id,
"scholar_id": "favoriteScholar01",
"display_name": "Favorite Scholar",
},
)
scholar_profile_id = int(scholar_result.scalar_one())
publication_a_result = await db_session.execute(
text(
"""
INSERT INTO publications (fingerprint_sha256, title_raw, title_normalized, citation_count)
VALUES (:fingerprint, :title_raw, :title_normalized, 9)
RETURNING id
"""
),
{
"fingerprint": f"{(user_id + 101):064x}",
"title_raw": "Favorite Target",
"title_normalized": "favorite target",
},
)
publication_a_id = int(publication_a_result.scalar_one())
publication_b_result = await db_session.execute(
text(
"""
INSERT INTO publications (fingerprint_sha256, title_raw, title_normalized, citation_count)
VALUES (:fingerprint, :title_raw, :title_normalized, 2)
RETURNING id
"""
),
{
"fingerprint": f"{(user_id + 102):064x}",
"title_raw": "Not Favorite",
"title_normalized": "not favorite",
},
)
publication_b_id = int(publication_b_result.scalar_one())
await db_session.execute(
text(
"""
INSERT INTO scholar_publications (scholar_profile_id, publication_id, is_read, is_favorite)
VALUES
(:scholar_profile_id, :publication_a_id, false, false),
(:scholar_profile_id, :publication_b_id, false, false)
"""
),
{
"scholar_profile_id": scholar_profile_id,
"publication_a_id": publication_a_id,
"publication_b_id": publication_b_id,
},
)
await db_session.commit()
client = TestClient(app)
login_user(client, email="api-pubs-favorites@example.com", password="api-password")
headers = _api_csrf_headers(client)
set_response = client.post(
f"/api/v1/publications/{publication_a_id}/favorite",
json={"scholar_profile_id": scholar_profile_id, "is_favorite": True},
headers=headers,
)
assert set_response.status_code == 200
set_data = set_response.json()["data"]
assert set_data["publication"]["publication_id"] == publication_a_id
assert set_data["publication"]["is_favorite"] is True
favorite_only_response = client.get("/api/v1/publications?mode=all&favorite_only=true")
assert favorite_only_response.status_code == 200
favorite_only_data = favorite_only_response.json()["data"]
assert favorite_only_data["favorite_only"] is True
assert favorite_only_data["favorites_count"] == 1
assert favorite_only_data["total_count"] == 1
assert len(favorite_only_data["publications"]) == 1
assert int(favorite_only_data["publications"][0]["publication_id"]) == publication_a_id
clear_response = client.post(
f"/api/v1/publications/{publication_a_id}/favorite",
json={"scholar_profile_id": scholar_profile_id, "is_favorite": False},
headers=headers,
)
assert clear_response.status_code == 200
clear_data = clear_response.json()["data"]
assert clear_data["publication"]["publication_id"] == publication_a_id
assert clear_data["publication"]["is_favorite"] is False
favorite_only_after_clear_response = client.get("/api/v1/publications?mode=all&favorite_only=true")
assert favorite_only_after_clear_response.status_code == 200
favorite_only_after_clear_data = favorite_only_after_clear_response.json()["data"]
assert favorite_only_after_clear_data["favorites_count"] == 0
assert favorite_only_after_clear_data["total_count"] == 0
assert favorite_only_after_clear_data["publications"] == []
favorite_state_result = await db_session.execute(
text(
"""
SELECT is_favorite
FROM scholar_publications
WHERE scholar_profile_id = :scholar_profile_id
AND publication_id = :publication_id
"""
),
{
"scholar_profile_id": scholar_profile_id,
"publication_id": publication_a_id,
},
)
assert bool(favorite_state_result.scalar_one()) is False
@pytest.mark.integration
@pytest.mark.db
@pytest.mark.asyncio
@ -1521,20 +2045,14 @@ async def test_api_publications_list_schedules_background_enrichment(
)
await db_session.commit()
async def _fail_resolver(*args, **kwargs):
raise AssertionError("list endpoint should not resolve OA metadata inline")
async def _fake_schedule(*, user_id: int, request_email: str | None, items, max_items: int):
async def _fake_schedule(db_session, *, user_id: int, request_email: str | None, items, max_items: int):
assert db_session is not None
assert user_id > 0
assert request_email == "api-pubs-bg@example.com"
assert int(max_items) > 0
assert any(int(item.publication_id) == publication_id for item in items)
return 1
monkeypatch.setattr(
"app.services.domains.publications.listing.resolve_publication_oa_metadata",
_fail_resolver,
)
monkeypatch.setattr(
"app.services.domains.publications.application.schedule_missing_pdf_enrichment_for_user",
_fake_schedule,
@ -1549,12 +2067,13 @@ async def test_api_publications_list_schedules_background_enrichment(
assert len(data["publications"]) == 1
assert int(data["publications"][0]["publication_id"]) == publication_id
assert data["publications"][0]["pdf_url"] is None
assert data["publications"][0]["pdf_status"] in {"untracked", "queued", "running", "failed"}
@pytest.mark.integration
@pytest.mark.db
@pytest.mark.asyncio
async def test_api_publication_retry_pdf_updates_publication(
async def test_api_publication_retry_pdf_queues_resolution_job(
db_session: AsyncSession,
monkeypatch: pytest.MonkeyPatch,
) -> None:
@ -1607,13 +2126,47 @@ async def test_api_publication_retry_pdf_updates_publication(
)
await db_session.commit()
async def _fake_resolver(items, *, request_email=None):
async def _fake_retry_scheduler(db_session, *, user_id: int, request_email: str | None, item: PublicationListItem):
assert db_session is not None
assert user_id > 0
assert request_email == "api-pubs-retry@example.com"
return {int(items[0].publication_id): ("10.1000/retry-target", "https://oa.example/retry.pdf")}
assert int(item.publication_id) == publication_id
return True
monkeypatch.setattr(
"app.services.domains.publications.listing.resolve_publication_oa_metadata",
_fake_resolver,
"app.services.domains.publications.application.schedule_retry_pdf_enrichment_for_row",
_fake_retry_scheduler,
)
async def _fake_hydrate(db_session, *, items: list[PublicationListItem]):
assert db_session is not None
assert len(items) == 1
item = items[0]
return [
PublicationListItem(
publication_id=item.publication_id,
scholar_profile_id=item.scholar_profile_id,
scholar_label=item.scholar_label,
title=item.title,
year=item.year,
citation_count=item.citation_count,
venue_text=item.venue_text,
pub_url=item.pub_url,
doi=item.doi,
pdf_url=item.pdf_url,
is_read=item.is_read,
first_seen_at=item.first_seen_at,
is_new_in_latest_run=item.is_new_in_latest_run,
pdf_status="queued",
pdf_attempt_count=1,
pdf_failure_reason="no_pdf_found",
pdf_failure_detail="no_pdf_found",
)
]
monkeypatch.setattr(
"app.services.domains.publications.application.hydrate_pdf_enrichment_state",
_fake_hydrate,
)
client = TestClient(app)
@ -1627,18 +2180,21 @@ async def test_api_publication_retry_pdf_updates_publication(
)
assert response.status_code == 200
payload = response.json()["data"]
assert payload["resolved_pdf"] is True
assert payload["queued"] is True
assert payload["resolved_pdf"] is False
assert payload["publication"]["publication_id"] == publication_id
assert payload["publication"]["pdf_url"] == "https://oa.example/retry.pdf"
assert payload["publication"]["doi"] == "10.1000/retry-target"
assert payload["publication"]["pdf_url"] is None
assert payload["publication"]["pdf_status"] == "queued"
assert payload["publication"]["pdf_attempt_count"] == 1
assert payload["publication"]["pdf_failure_reason"] == "no_pdf_found"
stored = await db_session.execute(
text("SELECT doi, pdf_url FROM publications WHERE id = :publication_id"),
{"publication_id": publication_id},
)
stored_doi, stored_pdf_url = stored.one()
assert stored_doi == "10.1000/retry-target"
assert stored_pdf_url == "https://oa.example/retry.pdf"
assert stored_doi is None
assert stored_pdf_url is None
@pytest.mark.integration

252
tests/integration/test_data_repair_jobs.py Normal file
View file

@ -0,0 +1,252 @@
from __future__ import annotations
import pytest
from sqlalchemy import text
from sqlalchemy.ext.asyncio import AsyncSession
from app.services.domains.dbops import run_publication_link_repair
from tests.integration.helpers import insert_user
async def _insert_scholar_profile(
db_session: AsyncSession,
*,
user_id: int,
scholar_id: str,
display_name: str,
baseline_completed: bool = False,
) -> int:
result = await db_session.execute(
text(
"""
INSERT INTO scholar_profiles (user_id, scholar_id, display_name, is_enabled, baseline_completed)
VALUES (:user_id, :scholar_id, :display_name, true, :baseline_completed)
RETURNING id
"""
),
{
"user_id": user_id,
"scholar_id": scholar_id,
"display_name": display_name,
"baseline_completed": baseline_completed,
},
)
return int(result.scalar_one())
async def _insert_publication(
db_session: AsyncSession,
*,
fingerprint: str,
title_raw: str,
title_normalized: str,
citation_count: int,
) -> int:
result = await db_session.execute(
text(
"""
INSERT INTO publications (fingerprint_sha256, title_raw, title_normalized, citation_count)
VALUES (:fingerprint, :title_raw, :title_normalized, :citation_count)
RETURNING id
"""
),
{
"fingerprint": fingerprint,
"title_raw": title_raw,
"title_normalized": title_normalized,
"citation_count": citation_count,
},
)
return int(result.scalar_one())
async def _insert_scholar_publication_link(
db_session: AsyncSession,
*,
scholar_profile_id: int,
publication_id: int,
) -> None:
await db_session.execute(
text(
"""
INSERT INTO scholar_publications (scholar_profile_id, publication_id, is_read)
VALUES (:scholar_profile_id, :publication_id, false)
"""
),
{
"scholar_profile_id": scholar_profile_id,
"publication_id": publication_id,
},
)
async def _insert_queue_item(
db_session: AsyncSession,
*,
user_id: int,
scholar_profile_id: int,
reason: str,
) -> None:
await db_session.execute(
text(
"""
INSERT INTO ingestion_queue_items (user_id, scholar_profile_id, reason)
VALUES (:user_id, :scholar_profile_id, :reason)
"""
),
{
"user_id": user_id,
"scholar_profile_id": scholar_profile_id,
"reason": reason,
},
)
async def _count_rows(
db_session: AsyncSession,
*,
sql: str,
params: dict[str, int],
) -> int:
result = await db_session.execute(text(sql), params)
return int(result.scalar_one())
async def _seed_apply_case(db_session: AsyncSession, *, user_id: int) -> tuple[int, int]:
scholar_profile_id = await _insert_scholar_profile(
db_session,
user_id=user_id,
scholar_id="repairApply01",
display_name="Repair Apply",
baseline_completed=True,
)
publication_id = await _insert_publication(
db_session,
fingerprint=f"{(user_id + 1):064x}",
title_raw="Repair Apply Paper",
title_normalized="repair apply paper",
citation_count=9,
)
await _insert_scholar_publication_link(
db_session,
scholar_profile_id=scholar_profile_id,
publication_id=publication_id,
)
await _insert_queue_item(
db_session,
user_id=user_id,
scholar_profile_id=scholar_profile_id,
reason="manual_repair_test",
)
await db_session.commit()
return scholar_profile_id, publication_id
async def _assert_apply_effects(
db_session: AsyncSession,
*,
scholar_profile_id: int,
publication_id: int,
) -> None:
links_count = await _count_rows(
db_session,
sql="SELECT count(*) FROM scholar_publications WHERE scholar_profile_id = :scholar_profile_id",
params={"scholar_profile_id": scholar_profile_id},
)
queue_count = await _count_rows(
db_session,
sql="SELECT count(*) FROM ingestion_queue_items WHERE scholar_profile_id = :scholar_profile_id",
params={"scholar_profile_id": scholar_profile_id},
)
baseline_completed = await _count_rows(
db_session,
sql=(
"SELECT count(*) FROM scholar_profiles "
"WHERE id = :scholar_profile_id AND baseline_completed = false"
),
params={"scholar_profile_id": scholar_profile_id},
)
publication_count = await _count_rows(
db_session,
sql="SELECT count(*) FROM publications WHERE id = :publication_id",
params={"publication_id": publication_id},
)
assert links_count == 0
assert queue_count == 0
assert baseline_completed == 1
assert publication_count == 0
@pytest.mark.integration
@pytest.mark.db
@pytest.mark.asyncio
async def test_publication_link_repair_dry_run_records_job_without_mutation(
db_session: AsyncSession,
) -> None:
user_id = await insert_user(db_session, email="repair-dry@example.com", password="api-password")
scholar_profile_id = await _insert_scholar_profile(
db_session,
user_id=user_id,
scholar_id="repairDry001",
display_name="Repair Dry",
)
publication_id = await _insert_publication(
db_session,
fingerprint=f"{user_id:064x}",
title_raw="Repair Dry Paper",
title_normalized="repair dry paper",
citation_count=1,
)
await _insert_scholar_publication_link(
db_session,
scholar_profile_id=scholar_profile_id,
publication_id=publication_id,
)
await db_session.commit()
result = await run_publication_link_repair(
db_session,
user_id=user_id,
dry_run=True,
requested_by="test-suite",
)
assert result["status"] == "completed"
assert bool(result["summary"]["dry_run"]) is True
assert int(result["summary"]["links_deleted"]) == 0
links_count = await _count_rows(
db_session,
sql="SELECT count(*) FROM scholar_publications WHERE scholar_profile_id = :scholar_profile_id",
params={"scholar_profile_id": scholar_profile_id},
)
assert links_count == 1
@pytest.mark.integration
@pytest.mark.db
@pytest.mark.asyncio
async def test_publication_link_repair_apply_clears_links_and_can_gc_orphans(
db_session: AsyncSession,
) -> None:
user_id = await insert_user(db_session, email="repair-apply@example.com", password="api-password")
scholar_profile_id, publication_id = await _seed_apply_case(db_session, user_id=user_id)
result = await run_publication_link_repair(
db_session,
user_id=user_id,
dry_run=False,
gc_orphan_publications=True,
requested_by="test-suite",
)
assert result["status"] == "completed"
assert bool(result["summary"]["dry_run"]) is False
assert int(result["summary"]["links_deleted"]) == 1
assert int(result["summary"]["queue_items_deleted"]) == 1
assert int(result["summary"]["scholars_reset"]) == 1
await _assert_apply_effects(
db_session,
scholar_profile_id=scholar_profile_id,
publication_id=publication_id,
)

70
tests/integration/test_db_integrity.py Normal file
View file

@ -0,0 +1,70 @@
from __future__ import annotations
import pytest
from sqlalchemy import text
from sqlalchemy.ext.asyncio import AsyncSession
from app.services.domains.dbops.integrity import collect_integrity_report
def _check_counts(report: dict) -> dict[str, int]:
return {row["name"]: int(row["count"]) for row in report["checks"]}
@pytest.mark.integration
@pytest.mark.db
@pytest.mark.asyncio
async def test_collect_integrity_report_ok_on_clean_database(db_session: AsyncSession) -> None:
report = await collect_integrity_report(db_session)
assert report["status"] == "ok"
assert report["failures"] == []
assert report["warnings"] == []
check_counts = _check_counts(report)
assert all(count == 0 for count in check_counts.values())
@pytest.mark.integration
@pytest.mark.db
@pytest.mark.asyncio
async def test_collect_integrity_report_detects_warning_and_failure(db_session: AsyncSession) -> None:
await db_session.execute(
text(
"""
INSERT INTO publications (
fingerprint_sha256,
cluster_id,
title_raw,
title_normalized,
citation_count
)
VALUES (
:fingerprint_sha256,
:cluster_id,
:title_raw,
:title_normalized,
:citation_count
)
"""
),
{
"fingerprint_sha256": f"{1:064x}",
"cluster_id": "legacy-cluster-123",
"title_raw": "Integrity test paper",
"title_normalized": "integrity test paper",
"citation_count": -5,
},
)
await db_session.commit()
report = await collect_integrity_report(db_session)
check_counts = _check_counts(report)
assert report["status"] == "failed"
assert "negative_citation_count" in report["failures"]
assert "legacy_cluster_id_format" in report["warnings"]
assert "orphan_publications" in report["warnings"]
assert check_counts["negative_citation_count"] == 1
assert check_counts["legacy_cluster_id_format"] == 1
assert check_counts["orphan_publications"] == 1

46
tests/integration/test_migrations.py
View file

@ -13,10 +13,13 @@ EXPECTED_TABLES = {
"ingestion_queue_items",
"author_search_runtime_state",
"author_search_cache_entries",
"data_repair_jobs",
"publication_pdf_jobs",
"publication_pdf_job_events",
}
EXPECTED_ENUMS = {"run_status", "run_trigger_type"}
EXPECTED_REVISION = "20260220_0011"
EXPECTED_REVISION = "20260221_0015"
@pytest.mark.integration
@ -220,3 +223,44 @@ async def test_user_settings_has_scrape_safety_columns(db_session: AsyncSession)
)
columns = {row[0] for row in result}
assert columns == {"scrape_safety_state", "scrape_cooldown_until", "scrape_cooldown_reason"}
@pytest.mark.integration
@pytest.mark.db
@pytest.mark.migrations
@pytest.mark.asyncio
async def test_scholar_publications_has_is_favorite_column(db_session: AsyncSession) -> None:
result = await db_session.execute(
text(
"""
SELECT 1
FROM information_schema.columns
WHERE table_name = 'scholar_publications'
AND column_name = 'is_favorite'
"""
)
)
assert result.scalar_one() == 1
@pytest.mark.integration
@pytest.mark.db
@pytest.mark.migrations
@pytest.mark.asyncio
async def test_user_settings_request_delay_constraint_enforces_two_second_floor(
db_session: AsyncSession,
) -> None:
result = await db_session.execute(
text(
"""
SELECT pg_get_constraintdef(c.oid)
FROM pg_constraint c
JOIN pg_class t ON t.oid = c.conrelid
WHERE t.relname = 'user_settings'
AND c.contype = 'c'
AND pg_get_constraintdef(c.oid) ILIKE '%request_delay_seconds >= 2%'
"""
)
)
definition = result.scalar_one()
assert "request_delay_seconds >= 2" in definition

109
tests/unit/test_publication_pdf_queue_policy.py Normal file
View file

@ -0,0 +1,109 @@
from __future__ import annotations
from datetime import datetime, timedelta, timezone
import pytest
from app.db.models import PublicationPdfJob
from app.services.domains.publications import pdf_queue
def _job(
*,
status: str,
attempt_count: int,
last_attempt_at: datetime | None,
) -> PublicationPdfJob:
return PublicationPdfJob(
publication_id=1,
status=status,
attempt_count=attempt_count,
last_attempt_at=last_attempt_at,
)
def test_pdf_queue_auto_enqueue_blocks_recent_attempt(monkeypatch: pytest.MonkeyPatch) -> None:
now = datetime(2026, 2, 21, 12, 0, tzinfo=timezone.utc)
monkeypatch.setattr(pdf_queue, "_utcnow", lambda: now)
monkeypatch.setattr(pdf_queue, "_auto_retry_first_interval_seconds", lambda: 3_600)
monkeypatch.setattr(pdf_queue, "_auto_retry_interval_seconds", lambda: 86_400)
monkeypatch.setattr(pdf_queue, "_auto_retry_max_attempts", lambda: 3)
job = _job(
status=pdf_queue.PDF_STATUS_FAILED,
attempt_count=1,
last_attempt_at=now - timedelta(hours=2),
)
assert pdf_queue._can_enqueue_job(job, force_retry=False) is True
def test_pdf_queue_auto_enqueue_blocks_recent_first_retry(monkeypatch: pytest.MonkeyPatch) -> None:
now = datetime(2026, 2, 21, 12, 0, tzinfo=timezone.utc)
monkeypatch.setattr(pdf_queue, "_utcnow", lambda: now)
monkeypatch.setattr(pdf_queue, "_auto_retry_first_interval_seconds", lambda: 3_600)
monkeypatch.setattr(pdf_queue, "_auto_retry_interval_seconds", lambda: 86_400)
monkeypatch.setattr(pdf_queue, "_auto_retry_max_attempts", lambda: 3)
job = _job(
status=pdf_queue.PDF_STATUS_FAILED,
attempt_count=1,
last_attempt_at=now - timedelta(minutes=20),
)
assert pdf_queue._can_enqueue_job(job, force_retry=False) is False
def test_pdf_queue_auto_enqueue_blocks_after_max_attempts(monkeypatch: pytest.MonkeyPatch) -> None:
now = datetime(2026, 2, 21, 12, 0, tzinfo=timezone.utc)
monkeypatch.setattr(pdf_queue, "_utcnow", lambda: now)
monkeypatch.setattr(pdf_queue, "_auto_retry_first_interval_seconds", lambda: 3_600)
monkeypatch.setattr(pdf_queue, "_auto_retry_interval_seconds", lambda: 86_400)
monkeypatch.setattr(pdf_queue, "_auto_retry_max_attempts", lambda: 3)
job = _job(
status=pdf_queue.PDF_STATUS_FAILED,
attempt_count=3,
last_attempt_at=now - timedelta(days=2),
)
assert pdf_queue._can_enqueue_job(job, force_retry=False) is False
def test_pdf_queue_auto_enqueue_blocks_second_retry_within_day(monkeypatch: pytest.MonkeyPatch) -> None:
now = datetime(2026, 2, 21, 12, 0, tzinfo=timezone.utc)
monkeypatch.setattr(pdf_queue, "_utcnow", lambda: now)
monkeypatch.setattr(pdf_queue, "_auto_retry_first_interval_seconds", lambda: 3_600)
monkeypatch.setattr(pdf_queue, "_auto_retry_interval_seconds", lambda: 86_400)
monkeypatch.setattr(pdf_queue, "_auto_retry_max_attempts", lambda: 3)
job = _job(
status=pdf_queue.PDF_STATUS_FAILED,
attempt_count=2,
last_attempt_at=now - timedelta(hours=2),
)
assert pdf_queue._can_enqueue_job(job, force_retry=False) is False
def test_pdf_queue_manual_requeue_bypasses_cooldown_and_max_attempts(
monkeypatch: pytest.MonkeyPatch,
) -> None:
now = datetime(2026, 2, 21, 12, 0, tzinfo=timezone.utc)
monkeypatch.setattr(pdf_queue, "_utcnow", lambda: now)
monkeypatch.setattr(pdf_queue, "_auto_retry_first_interval_seconds", lambda: 3_600)
monkeypatch.setattr(pdf_queue, "_auto_retry_interval_seconds", lambda: 86_400)
monkeypatch.setattr(pdf_queue, "_auto_retry_max_attempts", lambda: 3)
job = _job(
status=pdf_queue.PDF_STATUS_FAILED,
attempt_count=5,
last_attempt_at=now - timedelta(minutes=10),
)
assert pdf_queue._can_enqueue_job(job, force_retry=True) is True
def test_pdf_queue_manual_requeue_still_blocks_when_inflight() -> None:
running = _job(
status=pdf_queue.PDF_STATUS_RUNNING,
attempt_count=1,
last_attempt_at=None,
)
queued = _job(
status=pdf_queue.PDF_STATUS_QUEUED,
attempt_count=1,
last_attempt_at=None,
)
assert pdf_queue._can_enqueue_job(running, force_retry=True) is False
assert pdf_queue._can_enqueue_job(queued, force_retry=True) is False

45
tests/unit/test_request_delay_policy.py Normal file
View file

@ -0,0 +1,45 @@
from __future__ import annotations
from datetime import datetime, timezone
from app.services.domains.ingestion.application import ScholarIngestionService
from app.services.domains.ingestion import scheduler as scheduler_module
from app.settings import settings
def _set_policy_minimum(value: int) -> int:
previous = int(settings.ingestion_min_request_delay_seconds)
object.__setattr__(settings, "ingestion_min_request_delay_seconds", int(value))
return previous
def test_ingestion_effective_request_delay_respects_policy_minimum() -> None:
previous = _set_policy_minimum(8)
try:
assert ScholarIngestionService._effective_request_delay_seconds(7) == 8
assert ScholarIngestionService._effective_request_delay_seconds(12) == 12
finally:
object.__setattr__(settings, "ingestion_min_request_delay_seconds", previous)
def test_scheduler_effective_request_delay_respects_policy_minimum() -> None:
previous = _set_policy_minimum(9)
try:
assert scheduler_module._effective_request_delay_seconds(None) == 9
assert scheduler_module._effective_request_delay_seconds(6) == 9
assert scheduler_module._effective_request_delay_seconds(11) == 11
finally:
object.__setattr__(settings, "ingestion_min_request_delay_seconds", previous)
def test_scheduler_candidate_row_clamps_request_delay() -> None:
previous = _set_policy_minimum(6)
try:
candidate = scheduler_module.SchedulerService._candidate_from_row(
(1, 15, 1, None, None),
now_utc=datetime(2026, 2, 21, tzinfo=timezone.utc),
)
assert candidate is not None
assert candidate.request_delay_seconds == 6
finally:
object.__setattr__(settings, "ingestion_min_request_delay_seconds", previous)

94
tests/unit/test_unpaywall_pdf_discovery.py Normal file
View file

@ -0,0 +1,94 @@
from __future__ import annotations
import pytest
from app.services.domains.unpaywall import pdf_discovery
def test_looks_like_pdf_url_detects_path_and_query_variants() -> None:
assert pdf_discovery.looks_like_pdf_url("https://example.org/file.pdf")
assert pdf_discovery.looks_like_pdf_url("https://example.org/download?target=paper.pdf")
assert not pdf_discovery.looks_like_pdf_url("https://example.org/landing")
assert not pdf_discovery.looks_like_pdf_url(None)
def test_normalized_candidate_urls_extracts_and_resolves_relative_links() -> None:
html = """
<html>
<head>
<base href="https://publisher.example.org/articles/42/" />
<meta name="citation_pdf_url" content="/pdfs/42-main.pdf" />
<link rel="alternate" type="application/pdf" href="https://cdn.example.org/42-supp.pdf" />
</head>
<body>
<a href="appendix.pdf">Appendix</a>
</body>
</html>
"""
candidates = pdf_discovery._normalized_candidate_urls(
page_url="https://publisher.example.org/articles/42",
html=html,
)
assert len(candidates) == 3
assert "https://cdn.example.org/42-supp.pdf" in candidates
assert "https://publisher.example.org/pdfs/42-main.pdf" in candidates
assert "https://publisher.example.org/articles/42/appendix.pdf" in candidates
def test_normalized_candidate_urls_extracts_plain_text_urls() -> None:
html = """
<html>
<body>
Footnote URL: http://www.mext.go.jp/component/english/file.pdf
</body>
</html>
"""
candidates = pdf_discovery._normalized_candidate_urls(
page_url="https://www.science.org/doi/10.1126/science.1239057",
html=html,
)
assert "http://www.mext.go.jp/component/english/file.pdf" in candidates
class _FakeResponse:
def __init__(self, *, status_code: int, content_type: str, text: str) -> None:
self.status_code = status_code
self.headers = {"content-type": content_type}
self.text = text
class _FakeClient:
def __init__(self, pages: dict[str, _FakeResponse]) -> None:
self._pages = pages
async def get(self, url: str, follow_redirects: bool = True): # noqa: ARG002
return self._pages[url]
@pytest.mark.asyncio
async def test_resolve_pdf_from_landing_page_follows_one_hop_html_candidate(
monkeypatch: pytest.MonkeyPatch,
) -> None:
async def _skip_wait(*args, **kwargs): # noqa: ARG001, ANN002
return None
monkeypatch.setattr(pdf_discovery, "wait_for_unpaywall_slot", _skip_wait)
landing_url = "https://example.org/article"
hop_url = "https://example.org/doi/full/abc"
pdf_url = "https://downloads.example.org/archive/paper.pdf"
client = _FakeClient(
{
landing_url: _FakeResponse(
status_code=200,
content_type="text/html",
text=f"<html><body><a href=\"{hop_url}\">View article</a></body></html>",
),
hop_url: _FakeResponse(
status_code=200,
content_type="text/html",
text=f"<html><body>Download here: {pdf_url}</body></html>",
),
}
)
resolved = await pdf_discovery.resolve_pdf_from_landing_page(client, page_url=landing_url)
assert resolved == pdf_url

89
tests/unit/test_unpaywall_resolution.py Normal file
View file

@ -0,0 +1,89 @@
from __future__ import annotations
from datetime import datetime, timezone
import pytest
from app.services.domains.publications.types import PublicationListItem
from app.services.domains.unpaywall import application as unpaywall_app
class _DummyAsyncClient:
def __init__(self, *args, **kwargs) -> None:
self.args = args
self.kwargs = kwargs
async def __aenter__(self):
return self
async def __aexit__(self, exc_type, exc, tb):
return None
def _item(publication_id: int) -> PublicationListItem:
return PublicationListItem(
publication_id=publication_id,
scholar_profile_id=1,
scholar_label="Shinya Yamanaka",
title="Induction of pluripotent stem cells",
year=2007,
citation_count=1000,
venue_text="Cell",
pub_url="https://doi.org/10.1016/j.cell.2007.11.019",
doi=None,
pdf_url=None,
is_read=False,
first_seen_at=datetime.now(timezone.utc),
is_new_in_latest_run=True,
)
@pytest.mark.asyncio
async def test_unpaywall_resolve_prefers_direct_pdf_without_landing_crawl(monkeypatch: pytest.MonkeyPatch) -> None:
payload = {
"doi": "10.1016/j.cell.2007.11.019",
"best_oa_location": {
"url_for_pdf": "https://oa.example.org/article.pdf",
"url": "https://oa.example.org/landing",
},
}
async def _fake_resolve_item_payload(**_kwargs):
return payload, False
async def _fail_crawl(_client, *, page_url: str):
raise AssertionError(f"unexpected landing crawl: {page_url}")
monkeypatch.setattr(unpaywall_app, "_resolve_item_payload", _fake_resolve_item_payload)
monkeypatch.setattr(unpaywall_app, "resolve_pdf_from_landing_page", _fail_crawl)
monkeypatch.setattr("httpx.AsyncClient", _DummyAsyncClient)
resolved = await unpaywall_app.resolve_publication_oa_metadata([_item(1)], request_email="user@example.com")
assert resolved == {1: ("10.1016/j.cell.2007.11.019", "https://oa.example.org/article.pdf")}
@pytest.mark.asyncio
async def test_unpaywall_resolve_crawls_landing_page_when_pdf_url_is_not_direct(
monkeypatch: pytest.MonkeyPatch,
) -> None:
payload = {
"doi": "10.1016/j.cell.2007.11.019",
"best_oa_location": {
"url_for_pdf": "https://oa.example.org/view?paper=42",
"url": "https://oa.example.org/landing/42",
},
}
crawled_pages: list[str] = []
async def _fake_resolve_item_payload(**_kwargs):
return payload, False
async def _fake_crawl(_client, *, page_url: str):
crawled_pages.append(page_url)
return "https://oa.example.org/files/paper-42.pdf"
monkeypatch.setattr(unpaywall_app, "_resolve_item_payload", _fake_resolve_item_payload)
monkeypatch.setattr(unpaywall_app, "resolve_pdf_from_landing_page", _fake_crawl)
monkeypatch.setattr("httpx.AsyncClient", _DummyAsyncClient)
resolved = await unpaywall_app.resolve_publication_oa_metadata([_item(2)], request_email="user@example.com")
assert resolved == {2: ("10.1016/j.cell.2007.11.019", "https://oa.example.org/files/paper-42.pdf")}
assert "https://oa.example.org/landing/42" in crawled_pages