justinzeus/scholarr
1
0
Fork 0
Code Issues 1 Pull requests 18 Projects Releases Packages Wiki Activity Actions

temp commit

Browse source
This commit is contained in:
Justin Visser 2026-02-26 12:54:19 +01:00
parent 8760f27b51
commit 0e9e49df16
193 changed files with 23228 additions and 935 deletions
Download patch file Download diff file Expand all files Collapse all files

238
tests/integration/test_api_v1.py
View file

@ -1,5 +1,6 @@
from __future__ import annotations
import asyncio
from datetime import datetime, timedelta, timezone
from pathlib import Path
@ -66,6 +67,31 @@ def _assert_safety_state_contract(payload: dict[str, object]) -> None:
assert set(counters.keys()) == SAFETY_COUNTER_KEYS
_ACTIVE_RUN_STATUSES = {"running", "resolving"}
async def _wait_for_run_complete(
client: TestClient,
run_id: int,
*,
max_retries: int = 300,
poll_interval: float = 0.2,
) -> dict:
"""Poll GET /api/v1/runs/{run_id} until the run is in a terminal state.
Returns the final run detail data dict.
"""
for _ in range(max_retries):
await asyncio.sleep(poll_interval)
r = client.get(f"/api/v1/runs/{run_id}")
assert r.status_code == 200
data = r.json()["data"]
if data["run"]["status"] not in _ACTIVE_RUN_STATUSES:
return data
r = client.get(f"/api/v1/runs/{run_id}")
return r.json()["data"]
async def _seed_publication_link_for_user(
db_session: AsyncSession,
*,
@ -1041,44 +1067,42 @@ async def test_api_manual_run_skips_unchanged_initial_page_for_scholar(
app.dependency_overrides[get_scholar_source] = lambda: StubScholarSource()
try:
client = TestClient(app)
login_user(client, email="api-skip-unchanged@example.com", password="api-password")
headers = _api_csrf_headers(client)
with TestClient(app) as client:
login_user(client, email="api-skip-unchanged@example.com", password="api-password")
headers = _api_csrf_headers(client)
create_response = client.post(
"/api/v1/scholars",
json={"scholar_id": "abcDEF123456"},
headers=headers,
)
assert create_response.status_code == 201
create_response = client.post(
"/api/v1/scholars",
json={"scholar_id": "abcDEF123456"},
headers=headers,
)
assert create_response.status_code == 201
first_run_response = client.post(
"/api/v1/runs/manual",
headers={**headers, "Idempotency-Key": "skip-unchanged-run-001"},
)
assert first_run_response.status_code == 200
first_run_id = int(first_run_response.json()["data"]["run_id"])
first_run_response = client.post(
"/api/v1/runs/manual",
headers={**headers, "Idempotency-Key": "skip-unchanged-run-001"},
)
assert first_run_response.status_code == 200
first_run_id = int(first_run_response.json()["data"]["run_id"])
first_run_detail = client.get(f"/api/v1/runs/{first_run_id}")
assert first_run_detail.status_code == 200
first_results = first_run_detail.json()["data"]["scholar_results"]
assert len(first_results) == 1
assert first_results[0]["state_reason"] != "no_change_initial_page_signature"
first_detail_data = await _wait_for_run_complete(client, first_run_id)
first_results = first_detail_data["scholar_results"]
assert len(first_results) == 1
assert first_results[0]["state_reason"] != "no_change_initial_page_signature"
second_run_response = client.post(
"/api/v1/runs/manual",
headers={**headers, "Idempotency-Key": "skip-unchanged-run-002"},
)
assert second_run_response.status_code == 200
second_run_id = int(second_run_response.json()["data"]["run_id"])
second_run_response = client.post(
"/api/v1/runs/manual",
headers={**headers, "Idempotency-Key": "skip-unchanged-run-002"},
)
assert second_run_response.status_code == 200
second_run_id = int(second_run_response.json()["data"]["run_id"])
second_run_detail = client.get(f"/api/v1/runs/{second_run_id}")
assert second_run_detail.status_code == 200
second_results = second_run_detail.json()["data"]["scholar_results"]
assert len(second_results) == 1
assert second_results[0]["state_reason"] == "no_change_initial_page_signature"
assert second_results[0]["publication_count"] == 0
assert second_results[0]["outcome"] == "success"
second_detail_data = await _wait_for_run_complete(client, second_run_id)
second_results = second_detail_data["scholar_results"]
assert len(second_results) == 1
assert second_results[0]["state_reason"] == "no_change_initial_page_signature"
assert second_results[0]["publication_count"] == 0
assert second_results[0]["outcome"] == "success"
finally:
app.dependency_overrides.pop(get_scholar_source, None)
@ -1227,7 +1251,7 @@ async def test_api_runs_manual_and_queue_actions(db_session: AsyncSession) -> No
assert run_response.status_code == 200
run_payload = run_response.json()["data"]
assert "run_id" in run_payload
assert run_payload["status"] in {"success", "partial_failure", "failed"}
assert run_payload["status"] in {"running", "resolving", "success", "partial_failure", "failed"}
assert run_payload["reused_existing_run"] is False
assert run_payload["idempotency_key"] == "manual-run-0001"
_assert_safety_state_contract(run_payload["safety_state"])
@ -1243,11 +1267,15 @@ async def test_api_runs_manual_and_queue_actions(db_session: AsyncSession) -> No
"/api/v1/runs/manual",
headers={**headers, "Idempotency-Key": "manual-run-0001"},
)
assert replay_response.status_code == 200
replay_payload = replay_response.json()["data"]
assert replay_payload["run_id"] == run_payload["run_id"]
assert replay_payload["reused_existing_run"] is True
_assert_safety_state_contract(replay_payload["safety_state"])
assert replay_response.status_code in {200, 409}
if replay_response.status_code == 200:
replay_payload = replay_response.json()["data"]
assert replay_payload["run_id"] == run_payload["run_id"]
assert replay_payload["reused_existing_run"] is True
_assert_safety_state_contract(replay_payload["safety_state"])
else:
replay_error = replay_response.json()["error"]
assert replay_error["code"] == "run_in_progress"
runs_response = client.get("/api/v1/runs")
assert runs_response.status_code == 200
@ -1429,43 +1457,45 @@ async def test_api_manual_run_enforces_scrape_safety_cooldown(db_session: AsyncS
object.__setattr__(settings, "ingestion_safety_cooldown_blocked_seconds", 600)
try:
client = TestClient(app)
login_user(client, email="api-runs-safety@example.com", password="api-password")
headers = _api_csrf_headers(client)
with TestClient(app) as client:
login_user(client, email="api-runs-safety@example.com", password="api-password")
headers = _api_csrf_headers(client)
first_run_response = client.post(
"/api/v1/runs/manual",
headers={**headers, "Idempotency-Key": "safety-cooldown-run-1"},
)
assert first_run_response.status_code == 200
first_run_response = client.post(
"/api/v1/runs/manual",
headers={**headers, "Idempotency-Key": "safety-cooldown-run-1"},
)
assert first_run_response.status_code == 200
first_run_id = int(first_run_response.json()["data"]["run_id"])
await _wait_for_run_complete(client, first_run_id)
settings_response = client.get("/api/v1/settings")
assert settings_response.status_code == 200
safety_state = settings_response.json()["data"]["safety_state"]
_assert_safety_state_contract(safety_state)
assert safety_state["cooldown_active"] is True
assert safety_state["cooldown_reason"] == "blocked_failure_threshold_exceeded"
assert int(safety_state["cooldown_remaining_seconds"]) > 0
assert int(safety_state["counters"]["cooldown_entry_count"]) >= 1
assert int(safety_state["counters"]["last_blocked_failure_count"]) >= 1
settings_response = client.get("/api/v1/settings")
assert settings_response.status_code == 200
safety_state = settings_response.json()["data"]["safety_state"]
_assert_safety_state_contract(safety_state)
assert safety_state["cooldown_active"] is True
assert safety_state["cooldown_reason"] == "blocked_failure_threshold_exceeded"
assert int(safety_state["cooldown_remaining_seconds"]) > 0
assert int(safety_state["counters"]["cooldown_entry_count"]) >= 1
assert int(safety_state["counters"]["last_blocked_failure_count"]) >= 1
blocked_start_response = client.post(
"/api/v1/runs/manual",
headers={**headers, "Idempotency-Key": "safety-cooldown-run-2"},
)
assert blocked_start_response.status_code == 429
blocked_payload = blocked_start_response.json()
assert blocked_payload["error"]["code"] == "scrape_cooldown_active"
blocked_state = blocked_payload["error"]["details"]["safety_state"]
_assert_safety_state_contract(blocked_state)
assert blocked_state["cooldown_active"] is True
assert blocked_state["cooldown_reason"] == "blocked_failure_threshold_exceeded"
assert int(blocked_state["counters"]["blocked_start_count"]) >= 1
blocked_start_response = client.post(
"/api/v1/runs/manual",
headers={**headers, "Idempotency-Key": "safety-cooldown-run-2"},
)
assert blocked_start_response.status_code == 429
blocked_payload = blocked_start_response.json()
assert blocked_payload["error"]["code"] == "scrape_cooldown_active"
blocked_state = blocked_payload["error"]["details"]["safety_state"]
_assert_safety_state_contract(blocked_state)
assert blocked_state["cooldown_active"] is True
assert blocked_state["cooldown_reason"] == "blocked_failure_threshold_exceeded"
assert int(blocked_state["counters"]["blocked_start_count"]) >= 1
runs_response = client.get("/api/v1/runs")
assert runs_response.status_code == 200
_assert_safety_state_contract(runs_response.json()["data"]["safety_state"])
assert runs_response.json()["data"]["safety_state"]["cooldown_active"] is True
runs_response = client.get("/api/v1/runs")
assert runs_response.status_code == 200
_assert_safety_state_contract(runs_response.json()["data"]["safety_state"])
assert runs_response.json()["data"]["safety_state"]["cooldown_active"] is True
finally:
object.__setattr__(settings, "ingestion_alert_blocked_failure_threshold", previous_blocked_threshold)
object.__setattr__(settings, "ingestion_safety_cooldown_blocked_seconds", previous_blocked_cooldown_seconds)
@ -1535,37 +1565,39 @@ async def test_api_manual_run_enforces_network_failure_safety_cooldown(
object.__setattr__(settings, "ingestion_retry_backoff_seconds", 0.0)
try:
client = TestClient(app)
login_user(client, email="api-runs-safety-network@example.com", password="api-password")
headers = _api_csrf_headers(client)
with TestClient(app) as client:
login_user(client, email="api-runs-safety-network@example.com", password="api-password")
headers = _api_csrf_headers(client)
first_run_response = client.post(
"/api/v1/runs/manual",
headers={**headers, "Idempotency-Key": "safety-network-cooldown-run-1"},
)
assert first_run_response.status_code == 200
first_run_response = client.post(
"/api/v1/runs/manual",
headers={**headers, "Idempotency-Key": "safety-network-cooldown-run-1"},
)
assert first_run_response.status_code == 200
first_run_id = int(first_run_response.json()["data"]["run_id"])
await _wait_for_run_complete(client, first_run_id)
settings_response = client.get("/api/v1/settings")
assert settings_response.status_code == 200
safety_state = settings_response.json()["data"]["safety_state"]
_assert_safety_state_contract(safety_state)
assert safety_state["cooldown_active"] is True
assert safety_state["cooldown_reason"] == "network_failure_threshold_exceeded"
assert int(safety_state["cooldown_remaining_seconds"]) > 0
assert int(safety_state["counters"]["last_network_failure_count"]) >= 1
settings_response = client.get("/api/v1/settings")
assert settings_response.status_code == 200
safety_state = settings_response.json()["data"]["safety_state"]
_assert_safety_state_contract(safety_state)
assert safety_state["cooldown_active"] is True
assert safety_state["cooldown_reason"] == "network_failure_threshold_exceeded"
assert int(safety_state["cooldown_remaining_seconds"]) > 0
assert int(safety_state["counters"]["last_network_failure_count"]) >= 1
blocked_start_response = client.post(
"/api/v1/runs/manual",
headers={**headers, "Idempotency-Key": "safety-network-cooldown-run-2"},
)
assert blocked_start_response.status_code == 429
blocked_payload = blocked_start_response.json()
assert blocked_payload["error"]["code"] == "scrape_cooldown_active"
blocked_state = blocked_payload["error"]["details"]["safety_state"]
_assert_safety_state_contract(blocked_state)
assert blocked_state["cooldown_active"] is True
assert blocked_state["cooldown_reason"] == "network_failure_threshold_exceeded"
assert int(blocked_state["counters"]["blocked_start_count"]) >= 1
blocked_start_response = client.post(
"/api/v1/runs/manual",
headers={**headers, "Idempotency-Key": "safety-network-cooldown-run-2"},
)
assert blocked_start_response.status_code == 429
blocked_payload = blocked_start_response.json()
assert blocked_payload["error"]["code"] == "scrape_cooldown_active"
blocked_state = blocked_payload["error"]["details"]["safety_state"]
_assert_safety_state_contract(blocked_state)
assert blocked_state["cooldown_active"] is True
assert blocked_state["cooldown_reason"] == "network_failure_threshold_exceeded"
assert int(blocked_state["counters"]["blocked_start_count"]) >= 1
finally:
object.__setattr__(settings, "ingestion_alert_network_failure_threshold", previous_network_threshold)
object.__setattr__(settings, "ingestion_safety_cooldown_network_seconds", previous_network_cooldown_seconds)
@ -2152,7 +2184,6 @@ async def test_api_publication_retry_pdf_queues_resolution_job(
citation_count=item.citation_count,
venue_text=item.venue_text,
pub_url=item.pub_url,
doi=item.doi,
pdf_url=item.pdf_url,
is_read=item.is_read,
first_seen_at=item.first_seen_at,
@ -2189,11 +2220,10 @@ async def test_api_publication_retry_pdf_queues_resolution_job(
assert payload["publication"]["pdf_failure_reason"] == "no_pdf_found"
stored = await db_session.execute(
text("SELECT doi, pdf_url FROM publications WHERE id = :publication_id"),
text("SELECT pdf_url FROM publications WHERE id = :publication_id"),
{"publication_id": publication_id},
)
stored_doi, stored_pdf_url = stored.one()
assert stored_doi is None
stored_pdf_url = stored.scalar_one()
assert stored_pdf_url is None

103
tests/integration/test_deferred_enrichment.py Normal file
View file

@ -0,0 +1,103 @@
from __future__ import annotations
import pytest
from unittest.mock import patch, AsyncMock, MagicMock
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from datetime import datetime, timezone
from app.db.models import CrawlRun, Publication, ScholarProfile, ScholarPublication, RunStatus, RunTriggerType
from app.services.domains.ingestion.application import ScholarIngestionService
from app.services.domains.openalex.types import OpenAlexWork
from tests.integration.helpers import insert_user
@pytest.mark.integration
@pytest.mark.asyncio
async def test_deferred_enrichment_sweeps_previous_runs(db_session: AsyncSession) -> None:
# 1. Setup: Create user and scholar
user_id = await insert_user(db_session, email="test@example.com", password="password123")
scholar = ScholarProfile(
user_id=user_id,
scholar_id="SaiiI5MAAAAJ",
display_name="Test Scholar",
is_enabled=True,
)
db_session.add(scholar)
await db_session.flush()
# 2. Simulate a previous FAILED run that left an un-enriched publication
failed_run = CrawlRun(
user_id=user_id,
status=RunStatus.FAILED,
trigger_type=RunTriggerType.MANUAL,
start_dt=datetime.now(timezone.utc),
)
db_session.add(failed_run)
await db_session.flush()
pub = Publication(
title_raw="A fast quantum mechanical algorithm for database search",
title_normalized="a fast quantum mechanical algorithm for database search",
fingerprint_sha256="dummy_fingerprint_for_test",
author_text="LK Grover",
openalex_enriched=False,
)
db_session.add(pub)
await db_session.flush()
link = ScholarPublication(
scholar_profile_id=scholar.id,
publication_id=pub.id,
first_seen_run_id=failed_run.id,
)
db_session.add(link)
await db_session.commit()
# 3. Create a NEW run
new_run = CrawlRun(
user_id=user_id,
status=RunStatus.RUNNING,
trigger_type=RunTriggerType.MANUAL,
start_dt=datetime.now(timezone.utc),
)
db_session.add(new_run)
await db_session.commit()
# 4. Mock OpenAlex client to return enrichment data
mock_work = OpenAlexWork(
openalex_id="W1234567",
doi="10.1145/237814.237866",
pmid=None,
pmcid=None,
title="A fast quantum mechanical algorithm for database search",
publication_year=1996,
cited_by_count=1000,
is_oa=True,
oa_url="http://example.com/grover.pdf",
)
mock_source = MagicMock()
service = ScholarIngestionService(source=mock_source)
# We patch the client at its source, and also mock arXiv to avoid real HTTP calls
with patch("app.services.domains.openalex.client.OpenAlexClient") as MockClient, \
patch("app.services.domains.arxiv.application.discover_arxiv_id_for_publication", new=AsyncMock(return_value=None)):
mock_instance = MockClient.return_value
mock_instance.get_works_by_filter = AsyncMock(return_value=[mock_work])
# 5. Execute the enrichment pass for the NEW run
await service._enrich_pending_publications(db_session, run_id=new_run.id)
await db_session.commit()
# 6. Verification: The publication from the FAILED run should now be enriched
await db_session.refresh(pub)
assert pub.openalex_enriched is True
assert pub.citation_count == 1000
assert pub.pdf_url == "http://example.com/grover.pdf"
# Double check it was indeed processed
stmt = select(Publication).where(Publication.id == pub.id)
result = await db_session.execute(stmt)
enriched_pub = result.scalar_one()
assert enriched_pub.openalex_enriched is True

4
tests/integration/test_fixture_probe_runs.py
View file

@ -144,7 +144,9 @@ async def test_fixture_probe_run_emits_failure_and_retry_summary_metrics(
request_delay_seconds=0,
network_error_retries=1,
retry_backoff_seconds=0.0,
max_pages_per_scholar=1,
rate_limit_retries=0,
rate_limit_backoff_seconds=0.0,
max_pages_per_scholar=10,
page_size=100,
auto_queue_continuations=False,
alert_blocked_failure_threshold=1,

24
tests/integration/test_migrations.py
View file

@ -19,7 +19,7 @@ EXPECTED_TABLES = {
}
EXPECTED_ENUMS = {"run_status", "run_trigger_type"}
EXPECTED_REVISION = "20260221_0015"
EXPECTED_REVISION = "20260225_0022"
@pytest.mark.integration
@ -137,6 +137,28 @@ async def test_crawl_runs_has_manual_idempotency_unique_index(db_session: AsyncS
assert "WHERE" in indexdef
@pytest.mark.integration
@pytest.mark.db
@pytest.mark.migrations
@pytest.mark.asyncio
async def test_crawl_runs_has_single_active_run_unique_index(db_session: AsyncSession) -> None:
result = await db_session.execute(
text(
"""
SELECT indexdef
FROM pg_indexes
WHERE schemaname = 'public'
AND tablename = 'crawl_runs'
AND indexname = 'uq_crawl_runs_user_active'
"""
)
)
indexdef = result.scalar_one()
assert "UNIQUE INDEX" in indexdef
assert "(user_id)" in indexdef
assert "WHERE" in indexdef
@pytest.mark.integration
@pytest.mark.db
@pytest.mark.migrations

514
tests/integration/test_run_lifecycle_consistency.py Normal file
View file

@ -0,0 +1,514 @@
from __future__ import annotations
import asyncio
from typing import Any
import pytest
from fastapi.testclient import TestClient
from sqlalchemy import text
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from app.api.runtime_deps import get_ingestion_service
from app.db.models import CrawlRun, Publication, RunStatus, RunTriggerType
from app.main import app
from app.services.domains.ingestion.application import ScholarIngestionService
from tests.integration.helpers import insert_user, login_user
def _csrf_headers(client: TestClient) -> dict[str, str]:
response = client.get("/api/v1/auth/me")
assert response.status_code == 200
csrf_token = response.json()["data"]["csrf_token"]
assert isinstance(csrf_token, str) and csrf_token
return {"X-CSRF-Token": csrf_token}
@pytest.mark.integration
@pytest.mark.db
@pytest.mark.asyncio
async def test_crawl_runs_enforce_single_active_run_per_user(db_session: AsyncSession) -> None:
user_id = await insert_user(
db_session,
email="api-single-active@example.com",
password="api-password",
)
await db_session.execute(
text(
"""
INSERT INTO crawl_runs (user_id, trigger_type, status, scholar_count, new_pub_count)
VALUES (:user_id, 'manual', 'running', 1, 0)
"""
),
{"user_id": user_id},
)
await db_session.commit()
with pytest.raises(IntegrityError):
await db_session.execute(
text(
"""
INSERT INTO crawl_runs (user_id, trigger_type, status, scholar_count, new_pub_count)
VALUES (:user_id, 'scheduled', 'resolving', 1, 0)
"""
),
{"user_id": user_id},
)
await db_session.commit()
await db_session.rollback()
@pytest.mark.integration
@pytest.mark.db
@pytest.mark.asyncio
async def test_api_manual_run_conflicts_when_an_active_run_exists(
db_session: AsyncSession,
monkeypatch: pytest.MonkeyPatch,
) -> None:
user_id = await insert_user(
db_session,
email="api-run-conflict@example.com",
password="api-password",
)
await db_session.execute(
text(
"""
INSERT INTO scholar_profiles (user_id, scholar_id, display_name, is_enabled)
VALUES (:user_id, :scholar_id, :display_name, true)
"""
),
{"user_id": user_id, "scholar_id": "runConflict001", "display_name": "Run Conflict"},
)
await db_session.commit()
service = ScholarIngestionService(source=object())
async def _stall_execute_run(**_kwargs: Any) -> None:
await asyncio.sleep(0.4)
monkeypatch.setattr(service, "execute_run", _stall_execute_run)
app.dependency_overrides[get_ingestion_service] = lambda: service
try:
client = TestClient(app)
login_user(client, email="api-run-conflict@example.com", password="api-password")
headers = _csrf_headers(client)
first_response = client.post("/api/v1/runs/manual", headers=headers)
assert first_response.status_code == 200
assert first_response.json()["data"]["status"] == "running"
second_response = client.post("/api/v1/runs/manual", headers=headers)
assert second_response.status_code == 409
payload = second_response.json()
assert payload["error"]["code"] == "run_in_progress"
await asyncio.sleep(0.45)
finally:
app.dependency_overrides.pop(get_ingestion_service, None)
@pytest.mark.integration
@pytest.mark.db
@pytest.mark.asyncio
async def test_api_cancel_allows_resolving_and_rejects_terminal_run(db_session: AsyncSession) -> None:
user_id = await insert_user(
db_session,
email="api-cancel-resolving@example.com",
password="api-password",
)
run_result = await db_session.execute(
text(
"""
INSERT INTO crawl_runs (user_id, trigger_type, status, scholar_count, new_pub_count)
VALUES (:user_id, 'manual', 'resolving', 1, 2)
RETURNING id
"""
),
{"user_id": user_id},
)
run_id = int(run_result.scalar_one())
await db_session.commit()
client = TestClient(app)
login_user(client, email="api-cancel-resolving@example.com", password="api-password")
headers = _csrf_headers(client)
cancel_response = client.post(f"/api/v1/runs/{run_id}/cancel", headers=headers)
assert cancel_response.status_code == 200
assert cancel_response.json()["data"]["run"]["status"] == "canceled"
cancel_again_response = client.post(f"/api/v1/runs/{run_id}/cancel", headers=headers)
assert cancel_again_response.status_code == 409
assert cancel_again_response.json()["error"]["code"] == "run_not_cancelable"
@pytest.mark.integration
@pytest.mark.db
@pytest.mark.asyncio
async def test_background_enrichment_preserves_canceled_status(
db_session: AsyncSession,
monkeypatch: pytest.MonkeyPatch,
) -> None:
user_id = await insert_user(
db_session,
email="api-cancel-enrichment@example.com",
password="api-password",
)
scholar_result = await db_session.execute(
text(
"""
INSERT INTO scholar_profiles (user_id, scholar_id, display_name, is_enabled)
VALUES (:user_id, 'cancelResolve001', 'Cancel Resolve', true)
RETURNING id
"""
),
{"user_id": user_id},
)
scholar_profile_id = int(scholar_result.scalar_one())
run = CrawlRun(
user_id=user_id,
trigger_type=RunTriggerType.MANUAL,
status=RunStatus.RESOLVING,
scholar_count=1,
)
publication = Publication(
fingerprint_sha256=f"{(user_id + 9000):064x}",
title_raw="Cancel During Resolving",
title_normalized="cancel during resolving",
citation_count=0,
openalex_enriched=False,
)
db_session.add_all([run, publication])
await db_session.flush()
await db_session.execute(
text(
"""
INSERT INTO scholar_publications (
scholar_profile_id,
publication_id,
first_seen_run_id,
is_read
)
VALUES (:scholar_profile_id, :publication_id, :run_id, false)
"""
),
{
"scholar_profile_id": scholar_profile_id,
"publication_id": int(publication.id),
"run_id": int(run.id),
},
)
await db_session.commit()
session_factory = async_sessionmaker(db_session.bind, expire_on_commit=False)
class _OpenAlexClientStub:
def __init__(self, *args: Any, **kwargs: Any) -> None:
_ = (args, kwargs)
async def get_works_by_filter(self, *_args: Any, **_kwargs: Any) -> list[Any]:
async with session_factory() as cancel_session:
run_to_cancel = await cancel_session.get(CrawlRun, int(run.id))
assert run_to_cancel is not None
run_to_cancel.status = RunStatus.CANCELED
await cancel_session.commit()
return []
monkeypatch.setattr(
"app.services.domains.openalex.client.OpenAlexClient",
_OpenAlexClientStub,
)
service = ScholarIngestionService(source=object())
await service._background_enrich(
session_factory,
run_id=int(run.id),
intended_final_status=RunStatus.SUCCESS,
)
await db_session.refresh(run)
await db_session.refresh(publication)
assert run.status == RunStatus.CANCELED
assert publication.openalex_enriched is False
@pytest.mark.integration
@pytest.mark.db
@pytest.mark.asyncio
async def test_publications_list_endpoint_does_not_raise_name_error(
db_session: AsyncSession,
) -> None:
user_id = await insert_user(
db_session,
email="api-publications-nameerror@example.com",
password="api-password",
)
scholar_result = await db_session.execute(
text(
"""
INSERT INTO scholar_profiles (user_id, scholar_id, display_name, is_enabled)
VALUES (:user_id, 'pubNameError001', 'Pub NameError', true)
RETURNING id
"""
),
{"user_id": user_id},
)
scholar_profile_id = int(scholar_result.scalar_one())
publication_result = await db_session.execute(
text(
"""
INSERT INTO publications (fingerprint_sha256, title_raw, title_normalized, citation_count)
VALUES (:fingerprint, :title_raw, :title_normalized, 1)
RETURNING id
"""
),
{
"fingerprint": f"{(user_id + 2222):064x}",
"title_raw": "NameError Regression",
"title_normalized": "nameerror regression",
},
)
publication_id = int(publication_result.scalar_one())
await db_session.execute(
text(
"""
INSERT INTO scholar_publications (scholar_profile_id, publication_id, is_read, is_favorite)
VALUES (:scholar_profile_id, :publication_id, false, false)
"""
),
{"scholar_profile_id": scholar_profile_id, "publication_id": publication_id},
)
await db_session.commit()
client = TestClient(app)
login_user(client, email="api-publications-nameerror@example.com", password="api-password")
response = client.get("/api/v1/publications?mode=all&limit=10&offset=0")
assert response.status_code == 200
assert len(response.json()["data"]["publications"]) == 1
@pytest.mark.integration
@pytest.mark.db
@pytest.mark.asyncio
async def test_partial_discovery_exception_keeps_new_pub_count_consistent(
db_session: AsyncSession,
monkeypatch: pytest.MonkeyPatch,
) -> None:
user_id = await insert_user(
db_session,
email="api-pub-count-consistency@example.com",
password="api-password",
)
scholar_result = await db_session.execute(
text(
"""
INSERT INTO scholar_profiles (user_id, scholar_id, display_name, is_enabled)
VALUES (:user_id, 'countConsistency001', 'Count Consistency', true)
RETURNING id
"""
),
{"user_id": user_id},
)
scholar_profile_id = int(scholar_result.scalar_one())
run = CrawlRun(
user_id=user_id,
trigger_type=RunTriggerType.MANUAL,
status=RunStatus.RUNNING,
scholar_count=1,
new_pub_count=0,
)
publication = Publication(
fingerprint_sha256=f"{(user_id + 777):064x}",
title_raw="Persisted Discovery",
title_normalized="persisted discovery",
citation_count=0,
)
db_session.add_all([run, publication])
await db_session.commit()
service = ScholarIngestionService(source=object())
call_count = 0
async def _resolve_publication_stub(*_args: Any, **_kwargs: Any) -> Publication:
nonlocal call_count
call_count += 1
if call_count == 1:
return publication
raise RuntimeError("mid_page_failure")
monkeypatch.setattr(service, "_resolve_publication", _resolve_publication_stub)
from app.services.domains.scholar.parser_types import PublicationCandidate
from app.db.models import ScholarProfile
scholar = await db_session.get(ScholarProfile, scholar_profile_id)
assert scholar is not None
publications = [
PublicationCandidate(
title="Persisted Discovery",
title_url=None,
cluster_id=None,
year=None,
citation_count=0,
authors_text=None,
venue_text=None,
pdf_url=None,
),
PublicationCandidate(
title="Will Fail",
title_url=None,
cluster_id=None,
year=None,
citation_count=0,
authors_text=None,
venue_text=None,
pdf_url=None,
),
]
with pytest.raises(RuntimeError, match="mid_page_failure"):
await service._upsert_profile_publications(
db_session,
run=run,
scholar=scholar,
publications=publications,
)
refreshed_run = await db_session.get(CrawlRun, int(run.id))
assert refreshed_run is not None
assert int(refreshed_run.new_pub_count) == 1
link_count_result = await db_session.execute(
text(
"""
SELECT count(*)
FROM scholar_publications
WHERE scholar_profile_id = :scholar_profile_id
AND first_seen_run_id = :run_id
"""
),
{"scholar_profile_id": scholar_profile_id, "run_id": int(run.id)},
)
assert int(link_count_result.scalar_one()) == 1
@pytest.mark.integration
@pytest.mark.db
@pytest.mark.asyncio
async def test_publications_pagination_snapshot_stays_stable_across_inserts(
db_session: AsyncSession,
) -> None:
user_id = await insert_user(
db_session,
email="api-publications-snapshot@example.com",
password="api-password",
)
scholar_result = await db_session.execute(
text(
"""
INSERT INTO scholar_profiles (user_id, scholar_id, display_name, is_enabled)
VALUES (:user_id, 'pagingSnapshot001', 'Paging Snapshot', true)
RETURNING id
"""
),
{"user_id": user_id},
)
scholar_profile_id = int(scholar_result.scalar_one())
publication_ids: list[int] = []
for index in range(4):
created = await db_session.execute(
text(
"""
INSERT INTO publications (fingerprint_sha256, title_raw, title_normalized, citation_count)
VALUES (:fingerprint, :title_raw, :title_normalized, 1)
RETURNING id
"""
),
{
"fingerprint": f"{(user_id + 1000 + index):064x}",
"title_raw": f"Stable Page {index}",
"title_normalized": f"stable page {index}",
},
)
publication_ids.append(int(created.scalar_one()))
for publication_id in publication_ids:
await db_session.execute(
text(
"""
INSERT INTO scholar_publications (scholar_profile_id, publication_id, is_read, is_favorite)
VALUES (:scholar_profile_id, :publication_id, false, false)
"""
),
{"scholar_profile_id": scholar_profile_id, "publication_id": publication_id},
)
await db_session.commit()
client = TestClient(app)
login_user(client, email="api-publications-snapshot@example.com", password="api-password")
first_page_response = client.get("/api/v1/publications?mode=all&page=1&page_size=2")
assert first_page_response.status_code == 200
first_page = first_page_response.json()["data"]
first_page_ids = [int(item["publication_id"]) for item in first_page["publications"]]
snapshot = first_page["snapshot"]
assert isinstance(snapshot, str) and snapshot
inserted = await db_session.execute(
text(
"""
INSERT INTO publications (fingerprint_sha256, title_raw, title_normalized, citation_count)
VALUES (:fingerprint, :title_raw, :title_normalized, 1)
RETURNING id
"""
),
{
"fingerprint": f"{(user_id + 5000):064x}",
"title_raw": "Inserted After Snapshot",
"title_normalized": "inserted after snapshot",
},
)
inserted_publication_id = int(inserted.scalar_one())
await db_session.execute(
text(
"""
INSERT INTO scholar_publications (scholar_profile_id, publication_id, is_read, is_favorite)
VALUES (:scholar_profile_id, :publication_id, false, false)
"""
),
{"scholar_profile_id": scholar_profile_id, "publication_id": inserted_publication_id},
)
await db_session.commit()
second_page_response = client.get(
"/api/v1/publications",
params={
"mode": "all",
"page": 2,
"page_size": 2,
"snapshot": snapshot,
},
)
assert second_page_response.status_code == 200
second_page = second_page_response.json()["data"]
second_page_ids = [int(item["publication_id"]) for item in second_page["publications"]]
first_page_again_response = client.get(
"/api/v1/publications",
params={
"mode": "all",
"page": 1,
"page_size": 2,
"snapshot": snapshot,
},
)
assert first_page_again_response.status_code == 200
first_page_again = first_page_again_response.json()["data"]
first_page_again_ids = [
int(item["publication_id"]) for item in first_page_again["publications"]
]
assert first_page_again_ids == first_page_ids
assert not (set(first_page_ids) & set(second_page_ids))
assert inserted_publication_id not in set(first_page_ids + second_page_ids)