from __future__ import annotations from datetime import datetime, timedelta, timezone from pathlib import Path import pytest from fastapi.testclient import TestClient from sqlalchemy import text from sqlalchemy.ext.asyncio import AsyncSession from app.api.runtime_deps import get_scholar_source from app.main import app from app.services.domains.publications.types import PublicationListItem from app.services.domains.settings import application as user_settings_service from app.services.domains.scholar.source import FetchResult from app.settings import settings from tests.integration.helpers import insert_user, login_user REGRESSION_FIXTURE_DIR = Path("tests/fixtures/scholar/regression") SAFETY_STATE_KEYS = { "cooldown_active", "cooldown_reason", "cooldown_reason_label", "cooldown_until", "cooldown_remaining_seconds", "recommended_action", "counters", } SAFETY_COUNTER_KEYS = { "consecutive_blocked_runs", "consecutive_network_runs", "cooldown_entry_count", "blocked_start_count", "last_blocked_failure_count", "last_network_failure_count", "last_evaluated_run_id", } def _api_bootstrap_csrf_headers(client: TestClient) -> dict[str, str]: bootstrap_response = client.get("/api/v1/auth/csrf") assert bootstrap_response.status_code == 200 payload = bootstrap_response.json()["data"] token = payload["csrf_token"] assert isinstance(token, str) and token return {"X-CSRF-Token": token} def _api_csrf_headers(client: TestClient) -> dict[str, str]: me_response = client.get("/api/v1/auth/me") assert me_response.status_code == 200 body = me_response.json() token = body["data"]["csrf_token"] assert isinstance(token, str) and token return {"X-CSRF-Token": token} def _regression_fixture(name: str) -> str: return (REGRESSION_FIXTURE_DIR / name).read_text(encoding="utf-8") def _assert_safety_state_contract(payload: dict[str, object]) -> None: assert set(payload.keys()) == SAFETY_STATE_KEYS counters = payload["counters"] assert isinstance(counters, dict) assert set(counters.keys()) == SAFETY_COUNTER_KEYS async def _seed_publication_link_for_user( db_session: AsyncSession, *, user_id: int, scholar_id: str, title: str, fingerprint: str, ) -> tuple[int, int]: scholar_result = await db_session.execute( text( """ INSERT INTO scholar_profiles (user_id, scholar_id, display_name, is_enabled) VALUES (:user_id, :scholar_id, :display_name, true) RETURNING id """ ), {"user_id": user_id, "scholar_id": scholar_id, "display_name": scholar_id}, ) scholar_profile_id = int(scholar_result.scalar_one()) publication_result = await db_session.execute( text( """ INSERT INTO publications (fingerprint_sha256, title_raw, title_normalized, citation_count) VALUES (:fingerprint, :title_raw, :title_normalized, 4) RETURNING id """ ), {"fingerprint": fingerprint, "title_raw": title, "title_normalized": title.lower()}, ) publication_id = int(publication_result.scalar_one()) await db_session.execute( text( """ INSERT INTO scholar_publications (scholar_profile_id, publication_id, is_read) VALUES (:scholar_profile_id, :publication_id, false) """ ), {"scholar_profile_id": scholar_profile_id, "publication_id": publication_id}, ) await db_session.commit() return scholar_profile_id, publication_id @pytest.mark.integration @pytest.mark.db @pytest.mark.asyncio async def test_api_me_requires_authentication() -> None: client = TestClient(app) response = client.get("/api/v1/auth/me") assert response.status_code == 401 payload = response.json() assert payload["error"]["code"] == "auth_required" assert payload["error"]["message"] == "Authentication required." assert "request_id" in payload["meta"] @pytest.mark.integration @pytest.mark.db @pytest.mark.asyncio async def test_api_csrf_bootstrap_returns_token_for_anonymous_and_authenticated( db_session: AsyncSession, ) -> None: await insert_user( db_session, email="api-bootstrap@example.com", password="api-password", ) client = TestClient(app) anonymous_response = client.get("/api/v1/auth/csrf") assert anonymous_response.status_code == 200 anonymous_payload = anonymous_response.json()["data"] assert isinstance(anonymous_payload["csrf_token"], str) assert anonymous_payload["authenticated"] is False login_user(client, email="api-bootstrap@example.com", password="api-password") authenticated_response = client.get("/api/v1/auth/csrf") assert authenticated_response.status_code == 200 authenticated_payload = authenticated_response.json()["data"] assert isinstance(authenticated_payload["csrf_token"], str) assert authenticated_payload["authenticated"] is True @pytest.mark.integration @pytest.mark.db @pytest.mark.asyncio async def test_api_me_returns_user_and_csrf_token(db_session: AsyncSession) -> None: await insert_user( db_session, email="api-me@example.com", password="api-password", ) client = TestClient(app) login_user(client, email="api-me@example.com", password="api-password") response = client.get("/api/v1/auth/me") assert response.status_code == 200 payload = response.json() assert payload["data"]["authenticated"] is True assert payload["data"]["user"]["email"] == "api-me@example.com" assert isinstance(payload["data"]["csrf_token"], str) assert payload["meta"]["request_id"] @pytest.mark.integration @pytest.mark.db @pytest.mark.asyncio async def test_api_login_and_change_password_flow(db_session: AsyncSession) -> None: await insert_user( db_session, email="api-login@example.com", password="old-password", ) client = TestClient(app) missing_csrf = client.post( "/api/v1/auth/login", json={"email": "api-login@example.com", "password": "old-password"}, ) assert missing_csrf.status_code == 403 assert missing_csrf.json()["error"]["code"] == "csrf_missing" login_headers = _api_bootstrap_csrf_headers(client) login_response = client.post( "/api/v1/auth/login", json={"email": "api-login@example.com", "password": "old-password"}, headers=login_headers, ) assert login_response.status_code == 200 login_payload = login_response.json()["data"] assert login_payload["authenticated"] is True assert login_payload["user"]["email"] == "api-login@example.com" assert isinstance(login_payload["csrf_token"], str) bad_change_response = client.post( "/api/v1/auth/change-password", json={ "current_password": "not-correct", "new_password": "new-password", "confirm_password": "new-password", }, headers={"X-CSRF-Token": login_payload["csrf_token"]}, ) assert bad_change_response.status_code == 400 assert bad_change_response.json()["error"]["code"] == "invalid_current_password" change_response = client.post( "/api/v1/auth/change-password", json={ "current_password": "old-password", "new_password": "new-password", "confirm_password": "new-password", }, headers={"X-CSRF-Token": login_payload["csrf_token"]}, ) assert change_response.status_code == 200 assert change_response.json()["data"]["message"] == "Password updated successfully." logout_response = client.post( "/api/v1/auth/logout", headers={"X-CSRF-Token": login_payload["csrf_token"]}, ) assert logout_response.status_code == 200 relogin_headers = _api_bootstrap_csrf_headers(client) old_password_login = client.post( "/api/v1/auth/login", json={"email": "api-login@example.com", "password": "old-password"}, headers=relogin_headers, ) assert old_password_login.status_code == 401 assert old_password_login.json()["error"]["code"] == "invalid_credentials" fresh_headers = _api_bootstrap_csrf_headers(client) new_password_login = client.post( "/api/v1/auth/login", json={"email": "api-login@example.com", "password": "new-password"}, headers=fresh_headers, ) assert new_password_login.status_code == 200 assert new_password_login.json()["data"]["authenticated"] is True @pytest.mark.integration @pytest.mark.db @pytest.mark.asyncio async def test_api_admin_user_management_endpoints(db_session: AsyncSession) -> None: admin_user_id = await insert_user( db_session, email="api-admin@example.com", password="admin-password", is_admin=True, ) target_user_id = await insert_user( db_session, email="api-target@example.com", password="target-password", is_admin=False, ) await insert_user( db_session, email="api-member@example.com", password="member-password", is_admin=False, ) client = TestClient(app) login_user(client, email="api-admin@example.com", password="admin-password") headers = _api_csrf_headers(client) list_response = client.get("/api/v1/admin/users") assert list_response.status_code == 200 users = list_response.json()["data"]["users"] assert any(item["email"] == "api-target@example.com" for item in users) create_response = client.post( "/api/v1/admin/users", json={ "email": "api-created@example.com", "password": "created-password", "is_admin": True, }, headers=headers, ) assert create_response.status_code == 201 created_user = create_response.json()["data"] assert created_user["email"] == "api-created@example.com" created_user_id = int(created_user["id"]) deactivate_response = client.patch( f"/api/v1/admin/users/{target_user_id}/active", json={"is_active": False}, headers=headers, ) assert deactivate_response.status_code == 200 assert deactivate_response.json()["data"]["is_active"] is False reactivate_response = client.patch( f"/api/v1/admin/users/{target_user_id}/active", json={"is_active": True}, headers=headers, ) assert reactivate_response.status_code == 200 assert reactivate_response.json()["data"]["is_active"] is True reset_response = client.post( f"/api/v1/admin/users/{target_user_id}/reset-password", json={"new_password": "target-password-updated"}, headers=headers, ) assert reset_response.status_code == 200 assert "Password reset" in reset_response.json()["data"]["message"] self_deactivate = client.patch( f"/api/v1/admin/users/{admin_user_id}/active", json={"is_active": False}, headers=headers, ) assert self_deactivate.status_code == 400 assert self_deactivate.json()["error"]["code"] == "cannot_deactivate_self" logout_response = client.post("/api/v1/auth/logout", headers=headers) assert logout_response.status_code == 200 target_headers = _api_bootstrap_csrf_headers(client) target_login = client.post( "/api/v1/auth/login", json={"email": "api-target@example.com", "password": "target-password-updated"}, headers=target_headers, ) assert target_login.status_code == 200 non_admin_headers = {"X-CSRF-Token": target_login.json()["data"]["csrf_token"]} forbidden_response = client.get("/api/v1/admin/users") assert forbidden_response.status_code == 403 assert forbidden_response.json()["error"]["code"] == "forbidden" forbidden_create = client.post( "/api/v1/admin/users", json={ "email": "should-not-work@example.com", "password": "password-123", "is_admin": False, }, headers=non_admin_headers, ) assert forbidden_create.status_code == 403 created_exists = await db_session.execute( text("SELECT COUNT(*) FROM users WHERE id = :user_id"), {"user_id": created_user_id}, ) assert created_exists.scalar_one() == 1 @pytest.mark.integration @pytest.mark.db @pytest.mark.asyncio async def test_api_admin_dbops_integrity_and_repair_flow(db_session: AsyncSession) -> None: await insert_user(db_session, email="api-admin-dbops@example.com", password="admin-password", is_admin=True) target_user_id = await insert_user(db_session, email="api-dbops-target@example.com", password="target-password") _scholar_id, publication_id = await _seed_publication_link_for_user( db_session, user_id=target_user_id, scholar_id="dbopsTarget01", title="DB Ops Target Paper", fingerprint=f"{(target_user_id + 31):064x}", ) client = TestClient(app) login_user(client, email="api-admin-dbops@example.com", password="admin-password") headers = _api_csrf_headers(client) integrity_response = client.get("/api/v1/admin/db/integrity") assert integrity_response.status_code == 200 integrity_payload = integrity_response.json()["data"] assert integrity_payload["status"] in {"ok", "warning", "failed"} assert any( check["name"] == "missing_pdf_url" for check in integrity_payload["checks"] ) repair_response = client.post( "/api/v1/admin/db/repairs/publication-links", json={"user_id": target_user_id, "dry_run": True}, headers=headers, ) assert repair_response.status_code == 200 repair_data = repair_response.json()["data"] assert repair_data["status"] == "completed" assert bool(repair_data["summary"]["dry_run"]) is True job_id = int(repair_data["job_id"]) jobs_response = client.get("/api/v1/admin/db/repair-jobs?limit=10") assert jobs_response.status_code == 200 jobs = jobs_response.json()["data"]["jobs"] assert any(int(job["id"]) == job_id for job in jobs) pdf_queue_response = client.get("/api/v1/admin/db/pdf-queue?limit=20") assert pdf_queue_response.status_code == 200 pdf_queue_payload = pdf_queue_response.json()["data"] assert isinstance(pdf_queue_payload["items"], list) assert int(pdf_queue_payload["page"]) == 1 assert int(pdf_queue_payload["page_size"]) == 20 assert int(pdf_queue_payload["total_count"]) >= 0 assert isinstance(pdf_queue_payload["has_next"], bool) assert isinstance(pdf_queue_payload["has_prev"], bool) page_two_response = client.get("/api/v1/admin/db/pdf-queue?page=2&page_size=1") assert page_two_response.status_code == 200 page_two_payload = page_two_response.json()["data"] assert int(page_two_payload["page"]) == 2 assert int(page_two_payload["page_size"]) == 1 assert page_two_payload["has_prev"] is True untracked_response = client.get("/api/v1/admin/db/pdf-queue?limit=20&status=untracked") assert untracked_response.status_code == 200 assert any( item["status"] == "untracked" for item in untracked_response.json()["data"]["items"] ) link_count_result = await db_session.execute( text("SELECT count(*) FROM scholar_publications WHERE publication_id = :publication_id"), {"publication_id": publication_id}, ) assert int(link_count_result.scalar_one()) == 1 @pytest.mark.integration @pytest.mark.db @pytest.mark.asyncio async def test_api_admin_dbops_pdf_queue_requeue_endpoint( db_session: AsyncSession, monkeypatch: pytest.MonkeyPatch, ) -> None: await insert_user(db_session, email="api-admin-pdf@example.com", password="admin-password", is_admin=True) target_user_id = await insert_user(db_session, email="api-pdf-target@example.com", password="target-password") _scholar_id, publication_id = await _seed_publication_link_for_user( db_session, user_id=target_user_id, scholar_id="dbopsPdf01", title="PDF Queue Target", fingerprint=f"{(target_user_id + 73):064x}", ) monkeypatch.setattr( "app.services.domains.publications.pdf_queue._schedule_rows", lambda **_kwargs: None, ) client = TestClient(app) login_user(client, email="api-admin-pdf@example.com", password="admin-password") headers = _api_csrf_headers(client) first_response = client.post( f"/api/v1/admin/db/pdf-queue/{publication_id}/requeue", headers=headers, ) assert first_response.status_code == 200 first_data = first_response.json()["data"] assert first_data["publication_id"] == publication_id assert first_data["queued"] is True assert first_data["status"] == "queued" second_response = client.post( f"/api/v1/admin/db/pdf-queue/{publication_id}/requeue", headers=headers, ) assert second_response.status_code == 200 second_data = second_response.json()["data"] assert second_data["queued"] is False assert second_data["status"] == "blocked" @pytest.mark.integration @pytest.mark.db @pytest.mark.asyncio async def test_api_admin_dbops_pdf_queue_requeue_all_endpoint( db_session: AsyncSession, monkeypatch: pytest.MonkeyPatch, ) -> None: await insert_user(db_session, email="api-admin-pdf-all@example.com", password="admin-password", is_admin=True) target_user_id = await insert_user(db_session, email="api-pdf-all-target@example.com", password="target-password") _scholar_id, _publication_id = await _seed_publication_link_for_user( db_session, user_id=target_user_id, scholar_id="dbopsPdfAll01", title="PDF Queue All Target", fingerprint=f"{(target_user_id + 83):064x}", ) monkeypatch.setattr( "app.services.domains.publications.pdf_queue._schedule_rows", lambda **_kwargs: None, ) client = TestClient(app) login_user(client, email="api-admin-pdf-all@example.com", password="admin-password") headers = _api_csrf_headers(client) response = client.post( "/api/v1/admin/db/pdf-queue/requeue-all?limit=500", headers=headers, ) assert response.status_code == 200 payload = response.json()["data"] assert int(payload["requested_count"]) >= 1 assert int(payload["queued_count"]) >= 1 @pytest.mark.integration @pytest.mark.db @pytest.mark.asyncio async def test_api_admin_dbops_forbidden_for_non_admin_and_validates_scope(db_session: AsyncSession) -> None: await insert_user(db_session, email="api-admin-dbops2@example.com", password="admin-password", is_admin=True) member_user_id = await insert_user(db_session, email="api-dbops-member@example.com", password="member-password") client = TestClient(app) login_user(client, email="api-dbops-member@example.com", password="member-password") headers = _api_csrf_headers(client) forbidden_integrity = client.get("/api/v1/admin/db/integrity") assert forbidden_integrity.status_code == 403 assert forbidden_integrity.json()["error"]["code"] == "forbidden" forbidden_pdf_queue = client.get("/api/v1/admin/db/pdf-queue") assert forbidden_pdf_queue.status_code == 403 assert forbidden_pdf_queue.json()["error"]["code"] == "forbidden" forbidden_requeue = client.post( "/api/v1/admin/db/pdf-queue/1/requeue", headers=headers, ) assert forbidden_requeue.status_code == 403 assert forbidden_requeue.json()["error"]["code"] == "forbidden" forbidden_requeue_all = client.post( "/api/v1/admin/db/pdf-queue/requeue-all?limit=100", headers=headers, ) assert forbidden_requeue_all.status_code == 403 assert forbidden_requeue_all.json()["error"]["code"] == "forbidden" forbidden_repair = client.post( "/api/v1/admin/db/repairs/publication-links", json={"user_id": member_user_id, "dry_run": True}, headers=headers, ) assert forbidden_repair.status_code == 403 assert forbidden_repair.json()["error"]["code"] == "forbidden" admin_headers = _api_bootstrap_csrf_headers(client) admin_login = client.post( "/api/v1/auth/login", json={"email": "api-admin-dbops2@example.com", "password": "admin-password"}, headers=admin_headers, ) assert admin_login.status_code == 200 post_login_headers = {"X-CSRF-Token": admin_login.json()["data"]["csrf_token"]} invalid_scope = client.post( "/api/v1/admin/db/repairs/publication-links", json={"user_id": member_user_id, "dry_run": True}, headers=post_login_headers, ) assert invalid_scope.status_code == 400 assert invalid_scope.json()["error"]["code"] == "invalid_repair_scope" @pytest.mark.integration @pytest.mark.db @pytest.mark.asyncio async def test_api_admin_dbops_all_users_apply_requires_confirmation(db_session: AsyncSession) -> None: await insert_user(db_session, email="api-admin-dbops3@example.com", password="admin-password", is_admin=True) first_user_id = await insert_user(db_session, email="api-dbops-a@example.com", password="user-password") second_user_id = await insert_user(db_session, email="api-dbops-b@example.com", password="user-password") await _seed_publication_link_for_user( db_session, user_id=first_user_id, scholar_id="dbopsAll01", title="DB Ops All User Paper One", fingerprint=f"{(first_user_id + 61):064x}", ) await _seed_publication_link_for_user( db_session, user_id=second_user_id, scholar_id="dbopsAll02", title="DB Ops All User Paper Two", fingerprint=f"{(second_user_id + 71):064x}", ) client = TestClient(app) login_user(client, email="api-admin-dbops3@example.com", password="admin-password") headers = _api_csrf_headers(client) missing_confirmation = client.post( "/api/v1/admin/db/repairs/publication-links", json={"scope_mode": "all_users", "dry_run": False}, headers=headers, ) assert missing_confirmation.status_code == 422 assert "confirmation_text" in str(missing_confirmation.json()) apply_response = client.post( "/api/v1/admin/db/repairs/publication-links", json={ "scope_mode": "all_users", "dry_run": False, "confirmation_text": "REPAIR ALL USERS", }, headers=headers, ) assert apply_response.status_code == 200 apply_data = apply_response.json()["data"] assert apply_data["status"] == "completed" assert apply_data["scope"]["scope_mode"] == "all_users" assert int(apply_data["summary"]["links_deleted"]) >= 2 remaining_links = await db_session.execute(text("SELECT count(*) FROM scholar_publications")) assert int(remaining_links.scalar_one()) == 0 @pytest.mark.integration @pytest.mark.db @pytest.mark.asyncio async def test_api_scholars_crud_flow(db_session: AsyncSession) -> None: await insert_user( db_session, email="api-scholars@example.com", password="api-password", ) client = TestClient(app) login_user(client, email="api-scholars@example.com", password="api-password") headers = _api_csrf_headers(client) missing_csrf = client.post( "/api/v1/scholars", json={"scholar_id": "abcDEF123456"}, ) assert missing_csrf.status_code == 403 assert missing_csrf.json()["error"]["code"] == "csrf_invalid" create_response = client.post( "/api/v1/scholars", json={"scholar_id": "abcDEF123456"}, headers=headers, ) assert create_response.status_code == 201 created = create_response.json()["data"] assert created["scholar_id"] == "abcDEF123456" scholar_profile_id = int(created["id"]) list_response = client.get("/api/v1/scholars") assert list_response.status_code == 200 scholars = list_response.json()["data"]["scholars"] assert any(int(item["id"]) == scholar_profile_id for item in scholars) toggle_response = client.patch( f"/api/v1/scholars/{scholar_profile_id}/toggle", headers=headers, ) assert toggle_response.status_code == 200 assert toggle_response.json()["data"]["is_enabled"] is False delete_response = client.delete( f"/api/v1/scholars/{scholar_profile_id}", headers=headers, ) assert delete_response.status_code == 200 assert delete_response.json()["data"]["message"] == "Scholar deleted." @pytest.mark.integration @pytest.mark.db @pytest.mark.asyncio async def test_api_scholars_search_and_profile_image_management( db_session: AsyncSession, tmp_path: Path, ) -> None: await insert_user( db_session, email="api-scholar-images@example.com", password="api-password", ) class StubScholarSource: async def fetch_profile_html(self, scholar_id: str) -> FetchResult: assert scholar_id == "abcDEF123456" return FetchResult( requested_url="https://scholar.google.com/citations?hl=en&user=abcDEF123456", status_code=200, final_url="https://scholar.google.com/citations?hl=en&user=abcDEF123456", body=( "" '' "" '
Ada Lovelace
' "" ), error=None, ) async def fetch_author_search_html(self, query: str, *, start: int) -> FetchResult: assert query == "Ada Lovelace" assert start == 0 return FetchResult( requested_url="https://scholar.google.com/citations?hl=en&view_op=search_authors&mauthors=ada", status_code=200, final_url="https://scholar.google.com/citations?hl=en&view_op=search_authors&mauthors=ada", body=( '
' '' 'Ada Lovelace' '
Analytical Engine
' '
Verified email at computing.example
' '
Cited by 42
' 'Mathematics' "
" ), error=None, ) previous_upload_dir = settings.scholar_image_upload_dir previous_upload_max_bytes = settings.scholar_image_upload_max_bytes app.dependency_overrides[get_scholar_source] = lambda: StubScholarSource() object.__setattr__(settings, "scholar_image_upload_dir", str(tmp_path / "scholar_images")) object.__setattr__(settings, "scholar_image_upload_max_bytes", 1_000_000) try: client = TestClient(app) login_user(client, email="api-scholar-images@example.com", password="api-password") headers = _api_csrf_headers(client) search_response = client.get("/api/v1/scholars/search", params={"query": "Ada Lovelace", "limit": 5}) assert search_response.status_code == 200 search_payload = search_response.json()["data"] assert search_payload["state"] == "ok" assert len(search_payload["candidates"]) == 1 candidate = search_payload["candidates"][0] assert candidate["scholar_id"] == "abcDEF123456" assert candidate["profile_image_url"] == "https://scholar.google.com/citations/images/avatar_scholar_256.png" create_response = client.post( "/api/v1/scholars", json={ "scholar_id": candidate["scholar_id"], }, headers=headers, ) assert create_response.status_code == 201 created = create_response.json()["data"] scholar_profile_id = int(created["id"]) assert created["profile_image_source"] == "scraped" assert created["profile_image_url"] == "https://images.example.com/ada.png" set_url_response = client.put( f"/api/v1/scholars/{scholar_profile_id}/image/url", json={"image_url": "https://cdn.example.com/custom-avatar.png"}, headers=headers, ) assert set_url_response.status_code == 200 set_url_data = set_url_response.json()["data"] assert set_url_data["profile_image_source"] == "override" assert set_url_data["profile_image_url"] == "https://cdn.example.com/custom-avatar.png" uploaded_bytes = b"\\x89PNG\\r\\n\\x1a\\n\\x00\\x00\\x00\\rIHDR\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01" upload_response = client.post( f"/api/v1/scholars/{scholar_profile_id}/image/upload", files={"image": ("avatar.png", uploaded_bytes, "image/png")}, headers=headers, ) assert upload_response.status_code == 200 upload_data = upload_response.json()["data"] assert upload_data["profile_image_source"] == "upload" assert upload_data["profile_image_url"] == f"/api/v1/scholars/{scholar_profile_id}/image/upload" uploaded_image_response = client.get(f"/api/v1/scholars/{scholar_profile_id}/image/upload") assert uploaded_image_response.status_code == 200 assert uploaded_image_response.headers["content-type"].startswith("image/png") assert uploaded_image_response.content == uploaded_bytes clear_response = client.delete( f"/api/v1/scholars/{scholar_profile_id}/image", headers=headers, ) assert clear_response.status_code == 200 clear_data = clear_response.json()["data"] assert clear_data["profile_image_source"] == "scraped" assert clear_data["profile_image_url"] == "https://images.example.com/ada.png" finally: app.dependency_overrides.pop(get_scholar_source, None) object.__setattr__(settings, "scholar_image_upload_dir", previous_upload_dir) object.__setattr__( settings, "scholar_image_upload_max_bytes", previous_upload_max_bytes, ) @pytest.mark.integration @pytest.mark.db @pytest.mark.asyncio async def test_api_scholar_import_export_round_trip( db_session: AsyncSession, ) -> None: user_id = await insert_user( db_session, email="api-import-export@example.com", password="api-password", ) scholar_result = await db_session.execute( text( """ INSERT INTO scholar_profiles (user_id, scholar_id, display_name, is_enabled) VALUES (:user_id, :scholar_id, :display_name, true) RETURNING id """ ), { "user_id": user_id, "scholar_id": "abcDEF123456", "display_name": "Existing Scholar", }, ) scholar_profile_id = int(scholar_result.scalar_one()) publication_result = await db_session.execute( text( """ INSERT INTO publications (fingerprint_sha256, title_raw, title_normalized, citation_count) VALUES (:fingerprint, :title_raw, :title_normalized, 1) RETURNING id """ ), { "fingerprint": f"{(user_id + 500):064x}", "title_raw": "Existing Publication", "title_normalized": "existingpublication", }, ) publication_id = int(publication_result.scalar_one()) await db_session.execute( text( """ INSERT INTO scholar_publications (scholar_profile_id, publication_id, is_read) VALUES (:scholar_profile_id, :publication_id, false) """ ), { "scholar_profile_id": scholar_profile_id, "publication_id": publication_id, }, ) await db_session.commit() client = TestClient(app) login_user(client, email="api-import-export@example.com", password="api-password") headers = _api_csrf_headers(client) export_response = client.get("/api/v1/scholars/export") assert export_response.status_code == 200 export_payload = export_response.json()["data"] assert export_payload["schema_version"] == 1 assert len(export_payload["scholars"]) == 1 assert len(export_payload["publications"]) == 1 import_response = client.post( "/api/v1/scholars/import", json={ "schema_version": 1, "scholars": [ { "scholar_id": "abcDEF123456", "display_name": "Updated Scholar", "is_enabled": False, "profile_image_override_url": "https://cdn.example.com/avatar.png", }, { "scholar_id": "zzzYYY111222", "display_name": "Imported Scholar", "is_enabled": True, "profile_image_override_url": None, }, ], "publications": [ { "scholar_id": "abcDEF123456", "title": "Existing Publication", "year": 2024, "citation_count": 8, "author_text": "A. Author", "venue_text": "Test Venue", "pub_url": "https://example.org/existing", "pdf_url": "https://example.org/existing.pdf", "is_read": True, }, { "scholar_id": "zzzYYY111222", "title": "Imported Publication", "year": 2025, "citation_count": 2, "author_text": "B. Author", "venue_text": "Another Venue", "pub_url": "https://example.org/imported", "pdf_url": "https://example.org/imported.pdf", "is_read": False, }, ], }, headers=headers, ) assert import_response.status_code == 200 import_data = import_response.json()["data"] assert int(import_data["scholars_created"]) == 1 assert int(import_data["scholars_updated"]) >= 1 assert int(import_data["publications_created"]) == 1 assert int(import_data["links_created"]) == 1 updated_scholar_result = await db_session.execute( text( """ SELECT display_name, is_enabled, profile_image_override_url FROM scholar_profiles WHERE user_id = :user_id AND scholar_id = :scholar_id """ ), { "user_id": user_id, "scholar_id": "abcDEF123456", }, ) updated_scholar = updated_scholar_result.one() assert updated_scholar[0] == "Updated Scholar" assert updated_scholar[1] is False assert updated_scholar[2] == "https://cdn.example.com/avatar.png" imported_pub_result = await db_session.execute( text( """ SELECT p.title_raw, p.pdf_url FROM publications p WHERE p.title_raw = :title """ ), {"title": "Imported Publication"}, ) imported_pub = imported_pub_result.one() assert imported_pub[0] == "Imported Publication" assert imported_pub[1] == "https://example.org/imported.pdf" updated_link_result = await db_session.execute( text( """ SELECT sp.is_read FROM scholar_publications sp JOIN scholar_profiles s ON s.id = sp.scholar_profile_id JOIN publications p ON p.id = sp.publication_id WHERE s.user_id = :user_id AND s.scholar_id = :scholar_id AND p.title_raw = :title """ ), { "user_id": user_id, "scholar_id": "abcDEF123456", "title": "Existing Publication", }, ) assert bool(updated_link_result.scalar_one()) is True @pytest.mark.integration @pytest.mark.db @pytest.mark.asyncio async def test_api_manual_run_skips_unchanged_initial_page_for_scholar( db_session: AsyncSession, ) -> None: await insert_user( db_session, email="api-skip-unchanged@example.com", password="api-password", ) profile_html = """
Skip Candidate
Articles 1-1
Stable Paper
A Author
Stable Venue
 5 2024
""" class StubScholarSource: async def fetch_profile_html(self, scholar_id: str) -> FetchResult: assert scholar_id == "abcDEF123456" return FetchResult( requested_url="https://scholar.google.com/citations?hl=en&user=abcDEF123456", status_code=200, final_url="https://scholar.google.com/citations?hl=en&user=abcDEF123456", body=profile_html, error=None, ) async def fetch_profile_page_html( self, scholar_id: str, *, cstart: int, pagesize: int, ) -> FetchResult: assert scholar_id == "abcDEF123456" assert cstart == 0 assert pagesize == settings.ingestion_page_size return FetchResult( requested_url="https://scholar.google.com/citations?hl=en&user=abcDEF123456", status_code=200, final_url="https://scholar.google.com/citations?hl=en&user=abcDEF123456", body=profile_html, error=None, ) app.dependency_overrides[get_scholar_source] = lambda: StubScholarSource() try: client = TestClient(app) login_user(client, email="api-skip-unchanged@example.com", password="api-password") headers = _api_csrf_headers(client) create_response = client.post( "/api/v1/scholars", json={"scholar_id": "abcDEF123456"}, headers=headers, ) assert create_response.status_code == 201 first_run_response = client.post( "/api/v1/runs/manual", headers={**headers, "Idempotency-Key": "skip-unchanged-run-001"}, ) assert first_run_response.status_code == 200 first_run_id = int(first_run_response.json()["data"]["run_id"]) first_run_detail = client.get(f"/api/v1/runs/{first_run_id}") assert first_run_detail.status_code == 200 first_results = first_run_detail.json()["data"]["scholar_results"] assert len(first_results) == 1 assert first_results[0]["state_reason"] != "no_change_initial_page_signature" second_run_response = client.post( "/api/v1/runs/manual", headers={**headers, "Idempotency-Key": "skip-unchanged-run-002"}, ) assert second_run_response.status_code == 200 second_run_id = int(second_run_response.json()["data"]["run_id"]) second_run_detail = client.get(f"/api/v1/runs/{second_run_id}") assert second_run_detail.status_code == 200 second_results = second_run_detail.json()["data"]["scholar_results"] assert len(second_results) == 1 assert second_results[0]["state_reason"] == "no_change_initial_page_signature" assert second_results[0]["publication_count"] == 0 assert second_results[0]["outcome"] == "success" finally: app.dependency_overrides.pop(get_scholar_source, None) @pytest.mark.integration @pytest.mark.db @pytest.mark.asyncio async def test_api_settings_get_and_update(db_session: AsyncSession) -> None: await insert_user( db_session, email="api-settings@example.com", password="api-password", ) client = TestClient(app) login_user(client, email="api-settings@example.com", password="api-password") headers = _api_csrf_headers(client) get_response = client.get("/api/v1/settings") assert get_response.status_code == 200 settings_payload = get_response.json()["data"] assert "request_delay_seconds" in settings_payload assert "nav_visible_pages" in settings_payload assert settings_payload["policy"]["min_run_interval_minutes"] == user_settings_service.resolve_run_interval_minimum( settings.ingestion_min_run_interval_minutes ) assert settings_payload["policy"]["min_request_delay_seconds"] == user_settings_service.resolve_request_delay_minimum( settings.ingestion_min_request_delay_seconds ) assert settings_payload["policy"]["automation_allowed"] is settings.ingestion_automation_allowed assert settings_payload["policy"]["manual_run_allowed"] is settings.ingestion_manual_run_allowed assert settings_payload["policy"]["blocked_failure_threshold"] == max( 1, int(settings.ingestion_alert_blocked_failure_threshold), ) assert settings_payload["policy"]["network_failure_threshold"] == max( 1, int(settings.ingestion_alert_network_failure_threshold), ) assert settings_payload["policy"]["cooldown_blocked_seconds"] == max( 60, int(settings.ingestion_safety_cooldown_blocked_seconds), ) assert settings_payload["policy"]["cooldown_network_seconds"] == max( 60, int(settings.ingestion_safety_cooldown_network_seconds), ) _assert_safety_state_contract(settings_payload["safety_state"]) assert settings_payload["safety_state"]["cooldown_active"] is False update_response = client.put( "/api/v1/settings", json={ "auto_run_enabled": True, "run_interval_minutes": 45, "request_delay_seconds": 6, "nav_visible_pages": ["dashboard", "scholars", "publications", "settings", "runs"], }, headers=headers, ) assert update_response.status_code == 200 updated = update_response.json()["data"] assert updated["auto_run_enabled"] is True assert updated["run_interval_minutes"] == 45 assert updated["request_delay_seconds"] == 6 assert updated["nav_visible_pages"] == [ "dashboard", "scholars", "publications", "settings", "runs", ] assert "policy" in updated _assert_safety_state_contract(updated["safety_state"]) @pytest.mark.integration @pytest.mark.db @pytest.mark.asyncio async def test_api_settings_enforce_env_minimums(db_session: AsyncSession) -> None: await insert_user( db_session, email="api-settings-policy@example.com", password="api-password", ) client = TestClient(app) login_user(client, email="api-settings-policy@example.com", password="api-password") headers = _api_csrf_headers(client) previous_min_interval = settings.ingestion_min_run_interval_minutes previous_min_delay = settings.ingestion_min_request_delay_seconds object.__setattr__(settings, "ingestion_min_run_interval_minutes", 30) object.__setattr__(settings, "ingestion_min_request_delay_seconds", 8) try: interval_response = client.put( "/api/v1/settings", json={ "auto_run_enabled": True, "run_interval_minutes": 29, "request_delay_seconds": 9, "nav_visible_pages": ["dashboard", "scholars", "settings"], }, headers=headers, ) assert interval_response.status_code == 400 assert interval_response.json()["error"]["code"] == "invalid_settings" assert interval_response.json()["error"]["message"] == "Check interval must be at least 30 minutes." delay_response = client.put( "/api/v1/settings", json={ "auto_run_enabled": True, "run_interval_minutes": 30, "request_delay_seconds": 7, "nav_visible_pages": ["dashboard", "scholars", "settings"], }, headers=headers, ) assert delay_response.status_code == 400 assert delay_response.json()["error"]["code"] == "invalid_settings" assert delay_response.json()["error"]["message"] == "Request delay must be at least 8 seconds." finally: object.__setattr__(settings, "ingestion_min_run_interval_minutes", previous_min_interval) object.__setattr__(settings, "ingestion_min_request_delay_seconds", previous_min_delay) @pytest.mark.integration @pytest.mark.db @pytest.mark.asyncio async def test_api_runs_manual_and_queue_actions(db_session: AsyncSession) -> None: user_id = await insert_user( db_session, email="api-runs@example.com", password="api-password", ) client = TestClient(app) login_user(client, email="api-runs@example.com", password="api-password") headers = _api_csrf_headers(client) run_response = client.post( "/api/v1/runs/manual", headers={**headers, "Idempotency-Key": "manual-run-0001"}, ) assert run_response.status_code == 200 run_payload = run_response.json()["data"] assert "run_id" in run_payload assert run_payload["status"] in {"success", "partial_failure", "failed"} assert run_payload["reused_existing_run"] is False assert run_payload["idempotency_key"] == "manual-run-0001" _assert_safety_state_contract(run_payload["safety_state"]) run_id = int(run_payload["run_id"]) stored_key = await db_session.execute( text("SELECT idempotency_key FROM crawl_runs WHERE id = :run_id"), {"run_id": run_id}, ) assert stored_key.scalar_one() == "manual-run-0001" replay_response = client.post( "/api/v1/runs/manual", headers={**headers, "Idempotency-Key": "manual-run-0001"}, ) assert replay_response.status_code == 200 replay_payload = replay_response.json()["data"] assert replay_payload["run_id"] == run_payload["run_id"] assert replay_payload["reused_existing_run"] is True _assert_safety_state_contract(replay_payload["safety_state"]) runs_response = client.get("/api/v1/runs") assert runs_response.status_code == 200 assert len(runs_response.json()["data"]["runs"]) >= 1 _assert_safety_state_contract(runs_response.json()["data"]["safety_state"]) run_detail_response = client.get(f"/api/v1/runs/{run_id}") assert run_detail_response.status_code == 200 detail_payload = run_detail_response.json()["data"] assert "summary" in detail_payload assert isinstance(detail_payload["scholar_results"], list) _assert_safety_state_contract(detail_payload["safety_state"]) scholar_result = await db_session.execute( text( """ INSERT INTO scholar_profiles (user_id, scholar_id, display_name, is_enabled) VALUES (:user_id, :scholar_id, :display_name, true) RETURNING id """ ), { "user_id": user_id, "scholar_id": "abcDEF123456", "display_name": "Queue Scholar", }, ) scholar_profile_id = int(scholar_result.scalar_one()) queue_result = await db_session.execute( text( """ INSERT INTO ingestion_queue_items ( user_id, scholar_profile_id, resume_cstart, reason, status, attempt_count, next_attempt_dt, dropped_reason, dropped_at ) VALUES ( :user_id, :scholar_profile_id, 7, 'dropped', 'dropped', 2, NOW(), 'manual_drop', NOW() ) RETURNING id """ ), {"user_id": user_id, "scholar_profile_id": scholar_profile_id}, ) queue_item_id = int(queue_result.scalar_one()) await db_session.commit() queue_list_response = client.get("/api/v1/runs/queue/items") assert queue_list_response.status_code == 200 assert any( int(item["id"]) == queue_item_id for item in queue_list_response.json()["data"]["queue_items"] ) retry_response = client.post(f"/api/v1/runs/queue/{queue_item_id}/retry", headers=headers) assert retry_response.status_code == 200 assert retry_response.json()["data"]["status"] == "queued" retry_again_response = client.post( f"/api/v1/runs/queue/{queue_item_id}/retry", headers=headers, ) assert retry_again_response.status_code == 409 assert retry_again_response.json()["error"]["code"] == "queue_item_already_queued" drop_response = client.post(f"/api/v1/runs/queue/{queue_item_id}/drop", headers=headers) assert drop_response.status_code == 200 assert drop_response.json()["data"]["status"] == "dropped" clear_response = client.request( "DELETE", f"/api/v1/runs/queue/{queue_item_id}", headers=headers, ) assert clear_response.status_code == 200 assert clear_response.json()["data"]["status"] == "cleared" assert clear_response.json()["data"]["message"] == "Queue item cleared." @pytest.mark.integration @pytest.mark.db @pytest.mark.asyncio async def test_api_manual_run_can_be_disabled_by_policy(db_session: AsyncSession) -> None: await insert_user( db_session, email="api-runs-policy@example.com", password="api-password", ) client = TestClient(app) login_user(client, email="api-runs-policy@example.com", password="api-password") headers = _api_csrf_headers(client) previous_manual_allowed = settings.ingestion_manual_run_allowed object.__setattr__(settings, "ingestion_manual_run_allowed", False) try: response = client.post("/api/v1/runs/manual", headers=headers) assert response.status_code == 403 payload = response.json() assert payload["error"]["code"] == "manual_runs_disabled" assert payload["error"]["details"]["policy"]["manual_run_allowed"] is False assert payload["error"]["details"]["safety_state"]["cooldown_active"] is False finally: object.__setattr__(settings, "ingestion_manual_run_allowed", previous_manual_allowed) @pytest.mark.integration @pytest.mark.db @pytest.mark.asyncio async def test_api_manual_run_enforces_scrape_safety_cooldown(db_session: AsyncSession) -> None: user_id = await insert_user( db_session, email="api-runs-safety@example.com", password="api-password", ) await db_session.execute( text( """ INSERT INTO scholar_profiles (user_id, scholar_id, display_name, is_enabled) VALUES (:user_id, :scholar_id, :display_name, true) """ ), { "user_id": user_id, "scholar_id": "A1B2C3D4E5F6", "display_name": "Safety Probe", }, ) await db_session.commit() blocked_fixture = _regression_fixture("profile_AAAAAAAAAAAA.html") class BlockedScholarSource: async def fetch_profile_page_html( self, scholar_id: str, *, cstart: int, pagesize: int, ) -> FetchResult: _ = (scholar_id, cstart, pagesize) return FetchResult( requested_url="https://scholar.google.com/citations?hl=en&user=A1B2C3D4E5F6", status_code=200, final_url=( "https://accounts.google.com/v3/signin/identifier" "?continue=https%3A%2F%2Fscholar.google.com%2Fcitations" ), body=blocked_fixture, error=None, ) async def fetch_profile_html(self, scholar_id: str) -> FetchResult: _ = scholar_id return await self.fetch_profile_page_html( "A1B2C3D4E5F6", cstart=0, pagesize=settings.ingestion_page_size, ) app.dependency_overrides[get_scholar_source] = lambda: BlockedScholarSource() previous_blocked_threshold = settings.ingestion_alert_blocked_failure_threshold previous_blocked_cooldown_seconds = settings.ingestion_safety_cooldown_blocked_seconds object.__setattr__(settings, "ingestion_alert_blocked_failure_threshold", 1) object.__setattr__(settings, "ingestion_safety_cooldown_blocked_seconds", 600) try: client = TestClient(app) login_user(client, email="api-runs-safety@example.com", password="api-password") headers = _api_csrf_headers(client) first_run_response = client.post( "/api/v1/runs/manual", headers={**headers, "Idempotency-Key": "safety-cooldown-run-1"}, ) assert first_run_response.status_code == 200 settings_response = client.get("/api/v1/settings") assert settings_response.status_code == 200 safety_state = settings_response.json()["data"]["safety_state"] _assert_safety_state_contract(safety_state) assert safety_state["cooldown_active"] is True assert safety_state["cooldown_reason"] == "blocked_failure_threshold_exceeded" assert int(safety_state["cooldown_remaining_seconds"]) > 0 assert int(safety_state["counters"]["cooldown_entry_count"]) >= 1 assert int(safety_state["counters"]["last_blocked_failure_count"]) >= 1 blocked_start_response = client.post( "/api/v1/runs/manual", headers={**headers, "Idempotency-Key": "safety-cooldown-run-2"}, ) assert blocked_start_response.status_code == 429 blocked_payload = blocked_start_response.json() assert blocked_payload["error"]["code"] == "scrape_cooldown_active" blocked_state = blocked_payload["error"]["details"]["safety_state"] _assert_safety_state_contract(blocked_state) assert blocked_state["cooldown_active"] is True assert blocked_state["cooldown_reason"] == "blocked_failure_threshold_exceeded" assert int(blocked_state["counters"]["blocked_start_count"]) >= 1 runs_response = client.get("/api/v1/runs") assert runs_response.status_code == 200 _assert_safety_state_contract(runs_response.json()["data"]["safety_state"]) assert runs_response.json()["data"]["safety_state"]["cooldown_active"] is True finally: object.__setattr__(settings, "ingestion_alert_blocked_failure_threshold", previous_blocked_threshold) object.__setattr__(settings, "ingestion_safety_cooldown_blocked_seconds", previous_blocked_cooldown_seconds) app.dependency_overrides.pop(get_scholar_source, None) @pytest.mark.integration @pytest.mark.db @pytest.mark.asyncio async def test_api_manual_run_enforces_network_failure_safety_cooldown( db_session: AsyncSession, ) -> None: user_id = await insert_user( db_session, email="api-runs-safety-network@example.com", password="api-password", ) await db_session.execute( text( """ INSERT INTO scholar_profiles (user_id, scholar_id, display_name, is_enabled) VALUES (:user_id, :scholar_id, :display_name, true) """ ), { "user_id": user_id, "scholar_id": "NNN111NNN111", "display_name": "Network Safety Probe", }, ) await db_session.commit() class NetworkFailureScholarSource: async def fetch_profile_page_html( self, scholar_id: str, *, cstart: int, pagesize: int, ) -> FetchResult: _ = (scholar_id, cstart, pagesize) return FetchResult( requested_url="https://scholar.google.com/citations?hl=en&user=NNN111NNN111", status_code=None, final_url=None, body="", error="timed out", ) async def fetch_profile_html(self, scholar_id: str) -> FetchResult: _ = scholar_id return await self.fetch_profile_page_html( "NNN111NNN111", cstart=0, pagesize=settings.ingestion_page_size, ) app.dependency_overrides[get_scholar_source] = lambda: NetworkFailureScholarSource() previous_network_threshold = settings.ingestion_alert_network_failure_threshold previous_network_cooldown_seconds = settings.ingestion_safety_cooldown_network_seconds previous_network_retries = settings.ingestion_network_error_retries previous_retry_backoff = settings.ingestion_retry_backoff_seconds object.__setattr__(settings, "ingestion_alert_network_failure_threshold", 1) object.__setattr__(settings, "ingestion_safety_cooldown_network_seconds", 600) object.__setattr__(settings, "ingestion_network_error_retries", 0) object.__setattr__(settings, "ingestion_retry_backoff_seconds", 0.0) try: client = TestClient(app) login_user(client, email="api-runs-safety-network@example.com", password="api-password") headers = _api_csrf_headers(client) first_run_response = client.post( "/api/v1/runs/manual", headers={**headers, "Idempotency-Key": "safety-network-cooldown-run-1"}, ) assert first_run_response.status_code == 200 settings_response = client.get("/api/v1/settings") assert settings_response.status_code == 200 safety_state = settings_response.json()["data"]["safety_state"] _assert_safety_state_contract(safety_state) assert safety_state["cooldown_active"] is True assert safety_state["cooldown_reason"] == "network_failure_threshold_exceeded" assert int(safety_state["cooldown_remaining_seconds"]) > 0 assert int(safety_state["counters"]["last_network_failure_count"]) >= 1 blocked_start_response = client.post( "/api/v1/runs/manual", headers={**headers, "Idempotency-Key": "safety-network-cooldown-run-2"}, ) assert blocked_start_response.status_code == 429 blocked_payload = blocked_start_response.json() assert blocked_payload["error"]["code"] == "scrape_cooldown_active" blocked_state = blocked_payload["error"]["details"]["safety_state"] _assert_safety_state_contract(blocked_state) assert blocked_state["cooldown_active"] is True assert blocked_state["cooldown_reason"] == "network_failure_threshold_exceeded" assert int(blocked_state["counters"]["blocked_start_count"]) >= 1 finally: object.__setattr__(settings, "ingestion_alert_network_failure_threshold", previous_network_threshold) object.__setattr__(settings, "ingestion_safety_cooldown_network_seconds", previous_network_cooldown_seconds) object.__setattr__(settings, "ingestion_network_error_retries", previous_network_retries) object.__setattr__(settings, "ingestion_retry_backoff_seconds", previous_retry_backoff) app.dependency_overrides.pop(get_scholar_source, None) @pytest.mark.integration @pytest.mark.db @pytest.mark.asyncio async def test_api_settings_clears_expired_scrape_safety_cooldown( db_session: AsyncSession, ) -> None: user_id = await insert_user( db_session, email="api-settings-safety-expired@example.com", password="api-password", ) user_settings = await user_settings_service.get_or_create_settings( db_session, user_id=user_id, ) user_settings.scrape_safety_state = {"blocked_start_count": 2} user_settings.scrape_cooldown_reason = "blocked_failure_threshold_exceeded" user_settings.scrape_cooldown_until = datetime.now(timezone.utc) - timedelta(seconds=15) await db_session.commit() client = TestClient(app) login_user(client, email="api-settings-safety-expired@example.com", password="api-password") settings_response = client.get("/api/v1/settings") assert settings_response.status_code == 200 settings_safety_state = settings_response.json()["data"]["safety_state"] _assert_safety_state_contract(settings_safety_state) assert settings_safety_state["cooldown_active"] is False assert settings_safety_state["cooldown_reason"] is None assert int(settings_safety_state["counters"]["blocked_start_count"]) == 2 runs_response = client.get("/api/v1/runs") assert runs_response.status_code == 200 runs_safety_state = runs_response.json()["data"]["safety_state"] _assert_safety_state_contract(runs_safety_state) assert runs_safety_state["cooldown_active"] is False assert runs_safety_state["cooldown_reason"] is None @pytest.mark.integration @pytest.mark.db @pytest.mark.asyncio async def test_api_publications_list_and_mark_read(db_session: AsyncSession) -> None: user_id = await insert_user( db_session, email="api-pubs@example.com", password="api-password", ) scholar_result = await db_session.execute( text( """ INSERT INTO scholar_profiles (user_id, scholar_id, display_name, is_enabled) VALUES (:user_id, :scholar_id, :display_name, true) RETURNING id """ ), { "user_id": user_id, "scholar_id": "abcDEF123456", "display_name": "Publication Owner", }, ) scholar_profile_id = int(scholar_result.scalar_one()) publication_a = await db_session.execute( text( """ INSERT INTO publications ( fingerprint_sha256, title_raw, title_normalized, citation_count, pdf_url ) VALUES (:fingerprint, :title_raw, :title_normalized, 10, :pdf_url) RETURNING id """ ), { "fingerprint": f"{user_id:064x}", "title_raw": "Paper A", "title_normalized": "paper a", "pdf_url": "https://example.org/paper-a.pdf", }, ) publication_a_id = int(publication_a.scalar_one()) publication_b = await db_session.execute( text( """ INSERT INTO publications (fingerprint_sha256, title_raw, title_normalized, citation_count) VALUES (:fingerprint, :title_raw, :title_normalized, 4) RETURNING id """ ), { "fingerprint": f"{(user_id + 1):064x}", "title_raw": "Paper B", "title_normalized": "paper b", }, ) publication_b_id = int(publication_b.scalar_one()) await db_session.execute( text( """ INSERT INTO scholar_publications (scholar_profile_id, publication_id, is_read) VALUES (:scholar_profile_id, :publication_a_id, false), (:scholar_profile_id, :publication_b_id, false) """ ), { "scholar_profile_id": scholar_profile_id, "publication_a_id": publication_a_id, "publication_b_id": publication_b_id, }, ) await db_session.commit() client = TestClient(app) login_user(client, email="api-pubs@example.com", password="api-password") headers = _api_csrf_headers(client) list_response = client.get("/api/v1/publications?mode=all") assert list_response.status_code == 200 data = list_response.json()["data"] assert data["mode"] == "all" assert data["favorite_only"] is False assert data["total_count"] == 2 assert data["unread_count"] == 2 assert data["favorites_count"] == 0 assert data["latest_count"] == 0 assert data["new_count"] == data["latest_count"] assert data["page"] == 1 assert data["page_size"] == 100 assert data["has_prev"] is False assert data["has_next"] is False assert isinstance(data["publications"], list) assert len(data["publications"]) == 2 assert all(bool(item["is_favorite"]) is False for item in data["publications"]) pdf_urls = {item["title"]: item["pdf_url"] for item in data["publications"]} assert pdf_urls["Paper A"] == "https://example.org/paper-a.pdf" assert pdf_urls["Paper B"] is None latest_response = client.get("/api/v1/publications?mode=latest") assert latest_response.status_code == 200 latest_data = latest_response.json()["data"] assert latest_data["mode"] == "latest" assert latest_data["latest_count"] == 0 unread_response = client.get("/api/v1/publications?mode=unread") assert unread_response.status_code == 200 unread_data = unread_response.json()["data"] assert unread_data["mode"] == "unread" assert unread_data["unread_count"] == 2 alias_response = client.get("/api/v1/publications?mode=new") assert alias_response.status_code == 200 alias_data = alias_response.json()["data"] assert alias_data["mode"] == "latest" assert alias_data["latest_count"] == latest_data["latest_count"] assert alias_data["publications"] == latest_data["publications"] mark_selected_response = client.post( "/api/v1/publications/mark-read", json={ "selections": [ { "scholar_profile_id": scholar_profile_id, "publication_id": publication_a_id, } ] }, headers=headers, ) assert mark_selected_response.status_code == 200 assert mark_selected_response.json()["data"]["requested_count"] == 1 assert mark_selected_response.json()["data"]["updated_count"] == 1 read_state = await db_session.execute( text( """ SELECT publication_id, is_read FROM scholar_publications WHERE scholar_profile_id = :scholar_profile_id ORDER BY publication_id """ ), {"scholar_profile_id": scholar_profile_id}, ) states = {int(row[0]): bool(row[1]) for row in read_state.all()} assert states[publication_a_id] is True assert states[publication_b_id] is False mark_response = client.post("/api/v1/publications/mark-all-read", headers=headers) assert mark_response.status_code == 200 assert mark_response.json()["data"]["updated_count"] == 1 @pytest.mark.integration @pytest.mark.db @pytest.mark.asyncio async def test_api_publications_list_supports_pagination(db_session: AsyncSession) -> None: user_id = await insert_user( db_session, email="api-pubs-paging@example.com", password="api-password", ) scholar_result = await db_session.execute( text( """ INSERT INTO scholar_profiles (user_id, scholar_id, display_name, is_enabled) VALUES (:user_id, :scholar_id, :display_name, true) RETURNING id """ ), { "user_id": user_id, "scholar_id": "pagingScholar01", "display_name": "Paging Scholar", }, ) scholar_profile_id = int(scholar_result.scalar_one()) publication_ids: list[int] = [] for index in range(3): created = await db_session.execute( text( """ INSERT INTO publications (fingerprint_sha256, title_raw, title_normalized, citation_count) VALUES (:fingerprint, :title_raw, :title_normalized, 1) RETURNING id """ ), { "fingerprint": f"{(user_id + 500 + index):064x}", "title_raw": f"Paged Paper {index}", "title_normalized": f"paged paper {index}", }, ) publication_ids.append(int(created.scalar_one())) await db_session.execute( text( """ INSERT INTO scholar_publications (scholar_profile_id, publication_id, is_read, is_favorite) VALUES (:scholar_profile_id, :publication_1, false, false), (:scholar_profile_id, :publication_2, false, false), (:scholar_profile_id, :publication_3, false, false) """ ), { "scholar_profile_id": scholar_profile_id, "publication_1": publication_ids[0], "publication_2": publication_ids[1], "publication_3": publication_ids[2], }, ) await db_session.commit() client = TestClient(app) login_user(client, email="api-pubs-paging@example.com", password="api-password") first_page = client.get("/api/v1/publications?mode=all&page=1&page_size=2") assert first_page.status_code == 200 first_data = first_page.json()["data"] assert first_data["total_count"] == 3 assert first_data["page"] == 1 assert first_data["page_size"] == 2 assert first_data["has_prev"] is False assert first_data["has_next"] is True assert len(first_data["publications"]) == 2 second_page = client.get("/api/v1/publications?mode=all&page=2&page_size=2") assert second_page.status_code == 200 second_data = second_page.json()["data"] assert second_data["total_count"] == 3 assert second_data["page"] == 2 assert second_data["page_size"] == 2 assert second_data["has_prev"] is True assert second_data["has_next"] is False assert len(second_data["publications"]) == 1 @pytest.mark.integration @pytest.mark.db @pytest.mark.asyncio async def test_api_publications_favorite_toggle_and_filter(db_session: AsyncSession) -> None: user_id = await insert_user( db_session, email="api-pubs-favorites@example.com", password="api-password", ) scholar_result = await db_session.execute( text( """ INSERT INTO scholar_profiles (user_id, scholar_id, display_name, is_enabled) VALUES (:user_id, :scholar_id, :display_name, true) RETURNING id """ ), { "user_id": user_id, "scholar_id": "favoriteScholar01", "display_name": "Favorite Scholar", }, ) scholar_profile_id = int(scholar_result.scalar_one()) publication_a_result = await db_session.execute( text( """ INSERT INTO publications (fingerprint_sha256, title_raw, title_normalized, citation_count) VALUES (:fingerprint, :title_raw, :title_normalized, 9) RETURNING id """ ), { "fingerprint": f"{(user_id + 101):064x}", "title_raw": "Favorite Target", "title_normalized": "favorite target", }, ) publication_a_id = int(publication_a_result.scalar_one()) publication_b_result = await db_session.execute( text( """ INSERT INTO publications (fingerprint_sha256, title_raw, title_normalized, citation_count) VALUES (:fingerprint, :title_raw, :title_normalized, 2) RETURNING id """ ), { "fingerprint": f"{(user_id + 102):064x}", "title_raw": "Not Favorite", "title_normalized": "not favorite", }, ) publication_b_id = int(publication_b_result.scalar_one()) await db_session.execute( text( """ INSERT INTO scholar_publications (scholar_profile_id, publication_id, is_read, is_favorite) VALUES (:scholar_profile_id, :publication_a_id, false, false), (:scholar_profile_id, :publication_b_id, false, false) """ ), { "scholar_profile_id": scholar_profile_id, "publication_a_id": publication_a_id, "publication_b_id": publication_b_id, }, ) await db_session.commit() client = TestClient(app) login_user(client, email="api-pubs-favorites@example.com", password="api-password") headers = _api_csrf_headers(client) set_response = client.post( f"/api/v1/publications/{publication_a_id}/favorite", json={"scholar_profile_id": scholar_profile_id, "is_favorite": True}, headers=headers, ) assert set_response.status_code == 200 set_data = set_response.json()["data"] assert set_data["publication"]["publication_id"] == publication_a_id assert set_data["publication"]["is_favorite"] is True favorite_only_response = client.get("/api/v1/publications?mode=all&favorite_only=true") assert favorite_only_response.status_code == 200 favorite_only_data = favorite_only_response.json()["data"] assert favorite_only_data["favorite_only"] is True assert favorite_only_data["favorites_count"] == 1 assert favorite_only_data["total_count"] == 1 assert len(favorite_only_data["publications"]) == 1 assert int(favorite_only_data["publications"][0]["publication_id"]) == publication_a_id clear_response = client.post( f"/api/v1/publications/{publication_a_id}/favorite", json={"scholar_profile_id": scholar_profile_id, "is_favorite": False}, headers=headers, ) assert clear_response.status_code == 200 clear_data = clear_response.json()["data"] assert clear_data["publication"]["publication_id"] == publication_a_id assert clear_data["publication"]["is_favorite"] is False favorite_only_after_clear_response = client.get("/api/v1/publications?mode=all&favorite_only=true") assert favorite_only_after_clear_response.status_code == 200 favorite_only_after_clear_data = favorite_only_after_clear_response.json()["data"] assert favorite_only_after_clear_data["favorites_count"] == 0 assert favorite_only_after_clear_data["total_count"] == 0 assert favorite_only_after_clear_data["publications"] == [] favorite_state_result = await db_session.execute( text( """ SELECT is_favorite FROM scholar_publications WHERE scholar_profile_id = :scholar_profile_id AND publication_id = :publication_id """ ), { "scholar_profile_id": scholar_profile_id, "publication_id": publication_a_id, }, ) assert bool(favorite_state_result.scalar_one()) is False @pytest.mark.integration @pytest.mark.db @pytest.mark.asyncio async def test_api_publications_list_schedules_background_enrichment( db_session: AsyncSession, monkeypatch: pytest.MonkeyPatch, ) -> None: user_id = await insert_user( db_session, email="api-pubs-bg@example.com", password="api-password", ) scholar_result = await db_session.execute( text( """ INSERT INTO scholar_profiles (user_id, scholar_id, display_name, is_enabled) VALUES (:user_id, :scholar_id, :display_name, true) RETURNING id """ ), { "user_id": user_id, "scholar_id": "background111", "display_name": "Background Scholar", }, ) scholar_profile_id = int(scholar_result.scalar_one()) publication_result = await db_session.execute( text( """ INSERT INTO publications (fingerprint_sha256, title_raw, title_normalized, citation_count) VALUES (:fingerprint, :title_raw, :title_normalized, 3) RETURNING id """ ), { "fingerprint": f"{(user_id + 17):064x}", "title_raw": "Background Target", "title_normalized": "background target", }, ) publication_id = int(publication_result.scalar_one()) await db_session.execute( text( """ INSERT INTO scholar_publications (scholar_profile_id, publication_id, is_read) VALUES (:scholar_profile_id, :publication_id, false) """ ), { "scholar_profile_id": scholar_profile_id, "publication_id": publication_id, }, ) await db_session.commit() async def _fake_schedule(db_session, *, user_id: int, request_email: str | None, items, max_items: int): assert db_session is not None assert user_id > 0 assert request_email == "api-pubs-bg@example.com" assert int(max_items) > 0 assert any(int(item.publication_id) == publication_id for item in items) return 1 monkeypatch.setattr( "app.services.domains.publications.application.schedule_missing_pdf_enrichment_for_user", _fake_schedule, ) client = TestClient(app) login_user(client, email="api-pubs-bg@example.com", password="api-password") response = client.get("/api/v1/publications?mode=all") assert response.status_code == 200 data = response.json()["data"] assert len(data["publications"]) == 1 assert int(data["publications"][0]["publication_id"]) == publication_id assert data["publications"][0]["pdf_url"] is None assert data["publications"][0]["pdf_status"] in {"untracked", "queued", "running", "failed"} @pytest.mark.integration @pytest.mark.db @pytest.mark.asyncio async def test_api_publication_retry_pdf_queues_resolution_job( db_session: AsyncSession, monkeypatch: pytest.MonkeyPatch, ) -> None: user_id = await insert_user( db_session, email="api-pubs-retry@example.com", password="api-password", ) scholar_result = await db_session.execute( text( """ INSERT INTO scholar_profiles (user_id, scholar_id, display_name, is_enabled) VALUES (:user_id, :scholar_id, :display_name, true) RETURNING id """ ), { "user_id": user_id, "scholar_id": "retryScholar01", "display_name": "Retry Scholar", }, ) scholar_profile_id = int(scholar_result.scalar_one()) publication_result = await db_session.execute( text( """ INSERT INTO publications (fingerprint_sha256, title_raw, title_normalized, citation_count) VALUES (:fingerprint, :title_raw, :title_normalized, 7) RETURNING id """ ), { "fingerprint": f"{(user_id + 9):064x}", "title_raw": "Retry Target", "title_normalized": "retry target", }, ) publication_id = int(publication_result.scalar_one()) await db_session.execute( text( """ INSERT INTO scholar_publications (scholar_profile_id, publication_id, is_read) VALUES (:scholar_profile_id, :publication_id, false) """ ), { "scholar_profile_id": scholar_profile_id, "publication_id": publication_id, }, ) await db_session.commit() async def _fake_retry_scheduler(db_session, *, user_id: int, request_email: str | None, item: PublicationListItem): assert db_session is not None assert user_id > 0 assert request_email == "api-pubs-retry@example.com" assert int(item.publication_id) == publication_id return True monkeypatch.setattr( "app.services.domains.publications.application.schedule_retry_pdf_enrichment_for_row", _fake_retry_scheduler, ) async def _fake_hydrate(db_session, *, items: list[PublicationListItem]): assert db_session is not None assert len(items) == 1 item = items[0] return [ PublicationListItem( publication_id=item.publication_id, scholar_profile_id=item.scholar_profile_id, scholar_label=item.scholar_label, title=item.title, year=item.year, citation_count=item.citation_count, venue_text=item.venue_text, pub_url=item.pub_url, doi=item.doi, pdf_url=item.pdf_url, is_read=item.is_read, first_seen_at=item.first_seen_at, is_new_in_latest_run=item.is_new_in_latest_run, pdf_status="queued", pdf_attempt_count=1, pdf_failure_reason="no_pdf_found", pdf_failure_detail="no_pdf_found", ) ] monkeypatch.setattr( "app.services.domains.publications.application.hydrate_pdf_enrichment_state", _fake_hydrate, ) client = TestClient(app) login_user(client, email="api-pubs-retry@example.com", password="api-password") headers = _api_csrf_headers(client) response = client.post( f"/api/v1/publications/{publication_id}/retry-pdf", json={"scholar_profile_id": scholar_profile_id}, headers=headers, ) assert response.status_code == 200 payload = response.json()["data"] assert payload["queued"] is True assert payload["resolved_pdf"] is False assert payload["publication"]["publication_id"] == publication_id assert payload["publication"]["pdf_url"] is None assert payload["publication"]["pdf_status"] == "queued" assert payload["publication"]["pdf_attempt_count"] == 1 assert payload["publication"]["pdf_failure_reason"] == "no_pdf_found" stored = await db_session.execute( text("SELECT doi, pdf_url FROM publications WHERE id = :publication_id"), {"publication_id": publication_id}, ) stored_doi, stored_pdf_url = stored.one() assert stored_doi is None assert stored_pdf_url is None @pytest.mark.integration @pytest.mark.db @pytest.mark.asyncio async def test_api_publications_unread_and_latest_modes_can_diverge( db_session: AsyncSession, ) -> None: user_id = await insert_user( db_session, email="api-pubs-mismatch@example.com", password="api-password", ) scholar_result = await db_session.execute( text( """ INSERT INTO scholar_profiles (user_id, scholar_id, display_name, is_enabled) VALUES (:user_id, :scholar_id, :display_name, true) RETURNING id """ ), { "user_id": user_id, "scholar_id": "mismatchScholar01", "display_name": "Mismatch Scholar", }, ) scholar_profile_id = int(scholar_result.scalar_one()) older_run_result = await db_session.execute( text( """ INSERT INTO crawl_runs (user_id, trigger_type, status, scholar_count, new_pub_count) VALUES (:user_id, 'manual', 'success', 1, 1) RETURNING id """ ), {"user_id": user_id}, ) older_run_id = int(older_run_result.scalar_one()) latest_run_result = await db_session.execute( text( """ INSERT INTO crawl_runs (user_id, trigger_type, status, scholar_count, new_pub_count) VALUES (:user_id, 'scheduled', 'success', 1, 0) RETURNING id """ ), {"user_id": user_id}, ) latest_run_id = int(latest_run_result.scalar_one()) assert latest_run_id > older_run_id publication_result = await db_session.execute( text( """ INSERT INTO publications (fingerprint_sha256, title_raw, title_normalized, citation_count) VALUES (:fingerprint, :title_raw, :title_normalized, 12) RETURNING id """ ), { "fingerprint": f"{(user_id + 99):064x}", "title_raw": "Unread But Not Latest", "title_normalized": "unread but not latest", }, ) publication_id = int(publication_result.scalar_one()) await db_session.execute( text( """ INSERT INTO scholar_publications ( scholar_profile_id, publication_id, is_read, first_seen_run_id ) VALUES (:scholar_profile_id, :publication_id, false, :first_seen_run_id) """ ), { "scholar_profile_id": scholar_profile_id, "publication_id": publication_id, "first_seen_run_id": older_run_id, }, ) await db_session.commit() client = TestClient(app) login_user(client, email="api-pubs-mismatch@example.com", password="api-password") unread_response = client.get("/api/v1/publications?mode=unread") assert unread_response.status_code == 200 unread_data = unread_response.json()["data"] assert unread_data["mode"] == "unread" assert unread_data["unread_count"] == 1 assert unread_data["latest_count"] == 0 assert unread_data["new_count"] == 0 assert len(unread_data["publications"]) == 1 assert int(unread_data["publications"][0]["publication_id"]) == publication_id assert unread_data["publications"][0]["is_new_in_latest_run"] is False latest_response = client.get("/api/v1/publications?mode=latest") assert latest_response.status_code == 200 latest_data = latest_response.json()["data"] assert latest_data["mode"] == "latest" assert latest_data["latest_count"] == 0 assert len(latest_data["publications"]) == 0 alias_response = client.get("/api/v1/publications?mode=new") assert alias_response.status_code == 200 alias_data = alias_response.json()["data"] assert alias_data["mode"] == "latest" assert alias_data["latest_count"] == latest_data["latest_count"] assert alias_data["publications"] == latest_data["publications"]