scholarr/tests/integration/test_api_v1.py

from __future__ import annotations

import asyncio
from datetime import UTC, datetime, timedelta
from pathlib import Path

import pytest
from fastapi.testclient import TestClient
from sqlalchemy import text
from sqlalchemy.ext.asyncio import AsyncSession

from app.api.runtime_deps import get_scholar_source
from app.main import app
from app.services.publications.types import PublicationListItem
from app.services.scholar.source import FetchResult
from app.services.settings import application as user_settings_service
from app.settings import settings
from tests.integration.helpers import insert_user, login_user

REGRESSION_FIXTURE_DIR = Path("tests/fixtures/scholar/regression")
SAFETY_STATE_KEYS = {
    "cooldown_active",
    "cooldown_reason",
    "cooldown_reason_label",
    "cooldown_until",
    "cooldown_remaining_seconds",
    "recommended_action",
    "counters",
}
SAFETY_COUNTER_KEYS = {
    "consecutive_blocked_runs",
    "consecutive_network_runs",
    "cooldown_entry_count",
    "blocked_start_count",
    "last_blocked_failure_count",
    "last_network_failure_count",
    "last_evaluated_run_id",
}


def _api_bootstrap_csrf_headers(client: TestClient) -> dict[str, str]:
    bootstrap_response = client.get("/api/v1/auth/csrf")
    assert bootstrap_response.status_code == 200
    payload = bootstrap_response.json()["data"]
    token = payload["csrf_token"]
    assert isinstance(token, str) and token
    return {"X-CSRF-Token": token}


def _api_csrf_headers(client: TestClient) -> dict[str, str]:
    me_response = client.get("/api/v1/auth/me")
    assert me_response.status_code == 200
    body = me_response.json()
    token = body["data"]["csrf_token"]
    assert isinstance(token, str) and token
    return {"X-CSRF-Token": token}


def _regression_fixture(name: str) -> str:
    return (REGRESSION_FIXTURE_DIR / name).read_text(encoding="utf-8")


def _assert_safety_state_contract(payload: dict[str, object]) -> None:
    assert set(payload.keys()) == SAFETY_STATE_KEYS
    counters = payload["counters"]
    assert isinstance(counters, dict)
    assert set(counters.keys()) == SAFETY_COUNTER_KEYS


_ACTIVE_RUN_STATUSES = {"running", "resolving"}


async def _wait_for_run_complete(
    client: TestClient,
    run_id: int,
    *,
    max_retries: int = 300,
    poll_interval: float = 0.2,
) -> dict:
    """Poll GET /api/v1/runs/{run_id} until the run is in a terminal state.

    Returns the final run detail data dict.
    """
    for _ in range(max_retries):
        await asyncio.sleep(poll_interval)
        r = client.get(f"/api/v1/runs/{run_id}")
        assert r.status_code == 200
        data = r.json()["data"]
        if data["run"]["status"] not in _ACTIVE_RUN_STATUSES:
            return data
    r = client.get(f"/api/v1/runs/{run_id}")
    return r.json()["data"]


async def _seed_publication_link_for_user(
    db_session: AsyncSession,
    *,
    user_id: int,
    scholar_id: str,
    title: str,
    fingerprint: str,
) -> tuple[int, int]:
    scholar_result = await db_session.execute(
        text(
            """
            INSERT INTO scholar_profiles (user_id, scholar_id, display_name, is_enabled)
            VALUES (:user_id, :scholar_id, :display_name, true)
            RETURNING id
            """
        ),
        {"user_id": user_id, "scholar_id": scholar_id, "display_name": scholar_id},
    )
    scholar_profile_id = int(scholar_result.scalar_one())
    publication_result = await db_session.execute(
        text(
            """
            INSERT INTO publications (fingerprint_sha256, title_raw, title_normalized, citation_count)
            VALUES (:fingerprint, :title_raw, :title_normalized, 4)
            RETURNING id
            """
        ),
        {"fingerprint": fingerprint, "title_raw": title, "title_normalized": title.lower()},
    )
    publication_id = int(publication_result.scalar_one())
    await db_session.execute(
        text(
            """
            INSERT INTO scholar_publications (scholar_profile_id, publication_id, is_read)
            VALUES (:scholar_profile_id, :publication_id, false)
            """
        ),
        {"scholar_profile_id": scholar_profile_id, "publication_id": publication_id},
    )
    await db_session.commit()
    return scholar_profile_id, publication_id


@pytest.mark.integration
@pytest.mark.db
@pytest.mark.asyncio
async def test_api_me_requires_authentication() -> None:
    client = TestClient(app)

    response = client.get("/api/v1/auth/me")

    assert response.status_code == 401
    payload = response.json()
    assert payload["error"]["code"] == "auth_required"
    assert payload["error"]["message"] == "Authentication required."
    assert "request_id" in payload["meta"]


@pytest.mark.integration
@pytest.mark.db
@pytest.mark.asyncio
async def test_api_csrf_bootstrap_returns_token_for_anonymous_and_authenticated(
    db_session: AsyncSession,
) -> None:
    await insert_user(
        db_session,
        email="api-bootstrap@example.com",
        password="api-password",
    )
    client = TestClient(app)

    anonymous_response = client.get("/api/v1/auth/csrf")
    assert anonymous_response.status_code == 200
    anonymous_payload = anonymous_response.json()["data"]
    assert isinstance(anonymous_payload["csrf_token"], str)
    assert anonymous_payload["authenticated"] is False

    login_user(client, email="api-bootstrap@example.com", password="api-password")
    authenticated_response = client.get("/api/v1/auth/csrf")
    assert authenticated_response.status_code == 200
    authenticated_payload = authenticated_response.json()["data"]
    assert isinstance(authenticated_payload["csrf_token"], str)
    assert authenticated_payload["authenticated"] is True


@pytest.mark.integration
@pytest.mark.db
@pytest.mark.asyncio
async def test_api_me_returns_user_and_csrf_token(db_session: AsyncSession) -> None:
    await insert_user(
        db_session,
        email="api-me@example.com",
        password="api-password",
    )

    client = TestClient(app)
    login_user(client, email="api-me@example.com", password="api-password")

    response = client.get("/api/v1/auth/me")

    assert response.status_code == 200
    payload = response.json()
    assert payload["data"]["authenticated"] is True
    assert payload["data"]["user"]["email"] == "api-me@example.com"
    assert isinstance(payload["data"]["csrf_token"], str)
    assert payload["meta"]["request_id"]


@pytest.mark.integration
@pytest.mark.db
@pytest.mark.asyncio
async def test_api_login_and_change_password_flow(db_session: AsyncSession) -> None:
    await insert_user(
        db_session,
        email="api-login@example.com",
        password="old-password",
    )
    client = TestClient(app)

    missing_csrf = client.post(
        "/api/v1/auth/login",
        json={"email": "api-login@example.com", "password": "old-password"},
    )
    assert missing_csrf.status_code == 403
    assert missing_csrf.json()["error"]["code"] == "csrf_missing"

    login_headers = _api_bootstrap_csrf_headers(client)
    login_response = client.post(
        "/api/v1/auth/login",
        json={"email": "api-login@example.com", "password": "old-password"},
        headers=login_headers,
    )
    assert login_response.status_code == 200
    login_payload = login_response.json()["data"]
    assert login_payload["authenticated"] is True
    assert login_payload["user"]["email"] == "api-login@example.com"
    assert isinstance(login_payload["csrf_token"], str)

    bad_change_response = client.post(
        "/api/v1/auth/change-password",
        json={
            "current_password": "not-correct",
            "new_password": "new-password",
            "confirm_password": "new-password",
        },
        headers={"X-CSRF-Token": login_payload["csrf_token"]},
    )
    assert bad_change_response.status_code == 400
    assert bad_change_response.json()["error"]["code"] == "invalid_current_password"

    change_response = client.post(
        "/api/v1/auth/change-password",
        json={
            "current_password": "old-password",
            "new_password": "new-password",
            "confirm_password": "new-password",
        },
        headers={"X-CSRF-Token": login_payload["csrf_token"]},
    )
    assert change_response.status_code == 200
    assert change_response.json()["data"]["message"] == "Password updated successfully."

    logout_response = client.post(
        "/api/v1/auth/logout",
        headers={"X-CSRF-Token": login_payload["csrf_token"]},
    )
    assert logout_response.status_code == 200

    relogin_headers = _api_bootstrap_csrf_headers(client)
    old_password_login = client.post(
        "/api/v1/auth/login",
        json={"email": "api-login@example.com", "password": "old-password"},
        headers=relogin_headers,
    )
    assert old_password_login.status_code == 401
    assert old_password_login.json()["error"]["code"] == "invalid_credentials"

    fresh_headers = _api_bootstrap_csrf_headers(client)
    new_password_login = client.post(
        "/api/v1/auth/login",
        json={"email": "api-login@example.com", "password": "new-password"},
        headers=fresh_headers,
    )
    assert new_password_login.status_code == 200
    assert new_password_login.json()["data"]["authenticated"] is True


@pytest.mark.integration
@pytest.mark.db
@pytest.mark.asyncio
async def test_api_admin_user_management_endpoints(db_session: AsyncSession) -> None:
    admin_user_id = await insert_user(
        db_session,
        email="api-admin@example.com",
        password="admin-password",
        is_admin=True,
    )
    target_user_id = await insert_user(
        db_session,
        email="api-target@example.com",
        password="target-password",
        is_admin=False,
    )
    await insert_user(
        db_session,
        email="api-member@example.com",
        password="member-password",
        is_admin=False,
    )

    client = TestClient(app)
    login_user(client, email="api-admin@example.com", password="admin-password")
    headers = _api_csrf_headers(client)

    list_response = client.get("/api/v1/admin/users")
    assert list_response.status_code == 200
    users = list_response.json()["data"]["users"]
    assert any(item["email"] == "api-target@example.com" for item in users)

    create_response = client.post(
        "/api/v1/admin/users",
        json={
            "email": "api-created@example.com",
            "password": "created-password",
            "is_admin": True,
        },
        headers=headers,
    )
    assert create_response.status_code == 201
    created_user = create_response.json()["data"]
    assert created_user["email"] == "api-created@example.com"
    created_user_id = int(created_user["id"])

    deactivate_response = client.patch(
        f"/api/v1/admin/users/{target_user_id}/active",
        json={"is_active": False},
        headers=headers,
    )
    assert deactivate_response.status_code == 200
    assert deactivate_response.json()["data"]["is_active"] is False

    reactivate_response = client.patch(
        f"/api/v1/admin/users/{target_user_id}/active",
        json={"is_active": True},
        headers=headers,
    )
    assert reactivate_response.status_code == 200
    assert reactivate_response.json()["data"]["is_active"] is True

    reset_response = client.post(
        f"/api/v1/admin/users/{target_user_id}/reset-password",
        json={"new_password": "target-password-updated"},
        headers=headers,
    )
    assert reset_response.status_code == 200
    assert "Password reset" in reset_response.json()["data"]["message"]

    self_deactivate = client.patch(
        f"/api/v1/admin/users/{admin_user_id}/active",
        json={"is_active": False},
        headers=headers,
    )
    assert self_deactivate.status_code == 400
    assert self_deactivate.json()["error"]["code"] == "cannot_deactivate_self"

    logout_response = client.post("/api/v1/auth/logout", headers=headers)
    assert logout_response.status_code == 200

    target_headers = _api_bootstrap_csrf_headers(client)
    target_login = client.post(
        "/api/v1/auth/login",
        json={"email": "api-target@example.com", "password": "target-password-updated"},
        headers=target_headers,
    )
    assert target_login.status_code == 200

    non_admin_headers = {"X-CSRF-Token": target_login.json()["data"]["csrf_token"]}
    forbidden_response = client.get("/api/v1/admin/users")
    assert forbidden_response.status_code == 403
    assert forbidden_response.json()["error"]["code"] == "forbidden"

    forbidden_create = client.post(
        "/api/v1/admin/users",
        json={
            "email": "should-not-work@example.com",
            "password": "password-123",
            "is_admin": False,
        },
        headers=non_admin_headers,
    )
    assert forbidden_create.status_code == 403

    created_exists = await db_session.execute(
        text("SELECT COUNT(*) FROM users WHERE id = :user_id"),
        {"user_id": created_user_id},
    )
    assert created_exists.scalar_one() == 1


@pytest.mark.integration
@pytest.mark.db
@pytest.mark.asyncio
async def test_api_admin_dbops_integrity_and_repair_flow(db_session: AsyncSession) -> None:
    await insert_user(db_session, email="api-admin-dbops@example.com", password="admin-password", is_admin=True)
    target_user_id = await insert_user(db_session, email="api-dbops-target@example.com", password="target-password")
    _scholar_id, publication_id = await _seed_publication_link_for_user(
        db_session,
        user_id=target_user_id,
        scholar_id="dbopsTarget01",
        title="DB Ops Target Paper",
        fingerprint=f"{(target_user_id + 31):064x}",
    )
    client = TestClient(app)
    login_user(client, email="api-admin-dbops@example.com", password="admin-password")
    headers = _api_csrf_headers(client)

    integrity_response = client.get("/api/v1/admin/db/integrity")
    assert integrity_response.status_code == 200
    integrity_payload = integrity_response.json()["data"]
    assert integrity_payload["status"] in {"ok", "warning", "failed"}
    assert any(check["name"] == "missing_pdf_url" for check in integrity_payload["checks"])

    repair_response = client.post(
        "/api/v1/admin/db/repairs/publication-links",
        json={"user_id": target_user_id, "dry_run": True},
        headers=headers,
    )
    assert repair_response.status_code == 200
    repair_data = repair_response.json()["data"]
    assert repair_data["status"] == "completed"
    assert bool(repair_data["summary"]["dry_run"]) is True
    job_id = int(repair_data["job_id"])

    jobs_response = client.get("/api/v1/admin/db/repair-jobs?limit=10")
    assert jobs_response.status_code == 200
    jobs = jobs_response.json()["data"]["jobs"]
    assert any(int(job["id"]) == job_id for job in jobs)

    pdf_queue_response = client.get("/api/v1/admin/db/pdf-queue?limit=20")
    assert pdf_queue_response.status_code == 200
    pdf_queue_payload = pdf_queue_response.json()["data"]
    assert isinstance(pdf_queue_payload["items"], list)
    assert int(pdf_queue_payload["page"]) == 1
    assert int(pdf_queue_payload["page_size"]) == 20
    assert int(pdf_queue_payload["total_count"]) >= 0
    assert isinstance(pdf_queue_payload["has_next"], bool)
    assert isinstance(pdf_queue_payload["has_prev"], bool)
    page_two_response = client.get("/api/v1/admin/db/pdf-queue?page=2&page_size=1")
    assert page_two_response.status_code == 200
    page_two_payload = page_two_response.json()["data"]
    assert int(page_two_payload["page"]) == 2
    assert int(page_two_payload["page_size"]) == 1
    assert page_two_payload["has_prev"] is True
    untracked_response = client.get("/api/v1/admin/db/pdf-queue?limit=20&status=untracked")
    assert untracked_response.status_code == 200
    assert any(item["status"] == "untracked" for item in untracked_response.json()["data"]["items"])

    link_count_result = await db_session.execute(
        text("SELECT count(*) FROM scholar_publications WHERE publication_id = :publication_id"),
        {"publication_id": publication_id},
    )
    assert int(link_count_result.scalar_one()) == 1


@pytest.mark.integration
@pytest.mark.db
@pytest.mark.asyncio
async def test_api_admin_dbops_pdf_queue_requeue_endpoint(
    db_session: AsyncSession,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    await insert_user(db_session, email="api-admin-pdf@example.com", password="admin-password", is_admin=True)
    target_user_id = await insert_user(db_session, email="api-pdf-target@example.com", password="target-password")
    _scholar_id, publication_id = await _seed_publication_link_for_user(
        db_session,
        user_id=target_user_id,
        scholar_id="dbopsPdf01",
        title="PDF Queue Target",
        fingerprint=f"{(target_user_id + 73):064x}",
    )
    monkeypatch.setattr(
        "app.services.publications.pdf_queue._schedule_rows",
        lambda **_kwargs: None,
    )
    client = TestClient(app)
    login_user(client, email="api-admin-pdf@example.com", password="admin-password")
    headers = _api_csrf_headers(client)

    first_response = client.post(
        f"/api/v1/admin/db/pdf-queue/{publication_id}/requeue",
        headers=headers,
    )
    assert first_response.status_code == 200
    first_data = first_response.json()["data"]
    assert first_data["publication_id"] == publication_id
    assert first_data["queued"] is True
    assert first_data["status"] == "queued"

    second_response = client.post(
        f"/api/v1/admin/db/pdf-queue/{publication_id}/requeue",
        headers=headers,
    )
    assert second_response.status_code == 200
    second_data = second_response.json()["data"]
    assert second_data["queued"] is False
    assert second_data["status"] == "blocked"


@pytest.mark.integration
@pytest.mark.db
@pytest.mark.asyncio
async def test_api_admin_dbops_pdf_queue_requeue_all_endpoint(
    db_session: AsyncSession,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    await insert_user(db_session, email="api-admin-pdf-all@example.com", password="admin-password", is_admin=True)
    target_user_id = await insert_user(db_session, email="api-pdf-all-target@example.com", password="target-password")
    _scholar_id, _publication_id = await _seed_publication_link_for_user(
        db_session,
        user_id=target_user_id,
        scholar_id="dbopsPdfAll01",
        title="PDF Queue All Target",
        fingerprint=f"{(target_user_id + 83):064x}",
    )
    monkeypatch.setattr(
        "app.services.publications.pdf_queue._schedule_rows",
        lambda **_kwargs: None,
    )
    client = TestClient(app)
    login_user(client, email="api-admin-pdf-all@example.com", password="admin-password")
    headers = _api_csrf_headers(client)

    response = client.post(
        "/api/v1/admin/db/pdf-queue/requeue-all?limit=500",
        headers=headers,
    )
    assert response.status_code == 200
    payload = response.json()["data"]
    assert int(payload["requested_count"]) >= 1
    assert int(payload["queued_count"]) >= 1


@pytest.mark.integration
@pytest.mark.db
@pytest.mark.asyncio
async def test_api_admin_dbops_forbidden_for_non_admin_and_validates_scope(db_session: AsyncSession) -> None:
    await insert_user(db_session, email="api-admin-dbops2@example.com", password="admin-password", is_admin=True)
    member_user_id = await insert_user(db_session, email="api-dbops-member@example.com", password="member-password")
    client = TestClient(app)
    login_user(client, email="api-dbops-member@example.com", password="member-password")
    headers = _api_csrf_headers(client)

    forbidden_integrity = client.get("/api/v1/admin/db/integrity")
    assert forbidden_integrity.status_code == 403
    assert forbidden_integrity.json()["error"]["code"] == "forbidden"

    forbidden_pdf_queue = client.get("/api/v1/admin/db/pdf-queue")
    assert forbidden_pdf_queue.status_code == 403
    assert forbidden_pdf_queue.json()["error"]["code"] == "forbidden"

    forbidden_requeue = client.post(
        "/api/v1/admin/db/pdf-queue/1/requeue",
        headers=headers,
    )
    assert forbidden_requeue.status_code == 403
    assert forbidden_requeue.json()["error"]["code"] == "forbidden"

    forbidden_requeue_all = client.post(
        "/api/v1/admin/db/pdf-queue/requeue-all?limit=100",
        headers=headers,
    )
    assert forbidden_requeue_all.status_code == 403
    assert forbidden_requeue_all.json()["error"]["code"] == "forbidden"

    forbidden_repair = client.post(
        "/api/v1/admin/db/repairs/publication-links",
        json={"user_id": member_user_id, "dry_run": True},
        headers=headers,
    )
    assert forbidden_repair.status_code == 403
    assert forbidden_repair.json()["error"]["code"] == "forbidden"

    forbidden_near_duplicate = client.post(
        "/api/v1/admin/db/repairs/publication-near-duplicates",
        json={"dry_run": True},
        headers=headers,
    )
    assert forbidden_near_duplicate.status_code == 403
    assert forbidden_near_duplicate.json()["error"]["code"] == "forbidden"

    admin_headers = _api_bootstrap_csrf_headers(client)
    admin_login = client.post(
        "/api/v1/auth/login",
        json={"email": "api-admin-dbops2@example.com", "password": "admin-password"},
        headers=admin_headers,
    )
    assert admin_login.status_code == 200
    post_login_headers = {"X-CSRF-Token": admin_login.json()["data"]["csrf_token"]}
    invalid_scope = client.post(
        "/api/v1/admin/db/repairs/publication-links",
        json={"user_id": member_user_id, "dry_run": True},
        headers=post_login_headers,
    )
    assert invalid_scope.status_code == 400
    assert invalid_scope.json()["error"]["code"] == "invalid_repair_scope"


@pytest.mark.integration
@pytest.mark.db
@pytest.mark.asyncio
async def test_api_admin_dbops_all_users_apply_requires_confirmation(db_session: AsyncSession) -> None:
    await insert_user(db_session, email="api-admin-dbops3@example.com", password="admin-password", is_admin=True)
    first_user_id = await insert_user(db_session, email="api-dbops-a@example.com", password="user-password")
    second_user_id = await insert_user(db_session, email="api-dbops-b@example.com", password="user-password")
    await _seed_publication_link_for_user(
        db_session,
        user_id=first_user_id,
        scholar_id="dbopsAll01",
        title="DB Ops All User Paper One",
        fingerprint=f"{(first_user_id + 61):064x}",
    )
    await _seed_publication_link_for_user(
        db_session,
        user_id=second_user_id,
        scholar_id="dbopsAll02",
        title="DB Ops All User Paper Two",
        fingerprint=f"{(second_user_id + 71):064x}",
    )

    client = TestClient(app)
    login_user(client, email="api-admin-dbops3@example.com", password="admin-password")
    headers = _api_csrf_headers(client)

    missing_confirmation = client.post(
        "/api/v1/admin/db/repairs/publication-links",
        json={"scope_mode": "all_users", "dry_run": False},
        headers=headers,
    )
    assert missing_confirmation.status_code == 422
    assert "confirmation_text" in str(missing_confirmation.json())

    apply_response = client.post(
        "/api/v1/admin/db/repairs/publication-links",
        json={
            "scope_mode": "all_users",
            "dry_run": False,
            "confirmation_text": "REPAIR ALL USERS",
        },
        headers=headers,
    )
    assert apply_response.status_code == 200
    apply_data = apply_response.json()["data"]
    assert apply_data["status"] == "completed"
    assert apply_data["scope"]["scope_mode"] == "all_users"
    assert int(apply_data["summary"]["links_deleted"]) >= 2

    remaining_links = await db_session.execute(text("SELECT count(*) FROM scholar_publications"))
    assert int(remaining_links.scalar_one()) == 0


@pytest.mark.integration
@pytest.mark.db
@pytest.mark.asyncio
async def test_api_admin_dbops_near_duplicate_repair_scan_and_apply(
    db_session: AsyncSession,
) -> None:
    await insert_user(
        db_session,
        email="api-admin-near-dup@example.com",
        password="admin-password",
        is_admin=True,
    )
    user_id = await insert_user(
        db_session,
        email="api-near-dup-target@example.com",
        password="user-password",
    )
    scholar_result = await db_session.execute(
        text(
            """
            INSERT INTO scholar_profiles (user_id, scholar_id, display_name, is_enabled)
            VALUES (:user_id, :scholar_id, :display_name, true)
            RETURNING id
            """
        ),
        {
            "user_id": user_id,
            "scholar_id": "nearDupScholar01",
            "display_name": "Near Dup Target",
        },
    )
    scholar_profile_id = int(scholar_result.scalar_one())
    first_pub = await db_session.execute(
        text(
            """
            INSERT INTO publications (fingerprint_sha256, title_raw, title_normalized, year, citation_count)
            VALUES (:fingerprint, :title_raw, :title_normalized, 2014, 100)
            RETURNING id
            """
        ),
        {
            "fingerprint": f"{(user_id + 1201):064x}",
            "title_raw": "Adam: A method for stochastic optimization",
            "title_normalized": "adam a method for stochastic optimization",
        },
    )
    first_publication_id = int(first_pub.scalar_one())
    second_pub = await db_session.execute(
        text(
            """
            INSERT INTO publications (fingerprint_sha256, title_raw, title_normalized, year, citation_count)
            VALUES (:fingerprint, :title_raw, :title_normalized, 2015, 10)
            RETURNING id
            """
        ),
        {
            "fingerprint": f"{(user_id + 1202):064x}",
            "title_raw": "†œAdam: A method for stochastic optimization, †3rd Int. Conf. Learn. Represent.",
            "title_normalized": "adam method for stochastic optimization",
        },
    )
    second_publication_id = int(second_pub.scalar_one())
    await db_session.execute(
        text(
            """
            INSERT INTO scholar_publications (scholar_profile_id, publication_id, is_read)
            VALUES (:scholar_profile_id, :first_publication_id, false),
                   (:scholar_profile_id, :second_publication_id, false)
            """
        ),
        {
            "scholar_profile_id": scholar_profile_id,
            "first_publication_id": first_publication_id,
            "second_publication_id": second_publication_id,
        },
    )
    await db_session.commit()

    client = TestClient(app)
    login_user(client, email="api-admin-near-dup@example.com", password="admin-password")
    headers = _api_csrf_headers(client)

    scan_response = client.post(
        "/api/v1/admin/db/repairs/publication-near-duplicates",
        json={"dry_run": True, "max_clusters": 20},
        headers=headers,
    )
    assert scan_response.status_code == 200
    scan_data = scan_response.json()["data"]
    assert scan_data["status"] == "completed"
    assert int(scan_data["summary"]["candidate_cluster_count"]) >= 1
    assert len(scan_data["clusters"]) >= 1
    selected_key = str(scan_data["clusters"][0]["cluster_key"])

    missing_confirmation = client.post(
        "/api/v1/admin/db/repairs/publication-near-duplicates",
        json={"dry_run": False, "selected_cluster_keys": [selected_key]},
        headers=headers,
    )
    assert missing_confirmation.status_code == 422
    assert "confirmation_text" in str(missing_confirmation.json())

    apply_response = client.post(
        "/api/v1/admin/db/repairs/publication-near-duplicates",
        json={
            "dry_run": False,
            "selected_cluster_keys": [selected_key],
            "confirmation_text": "MERGE SELECTED DUPLICATES",
        },
        headers=headers,
    )
    assert apply_response.status_code == 200
    apply_data = apply_response.json()["data"]
    assert apply_data["status"] == "completed"
    assert int(apply_data["summary"]["merged_publications"]) >= 1

    remaining = await db_session.execute(
        text(
            """
            SELECT count(*)
            FROM publications
            WHERE id IN (:first_publication_id, :second_publication_id)
            """
        ),
        {
            "first_publication_id": first_publication_id,
            "second_publication_id": second_publication_id,
        },
    )
    assert int(remaining.scalar_one()) == 1


@pytest.mark.integration
@pytest.mark.db
@pytest.mark.asyncio
async def test_api_scholars_crud_flow(db_session: AsyncSession) -> None:
    await insert_user(
        db_session,
        email="api-scholars@example.com",
        password="api-password",
    )

    client = TestClient(app)
    login_user(client, email="api-scholars@example.com", password="api-password")
    headers = _api_csrf_headers(client)

    missing_csrf = client.post(
        "/api/v1/scholars",
        json={"scholar_id": "abcDEF123456"},
    )
    assert missing_csrf.status_code == 403
    assert missing_csrf.json()["error"]["code"] == "csrf_invalid"

    create_response = client.post(
        "/api/v1/scholars",
        json={"scholar_id": "abcDEF123456"},
        headers=headers,
    )
    assert create_response.status_code == 201
    created = create_response.json()["data"]
    assert created["scholar_id"] == "abcDEF123456"
    scholar_profile_id = int(created["id"])

    list_response = client.get("/api/v1/scholars")
    assert list_response.status_code == 200
    scholars = list_response.json()["data"]["scholars"]
    assert any(int(item["id"]) == scholar_profile_id for item in scholars)

    toggle_response = client.patch(
        f"/api/v1/scholars/{scholar_profile_id}/toggle",
        headers=headers,
    )
    assert toggle_response.status_code == 200
    assert toggle_response.json()["data"]["is_enabled"] is False

    delete_response = client.delete(
        f"/api/v1/scholars/{scholar_profile_id}",
        headers=headers,
    )
    assert delete_response.status_code == 200
    assert delete_response.json()["data"]["message"] == "Scholar deleted."


@pytest.mark.integration
@pytest.mark.db
@pytest.mark.asyncio
async def test_api_scholars_search_and_profile_image_management(
    db_session: AsyncSession,
    tmp_path: Path,
) -> None:
    await insert_user(
        db_session,
        email="api-scholar-images@example.com",
        password="api-password",
    )

    class StubScholarSource:
        async def fetch_profile_html(self, scholar_id: str) -> FetchResult:
            assert scholar_id == "abcDEF123456"
            return FetchResult(
                requested_url="https://scholar.google.com/citations?hl=en&user=abcDEF123456",
                status_code=200,
                final_url="https://scholar.google.com/citations?hl=en&user=abcDEF123456",
                body=(
                    "<html><head>"
                    '<meta property="og:image" content="https://images.example.com/ada.png" />'
                    "</head><body>"
                    '<div id="gsc_prf_in">Ada Lovelace</div>'
                    "</body></html>"
                ),
                error=None,
            )

        async def fetch_author_search_html(self, query: str, *, start: int) -> FetchResult:
            assert query == "Ada Lovelace"
            assert start == 0
            return FetchResult(
                requested_url="https://scholar.google.com/citations?hl=en&view_op=search_authors&mauthors=ada",
                status_code=200,
                final_url="https://scholar.google.com/citations?hl=en&view_op=search_authors&mauthors=ada",
                body=(
                    '<div class="gsc_1usr">'
                    '<img src="/citations/images/avatar_scholar_256.png" />'
                    '<a class="gs_ai_name" href="/citations?hl=en&user=abcDEF123456">Ada Lovelace</a>'
                    '<div class="gs_ai_aff">Analytical Engine</div>'
                    '<div class="gs_ai_eml">Verified email at computing.example</div>'
                    '<div class="gs_ai_cby">Cited by 42</div>'
                    '<a class="gs_ai_one_int">Mathematics</a>'
                    "</div>"
                ),
                error=None,
            )

    previous_upload_dir = settings.scholar_image_upload_dir
    previous_upload_max_bytes = settings.scholar_image_upload_max_bytes
    previous_queue_enabled = settings.ingestion_continuation_queue_enabled
    app.dependency_overrides[get_scholar_source] = lambda: StubScholarSource()
    object.__setattr__(settings, "scholar_image_upload_dir", str(tmp_path / "scholar_images"))
    object.__setattr__(settings, "scholar_image_upload_max_bytes", 1_000_000)
    object.__setattr__(settings, "ingestion_continuation_queue_enabled", False)

    try:
        client = TestClient(app)
        login_user(client, email="api-scholar-images@example.com", password="api-password")
        headers = _api_csrf_headers(client)

        search_response = client.get("/api/v1/scholars/search", params={"query": "Ada Lovelace", "limit": 5})
        assert search_response.status_code == 200
        search_payload = search_response.json()["data"]
        assert search_payload["state"] == "ok"
        assert len(search_payload["candidates"]) == 1
        candidate = search_payload["candidates"][0]
        assert candidate["scholar_id"] == "abcDEF123456"
        assert candidate["profile_image_url"] == "https://scholar.google.com/citations/images/avatar_scholar_256.png"

        create_response = client.post(
            "/api/v1/scholars",
            json={
                "scholar_id": candidate["scholar_id"],
            },
            headers=headers,
        )
        assert create_response.status_code == 201
        created = create_response.json()["data"]
        scholar_profile_id = int(created["id"])
        assert created["profile_image_source"] == "scraped"
        assert created["profile_image_url"] == "https://images.example.com/ada.png"

        set_url_response = client.put(
            f"/api/v1/scholars/{scholar_profile_id}/image/url",
            json={"image_url": "https://cdn.example.com/custom-avatar.png"},
            headers=headers,
        )
        assert set_url_response.status_code == 200
        set_url_data = set_url_response.json()["data"]
        assert set_url_data["profile_image_source"] == "override"
        assert set_url_data["profile_image_url"] == "https://cdn.example.com/custom-avatar.png"

        uploaded_bytes = b"\\x89PNG\\r\\n\\x1a\\n\\x00\\x00\\x00\\rIHDR\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01"
        upload_response = client.post(
            f"/api/v1/scholars/{scholar_profile_id}/image/upload",
            files={"image": ("avatar.png", uploaded_bytes, "image/png")},
            headers=headers,
        )
        assert upload_response.status_code == 200
        upload_data = upload_response.json()["data"]
        assert upload_data["profile_image_source"] == "upload"
        assert upload_data["profile_image_url"] == f"/scholar-images/{scholar_profile_id}/upload"

        uploaded_image_response = client.get(f"/scholar-images/{scholar_profile_id}/upload")
        assert uploaded_image_response.status_code == 200
        assert uploaded_image_response.headers["content-type"].startswith("image/png")
        assert uploaded_image_response.content == uploaded_bytes

        clear_response = client.delete(
            f"/api/v1/scholars/{scholar_profile_id}/image",
            headers=headers,
        )
        assert clear_response.status_code == 200
        clear_data = clear_response.json()["data"]
        assert clear_data["profile_image_source"] == "scraped"
        assert clear_data["profile_image_url"] == "https://images.example.com/ada.png"
    finally:
        app.dependency_overrides.pop(get_scholar_source, None)
        object.__setattr__(settings, "scholar_image_upload_dir", previous_upload_dir)
        object.__setattr__(
            settings,
            "scholar_image_upload_max_bytes",
            previous_upload_max_bytes,
        )
        object.__setattr__(settings, "ingestion_continuation_queue_enabled", previous_queue_enabled)


@pytest.mark.integration
@pytest.mark.db
@pytest.mark.asyncio
async def test_api_scholar_import_export_round_trip(
    db_session: AsyncSession,
) -> None:
    user_id = await insert_user(
        db_session,
        email="api-import-export@example.com",
        password="api-password",
    )
    scholar_result = await db_session.execute(
        text(
            """
            INSERT INTO scholar_profiles (user_id, scholar_id, display_name, is_enabled)
            VALUES (:user_id, :scholar_id, :display_name, true)
            RETURNING id
            """
        ),
        {
            "user_id": user_id,
            "scholar_id": "abcDEF123456",
            "display_name": "Existing Scholar",
        },
    )
    scholar_profile_id = int(scholar_result.scalar_one())
    publication_result = await db_session.execute(
        text(
            """
            INSERT INTO publications (fingerprint_sha256, title_raw, title_normalized, citation_count)
            VALUES (:fingerprint, :title_raw, :title_normalized, 1)
            RETURNING id
            """
        ),
        {
            "fingerprint": f"{(user_id + 500):064x}",
            "title_raw": "Existing Publication",
            "title_normalized": "existingpublication",
        },
    )
    publication_id = int(publication_result.scalar_one())
    await db_session.execute(
        text(
            """
            INSERT INTO scholar_publications (scholar_profile_id, publication_id, is_read)
            VALUES (:scholar_profile_id, :publication_id, false)
            """
        ),
        {
            "scholar_profile_id": scholar_profile_id,
            "publication_id": publication_id,
        },
    )
    await db_session.commit()

    client = TestClient(app)
    login_user(client, email="api-import-export@example.com", password="api-password")
    headers = _api_csrf_headers(client)

    export_response = client.get("/api/v1/scholars/export")
    assert export_response.status_code == 200
    export_payload = export_response.json()["data"]
    assert export_payload["schema_version"] == 1
    assert len(export_payload["scholars"]) == 1
    assert len(export_payload["publications"]) == 1

    import_response = client.post(
        "/api/v1/scholars/import",
        json={
            "schema_version": 1,
            "scholars": [
                {
                    "scholar_id": "abcDEF123456",
                    "display_name": "Updated Scholar",
                    "is_enabled": False,
                    "profile_image_override_url": "https://cdn.example.com/avatar.png",
                },
                {
                    "scholar_id": "zzzYYY111222",
                    "display_name": "Imported Scholar",
                    "is_enabled": True,
                    "profile_image_override_url": None,
                },
            ],
            "publications": [
                {
                    "scholar_id": "abcDEF123456",
                    "title": "Existing Publication",
                    "year": 2024,
                    "citation_count": 8,
                    "author_text": "A. Author",
                    "venue_text": "Test Venue",
                    "pub_url": "https://example.org/existing",
                    "pdf_url": "https://example.org/existing.pdf",
                    "is_read": True,
                },
                {
                    "scholar_id": "zzzYYY111222",
                    "title": "Imported Publication",
                    "year": 2025,
                    "citation_count": 2,
                    "author_text": "B. Author",
                    "venue_text": "Another Venue",
                    "pub_url": "https://example.org/imported",
                    "pdf_url": "https://example.org/imported.pdf",
                    "is_read": False,
                },
            ],
        },
        headers=headers,
    )
    assert import_response.status_code == 200
    import_data = import_response.json()["data"]
    assert int(import_data["scholars_created"]) == 1
    assert int(import_data["scholars_updated"]) >= 1
    assert int(import_data["publications_created"]) == 1
    assert int(import_data["links_created"]) == 1

    updated_scholar_result = await db_session.execute(
        text(
            """
            SELECT display_name, is_enabled, profile_image_override_url
            FROM scholar_profiles
            WHERE user_id = :user_id AND scholar_id = :scholar_id
            """
        ),
        {
            "user_id": user_id,
            "scholar_id": "abcDEF123456",
        },
    )
    updated_scholar = updated_scholar_result.one()
    assert updated_scholar[0] == "Updated Scholar"
    assert updated_scholar[1] is False
    assert updated_scholar[2] == "https://cdn.example.com/avatar.png"

    imported_pub_result = await db_session.execute(
        text(
            """
            SELECT p.title_raw, p.pdf_url
            FROM publications p
            WHERE p.title_raw = :title
            """
        ),
        {"title": "Imported Publication"},
    )
    imported_pub = imported_pub_result.one()
    assert imported_pub[0] == "Imported Publication"
    assert imported_pub[1] == "https://example.org/imported.pdf"

    updated_link_result = await db_session.execute(
        text(
            """
            SELECT sp.is_read
            FROM scholar_publications sp
            JOIN scholar_profiles s ON s.id = sp.scholar_profile_id
            JOIN publications p ON p.id = sp.publication_id
            WHERE s.user_id = :user_id
              AND s.scholar_id = :scholar_id
              AND p.title_raw = :title
            """
        ),
        {
            "user_id": user_id,
            "scholar_id": "abcDEF123456",
            "title": "Existing Publication",
        },
    )
    assert bool(updated_link_result.scalar_one()) is True


@pytest.mark.integration
@pytest.mark.db
@pytest.mark.asyncio
async def test_api_manual_run_skips_unchanged_initial_page_for_scholar(
    db_session: AsyncSession,
) -> None:
    await insert_user(
        db_session,
        email="api-skip-unchanged@example.com",
        password="api-password",
    )

    profile_html = """
    <html>
      <head>
        <meta property="og:image" content="https://images.example.com/skip.png" />
      </head>
      <body>
        <div id="gsc_prf_in">Skip Candidate</div>
        <span id="gsc_a_nn">Articles 1-1</span>
        <table>
          <tbody id="gsc_a_b">
            <tr class="gsc_a_tr">
              <td class="gsc_a_t">
                <a class="gsc_a_at" href="/citations?view_op=view_citation&citation_for_view=abcDEF123456:xyz123">Stable Paper</a>
                <div class="gs_gray">A Author</div>
                <div class="gs_gray">Stable Venue</div>
              </td>
              <td class="gsc_a_c"><a class="gsc_a_ac">5</a></td>
              <td class="gsc_a_y"><span class="gsc_a_h">2024</span></td>
            </tr>
          </tbody>
        </table>
      </body>
    </html>
    """

    class StubScholarSource:
        async def fetch_profile_html(self, scholar_id: str) -> FetchResult:
            assert scholar_id == "abcDEF123456"
            return FetchResult(
                requested_url="https://scholar.google.com/citations?hl=en&user=abcDEF123456",
                status_code=200,
                final_url="https://scholar.google.com/citations?hl=en&user=abcDEF123456",
                body=profile_html,
                error=None,
            )

        async def fetch_profile_page_html(
            self,
            scholar_id: str,
            *,
            cstart: int,
            pagesize: int,
        ) -> FetchResult:
            assert scholar_id == "abcDEF123456"
            assert cstart == 0
            assert pagesize == settings.ingestion_page_size
            return FetchResult(
                requested_url="https://scholar.google.com/citations?hl=en&user=abcDEF123456",
                status_code=200,
                final_url="https://scholar.google.com/citations?hl=en&user=abcDEF123456",
                body=profile_html,
                error=None,
            )

    app.dependency_overrides[get_scholar_source] = lambda: StubScholarSource()
    try:
        with TestClient(app) as client:
            login_user(client, email="api-skip-unchanged@example.com", password="api-password")
            headers = _api_csrf_headers(client)

            create_response = client.post(
                "/api/v1/scholars",
                json={"scholar_id": "abcDEF123456"},
                headers=headers,
            )
            assert create_response.status_code == 201

            first_run_response = client.post(
                "/api/v1/runs/manual",
                headers={**headers, "Idempotency-Key": "skip-unchanged-run-001"},
            )
            assert first_run_response.status_code == 200
            first_run_id = int(first_run_response.json()["data"]["run_id"])

            first_detail_data = await _wait_for_run_complete(client, first_run_id)
            first_results = first_detail_data["scholar_results"]
            assert len(first_results) == 1
            assert first_results[0]["state_reason"] != "no_change_initial_page_signature"

            second_run_response = client.post(
                "/api/v1/runs/manual",
                headers={**headers, "Idempotency-Key": "skip-unchanged-run-002"},
            )
            assert second_run_response.status_code == 200
            second_run_id = int(second_run_response.json()["data"]["run_id"])

            second_detail_data = await _wait_for_run_complete(client, second_run_id)
            second_results = second_detail_data["scholar_results"]
            assert len(second_results) == 1
            assert second_results[0]["state_reason"] == "no_change_initial_page_signature"
            assert second_results[0]["publication_count"] == 0
            assert second_results[0]["outcome"] == "success"
    finally:
        app.dependency_overrides.pop(get_scholar_source, None)


@pytest.mark.integration
@pytest.mark.db
@pytest.mark.asyncio
async def test_api_settings_get_and_update(db_session: AsyncSession) -> None:
    await insert_user(
        db_session,
        email="api-settings@example.com",
        password="api-password",
    )

    client = TestClient(app)
    login_user(client, email="api-settings@example.com", password="api-password")
    headers = _api_csrf_headers(client)

    get_response = client.get("/api/v1/settings")
    assert get_response.status_code == 200
    settings_payload = get_response.json()["data"]
    assert "request_delay_seconds" in settings_payload
    assert "nav_visible_pages" in settings_payload
    assert settings_payload["policy"]["min_run_interval_minutes"] == user_settings_service.resolve_run_interval_minimum(
        settings.ingestion_min_run_interval_minutes
    )
    assert settings_payload["policy"][
        "min_request_delay_seconds"
    ] == user_settings_service.resolve_request_delay_minimum(settings.ingestion_min_request_delay_seconds)
    assert settings_payload["policy"]["automation_allowed"] is settings.ingestion_automation_allowed
    assert settings_payload["policy"]["manual_run_allowed"] is settings.ingestion_manual_run_allowed
    assert settings_payload["policy"]["blocked_failure_threshold"] == max(
        1,
        int(settings.ingestion_alert_blocked_failure_threshold),
    )
    assert settings_payload["policy"]["network_failure_threshold"] == max(
        1,
        int(settings.ingestion_alert_network_failure_threshold),
    )
    assert settings_payload["policy"]["cooldown_blocked_seconds"] == max(
        60,
        int(settings.ingestion_safety_cooldown_blocked_seconds),
    )
    assert settings_payload["policy"]["cooldown_network_seconds"] == max(
        60,
        int(settings.ingestion_safety_cooldown_network_seconds),
    )
    _assert_safety_state_contract(settings_payload["safety_state"])
    assert settings_payload["safety_state"]["cooldown_active"] is False

    policy = settings_payload["policy"]
    run_interval_minutes = max(45, int(policy["min_run_interval_minutes"]))
    request_delay_seconds = max(6, int(policy["min_request_delay_seconds"]))

    update_response = client.put(
        "/api/v1/settings",
        json={
            "auto_run_enabled": True,
            "run_interval_minutes": run_interval_minutes,
            "request_delay_seconds": request_delay_seconds,
            "nav_visible_pages": ["dashboard", "scholars", "publications", "settings", "runs"],
        },
        headers=headers,
    )
    assert update_response.status_code == 200
    updated = update_response.json()["data"]
    assert updated["auto_run_enabled"] is True
    assert updated["run_interval_minutes"] == run_interval_minutes
    assert updated["request_delay_seconds"] == request_delay_seconds
    assert updated["nav_visible_pages"] == [
        "dashboard",
        "scholars",
        "publications",
        "settings",
        "runs",
    ]
    assert "policy" in updated
    _assert_safety_state_contract(updated["safety_state"])


@pytest.mark.integration
@pytest.mark.db
@pytest.mark.asyncio
async def test_api_settings_enforce_env_minimums(db_session: AsyncSession) -> None:
    await insert_user(
        db_session,
        email="api-settings-policy@example.com",
        password="api-password",
    )

    client = TestClient(app)
    login_user(client, email="api-settings-policy@example.com", password="api-password")
    headers = _api_csrf_headers(client)

    previous_min_interval = settings.ingestion_min_run_interval_minutes
    previous_min_delay = settings.ingestion_min_request_delay_seconds
    object.__setattr__(settings, "ingestion_min_run_interval_minutes", 30)
    object.__setattr__(settings, "ingestion_min_request_delay_seconds", 8)
    try:
        interval_response = client.put(
            "/api/v1/settings",
            json={
                "auto_run_enabled": True,
                "run_interval_minutes": 29,
                "request_delay_seconds": 9,
                "nav_visible_pages": ["dashboard", "scholars", "settings"],
            },
            headers=headers,
        )
        assert interval_response.status_code == 400
        assert interval_response.json()["error"]["code"] == "invalid_settings"
        assert interval_response.json()["error"]["message"] == "Check interval must be at least 30 minutes."

        delay_response = client.put(
            "/api/v1/settings",
            json={
                "auto_run_enabled": True,
                "run_interval_minutes": 30,
                "request_delay_seconds": 7,
                "nav_visible_pages": ["dashboard", "scholars", "settings"],
            },
            headers=headers,
        )
        assert delay_response.status_code == 400
        assert delay_response.json()["error"]["code"] == "invalid_settings"
        assert delay_response.json()["error"]["message"] == "Request delay must be at least 8 seconds."
    finally:
        object.__setattr__(settings, "ingestion_min_run_interval_minutes", previous_min_interval)
        object.__setattr__(settings, "ingestion_min_request_delay_seconds", previous_min_delay)


@pytest.mark.integration
@pytest.mark.db
@pytest.mark.asyncio
async def test_api_runs_manual_and_queue_actions(db_session: AsyncSession) -> None:
    user_id = await insert_user(
        db_session,
        email="api-runs@example.com",
        password="api-password",
    )

    client = TestClient(app)
    login_user(client, email="api-runs@example.com", password="api-password")
    headers = _api_csrf_headers(client)

    run_response = client.post(
        "/api/v1/runs/manual",
        headers={**headers, "Idempotency-Key": "manual-run-0001"},
    )
    assert run_response.status_code == 200
    run_payload = run_response.json()["data"]
    assert "run_id" in run_payload
    assert run_payload["status"] in {"running", "resolving", "success", "partial_failure", "failed"}
    assert run_payload["reused_existing_run"] is False
    assert run_payload["idempotency_key"] == "manual-run-0001"
    _assert_safety_state_contract(run_payload["safety_state"])
    run_id = int(run_payload["run_id"])

    stored_key = await db_session.execute(
        text("SELECT idempotency_key FROM crawl_runs WHERE id = :run_id"),
        {"run_id": run_id},
    )
    assert stored_key.scalar_one() == "manual-run-0001"

    replay_response = client.post(
        "/api/v1/runs/manual",
        headers={**headers, "Idempotency-Key": "manual-run-0001"},
    )
    assert replay_response.status_code in {200, 409}
    if replay_response.status_code == 200:
        replay_payload = replay_response.json()["data"]
        assert replay_payload["run_id"] == run_payload["run_id"]
        assert replay_payload["reused_existing_run"] is True
        _assert_safety_state_contract(replay_payload["safety_state"])
    else:
        replay_error = replay_response.json()["error"]
        assert replay_error["code"] == "run_in_progress"

    runs_response = client.get("/api/v1/runs")
    assert runs_response.status_code == 200
    assert len(runs_response.json()["data"]["runs"]) >= 1
    _assert_safety_state_contract(runs_response.json()["data"]["safety_state"])

    run_detail_response = client.get(f"/api/v1/runs/{run_id}")
    assert run_detail_response.status_code == 200
    detail_payload = run_detail_response.json()["data"]
    assert "summary" in detail_payload
    assert isinstance(detail_payload["scholar_results"], list)
    _assert_safety_state_contract(detail_payload["safety_state"])

    scholar_result = await db_session.execute(
        text(
            """
            INSERT INTO scholar_profiles (user_id, scholar_id, display_name, is_enabled)
            VALUES (:user_id, :scholar_id, :display_name, true)
            RETURNING id
            """
        ),
        {
            "user_id": user_id,
            "scholar_id": "abcDEF123456",
            "display_name": "Queue Scholar",
        },
    )
    scholar_profile_id = int(scholar_result.scalar_one())
    queue_result = await db_session.execute(
        text(
            """
            INSERT INTO ingestion_queue_items (
                user_id,
                scholar_profile_id,
                resume_cstart,
                reason,
                status,
                attempt_count,
                next_attempt_dt,
                dropped_reason,
                dropped_at
            )
            VALUES (
                :user_id,
                :scholar_profile_id,
                7,
                'dropped',
                'dropped',
                2,
                NOW(),
                'manual_drop',
                NOW()
            )
            RETURNING id
            """
        ),
        {"user_id": user_id, "scholar_profile_id": scholar_profile_id},
    )
    queue_item_id = int(queue_result.scalar_one())
    await db_session.commit()

    queue_list_response = client.get("/api/v1/runs/queue/items")
    assert queue_list_response.status_code == 200
    assert any(int(item["id"]) == queue_item_id for item in queue_list_response.json()["data"]["queue_items"])

    retry_response = client.post(f"/api/v1/runs/queue/{queue_item_id}/retry", headers=headers)
    assert retry_response.status_code == 200
    assert retry_response.json()["data"]["status"] == "queued"

    retry_again_response = client.post(
        f"/api/v1/runs/queue/{queue_item_id}/retry",
        headers=headers,
    )
    assert retry_again_response.status_code == 409
    assert retry_again_response.json()["error"]["code"] == "queue_item_already_queued"

    drop_response = client.post(f"/api/v1/runs/queue/{queue_item_id}/drop", headers=headers)
    assert drop_response.status_code == 200
    assert drop_response.json()["data"]["status"] == "dropped"

    clear_response = client.request(
        "DELETE",
        f"/api/v1/runs/queue/{queue_item_id}",
        headers=headers,
    )
    assert clear_response.status_code == 200
    assert clear_response.json()["data"]["status"] == "cleared"
    assert clear_response.json()["data"]["message"] == "Queue item cleared."


@pytest.mark.integration
@pytest.mark.db
@pytest.mark.asyncio
async def test_api_manual_run_can_be_disabled_by_policy(db_session: AsyncSession) -> None:
    await insert_user(
        db_session,
        email="api-runs-policy@example.com",
        password="api-password",
    )
    client = TestClient(app)
    login_user(client, email="api-runs-policy@example.com", password="api-password")
    headers = _api_csrf_headers(client)

    previous_manual_allowed = settings.ingestion_manual_run_allowed
    object.__setattr__(settings, "ingestion_manual_run_allowed", False)
    try:
        response = client.post("/api/v1/runs/manual", headers=headers)
        assert response.status_code == 403
        payload = response.json()
        assert payload["error"]["code"] == "manual_runs_disabled"
        assert payload["error"]["details"]["policy"]["manual_run_allowed"] is False
        assert payload["error"]["details"]["safety_state"]["cooldown_active"] is False
    finally:
        object.__setattr__(settings, "ingestion_manual_run_allowed", previous_manual_allowed)


@pytest.mark.integration
@pytest.mark.db
@pytest.mark.asyncio
async def test_api_manual_run_enforces_scrape_safety_cooldown(db_session: AsyncSession) -> None:
    user_id = await insert_user(
        db_session,
        email="api-runs-safety@example.com",
        password="api-password",
    )
    await db_session.execute(
        text(
            """
            INSERT INTO scholar_profiles (user_id, scholar_id, display_name, is_enabled)
            VALUES (:user_id, :scholar_id, :display_name, true)
            """
        ),
        {
            "user_id": user_id,
            "scholar_id": "A1B2C3D4E5F6",
            "display_name": "Safety Probe",
        },
    )
    await db_session.commit()

    blocked_fixture = _regression_fixture("profile_AAAAAAAAAAAA.html")

    class BlockedScholarSource:
        async def fetch_profile_page_html(
            self,
            scholar_id: str,
            *,
            cstart: int,
            pagesize: int,
        ) -> FetchResult:
            _ = (scholar_id, cstart, pagesize)
            return FetchResult(
                requested_url="https://scholar.google.com/citations?hl=en&user=A1B2C3D4E5F6",
                status_code=200,
                final_url=(
                    "https://accounts.google.com/v3/signin/identifier"
                    "?continue=https%3A%2F%2Fscholar.google.com%2Fcitations"
                ),
                body=blocked_fixture,
                error=None,
            )

        async def fetch_profile_html(self, scholar_id: str) -> FetchResult:
            _ = scholar_id
            return await self.fetch_profile_page_html(
                "A1B2C3D4E5F6",
                cstart=0,
                pagesize=settings.ingestion_page_size,
            )

    app.dependency_overrides[get_scholar_source] = lambda: BlockedScholarSource()

    previous_blocked_threshold = settings.ingestion_alert_blocked_failure_threshold
    previous_blocked_cooldown_seconds = settings.ingestion_safety_cooldown_blocked_seconds
    object.__setattr__(settings, "ingestion_alert_blocked_failure_threshold", 1)
    object.__setattr__(settings, "ingestion_safety_cooldown_blocked_seconds", 600)

    try:
        with TestClient(app) as client:
            login_user(client, email="api-runs-safety@example.com", password="api-password")
            headers = _api_csrf_headers(client)

            first_run_response = client.post(
                "/api/v1/runs/manual",
                headers={**headers, "Idempotency-Key": "safety-cooldown-run-1"},
            )
            assert first_run_response.status_code == 200
            first_run_id = int(first_run_response.json()["data"]["run_id"])
            await _wait_for_run_complete(client, first_run_id)

            settings_response = client.get("/api/v1/settings")
            assert settings_response.status_code == 200
            safety_state = settings_response.json()["data"]["safety_state"]
            _assert_safety_state_contract(safety_state)
            assert safety_state["cooldown_active"] is True
            assert safety_state["cooldown_reason"] == "blocked_failure_threshold_exceeded"
            assert int(safety_state["cooldown_remaining_seconds"]) > 0
            assert int(safety_state["counters"]["cooldown_entry_count"]) >= 1
            assert int(safety_state["counters"]["last_blocked_failure_count"]) >= 1

            blocked_start_response = client.post(
                "/api/v1/runs/manual",
                headers={**headers, "Idempotency-Key": "safety-cooldown-run-2"},
            )
            assert blocked_start_response.status_code == 429
            blocked_payload = blocked_start_response.json()
            assert blocked_payload["error"]["code"] == "scrape_cooldown_active"
            blocked_state = blocked_payload["error"]["details"]["safety_state"]
            _assert_safety_state_contract(blocked_state)
            assert blocked_state["cooldown_active"] is True
            assert blocked_state["cooldown_reason"] == "blocked_failure_threshold_exceeded"
            assert int(blocked_state["counters"]["blocked_start_count"]) >= 1

            runs_response = client.get("/api/v1/runs")
            assert runs_response.status_code == 200
            _assert_safety_state_contract(runs_response.json()["data"]["safety_state"])
            assert runs_response.json()["data"]["safety_state"]["cooldown_active"] is True
    finally:
        object.__setattr__(settings, "ingestion_alert_blocked_failure_threshold", previous_blocked_threshold)
        object.__setattr__(settings, "ingestion_safety_cooldown_blocked_seconds", previous_blocked_cooldown_seconds)
        app.dependency_overrides.pop(get_scholar_source, None)


@pytest.mark.integration
@pytest.mark.db
@pytest.mark.asyncio
async def test_api_manual_run_enforces_network_failure_safety_cooldown(
    db_session: AsyncSession,
) -> None:
    user_id = await insert_user(
        db_session,
        email="api-runs-safety-network@example.com",
        password="api-password",
    )
    await db_session.execute(
        text(
            """
            INSERT INTO scholar_profiles (user_id, scholar_id, display_name, is_enabled)
            VALUES (:user_id, :scholar_id, :display_name, true)
            """
        ),
        {
            "user_id": user_id,
            "scholar_id": "NNN111NNN111",
            "display_name": "Network Safety Probe",
        },
    )
    await db_session.commit()

    class NetworkFailureScholarSource:
        async def fetch_profile_page_html(
            self,
            scholar_id: str,
            *,
            cstart: int,
            pagesize: int,
        ) -> FetchResult:
            _ = (scholar_id, cstart, pagesize)
            return FetchResult(
                requested_url="https://scholar.google.com/citations?hl=en&user=NNN111NNN111",
                status_code=None,
                final_url=None,
                body="",
                error="timed out",
            )

        async def fetch_profile_html(self, scholar_id: str) -> FetchResult:
            _ = scholar_id
            return await self.fetch_profile_page_html(
                "NNN111NNN111",
                cstart=0,
                pagesize=settings.ingestion_page_size,
            )

    app.dependency_overrides[get_scholar_source] = lambda: NetworkFailureScholarSource()

    previous_network_threshold = settings.ingestion_alert_network_failure_threshold
    previous_network_cooldown_seconds = settings.ingestion_safety_cooldown_network_seconds
    previous_network_retries = settings.ingestion_network_error_retries
    previous_retry_backoff = settings.ingestion_retry_backoff_seconds
    object.__setattr__(settings, "ingestion_alert_network_failure_threshold", 1)
    object.__setattr__(settings, "ingestion_safety_cooldown_network_seconds", 600)
    object.__setattr__(settings, "ingestion_network_error_retries", 0)
    object.__setattr__(settings, "ingestion_retry_backoff_seconds", 0.0)

    try:
        with TestClient(app) as client:
            login_user(client, email="api-runs-safety-network@example.com", password="api-password")
            headers = _api_csrf_headers(client)

            first_run_response = client.post(
                "/api/v1/runs/manual",
                headers={**headers, "Idempotency-Key": "safety-network-cooldown-run-1"},
            )
            assert first_run_response.status_code == 200
            first_run_id = int(first_run_response.json()["data"]["run_id"])
            await _wait_for_run_complete(client, first_run_id)

            settings_response = client.get("/api/v1/settings")
            assert settings_response.status_code == 200
            safety_state = settings_response.json()["data"]["safety_state"]
            _assert_safety_state_contract(safety_state)
            assert safety_state["cooldown_active"] is True
            assert safety_state["cooldown_reason"] == "network_failure_threshold_exceeded"
            assert int(safety_state["cooldown_remaining_seconds"]) > 0
            assert int(safety_state["counters"]["last_network_failure_count"]) >= 1

            blocked_start_response = client.post(
                "/api/v1/runs/manual",
                headers={**headers, "Idempotency-Key": "safety-network-cooldown-run-2"},
            )
            assert blocked_start_response.status_code == 429
            blocked_payload = blocked_start_response.json()
            assert blocked_payload["error"]["code"] == "scrape_cooldown_active"
            blocked_state = blocked_payload["error"]["details"]["safety_state"]
            _assert_safety_state_contract(blocked_state)
            assert blocked_state["cooldown_active"] is True
            assert blocked_state["cooldown_reason"] == "network_failure_threshold_exceeded"
            assert int(blocked_state["counters"]["blocked_start_count"]) >= 1
    finally:
        object.__setattr__(settings, "ingestion_alert_network_failure_threshold", previous_network_threshold)
        object.__setattr__(settings, "ingestion_safety_cooldown_network_seconds", previous_network_cooldown_seconds)
        object.__setattr__(settings, "ingestion_network_error_retries", previous_network_retries)
        object.__setattr__(settings, "ingestion_retry_backoff_seconds", previous_retry_backoff)
        app.dependency_overrides.pop(get_scholar_source, None)


@pytest.mark.integration
@pytest.mark.db
@pytest.mark.asyncio
async def test_api_settings_clears_expired_scrape_safety_cooldown(
    db_session: AsyncSession,
) -> None:
    user_id = await insert_user(
        db_session,
        email="api-settings-safety-expired@example.com",
        password="api-password",
    )
    user_settings = await user_settings_service.get_or_create_settings(
        db_session,
        user_id=user_id,
    )
    user_settings.scrape_safety_state = {"blocked_start_count": 2}
    user_settings.scrape_cooldown_reason = "blocked_failure_threshold_exceeded"
    user_settings.scrape_cooldown_until = datetime.now(UTC) - timedelta(seconds=15)
    await db_session.commit()

    client = TestClient(app)
    login_user(client, email="api-settings-safety-expired@example.com", password="api-password")

    settings_response = client.get("/api/v1/settings")
    assert settings_response.status_code == 200
    settings_safety_state = settings_response.json()["data"]["safety_state"]
    _assert_safety_state_contract(settings_safety_state)
    assert settings_safety_state["cooldown_active"] is False
    assert settings_safety_state["cooldown_reason"] is None
    assert int(settings_safety_state["counters"]["blocked_start_count"]) == 2

    runs_response = client.get("/api/v1/runs")
    assert runs_response.status_code == 200
    runs_safety_state = runs_response.json()["data"]["safety_state"]
    _assert_safety_state_contract(runs_safety_state)
    assert runs_safety_state["cooldown_active"] is False
    assert runs_safety_state["cooldown_reason"] is None


@pytest.mark.integration
@pytest.mark.db
@pytest.mark.asyncio
async def test_api_publications_list_and_mark_read(db_session: AsyncSession) -> None:
    user_id = await insert_user(
        db_session,
        email="api-pubs@example.com",
        password="api-password",
    )

    scholar_result = await db_session.execute(
        text(
            """
            INSERT INTO scholar_profiles (user_id, scholar_id, display_name, is_enabled)
            VALUES (:user_id, :scholar_id, :display_name, true)
            RETURNING id
            """
        ),
        {
            "user_id": user_id,
            "scholar_id": "abcDEF123456",
            "display_name": "Publication Owner",
        },
    )
    scholar_profile_id = int(scholar_result.scalar_one())

    publication_a = await db_session.execute(
        text(
            """
            INSERT INTO publications (
                fingerprint_sha256,
                title_raw,
                title_normalized,
                citation_count,
                pdf_url
            )
            VALUES (:fingerprint, :title_raw, :title_normalized, 10, :pdf_url)
            RETURNING id
            """
        ),
        {
            "fingerprint": f"{user_id:064x}",
            "title_raw": "Paper A",
            "title_normalized": "paper a",
            "pdf_url": "https://example.org/paper-a.pdf",
        },
    )
    publication_a_id = int(publication_a.scalar_one())

    publication_b = await db_session.execute(
        text(
            """
            INSERT INTO publications (fingerprint_sha256, title_raw, title_normalized, citation_count)
            VALUES (:fingerprint, :title_raw, :title_normalized, 4)
            RETURNING id
            """
        ),
        {
            "fingerprint": f"{(user_id + 1):064x}",
            "title_raw": "Paper B",
            "title_normalized": "paper b",
        },
    )
    publication_b_id = int(publication_b.scalar_one())

    await db_session.execute(
        text(
            """
            INSERT INTO scholar_publications (scholar_profile_id, publication_id, is_read)
            VALUES
              (:scholar_profile_id, :publication_a_id, false),
              (:scholar_profile_id, :publication_b_id, false)
            """
        ),
        {
            "scholar_profile_id": scholar_profile_id,
            "publication_a_id": publication_a_id,
            "publication_b_id": publication_b_id,
        },
    )
    await db_session.commit()

    client = TestClient(app)
    login_user(client, email="api-pubs@example.com", password="api-password")
    headers = _api_csrf_headers(client)

    list_response = client.get("/api/v1/publications?mode=all")
    assert list_response.status_code == 200
    data = list_response.json()["data"]
    assert data["mode"] == "all"
    assert data["favorite_only"] is False
    assert data["total_count"] == 2
    assert data["unread_count"] == 2
    assert data["favorites_count"] == 0
    assert data["latest_count"] == 0
    assert data["new_count"] == data["latest_count"]
    assert data["page"] == 1
    assert data["page_size"] == 100
    assert data["has_prev"] is False
    assert data["has_next"] is False
    assert isinstance(data["publications"], list)
    assert len(data["publications"]) == 2
    assert all(bool(item["is_favorite"]) is False for item in data["publications"])
    pdf_urls = {item["title"]: item["pdf_url"] for item in data["publications"]}
    assert pdf_urls["Paper A"] == "https://example.org/paper-a.pdf"
    assert pdf_urls["Paper B"] is None

    latest_response = client.get("/api/v1/publications?mode=latest")
    assert latest_response.status_code == 200
    latest_data = latest_response.json()["data"]
    assert latest_data["mode"] == "latest"
    assert latest_data["latest_count"] == 0

    unread_response = client.get("/api/v1/publications?mode=unread")
    assert unread_response.status_code == 200
    unread_data = unread_response.json()["data"]
    assert unread_data["mode"] == "unread"
    assert unread_data["unread_count"] == 2

    alias_response = client.get("/api/v1/publications?mode=new")
    assert alias_response.status_code == 200
    alias_data = alias_response.json()["data"]
    assert alias_data["mode"] == "latest"
    assert alias_data["latest_count"] == latest_data["latest_count"]
    assert alias_data["publications"] == latest_data["publications"]

    mark_selected_response = client.post(
        "/api/v1/publications/mark-read",
        json={
            "selections": [
                {
                    "scholar_profile_id": scholar_profile_id,
                    "publication_id": publication_a_id,
                }
            ]
        },
        headers=headers,
    )
    assert mark_selected_response.status_code == 200
    assert mark_selected_response.json()["data"]["requested_count"] == 1
    assert mark_selected_response.json()["data"]["updated_count"] == 1

    read_state = await db_session.execute(
        text(
            """
            SELECT publication_id, is_read
            FROM scholar_publications
            WHERE scholar_profile_id = :scholar_profile_id
            ORDER BY publication_id
            """
        ),
        {"scholar_profile_id": scholar_profile_id},
    )
    states = {int(row[0]): bool(row[1]) for row in read_state.all()}
    assert states[publication_a_id] is True
    assert states[publication_b_id] is False

    mark_response = client.post("/api/v1/publications/mark-all-read", headers=headers)
    assert mark_response.status_code == 200
    assert mark_response.json()["data"]["updated_count"] == 1


@pytest.mark.integration
@pytest.mark.db
@pytest.mark.asyncio
async def test_api_publications_list_supports_pagination(db_session: AsyncSession) -> None:
    user_id = await insert_user(
        db_session,
        email="api-pubs-paging@example.com",
        password="api-password",
    )
    scholar_result = await db_session.execute(
        text(
            """
            INSERT INTO scholar_profiles (user_id, scholar_id, display_name, is_enabled)
            VALUES (:user_id, :scholar_id, :display_name, true)
            RETURNING id
            """
        ),
        {
            "user_id": user_id,
            "scholar_id": "pagingScholar01",
            "display_name": "Paging Scholar",
        },
    )
    scholar_profile_id = int(scholar_result.scalar_one())

    publication_ids: list[int] = []
    for index in range(3):
        created = await db_session.execute(
            text(
                """
                INSERT INTO publications (fingerprint_sha256, title_raw, title_normalized, citation_count)
                VALUES (:fingerprint, :title_raw, :title_normalized, 1)
                RETURNING id
                """
            ),
            {
                "fingerprint": f"{(user_id + 500 + index):064x}",
                "title_raw": f"Paged Paper {index}",
                "title_normalized": f"paged paper {index}",
            },
        )
        publication_ids.append(int(created.scalar_one()))

    await db_session.execute(
        text(
            """
            INSERT INTO scholar_publications (scholar_profile_id, publication_id, is_read, is_favorite)
            VALUES
              (:scholar_profile_id, :publication_1, false, false),
              (:scholar_profile_id, :publication_2, false, false),
              (:scholar_profile_id, :publication_3, false, false)
            """
        ),
        {
            "scholar_profile_id": scholar_profile_id,
            "publication_1": publication_ids[0],
            "publication_2": publication_ids[1],
            "publication_3": publication_ids[2],
        },
    )
    await db_session.commit()

    client = TestClient(app)
    login_user(client, email="api-pubs-paging@example.com", password="api-password")

    first_page = client.get("/api/v1/publications?mode=all&page=1&page_size=2")
    assert first_page.status_code == 200
    first_data = first_page.json()["data"]
    assert first_data["total_count"] == 3
    assert first_data["page"] == 1
    assert first_data["page_size"] == 2
    assert first_data["has_prev"] is False
    assert first_data["has_next"] is True
    assert len(first_data["publications"]) == 2

    second_page = client.get("/api/v1/publications?mode=all&page=2&page_size=2")
    assert second_page.status_code == 200
    second_data = second_page.json()["data"]
    assert second_data["total_count"] == 3
    assert second_data["page"] == 2
    assert second_data["page_size"] == 2
    assert second_data["has_prev"] is True
    assert second_data["has_next"] is False
    assert len(second_data["publications"]) == 1


@pytest.mark.integration
@pytest.mark.db
@pytest.mark.asyncio
async def test_api_publications_search_pagination_uses_filtered_total_count(
    db_session: AsyncSession,
) -> None:
    user_id = await insert_user(
        db_session,
        email="api-pubs-search-page@example.com",
        password="api-password",
    )
    scholar_result = await db_session.execute(
        text(
            """
            INSERT INTO scholar_profiles (user_id, scholar_id, display_name, is_enabled)
            VALUES (:user_id, :scholar_id, :display_name, true)
            RETURNING id
            """
        ),
        {
            "user_id": user_id,
            "scholar_id": "searchPagingScholar01",
            "display_name": "Search Paging Scholar",
        },
    )
    scholar_profile_id = int(scholar_result.scalar_one())
    titles = ["Alpha Optimization", "Alpha Learning", "Beta Methods"]
    publication_ids: list[int] = []
    for index, title in enumerate(titles):
        created = await db_session.execute(
            text(
                """
                INSERT INTO publications (fingerprint_sha256, title_raw, title_normalized, citation_count)
                VALUES (:fingerprint, :title_raw, :title_normalized, 1)
                RETURNING id
                """
            ),
            {
                "fingerprint": f"{(user_id + 900 + index):064x}",
                "title_raw": title,
                "title_normalized": title.lower(),
            },
        )
        publication_ids.append(int(created.scalar_one()))
    for publication_id in publication_ids:
        await db_session.execute(
            text(
                """
                INSERT INTO scholar_publications (scholar_profile_id, publication_id, is_read, is_favorite)
                VALUES (:scholar_profile_id, :publication_id, false, false)
                """
            ),
            {"scholar_profile_id": scholar_profile_id, "publication_id": publication_id},
        )
    await db_session.commit()

    client = TestClient(app)
    login_user(client, email="api-pubs-search-page@example.com", password="api-password")

    response = client.get("/api/v1/publications?mode=all&page=1&page_size=2&search=alpha")
    assert response.status_code == 200
    data = response.json()["data"]
    assert int(data["total_count"]) == 2
    assert data["has_next"] is False
    assert len(data["publications"]) == 2
    assert all("alpha" in str(item["title"]).lower() for item in data["publications"])


@pytest.mark.integration
@pytest.mark.db
@pytest.mark.asyncio
async def test_api_publications_supports_sort_by_pdf_status(
    db_session: AsyncSession,
) -> None:
    user_id = await insert_user(
        db_session,
        email="api-pubs-pdf-sort@example.com",
        password="api-password",
    )
    scholar_result = await db_session.execute(
        text(
            """
            INSERT INTO scholar_profiles (user_id, scholar_id, display_name, is_enabled)
            VALUES (:user_id, :scholar_id, :display_name, true)
            RETURNING id
            """
        ),
        {
            "user_id": user_id,
            "scholar_id": "pdfSortScholar01",
            "display_name": "PDF Sort Scholar",
        },
    )
    scholar_profile_id = int(scholar_result.scalar_one())

    resolved_result = await db_session.execute(
        text(
            """
            INSERT INTO publications (fingerprint_sha256, title_raw, title_normalized, citation_count, pdf_url)
            VALUES (:fingerprint, :title_raw, :title_normalized, 1, :pdf_url)
            RETURNING id
            """
        ),
        {
            "fingerprint": f"{(user_id + 1300):064x}",
            "title_raw": "Resolved PDF",
            "title_normalized": "resolved pdf",
            "pdf_url": "https://example.org/resolved.pdf",
        },
    )
    resolved_publication_id = int(resolved_result.scalar_one())
    queued_result = await db_session.execute(
        text(
            """
            INSERT INTO publications (fingerprint_sha256, title_raw, title_normalized, citation_count)
            VALUES (:fingerprint, :title_raw, :title_normalized, 1)
            RETURNING id
            """
        ),
        {
            "fingerprint": f"{(user_id + 1301):064x}",
            "title_raw": "Queued PDF",
            "title_normalized": "queued pdf",
        },
    )
    queued_publication_id = int(queued_result.scalar_one())
    failed_result = await db_session.execute(
        text(
            """
            INSERT INTO publications (fingerprint_sha256, title_raw, title_normalized, citation_count)
            VALUES (:fingerprint, :title_raw, :title_normalized, 1)
            RETURNING id
            """
        ),
        {
            "fingerprint": f"{(user_id + 1302):064x}",
            "title_raw": "Failed PDF",
            "title_normalized": "failed pdf",
        },
    )
    failed_publication_id = int(failed_result.scalar_one())
    await db_session.execute(
        text(
            """
            INSERT INTO scholar_publications (scholar_profile_id, publication_id, is_read, is_favorite)
            VALUES
              (:scholar_profile_id, :resolved_publication_id, false, false),
              (:scholar_profile_id, :queued_publication_id, false, false),
              (:scholar_profile_id, :failed_publication_id, false, false)
            """
        ),
        {
            "scholar_profile_id": scholar_profile_id,
            "resolved_publication_id": resolved_publication_id,
            "queued_publication_id": queued_publication_id,
            "failed_publication_id": failed_publication_id,
        },
    )
    await db_session.execute(
        text(
            """
            INSERT INTO publication_pdf_jobs (publication_id, status, attempt_count)
            VALUES (:queued_publication_id, 'queued', 1),
                   (:failed_publication_id, 'failed', 2)
            """
        ),
        {
            "queued_publication_id": queued_publication_id,
            "failed_publication_id": failed_publication_id,
        },
    )
    await db_session.commit()

    client = TestClient(app)
    login_user(client, email="api-pubs-pdf-sort@example.com", password="api-password")

    response = client.get("/api/v1/publications?mode=all&sort_by=pdf_status&sort_dir=desc")
    assert response.status_code == 200
    publications = response.json()["data"]["publications"]
    publication_ids = [int(item["publication_id"]) for item in publications]
    assert publication_ids[0] == resolved_publication_id
    assert publication_ids[-1] == failed_publication_id


@pytest.mark.integration
@pytest.mark.db
@pytest.mark.asyncio
async def test_api_publications_favorite_toggle_and_filter(db_session: AsyncSession) -> None:
    user_id = await insert_user(
        db_session,
        email="api-pubs-favorites@example.com",
        password="api-password",
    )
    scholar_result = await db_session.execute(
        text(
            """
            INSERT INTO scholar_profiles (user_id, scholar_id, display_name, is_enabled)
            VALUES (:user_id, :scholar_id, :display_name, true)
            RETURNING id
            """
        ),
        {
            "user_id": user_id,
            "scholar_id": "favoriteScholar01",
            "display_name": "Favorite Scholar",
        },
    )
    scholar_profile_id = int(scholar_result.scalar_one())
    publication_a_result = await db_session.execute(
        text(
            """
            INSERT INTO publications (fingerprint_sha256, title_raw, title_normalized, citation_count)
            VALUES (:fingerprint, :title_raw, :title_normalized, 9)
            RETURNING id
            """
        ),
        {
            "fingerprint": f"{(user_id + 101):064x}",
            "title_raw": "Favorite Target",
            "title_normalized": "favorite target",
        },
    )
    publication_a_id = int(publication_a_result.scalar_one())
    publication_b_result = await db_session.execute(
        text(
            """
            INSERT INTO publications (fingerprint_sha256, title_raw, title_normalized, citation_count)
            VALUES (:fingerprint, :title_raw, :title_normalized, 2)
            RETURNING id
            """
        ),
        {
            "fingerprint": f"{(user_id + 102):064x}",
            "title_raw": "Not Favorite",
            "title_normalized": "not favorite",
        },
    )
    publication_b_id = int(publication_b_result.scalar_one())
    await db_session.execute(
        text(
            """
            INSERT INTO scholar_publications (scholar_profile_id, publication_id, is_read, is_favorite)
            VALUES
              (:scholar_profile_id, :publication_a_id, false, false),
              (:scholar_profile_id, :publication_b_id, false, false)
            """
        ),
        {
            "scholar_profile_id": scholar_profile_id,
            "publication_a_id": publication_a_id,
            "publication_b_id": publication_b_id,
        },
    )
    await db_session.commit()

    client = TestClient(app)
    login_user(client, email="api-pubs-favorites@example.com", password="api-password")
    headers = _api_csrf_headers(client)

    set_response = client.post(
        f"/api/v1/publications/{publication_a_id}/favorite",
        json={"scholar_profile_id": scholar_profile_id, "is_favorite": True},
        headers=headers,
    )
    assert set_response.status_code == 200
    set_data = set_response.json()["data"]
    assert set_data["publication"]["publication_id"] == publication_a_id
    assert set_data["publication"]["is_favorite"] is True

    favorite_only_response = client.get("/api/v1/publications?mode=all&favorite_only=true")
    assert favorite_only_response.status_code == 200
    favorite_only_data = favorite_only_response.json()["data"]
    assert favorite_only_data["favorite_only"] is True
    assert favorite_only_data["favorites_count"] == 1
    assert favorite_only_data["total_count"] == 1
    assert len(favorite_only_data["publications"]) == 1
    assert int(favorite_only_data["publications"][0]["publication_id"]) == publication_a_id

    clear_response = client.post(
        f"/api/v1/publications/{publication_a_id}/favorite",
        json={"scholar_profile_id": scholar_profile_id, "is_favorite": False},
        headers=headers,
    )
    assert clear_response.status_code == 200
    clear_data = clear_response.json()["data"]
    assert clear_data["publication"]["publication_id"] == publication_a_id
    assert clear_data["publication"]["is_favorite"] is False

    favorite_only_after_clear_response = client.get("/api/v1/publications?mode=all&favorite_only=true")
    assert favorite_only_after_clear_response.status_code == 200
    favorite_only_after_clear_data = favorite_only_after_clear_response.json()["data"]
    assert favorite_only_after_clear_data["favorites_count"] == 0
    assert favorite_only_after_clear_data["total_count"] == 0
    assert favorite_only_after_clear_data["publications"] == []

    favorite_state_result = await db_session.execute(
        text(
            """
            SELECT is_favorite
            FROM scholar_publications
            WHERE scholar_profile_id = :scholar_profile_id
              AND publication_id = :publication_id
            """
        ),
        {
            "scholar_profile_id": scholar_profile_id,
            "publication_id": publication_a_id,
        },
    )
    assert bool(favorite_state_result.scalar_one()) is False


@pytest.mark.integration
@pytest.mark.db
@pytest.mark.asyncio
async def test_api_publications_list_schedules_background_enrichment(
    db_session: AsyncSession,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    user_id = await insert_user(
        db_session,
        email="api-pubs-bg@example.com",
        password="api-password",
    )
    scholar_result = await db_session.execute(
        text(
            """
            INSERT INTO scholar_profiles (user_id, scholar_id, display_name, is_enabled)
            VALUES (:user_id, :scholar_id, :display_name, true)
            RETURNING id
            """
        ),
        {
            "user_id": user_id,
            "scholar_id": "background111",
            "display_name": "Background Scholar",
        },
    )
    scholar_profile_id = int(scholar_result.scalar_one())
    publication_result = await db_session.execute(
        text(
            """
            INSERT INTO publications (fingerprint_sha256, title_raw, title_normalized, citation_count)
            VALUES (:fingerprint, :title_raw, :title_normalized, 3)
            RETURNING id
            """
        ),
        {
            "fingerprint": f"{(user_id + 17):064x}",
            "title_raw": "Background Target",
            "title_normalized": "background target",
        },
    )
    publication_id = int(publication_result.scalar_one())
    await db_session.execute(
        text(
            """
            INSERT INTO scholar_publications (scholar_profile_id, publication_id, is_read)
            VALUES (:scholar_profile_id, :publication_id, false)
            """
        ),
        {
            "scholar_profile_id": scholar_profile_id,
            "publication_id": publication_id,
        },
    )
    await db_session.commit()

    async def _fake_schedule(db_session, *, user_id: int, request_email: str | None, items, max_items: int):
        assert db_session is not None
        assert user_id > 0
        assert request_email == "api-pubs-bg@example.com"
        assert int(max_items) > 0
        assert any(int(item.publication_id) == publication_id for item in items)
        return 1

    monkeypatch.setattr(
        "app.services.publications.application.schedule_missing_pdf_enrichment_for_user",
        _fake_schedule,
    )

    client = TestClient(app)
    login_user(client, email="api-pubs-bg@example.com", password="api-password")

    response = client.get("/api/v1/publications?mode=all")
    assert response.status_code == 200
    data = response.json()["data"]
    assert len(data["publications"]) == 1
    assert int(data["publications"][0]["publication_id"]) == publication_id
    assert data["publications"][0]["pdf_url"] is None
    assert data["publications"][0]["pdf_status"] in {"untracked", "queued", "running", "failed"}


@pytest.mark.integration
@pytest.mark.db
@pytest.mark.asyncio
async def test_api_publication_retry_pdf_queues_resolution_job(
    db_session: AsyncSession,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    user_id = await insert_user(
        db_session,
        email="api-pubs-retry@example.com",
        password="api-password",
    )
    scholar_result = await db_session.execute(
        text(
            """
            INSERT INTO scholar_profiles (user_id, scholar_id, display_name, is_enabled)
            VALUES (:user_id, :scholar_id, :display_name, true)
            RETURNING id
            """
        ),
        {
            "user_id": user_id,
            "scholar_id": "retryScholar01",
            "display_name": "Retry Scholar",
        },
    )
    scholar_profile_id = int(scholar_result.scalar_one())
    publication_result = await db_session.execute(
        text(
            """
            INSERT INTO publications (fingerprint_sha256, title_raw, title_normalized, citation_count)
            VALUES (:fingerprint, :title_raw, :title_normalized, 7)
            RETURNING id
            """
        ),
        {
            "fingerprint": f"{(user_id + 9):064x}",
            "title_raw": "Retry Target",
            "title_normalized": "retry target",
        },
    )
    publication_id = int(publication_result.scalar_one())
    await db_session.execute(
        text(
            """
            INSERT INTO scholar_publications (scholar_profile_id, publication_id, is_read)
            VALUES (:scholar_profile_id, :publication_id, false)
            """
        ),
        {
            "scholar_profile_id": scholar_profile_id,
            "publication_id": publication_id,
        },
    )
    await db_session.commit()

    async def _fake_retry_scheduler(db_session, *, user_id: int, request_email: str | None, item: PublicationListItem):
        assert db_session is not None
        assert user_id > 0
        assert request_email == "api-pubs-retry@example.com"
        assert int(item.publication_id) == publication_id
        return True

    monkeypatch.setattr(
        "app.services.publications.application.schedule_retry_pdf_enrichment_for_row",
        _fake_retry_scheduler,
    )

    async def _fake_hydrate(db_session, *, items: list[PublicationListItem]):
        assert db_session is not None
        assert len(items) == 1
        item = items[0]
        return [
            PublicationListItem(
                publication_id=item.publication_id,
                scholar_profile_id=item.scholar_profile_id,
                scholar_label=item.scholar_label,
                title=item.title,
                year=item.year,
                citation_count=item.citation_count,
                venue_text=item.venue_text,
                pub_url=item.pub_url,
                pdf_url=item.pdf_url,
                is_read=item.is_read,
                first_seen_at=item.first_seen_at,
                is_new_in_latest_run=item.is_new_in_latest_run,
                pdf_status="queued",
                pdf_attempt_count=1,
                pdf_failure_reason="no_pdf_found",
                pdf_failure_detail="no_pdf_found",
            )
        ]

    monkeypatch.setattr(
        "app.services.publications.application.hydrate_pdf_enrichment_state",
        _fake_hydrate,
    )

    client = TestClient(app)
    login_user(client, email="api-pubs-retry@example.com", password="api-password")
    headers = _api_csrf_headers(client)

    response = client.post(
        f"/api/v1/publications/{publication_id}/retry-pdf",
        json={"scholar_profile_id": scholar_profile_id},
        headers=headers,
    )
    assert response.status_code == 200
    payload = response.json()["data"]
    assert payload["queued"] is True
    assert payload["resolved_pdf"] is False
    assert payload["publication"]["publication_id"] == publication_id
    assert payload["publication"]["pdf_url"] is None
    assert payload["publication"]["pdf_status"] == "queued"
    assert payload["publication"]["pdf_attempt_count"] == 1
    assert payload["publication"]["pdf_failure_reason"] == "no_pdf_found"

    stored = await db_session.execute(
        text("SELECT pdf_url FROM publications WHERE id = :publication_id"),
        {"publication_id": publication_id},
    )
    stored_pdf_url = stored.scalar_one()
    assert stored_pdf_url is None


@pytest.mark.integration
@pytest.mark.db
@pytest.mark.asyncio
async def test_api_publications_unread_and_latest_modes_can_diverge(
    db_session: AsyncSession,
) -> None:
    user_id = await insert_user(
        db_session,
        email="api-pubs-mismatch@example.com",
        password="api-password",
    )
    scholar_result = await db_session.execute(
        text(
            """
            INSERT INTO scholar_profiles (user_id, scholar_id, display_name, is_enabled)
            VALUES (:user_id, :scholar_id, :display_name, true)
            RETURNING id
            """
        ),
        {
            "user_id": user_id,
            "scholar_id": "mismatchScholar01",
            "display_name": "Mismatch Scholar",
        },
    )
    scholar_profile_id = int(scholar_result.scalar_one())

    older_run_result = await db_session.execute(
        text(
            """
            INSERT INTO crawl_runs (user_id, trigger_type, status, scholar_count, new_pub_count)
            VALUES (:user_id, 'manual', 'success', 1, 1)
            RETURNING id
            """
        ),
        {"user_id": user_id},
    )
    older_run_id = int(older_run_result.scalar_one())

    latest_run_result = await db_session.execute(
        text(
            """
            INSERT INTO crawl_runs (user_id, trigger_type, status, scholar_count, new_pub_count)
            VALUES (:user_id, 'scheduled', 'success', 1, 0)
            RETURNING id
            """
        ),
        {"user_id": user_id},
    )
    latest_run_id = int(latest_run_result.scalar_one())
    assert latest_run_id > older_run_id

    publication_result = await db_session.execute(
        text(
            """
            INSERT INTO publications (fingerprint_sha256, title_raw, title_normalized, citation_count)
            VALUES (:fingerprint, :title_raw, :title_normalized, 12)
            RETURNING id
            """
        ),
        {
            "fingerprint": f"{(user_id + 99):064x}",
            "title_raw": "Unread But Not Latest",
            "title_normalized": "unread but not latest",
        },
    )
    publication_id = int(publication_result.scalar_one())

    await db_session.execute(
        text(
            """
            INSERT INTO scholar_publications (
                scholar_profile_id,
                publication_id,
                is_read,
                first_seen_run_id
            )
            VALUES (:scholar_profile_id, :publication_id, false, :first_seen_run_id)
            """
        ),
        {
            "scholar_profile_id": scholar_profile_id,
            "publication_id": publication_id,
            "first_seen_run_id": older_run_id,
        },
    )
    await db_session.commit()

    client = TestClient(app)
    login_user(client, email="api-pubs-mismatch@example.com", password="api-password")

    unread_response = client.get("/api/v1/publications?mode=unread")
    assert unread_response.status_code == 200
    unread_data = unread_response.json()["data"]
    assert unread_data["mode"] == "unread"
    assert unread_data["unread_count"] == 1
    assert unread_data["latest_count"] == 0
    assert unread_data["new_count"] == 0
    assert len(unread_data["publications"]) == 1
    assert int(unread_data["publications"][0]["publication_id"]) == publication_id
    assert unread_data["publications"][0]["is_new_in_latest_run"] is False

    latest_response = client.get("/api/v1/publications?mode=latest")
    assert latest_response.status_code == 200
    latest_data = latest_response.json()["data"]
    assert latest_data["mode"] == "latest"
    assert latest_data["latest_count"] == 0
    assert len(latest_data["publications"]) == 0

    alias_response = client.get("/api/v1/publications?mode=new")
    assert alias_response.status_code == 200
    alias_data = alias_response.json()["data"]
    assert alias_data["mode"] == "latest"
    assert alias_data["latest_count"] == latest_data["latest_count"]
    assert alias_data["publications"] == latest_data["publications"]