diff --git a/.gitignore b/.gitignore index 26494bf..7c06050 100644 --- a/.gitignore +++ b/.gitignore @@ -19,4 +19,4 @@ frontend/.vite/ docs/website/node_modules/ docs/website/build/ docs/website/.docusaurus/ -.claude/settings.json +.claude/ diff --git a/tests/integration/test_new_publication_detection.py b/tests/integration/test_new_publication_detection.py new file mode 100644 index 0000000..585b0c4 --- /dev/null +++ b/tests/integration/test_new_publication_detection.py @@ -0,0 +1,176 @@ +from __future__ import annotations + +import pytest +from fastapi.testclient import TestClient +from sqlalchemy.ext.asyncio import AsyncSession + +from app.api.runtime_deps import get_scholar_source +from app.main import app +from app.services.scholar.source import FetchResult +from tests.integration.helpers import ( + api_csrf_headers, + insert_user, + login_user, + wait_for_run_complete, +) + +SCHOLAR_ID = "newPubDetc01" + + +def _build_profile_html(publications: list[dict[str, str]]) -> str: + rows = [] + for pub in publications: + rows.append( + f""" + + + {pub["title"]} +
{pub.get("authors", "A Author")}
+
{pub.get("venue", "Some Venue")}
+ + {pub.get("citations", "0")} + {pub.get("year", "2024")} + """ + ) + count = len(publications) + return f""" + + + + + +
New Pub Detector
+ Articles 1-{count} + + {"".join(rows)} + +
+ + + """ + + +INITIAL_PUBLICATIONS = [ + {"cluster": "aaa111", "title": "Existing Paper One", "year": "2023", "citations": "10"}, + {"cluster": "bbb222", "title": "Existing Paper Two", "year": "2022", "citations": "5"}, +] + +UPDATED_PUBLICATIONS = [ + *INITIAL_PUBLICATIONS, + {"cluster": "ccc333", "title": "Brand New Paper Three", "year": "2025", "citations": "0"}, +] + + +class _MutableScholarSource: + """Source that serves different HTML per phase to simulate page changes.""" + + def __init__(self, initial_html: str) -> None: + self.html = initial_html + + async def fetch_profile_html(self, scholar_id: str) -> FetchResult: + return self._result(scholar_id) + + async def fetch_profile_page_html( + self, + scholar_id: str, + *, + cstart: int, + pagesize: int, + ) -> FetchResult: + return self._result(scholar_id) + + def _result(self, scholar_id: str) -> FetchResult: + return FetchResult( + requested_url=f"https://scholar.google.com/citations?hl=en&user={scholar_id}", + status_code=200, + final_url=f"https://scholar.google.com/citations?hl=en&user={scholar_id}", + body=self.html, + error=None, + ) + + +@pytest.mark.integration +@pytest.mark.db +@pytest.mark.asyncio +async def test_new_publication_detected_after_page_change( + db_session: AsyncSession, +) -> None: + """Verify the fingerprint short-circuit lets new publications through. + + Phase 1: Initial scrape discovers 2 publications. + Phase 2: Same page → skipped via no_change fingerprint. + Phase 3: Page gains a 3rd publication → fingerprint differs, + scrape runs and discovers the new entry. + """ + await insert_user(db_session, email="new-pub-detect@example.com", password="api-password") + + source = _MutableScholarSource(_build_profile_html(INITIAL_PUBLICATIONS)) + app.dependency_overrides[get_scholar_source] = lambda: source + try: + with TestClient(app) as client: + login_user(client, email="new-pub-detect@example.com", password="api-password") + headers = api_csrf_headers(client) + + create_resp = client.post("/api/v1/scholars", json={"scholar_id": SCHOLAR_ID}, headers=headers) + assert create_resp.status_code == 201 + + # ── Phase 1: initial scrape ───────────────────────────── + run1 = client.post( + "/api/v1/runs/manual", + headers={**headers, "Idempotency-Key": "detect-run-001"}, + ) + assert run1.status_code == 200 + run1_id = int(run1.json()["data"]["run_id"]) + run1_data = await wait_for_run_complete(client, run1_id) + + assert run1_data["scholar_results"][0]["outcome"] == "success" + assert run1_data["scholar_results"][0]["publication_count"] == 2 + assert run1_data["scholar_results"][0].get("state_reason") != "no_change_initial_page_signature" + + pubs1 = client.get("/api/v1/publications?mode=latest").json()["data"] + assert pubs1["total_count"] == 2 + assert all(p["is_new_in_latest_run"] for p in pubs1["publications"]) + + # ── Phase 2: unchanged page → skipped ────────────────── + run2 = client.post( + "/api/v1/runs/manual", + headers={**headers, "Idempotency-Key": "detect-run-002"}, + ) + assert run2.status_code == 200 + run2_id = int(run2.json()["data"]["run_id"]) + run2_data = await wait_for_run_complete(client, run2_id) + + assert run2_data["scholar_results"][0]["state_reason"] == "no_change_initial_page_signature" + assert run2_data["scholar_results"][0]["publication_count"] == 0 + + # ── Phase 3: new publication appears ──────────────────── + source.html = _build_profile_html(UPDATED_PUBLICATIONS) + + run3 = client.post( + "/api/v1/runs/manual", + headers={**headers, "Idempotency-Key": "detect-run-003"}, + ) + assert run3.status_code == 200 + run3_id = int(run3.json()["data"]["run_id"]) + run3_data = await wait_for_run_complete(client, run3_id) + + assert run3_data["scholar_results"][0]["outcome"] == "success" + assert run3_data["scholar_results"][0].get("state_reason") != "no_change_initial_page_signature" + assert run3_data["scholar_results"][0]["publication_count"] == 3 + + pubs3 = client.get("/api/v1/publications?mode=latest").json()["data"] + assert pubs3["total_count"] == 3 + + titles = {p["title"] for p in pubs3["publications"]} + assert "Brand New Paper Three" in titles + + new_pub = next(p for p in pubs3["publications"] if p["title"] == "Brand New Paper Three") + assert new_pub["is_new_in_latest_run"] is True + + old_pubs = [p for p in pubs3["publications"] if p["title"] != "Brand New Paper Three"] + for p in old_pubs: + assert p["is_new_in_latest_run"] is False + finally: + app.dependency_overrides.pop(get_scholar_source, None)