from __future__ import annotations from pathlib import Path import pytest from app.services.scholar.parser import ( ParseState, ScholarDomInvariantError, parse_author_search_page, parse_profile_page, ) from app.services.scholar.source import FetchResult def _fixture(name: str) -> str: path = Path("tests/fixtures/scholar") / name return path.read_text(encoding="utf-8") def _regression_fixture(name: str) -> str: path = Path("tests/fixtures/scholar/regression") / name return path.read_text(encoding="utf-8") def test_parse_profile_page_extracts_core_fields_from_fixture() -> None: fetch_result = FetchResult( requested_url="https://scholar.google.com/citations?hl=en&user=amIMrIEAAAAJ", status_code=200, final_url="https://scholar.google.com/citations?hl=en&user=amIMrIEAAAAJ", body=_fixture("profile_ok_amIMrIEAAAAJ.html"), error=None, ) parsed = parse_profile_page(fetch_result) assert parsed.state == ParseState.OK assert parsed.state_reason == "publications_extracted" assert parsed.profile_name == "Bangar Raju Cherukuri" assert parsed.profile_image_url assert parsed.profile_image_url.startswith("http") assert len(parsed.publications) >= 10 assert parsed.has_show_more_button is True assert parsed.articles_range is not None first = parsed.publications[0] assert first.title assert first.cluster_id assert first.citation_count is not None def test_parse_profile_page_classifies_accounts_redirect_as_blocked() -> None: fetch_result = FetchResult( requested_url="https://scholar.google.com/citations?hl=en&user=AAAAAAAAAAAA", status_code=200, final_url="https://accounts.google.com/v3/signin/identifier?continue=...", body="Sign in", error=None, ) parsed = parse_profile_page(fetch_result) assert parsed.state == ParseState.BLOCKED_OR_CAPTCHA assert parsed.state_reason == "blocked_accounts_redirect" assert len(parsed.publications) == 0 def test_parse_profile_page_handles_missing_optional_metadata() -> None: html = """
Test Author
Articles 1-1
A Test Paper
A Person
 7
""" fetch_result = FetchResult( requested_url="https://scholar.google.com/citations?hl=en&user=abcDEF123456", status_code=200, final_url="https://scholar.google.com/citations?hl=en&user=abcDEF123456", body=html, error=None, ) parsed = parse_profile_page(fetch_result) assert parsed.state == ParseState.OK assert parsed.state_reason == "publications_extracted" assert len(parsed.publications) == 1 publication = parsed.publications[0] assert publication.cluster_id == "cfv:abc:def123" assert publication.year is None assert publication.venue_text is None def test_parse_profile_page_parses_comma_formatted_citation_counts() -> None: html = """
Citation Formatting Test
Articles 1-1
Paper Cited by 1,234 2024
""" fetch_result = FetchResult( requested_url="https://scholar.google.com/citations?hl=en&user=abcDEF123456", status_code=200, final_url="https://scholar.google.com/citations?hl=en&user=abcDEF123456", body=html, error=None, ) parsed = parse_profile_page(fetch_result) assert parsed.state == ParseState.OK assert len(parsed.publications) == 1 assert parsed.publications[0].citation_count == 1234 def test_parse_profile_page_ignores_direct_pdf_link_markup() -> None: html = """
Direct PDF Test
Articles 1-1
Paper
[PDF]
3 2025
""" fetch_result = FetchResult( requested_url="https://scholar.google.com/citations?hl=en&user=abcDEF123456", status_code=200, final_url="https://scholar.google.com/citations?hl=en&user=abcDEF123456", body=html, error=None, ) parsed = parse_profile_page(fetch_result) assert parsed.state == ParseState.OK assert len(parsed.publications) == 1 assert parsed.publications[0].pdf_url is None def test_parse_profile_page_fails_fast_when_citation_markup_is_unparseable() -> None: html = """
Citation Parse Drift
Articles 1-1
Paper Cited by none 2025
""" fetch_result = FetchResult( requested_url="https://scholar.google.com/citations?hl=en&user=abcDEF123456", status_code=200, final_url="https://scholar.google.com/citations?hl=en&user=abcDEF123456", body=html, error=None, ) with pytest.raises(ScholarDomInvariantError) as exc: parse_profile_page(fetch_result) assert exc.value.code == "layout_row_citation_unparseable" def test_parse_profile_page_detects_layout_change_when_markers_absent() -> None: fetch_result = FetchResult( requested_url="https://scholar.google.com/citations?hl=en&user=abcDEF123456", status_code=200, final_url="https://scholar.google.com/citations?hl=en&user=abcDEF123456", body="

Unexpected page

", error=None, ) with pytest.raises(ScholarDomInvariantError) as exc: parse_profile_page(fetch_result) assert exc.value.code == "layout_markers_missing" def test_parse_profile_page_reports_network_reason_when_status_missing() -> None: fetch_result = FetchResult( requested_url="https://scholar.google.com/citations?hl=en&user=abcDEF123456", status_code=None, final_url=None, body="", error="timed out", ) parsed = parse_profile_page(fetch_result) assert parsed.state == ParseState.NETWORK_ERROR assert parsed.state_reason == "network_timeout" def test_parse_profile_page_reports_dns_network_reason_when_status_missing() -> None: fetch_result = FetchResult( requested_url="https://scholar.google.com/citations?hl=en&user=abcDEF123456", status_code=None, final_url=None, body="", error="", ) parsed = parse_profile_page(fetch_result) assert parsed.state == ParseState.NETWORK_ERROR assert parsed.state_reason == "network_dns_resolution_failed" def test_parse_profile_page_ignores_no_results_keyword_inside_script_blocks() -> None: html = """
Scripted Author
""" fetch_result = FetchResult( requested_url="https://scholar.google.com/citations?hl=en&user=abcDEF123456", status_code=200, final_url="https://scholar.google.com/citations?hl=en&user=abcDEF123456", body=html, error=None, ) parsed = parse_profile_page(fetch_result) assert parsed.state == ParseState.OK assert parsed.state_reason == "no_rows_with_known_markers" def test_parse_profile_page_treats_disabled_show_more_button_as_absent() -> None: html = """
Disabled Show More
Articles 1-1
Paper 1 2024
""" fetch_result = FetchResult( requested_url="https://scholar.google.com/citations?hl=en&user=abcDEF123456", status_code=200, final_url="https://scholar.google.com/citations?hl=en&user=abcDEF123456", body=html, error=None, ) parsed = parse_profile_page(fetch_result) assert parsed.state == ParseState.OK assert parsed.has_show_more_button is False def test_parse_profile_page_regression_fixture_profile_p1rwlvo() -> None: fetch_result = FetchResult( requested_url="https://scholar.google.com/citations?hl=en&user=P1RwlvoAAAAJ", status_code=200, final_url="https://scholar.google.com/citations?hl=en&user=P1RwlvoAAAAJ", body=_regression_fixture("profile_P1RwlvoAAAAJ.html"), error=None, ) parsed = parse_profile_page(fetch_result) assert parsed.state == ParseState.OK assert parsed.state_reason == "publications_extracted" assert parsed.profile_name == "WENRUI ZUO" assert len(parsed.publications) == 5 assert parsed.has_show_more_button is False assert parsed.articles_range in {"Articles 1-5", "Articles 1–5"} assert "possible_partial_page_show_more_present" not in parsed.warnings assert all(item.cluster_id for item in parsed.publications) def test_parse_profile_page_regression_fixture_profile_lz5d() -> None: fetch_result = FetchResult( requested_url="https://scholar.google.com/citations?hl=en&user=LZ5D_p4AAAAJ", status_code=200, final_url="https://scholar.google.com/citations?hl=en&user=LZ5D_p4AAAAJ", body=_regression_fixture("profile_LZ5D_p4AAAAJ.html"), error=None, ) parsed = parse_profile_page(fetch_result) assert parsed.state == ParseState.OK assert parsed.state_reason == "publications_extracted" assert parsed.profile_name == "Doaa Elmatary" assert len(parsed.publications) == 12 assert parsed.has_show_more_button is False assert parsed.articles_range in {"Articles 1-12", "Articles 1–12"} assert "possible_partial_page_show_more_present" not in parsed.warnings assert any(item.venue_text is None for item in parsed.publications) def test_parse_profile_page_regression_fixture_blocked_redirect() -> None: fetch_result = FetchResult( requested_url="https://scholar.google.com/citations?hl=en&user=AAAAAAAAAAAA", status_code=200, final_url=( "https://accounts.google.com/v3/signin/identifier" "?continue=https%3A%2F%2Fscholar.google.com%2Fcitations%3Fhl%3Den%26user%3DAAAAAAAAAAAA" ), body=_regression_fixture("profile_AAAAAAAAAAAA.html"), error=None, ) parsed = parse_profile_page(fetch_result) assert parsed.state == ParseState.BLOCKED_OR_CAPTCHA assert parsed.state_reason == "blocked_accounts_redirect" assert parsed.profile_name is None assert len(parsed.publications) == 0 def test_parse_author_search_page_extracts_candidates_with_image() -> None: html = """
Ada Lovelace
Analytical Engine Lab
Verified email at computing.example
Cited by 128
Algorithms Mathematics
""" fetch_result = FetchResult( requested_url="https://scholar.google.com/citations?hl=en&view_op=search_authors&mauthors=ada", status_code=200, final_url="https://scholar.google.com/citations?hl=en&view_op=search_authors&mauthors=ada", body=html, error=None, ) parsed = parse_author_search_page(fetch_result) assert parsed.state == ParseState.OK assert parsed.state_reason == "author_candidates_extracted" assert len(parsed.candidates) == 1 candidate = parsed.candidates[0] assert candidate.scholar_id == "abcDEF123456" assert candidate.display_name == "Ada Lovelace" assert candidate.affiliation == "Analytical Engine Lab" assert candidate.email_domain == "computing.example" assert candidate.cited_by_count == 128 assert candidate.interests == ["Algorithms", "Mathematics"] assert candidate.profile_url.startswith("https://scholar.google.com/citations") assert candidate.profile_image_url == "https://scholar.google.com/citations/images/avatar_scholar_256.png" def test_parse_author_search_page_detects_no_results_keyword() -> None: fetch_result = FetchResult( requested_url="https://scholar.google.com/citations?hl=en&view_op=search_authors&mauthors=nope", status_code=200, final_url="https://scholar.google.com/citations?hl=en&view_op=search_authors&mauthors=nope", body="Your search didn't match any user profiles.", error=None, ) parsed = parse_author_search_page(fetch_result) assert parsed.state == ParseState.NO_RESULTS assert parsed.state_reason == "no_results_keyword_detected" assert len(parsed.candidates) == 0 def test_parse_author_search_page_classifies_http_429_as_blocked() -> None: fetch_result = FetchResult( requested_url="https://scholar.google.com/citations?hl=en&view_op=search_authors&mauthors=ada", status_code=429, final_url="https://scholar.google.com/citations?hl=en&view_op=search_authors&mauthors=ada", body="Too many requests", error=None, ) parsed = parse_author_search_page(fetch_result) assert parsed.state == ParseState.BLOCKED_OR_CAPTCHA assert parsed.state_reason == "blocked_http_429_rate_limited" def test_parse_author_search_page_prefers_sorry_challenge_reason_over_generic_429() -> None: fetch_result = FetchResult( requested_url="https://scholar.google.com/citations?hl=en&view_op=search_authors&mauthors=ada", status_code=429, final_url="https://www.google.com/sorry/index?continue=scholar", body="Our systems have detected unusual traffic.", error=None, ) parsed = parse_author_search_page(fetch_result) assert parsed.state == ParseState.BLOCKED_OR_CAPTCHA assert parsed.state_reason == "blocked_google_sorry_challenge"