modularize service layer, add mobile nav drawer, and sync docs

app/services/domains/scholar/__init__.py

@@ -0,0 +1 @@
+from __future__ import annotations

app/services/domains/scholar/author_rows.py

@@ -0,0 +1,202 @@
+from __future__ import annotations
+
+import re
+from html.parser import HTMLParser
+from typing import Any
+from urllib.parse import parse_qs, urlparse
+
+from app.services.domains.scholar.parser_constants import AUTHOR_SEARCH_MARKER_KEYS
+from app.services.domains.scholar.parser_types import ScholarSearchCandidate
+from app.services.domains.scholar.parser_utils import (
+    attr_class,
+    attr_href,
+    attr_src,
+    build_absolute_scholar_url,
+    normalize_space,
+)
+
+
+def parse_scholar_id_from_href(href: str | None) -> str | None:
+    if not href:
+        return None
+    parsed = urlparse(href)
+    query = parse_qs(parsed.query)
+    user_values = query.get("user")
+    if not user_values:
+        return None
+    candidate = user_values[0].strip()
+    return candidate or None
+
+
+def _extract_verified_email_domain(value: str | None) -> str | None:
+    if not value:
+        return None
+    match = re.search(r"verified email at\s+(.+)$", value.strip(), re.I)
+    if not match:
+        return None
+    domain = normalize_space(match.group(1))
+    return domain or None
+
+
+class ScholarAuthorSearchParser(HTMLParser):
+    def __init__(self) -> None:
+        super().__init__(convert_charrefs=True)
+        self.candidates: list[ScholarSearchCandidate] = []
+        self._candidate: dict[str, Any] | None = None
+
+    def _begin_candidate(self) -> None:
+        self._candidate = {
+            "depth": 1,
+            "name_href": None,
+            "name_parts": [],
+            "aff_depth": 0,
+            "aff_parts": [],
+            "name_depth": 0,
+            "eml_depth": 0,
+            "eml_parts": [],
+            "cby_depth": 0,
+            "cby_parts": [],
+            "interest_depth": 0,
+            "interest_parts": [],
+            "interests": [],
+            "image_src": None,
+        }
+
+    def _increment_capture_depths(self) -> None:
+        if self._candidate is None:
+            return
+        for key in ("name_depth", "aff_depth", "eml_depth", "cby_depth", "interest_depth"):
+            if self._candidate[key] > 0:
+                self._candidate[key] += 1
+
+    def _finalize_candidate(self) -> None:
+        if self._candidate is None:
+            return
+
+        name = normalize_space("".join(self._candidate["name_parts"]))
+        scholar_id = parse_scholar_id_from_href(self._candidate["name_href"])
+        if not name or not scholar_id:
+            return
+
+        affiliation = normalize_space("".join(self._candidate["aff_parts"])) or None
+        email_domain = _extract_verified_email_domain(
+            normalize_space("".join(self._candidate["eml_parts"])) or None
+        )
+        cited_by_text = normalize_space("".join(self._candidate["cby_parts"]))
+        cited_by_match = re.search(r"\d+", cited_by_text)
+        cited_by_count = int(cited_by_match.group(0)) if cited_by_match else None
+
+        seen_interests: set[str] = set()
+        interests: list[str] = []
+        for interest in self._candidate["interests"]:
+            normalized = normalize_space(interest)
+            if not normalized or normalized in seen_interests:
+                continue
+            seen_interests.add(normalized)
+            interests.append(normalized)
+
+        profile_url = build_absolute_scholar_url(self._candidate["name_href"])
+        if not profile_url:
+            profile_url = (
+                "https://scholar.google.com/citations"
+                f"?hl=en&user={scholar_id}"
+            )
+
+        self.candidates.append(
+            ScholarSearchCandidate(
+                scholar_id=scholar_id,
+                display_name=name,
+                affiliation=affiliation,
+                email_domain=email_domain,
+                cited_by_count=cited_by_count,
+                interests=interests,
+                profile_url=profile_url,
+                profile_image_url=build_absolute_scholar_url(self._candidate["image_src"]),
+            )
+        )
+
+    def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
+        classes = attr_class(attrs)
+
+        if self._candidate is None:
+            if tag == "div" and "gsc_1usr" in classes:
+                self._begin_candidate()
+            return
+
+        self._candidate["depth"] += 1
+        self._increment_capture_depths()
+
+        if tag == "a" and "gs_ai_name" in classes:
+            self._candidate["name_depth"] = 1
+            self._candidate["name_href"] = attr_href(attrs)
+            return
+
+        if tag == "div" and "gs_ai_aff" in classes:
+            self._candidate["aff_depth"] = 1
+            return
+
+        if tag == "div" and "gs_ai_eml" in classes:
+            self._candidate["eml_depth"] = 1
+            return
+
+        if tag == "div" and "gs_ai_cby" in classes:
+            self._candidate["cby_depth"] = 1
+            return
+
+        if tag == "a" and "gs_ai_one_int" in classes:
+            self._candidate["interest_depth"] = 1
+            self._candidate["interest_parts"] = []
+            return
+
+        if tag == "img" and self._candidate["image_src"] is None:
+            self._candidate["image_src"] = attr_src(attrs)
+
+    def handle_data(self, data: str) -> None:
+        if self._candidate is None:
+            return
+        if self._candidate["name_depth"] > 0:
+            self._candidate["name_parts"].append(data)
+        if self._candidate["aff_depth"] > 0:
+            self._candidate["aff_parts"].append(data)
+        if self._candidate["eml_depth"] > 0:
+            self._candidate["eml_parts"].append(data)
+        if self._candidate["cby_depth"] > 0:
+            self._candidate["cby_parts"].append(data)
+        if self._candidate["interest_depth"] > 0:
+            self._candidate["interest_parts"].append(data)
+
+    def _decrement_capture_depth(self, key: str) -> bool:
+        if self._candidate is None:
+            return False
+        if self._candidate[key] <= 0:
+            return False
+        self._candidate[key] -= 1
+        return self._candidate[key] == 0
+
+    def handle_endtag(self, _tag: str) -> None:
+        if self._candidate is None:
+            return
+
+        interest_closed = self._decrement_capture_depth("interest_depth")
+        self._decrement_capture_depth("name_depth")
+        self._decrement_capture_depth("aff_depth")
+        self._decrement_capture_depth("eml_depth")
+        self._decrement_capture_depth("cby_depth")
+
+        if interest_closed:
+            interest_text = normalize_space("".join(self._candidate["interest_parts"]))
+            if interest_text:
+                self._candidate["interests"].append(interest_text)
+            self._candidate["interest_parts"] = []
+
+        self._candidate["depth"] -= 1
+        if self._candidate["depth"] > 0:
+            return
+
+        self._finalize_candidate()
+        self._candidate = None
+
+
+def count_author_search_markers(html: str) -> dict[str, int]:
+    lowered = html.lower()
+    return {key: lowered.count(key.lower()) for key in AUTHOR_SEARCH_MARKER_KEYS}

app/services/domains/scholar/parser.py

@@ -0,0 +1,110 @@
+from __future__ import annotations
+
+from app.services.domains.scholar.author_rows import (
+    ScholarAuthorSearchParser,
+    count_author_search_markers,
+    parse_scholar_id_from_href,
+)
+from app.services.domains.scholar.parser_constants import SCRIPT_STYLE_RE
+from app.services.domains.scholar.parser_types import (
+    ParseState,
+    ParsedAuthorSearchPage,
+    ParsedProfilePage,
+    PublicationCandidate,
+    ScholarSearchCandidate,
+)
+from app.services.domains.scholar.parser_utils import (
+    attr_class,
+    attr_href,
+    attr_src,
+    build_absolute_scholar_url,
+    normalize_space,
+    strip_tags,
+)
+from app.services.domains.scholar.profile_rows import (
+    ScholarRowParser,
+    count_markers,
+    extract_articles_range,
+    extract_profile_image_url,
+    extract_profile_name,
+    extract_rows,
+    has_operation_error_banner,
+    has_show_more_button,
+    parse_citation_count,
+    parse_cluster_id_from_href,
+    parse_publications,
+    parse_year,
+)
+from app.services.domains.scholar.source import FetchResult
+from app.services.domains.scholar.state_detection import (
+    classify_block_or_captcha_reason,
+    classify_network_error_reason,
+    detect_author_search_state,
+    detect_state,
+)
+
+
+def parse_profile_page(fetch_result: FetchResult) -> ParsedProfilePage:
+    publications, warnings = parse_publications(fetch_result.body)
+    marker_counts = count_markers(fetch_result.body)
+    visible_text = strip_tags(SCRIPT_STYLE_RE.sub(" ", fetch_result.body)).lower()
+
+    show_more = has_show_more_button(fetch_result.body)
+    operation_error_banner = has_operation_error_banner(fetch_result.body)
+    articles_range = extract_articles_range(fetch_result.body)
+
+    if show_more:
+        warnings.append("possible_partial_page_show_more_present")
+    if operation_error_banner:
+        warnings.append("operation_error_banner_present")
+
+    warnings = sorted(set(warnings))
+
+    state, state_reason = detect_state(
+        fetch_result,
+        publications,
+        marker_counts,
+        warnings=warnings,
+        has_show_more_button_flag=show_more,
+        articles_range=articles_range,
+        visible_text=visible_text,
+    )
+
+    return ParsedProfilePage(
+        state=state,
+        state_reason=state_reason,
+        profile_name=extract_profile_name(fetch_result.body),
+        profile_image_url=extract_profile_image_url(fetch_result.body),
+        publications=publications,
+        marker_counts=marker_counts,
+        warnings=warnings,
+        has_show_more_button=show_more,
+        has_operation_error_banner=operation_error_banner,
+        articles_range=articles_range,
+    )
+
+
+def parse_author_search_page(fetch_result: FetchResult) -> ParsedAuthorSearchPage:
+    parser = ScholarAuthorSearchParser()
+    parser.feed(fetch_result.body)
+
+    marker_counts = count_author_search_markers(fetch_result.body)
+    visible_text = strip_tags(SCRIPT_STYLE_RE.sub(" ", fetch_result.body)).lower()
+    warnings: list[str] = []
+    if not parser.candidates:
+        warnings.append("no_author_candidates_detected")
+
+    state, state_reason = detect_author_search_state(
+        fetch_result,
+        parser.candidates,
+        marker_counts,
+        visible_text=visible_text,
+    )
+
+    return ParsedAuthorSearchPage(
+        state=state,
+        state_reason=state_reason,
+        candidates=parser.candidates,
+        marker_counts=marker_counts,
+        warnings=warnings,
+    )

app/services/domains/scholar/parser_constants.py

@@ -0,0 +1,87 @@
+from __future__ import annotations
+
+import re
+
+BLOCKED_KEYWORDS = [
+    "unusual traffic",
+    "sorry/index",
+    "not a robot",
+    "our systems have detected",
+    "automated queries",
+    "recaptcha",
+    "captcha",
+]
+
+NO_RESULTS_KEYWORDS = [
+    "didn't match any articles",
+    "did not match any articles",
+    "no articles",
+    "no documents",
+]
+
+NO_AUTHOR_RESULTS_KEYWORDS = [
+    "didn't match any user profiles",
+    "did not match any user profiles",
+    "didn't match any scholars",
+    "did not match any scholars",
+    "no user profiles",
+]
+
+MARKER_KEYS = [
+    "gsc_a_tr",
+    "gsc_a_at",
+    "gsc_a_ac",
+    "gsc_a_h",
+    "gsc_a_y",
+    "gs_gray",
+    "gsc_prf_in",
+    "gsc_rsb_st",
+]
+
+AUTHOR_SEARCH_MARKER_KEYS = [
+    "gsc_1usr",
+    "gs_ai_name",
+    "gs_ai_aff",
+    "gs_ai_eml",
+    "gs_ai_cby",
+    "gs_ai_one_int",
+]
+
+NETWORK_DNS_ERROR_KEYWORDS = [
+    "temporary failure in name resolution",
+    "name or service not known",
+    "nodename nor servname provided",
+    "getaddrinfo failed",
+]
+
+NETWORK_TIMEOUT_KEYWORDS = [
+    "timed out",
+    "timeout",
+]
+
+NETWORK_TLS_ERROR_KEYWORDS = [
+    "ssl",
+    "tls",
+    "certificate verify failed",
+]
+
+TAG_RE = re.compile(r"<[^>]+>", re.S)
+SCRIPT_STYLE_RE = re.compile(r"<(script|style)\b[^>]*>.*?</\1>", re.I | re.S)
+SHOW_MORE_BUTTON_RE = re.compile(
+    r"<button\b[^>]*\bid\s*=\s*['\"]gsc_bpf_more['\"][^>]*>",
+    re.I | re.S,
+)
+
+PROFILE_ROW_PARSER_DIRECT_MARKERS = (
+    "gs_ggs",
+    "gs_ggsd",
+    "gs_ggsa",
+    "gs_or_ggsm",
+)
+
+PROFILE_ROW_DIRECT_LABEL_TOKENS = (
+    "pdf",
+    "[pdf]",
+    "full text",
+    "download",
+)

app/services/domains/scholar/parser_types.py

@@ -0,0 +1,59 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from enum import StrEnum
+
+
+class ParseState(StrEnum):
+    OK = "ok"
+    NO_RESULTS = "no_results"
+    BLOCKED_OR_CAPTCHA = "blocked_or_captcha"
+    LAYOUT_CHANGED = "layout_changed"
+    NETWORK_ERROR = "network_error"
+
+
+@dataclass(frozen=True)
+class PublicationCandidate:
+    title: str
+    title_url: str | None
+    cluster_id: str | None
+    year: int | None
+    citation_count: int | None
+    authors_text: str | None
+    venue_text: str | None
+    pdf_url: str | None
+
+
+@dataclass(frozen=True)
+class ScholarSearchCandidate:
+    scholar_id: str
+    display_name: str
+    affiliation: str | None
+    email_domain: str | None
+    cited_by_count: int | None
+    interests: list[str]
+    profile_url: str
+    profile_image_url: str | None
+
+
+@dataclass(frozen=True)
+class ParsedProfilePage:
+    state: ParseState
+    state_reason: str
+    profile_name: str | None
+    profile_image_url: str | None
+    publications: list[PublicationCandidate]
+    marker_counts: dict[str, int]
+    warnings: list[str]
+    has_show_more_button: bool
+    has_operation_error_banner: bool
+    articles_range: str | None
+
+
+@dataclass(frozen=True)
+class ParsedAuthorSearchPage:
+    state: ParseState
+    state_reason: str
+    candidates: list[ScholarSearchCandidate]
+    marker_counts: dict[str, int]
+    warnings: list[str]

app/services/domains/scholar/parser_utils.py

@@ -0,0 +1,42 @@
+from __future__ import annotations
+
+from html import unescape
+
+from app.services.domains.scholar.parser_constants import TAG_RE
+
+
+def normalize_space(value: str) -> str:
+    return " ".join(unescape(value).split())
+
+
+def strip_tags(value: str) -> str:
+    return normalize_space(TAG_RE.sub(" ", value))
+
+
+def attr_class(attrs: list[tuple[str, str | None]]) -> str:
+    for name, raw_value in attrs:
+        if name.lower() == "class":
+            return raw_value or ""
+    return ""
+
+
+def attr_href(attrs: list[tuple[str, str | None]]) -> str | None:
+    for name, raw_value in attrs:
+        if name.lower() == "href":
+            return raw_value
+    return None
+
+
+def attr_src(attrs: list[tuple[str, str | None]]) -> str | None:
+    for name, raw_value in attrs:
+        if name.lower() == "src":
+            return raw_value
+    return None
+
+
+def build_absolute_scholar_url(path_or_url: str | None) -> str | None:
+    if not path_or_url:
+        return None
+    from urllib.parse import urljoin
+
+    return urljoin("https://scholar.google.com", path_or_url)

app/services/domains/scholar/profile_rows.py

@@ -0,0 +1,324 @@
+from __future__ import annotations
+
+import re
+from html.parser import HTMLParser
+from typing import Any
+from urllib.parse import parse_qs, urlparse
+
+from app.services.domains.scholar.parser_constants import (
+    MARKER_KEYS,
+    PROFILE_ROW_DIRECT_LABEL_TOKENS,
+    PROFILE_ROW_PARSER_DIRECT_MARKERS,
+    SHOW_MORE_BUTTON_RE,
+)
+from app.services.domains.scholar.parser_types import PublicationCandidate
+from app.services.domains.scholar.parser_utils import (
+    attr_class,
+    attr_href,
+    attr_src,
+    build_absolute_scholar_url,
+    normalize_space,
+    strip_tags,
+)
+
+
+class ScholarRowParser(HTMLParser):
+    def __init__(self) -> None:
+        super().__init__(convert_charrefs=True)
+        self.title_href: str | None = None
+        self.direct_download_href: str | None = None
+        self.title_parts: list[str] = []
+        self.citation_parts: list[str] = []
+        self.year_parts: list[str] = []
+        self.gray_texts: list[str] = []
+
+        self._title_depth = 0
+        self._citation_depth = 0
+        self._year_depth = 0
+        self._gray_stack: list[dict[str, Any]] = []
+        self._direct_marker_depth = 0
+        self._aux_link_stack: list[dict[str, Any]] = []
+
+    @staticmethod
+    def _contains_direct_marker(classes: str) -> bool:
+        lowered = classes.lower()
+        return any(marker in lowered for marker in PROFILE_ROW_PARSER_DIRECT_MARKERS)
+
+    def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
+        if self._title_depth > 0:
+            self._title_depth += 1
+        if self._citation_depth > 0:
+            self._citation_depth += 1
+        if self._year_depth > 0:
+            self._year_depth += 1
+        if self._gray_stack:
+            self._gray_stack[-1]["depth"] += 1
+        if self._direct_marker_depth > 0:
+            self._direct_marker_depth += 1
+        if self._aux_link_stack:
+            self._aux_link_stack[-1]["depth"] += 1
+
+        classes = attr_class(attrs)
+        if tag in {"div", "span"} and self._contains_direct_marker(classes):
+            self._direct_marker_depth = 1
+
+        if tag == "a" and "gsc_a_at" in classes:
+            self._title_depth = 1
+            self.title_href = attr_href(attrs)
+            return
+
+        if tag == "a" and "gsc_a_ac" in classes:
+            self._citation_depth = 1
+            return
+
+        if tag in {"span", "a"} and ("gsc_a_h" in classes or "gsc_a_y" in classes):
+            self._year_depth = 1
+            return
+
+        if tag == "div" and "gs_gray" in classes:
+            self._gray_stack.append({"depth": 1, "parts": []})
+            return
+
+        if tag == "a":
+            href = attr_href(attrs)
+            self._aux_link_stack.append(
+                {
+                    "depth": 1,
+                    "classes": classes,
+                    "href": href,
+                    "parts": [],
+                }
+            )
+
+    def handle_data(self, data: str) -> None:
+        if self._title_depth > 0:
+            self.title_parts.append(data)
+        if self._citation_depth > 0:
+            self.citation_parts.append(data)
+        if self._year_depth > 0:
+            self.year_parts.append(data)
+        if self._gray_stack:
+            self._gray_stack[-1]["parts"].append(data)
+        if self._aux_link_stack:
+            self._aux_link_stack[-1]["parts"].append(data)
+
+    def _capture_direct_download_href(self, link_info: dict[str, Any]) -> None:
+        if self.direct_download_href:
+            return
+        href = link_info.get("href")
+        if not href:
+            return
+        classes = str(link_info.get("classes") or "")
+        label = normalize_space("".join(link_info.get("parts") or [])).lower()
+        if "gs_ggsd" in classes or "gs_ggs" in classes or "gs_ggsa" in classes:
+            self.direct_download_href = href
+            return
+        label_match = any(token in label for token in PROFILE_ROW_DIRECT_LABEL_TOKENS)
+        if self._direct_marker_depth > 0 and label_match:
+            self.direct_download_href = href
+
+    def handle_endtag(self, tag: str) -> None:
+        if self._title_depth > 0:
+            self._title_depth -= 1
+        if self._citation_depth > 0:
+            self._citation_depth -= 1
+        if self._year_depth > 0:
+            self._year_depth -= 1
+        if self._gray_stack:
+            self._gray_stack[-1]["depth"] -= 1
+            if self._gray_stack[-1]["depth"] == 0:
+                text_value = normalize_space("".join(self._gray_stack[-1]["parts"]))
+                if text_value:
+                    self.gray_texts.append(text_value)
+                self._gray_stack.pop()
+        if self._aux_link_stack:
+            self._aux_link_stack[-1]["depth"] -= 1
+            if self._aux_link_stack[-1]["depth"] == 0:
+                link_info = self._aux_link_stack.pop()
+                self._capture_direct_download_href(link_info)
+        if self._direct_marker_depth > 0:
+            self._direct_marker_depth -= 1
+
+
+def extract_rows(html: str) -> list[str]:
+    pattern = re.compile(
+        r"<tr\b(?=[^>]*\bclass\s*=\s*['\"][^'\"]*\bgsc_a_tr\b[^'\"]*['\"])[^>]*>(.*?)</tr>",
+        re.I | re.S,
+    )
+    return [match.group(1) for match in pattern.finditer(html)]
+
+
+def parse_cluster_id_from_href(href: str | None) -> str | None:
+    if not href:
+        return None
+    parsed = urlparse(href)
+    query = parse_qs(parsed.query)
+
+    citation_for_view = query.get("citation_for_view")
+    if citation_for_view:
+        token = citation_for_view[0].strip()
+        if token:
+            if ":" in token:
+                return token.rsplit(":", 1)[-1] or None
+            return token
+
+    cluster = query.get("cluster")
+    if cluster:
+        token = cluster[0].strip()
+        if token:
+            return token
+    return None
+
+
+def parse_year(parts: list[str]) -> int | None:
+    text = normalize_space(" ".join(parts))
+    match = re.search(r"\b(19|20)\d{2}\b", text)
+    if not match:
+        return None
+    try:
+        return int(match.group(0))
+    except ValueError:
+        return None
+
+
+def parse_citation_count(parts: list[str]) -> int | None:
+    text = normalize_space(" ".join(parts))
+    if not text:
+        return 0
+    match = re.search(r"\d+", text)
+    if not match:
+        return None
+    return int(match.group(0))
+
+
+def _parse_publication_row(row_html: str) -> tuple[PublicationCandidate | None, list[str]]:
+    parser = ScholarRowParser()
+    parser.feed(row_html)
+    warnings: list[str] = []
+    title = normalize_space("".join(parser.title_parts))
+    if not title:
+        warnings.append("row_missing_title")
+        return None, warnings
+    if not parser.title_href:
+        warnings.append("row_missing_title_href")
+
+    citation_text = normalize_space(" ".join(parser.citation_parts))
+    citation_count = parse_citation_count(parser.citation_parts)
+    if citation_text and citation_count is None:
+        warnings.append("layout_row_citation_unparseable")
+
+    year_text = normalize_space(" ".join(parser.year_parts))
+    year = parse_year(parser.year_parts)
+    if year_text and year is None:
+        warnings.append("layout_row_year_unparseable")
+
+    authors_text = parser.gray_texts[0] if len(parser.gray_texts) > 0 else None
+    venue_text = parser.gray_texts[1] if len(parser.gray_texts) > 1 else None
+    return (
+        PublicationCandidate(
+            title=title,
+            title_url=parser.title_href,
+            cluster_id=parse_cluster_id_from_href(parser.title_href),
+            year=year,
+            citation_count=citation_count,
+            authors_text=authors_text,
+            venue_text=venue_text,
+            pdf_url=build_absolute_scholar_url(parser.direct_download_href),
+        ),
+        warnings,
+    )
+
+
+def parse_publications(html: str) -> tuple[list[PublicationCandidate], list[str]]:
+    rows = extract_rows(html)
+    warnings: list[str] = []
+    publications: list[PublicationCandidate] = []
+
+    for row_html in rows:
+        publication, row_warnings = _parse_publication_row(row_html)
+        warnings.extend(row_warnings)
+        if publication is None:
+            continue
+        publications.append(publication)
+
+    if not rows:
+        warnings.append("no_rows_detected")
+    if rows and not publications:
+        warnings.append("layout_all_rows_unparseable")
+
+    return publications, sorted(set(warnings))
+
+
+def extract_profile_name(html: str) -> str | None:
+    pattern = re.compile(
+        r"<[^>]*\bid\s*=\s*['\"]gsc_prf_in['\"][^>]*>(.*?)</[^>]+>",
+        re.I | re.S,
+    )
+    match = pattern.search(html)
+    if not match:
+        return None
+    value = strip_tags(match.group(1))
+    return value or None
+
+
+def extract_profile_image_url(html: str) -> str | None:
+    og_image_pattern = re.compile(
+        r"<meta[^>]+property=['\"]og:image['\"][^>]+content=['\"]([^'\"]+)['\"][^>]*>",
+        re.I | re.S,
+    )
+    og_match = og_image_pattern.search(html)
+    if og_match:
+        value = normalize_space(og_match.group(1))
+        absolute = build_absolute_scholar_url(value)
+        if absolute:
+            return absolute
+
+    image_pattern = re.compile(
+        r"<img[^>]*\bid=['\"]gsc_prf_pup-img['\"][^>]*\bsrc=['\"]([^'\"]+)['\"][^>]*>",
+        re.I | re.S,
+    )
+    image_match = image_pattern.search(html)
+    if not image_match:
+        return None
+
+    value = normalize_space(image_match.group(1))
+    return build_absolute_scholar_url(value)
+
+
+def extract_articles_range(html: str) -> str | None:
+    pattern = re.compile(
+        r"<[^>]*\bid\s*=\s*['\"]gsc_a_nn['\"][^>]*>(.*?)</[^>]+>",
+        re.I | re.S,
+    )
+    match = pattern.search(html)
+    if not match:
+        return None
+    value = strip_tags(match.group(1))
+    return value or None
+
+
+def has_show_more_button(html: str) -> bool:
+    match = SHOW_MORE_BUTTON_RE.search(html)
+    if match is None:
+        return False
+
+    button_tag = match.group(0).lower()
+    if "disabled" in button_tag:
+        return False
+    if 'aria-disabled="true"' in button_tag or "aria-disabled='true'" in button_tag:
+        return False
+    if "gs_dis" in button_tag:
+        return False
+    return True
+
+
+def has_operation_error_banner(html: str) -> bool:
+    lowered = html.lower()
+    if "id=\"gsc_a_err\"" not in lowered and "id='gsc_a_err'" not in lowered:
+        return False
+    return "can't perform the operation now" in lowered or "cannot perform the operation now" in lowered
+
+
+def count_markers(html: str) -> dict[str, int]:
+    lowered = html.lower()
+    return {key: lowered.count(key.lower()) for key in MARKER_KEYS}

app/services/domains/scholar/source.py

@@ -0,0 +1,225 @@
+from __future__ import annotations
+
+import asyncio
+import logging
+import random
+from dataclasses import dataclass
+from typing import Protocol
+from urllib.error import HTTPError, URLError
+from urllib.parse import urlencode
+from urllib.request import Request, urlopen
+
+SCHOLAR_PROFILE_URL = "https://scholar.google.com/citations"
+DEFAULT_PAGE_SIZE = 100
+
+DEFAULT_USER_AGENTS = [
+    (
+        "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
+        "(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
+    ),
+    (
+        "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_7) AppleWebKit/605.1.15 "
+        "(KHTML, like Gecko) Version/18.1 Safari/605.1.15"
+    ),
+    (
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:131.0) "
+        "Gecko/20100101 Firefox/131.0"
+    ),
+]
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass(frozen=True)
+class FetchResult:
+    requested_url: str
+    status_code: int | None
+    final_url: str | None
+    body: str
+    error: str | None
+
+
+class ScholarSource(Protocol):
+    async def fetch_profile_html(self, scholar_id: str) -> FetchResult:
+        ...
+
+    async def fetch_profile_page_html(
+        self,
+        scholar_id: str,
+        *,
+        cstart: int,
+        pagesize: int,
+    ) -> FetchResult:
+        ...
+
+    async def fetch_author_search_html(
+        self,
+        query: str,
+        *,
+        start: int,
+    ) -> FetchResult:
+        ...
+
+
+class LiveScholarSource:
+    def __init__(
+        self,
+        *,
+        timeout_seconds: float = 25.0,
+        user_agents: list[str] | None = None,
+    ) -> None:
+        self._timeout_seconds = timeout_seconds
+        self._user_agents = user_agents or DEFAULT_USER_AGENTS
+
+    async def fetch_profile_html(self, scholar_id: str) -> FetchResult:
+        return await self.fetch_profile_page_html(
+            scholar_id,
+            cstart=0,
+            pagesize=DEFAULT_PAGE_SIZE,
+        )
+
+    async def fetch_profile_page_html(
+        self,
+        scholar_id: str,
+        *,
+        cstart: int,
+        pagesize: int = DEFAULT_PAGE_SIZE,
+    ) -> FetchResult:
+        requested_url = _build_profile_url(
+            scholar_id=scholar_id,
+            cstart=cstart,
+            pagesize=pagesize,
+        )
+        logger.debug(
+            "scholar_source.fetch_started",
+            extra={
+                "event": "scholar_source.fetch_started",
+                "scholar_id": scholar_id,
+                "requested_url": requested_url,
+                "cstart": cstart,
+                "pagesize": pagesize,
+            },
+        )
+        return await asyncio.to_thread(self._fetch_sync, requested_url)
+
+    async def fetch_author_search_html(
+        self,
+        query: str,
+        *,
+        start: int = 0,
+    ) -> FetchResult:
+        requested_url = _build_author_search_url(
+            query=query,
+            start=start,
+        )
+        logger.debug(
+            "scholar_source.search_fetch_started",
+            extra={
+                "event": "scholar_source.search_fetch_started",
+                "query": query,
+                "requested_url": requested_url,
+                "start": start,
+            },
+        )
+        return await asyncio.to_thread(self._fetch_sync, requested_url)
+
+    def _build_request(self, requested_url: str) -> Request:
+        return Request(
+            requested_url,
+            headers={
+                "User-Agent": random.choice(self._user_agents),
+                "Accept": "text/html,application/xhtml+xml",
+                "Accept-Language": "en-US,en;q=0.9",
+                "Connection": "close",
+            },
+        )
+
+    @staticmethod
+    def _http_error_body(exc: HTTPError) -> str:
+        try:
+            return exc.read().decode("utf-8", errors="replace")
+        except Exception:
+            return ""
+
+    @staticmethod
+    def _network_error_result(requested_url: str, exc: URLError) -> FetchResult:
+        logger.warning(
+            "scholar_source.fetch_network_error",
+            extra={"event": "scholar_source.fetch_network_error", "requested_url": requested_url},
+        )
+        return FetchResult(
+            requested_url=requested_url,
+            status_code=None,
+            final_url=None,
+            body="",
+            error=str(exc),
+        )
+
+    @staticmethod
+    def _http_error_result(requested_url: str, exc: HTTPError) -> FetchResult:
+        logger.warning(
+            "scholar_source.fetch_http_error",
+            extra={
+                "event": "scholar_source.fetch_http_error",
+                "requested_url": requested_url,
+                "status_code": exc.code,
+            },
+        )
+        return FetchResult(
+            requested_url=requested_url,
+            status_code=exc.code,
+            final_url=exc.geturl(),
+            body=LiveScholarSource._http_error_body(exc),
+            error=str(exc),
+        )
+
+    @staticmethod
+    def _success_result(requested_url: str, response) -> FetchResult:
+        body = response.read().decode("utf-8", errors="replace")
+        status_code = getattr(response, "status", 200)
+        logger.debug(
+            "scholar_source.fetch_succeeded",
+            extra={
+                "event": "scholar_source.fetch_succeeded",
+                "requested_url": requested_url,
+                "status_code": status_code,
+            },
+        )
+        return FetchResult(
+            requested_url=requested_url,
+            status_code=status_code,
+            final_url=response.geturl(),
+            body=body,
+            error=None,
+        )
+
+    def _fetch_sync(self, requested_url: str) -> FetchResult:
+        request = self._build_request(requested_url)
+
+        try:
+            with urlopen(request, timeout=self._timeout_seconds) as response:
+                return self._success_result(requested_url, response)
+        except HTTPError as exc:
+            return self._http_error_result(requested_url, exc)
+        except URLError as exc:
+            return self._network_error_result(requested_url, exc)
+
+
+def _build_profile_url(*, scholar_id: str, cstart: int, pagesize: int) -> str:
+    query: dict[str, int | str] = {"hl": "en", "user": scholar_id}
+    if cstart > 0:
+        query["cstart"] = int(cstart)
+    if pagesize > 0:
+        query["pagesize"] = int(pagesize)
+    return f"{SCHOLAR_PROFILE_URL}?{urlencode(query)}"
+
+
+def _build_author_search_url(*, query: str, start: int) -> str:
+    params: dict[str, int | str] = {
+        "hl": "en",
+        "view_op": "search_authors",
+        "mauthors": query,
+    }
+    if start > 0:
+        params["astart"] = int(start)
+    return f"{SCHOLAR_PROFILE_URL}?{urlencode(params)}"

app/services/domains/scholar/state_detection.py

@@ -0,0 +1,164 @@
+from __future__ import annotations
+
+from app.services.domains.scholar.parser_constants import (
+    BLOCKED_KEYWORDS,
+    NETWORK_DNS_ERROR_KEYWORDS,
+    NETWORK_TIMEOUT_KEYWORDS,
+    NETWORK_TLS_ERROR_KEYWORDS,
+    NO_AUTHOR_RESULTS_KEYWORDS,
+    NO_RESULTS_KEYWORDS,
+)
+from app.services.domains.scholar.parser_types import ParseState, PublicationCandidate, ScholarSearchCandidate
+from app.services.domains.scholar.source import FetchResult
+
+
+def classify_network_error_reason(fetch_error: str | None) -> str:
+    lowered = (fetch_error or "").lower()
+    if lowered:
+        if any(keyword in lowered for keyword in NETWORK_DNS_ERROR_KEYWORDS):
+            return "network_dns_resolution_failed"
+        if any(keyword in lowered for keyword in NETWORK_TIMEOUT_KEYWORDS):
+            return "network_timeout"
+        if any(keyword in lowered for keyword in NETWORK_TLS_ERROR_KEYWORDS):
+            return "network_tls_error"
+        if "connection reset" in lowered:
+            return "network_connection_reset"
+        if "connection refused" in lowered:
+            return "network_connection_refused"
+        if "network is unreachable" in lowered:
+            return "network_unreachable"
+    return "network_error_missing_status_code"
+
+
+def classify_block_or_captcha_reason(
+    *,
+    status_code: int,
+    final_url: str,
+    body_lowered: str,
+) -> str | None:
+    if "accounts.google.com" in final_url and ("signin" in final_url or "servicelogin" in final_url):
+        return "blocked_accounts_redirect"
+    if status_code == 429:
+        return "blocked_http_429_rate_limited"
+    if status_code == 403:
+        if "recaptcha" in body_lowered or "captcha" in body_lowered or "sorry/index" in final_url:
+            return "blocked_http_403_captcha_challenge"
+        return "blocked_http_403_forbidden"
+    if "sorry/index" in final_url or "sorry/index" in body_lowered:
+        return "blocked_google_sorry_challenge"
+    if "our systems have detected" in body_lowered or "unusual traffic" in body_lowered:
+        return "blocked_unusual_traffic_detected"
+    if "automated queries" in body_lowered:
+        return "blocked_automated_queries_detected"
+    if "not a robot" in body_lowered:
+        return "blocked_not_a_robot_challenge"
+    if "recaptcha" in body_lowered:
+        return "blocked_recaptcha_challenge"
+    if "captcha" in body_lowered:
+        return "blocked_captcha_challenge"
+    if any(keyword in body_lowered for keyword in BLOCKED_KEYWORDS):
+        return "blocked_keyword_detected"
+    return None
+
+
+def _warnings_contain(warnings: list[str], code: str) -> bool:
+    return any(item == code for item in warnings)
+
+
+def _has_layout_row_failure(marker_counts: dict[str, int], warnings: list[str]) -> bool:
+    if _warnings_contain(warnings, "layout_all_rows_unparseable"):
+        return True
+    if marker_counts.get("gsc_a_tr", 0) <= 0:
+        return False
+    if _warnings_contain(warnings, "row_missing_title"):
+        return True
+    return marker_counts.get("gsc_a_at", 0) <= 0
+
+
+def _first_layout_warning(warnings: list[str]) -> str | None:
+    for warning in warnings:
+        if warning.startswith("layout_"):
+            return warning
+    return None
+
+
+def detect_state(
+    fetch_result: FetchResult,
+    publications: list[PublicationCandidate],
+    marker_counts: dict[str, int],
+    *,
+    warnings: list[str],
+    has_show_more_button_flag: bool,
+    articles_range: str | None,
+    visible_text: str,
+) -> tuple[ParseState, str]:
+    if fetch_result.status_code is None:
+        return ParseState.NETWORK_ERROR, classify_network_error_reason(fetch_result.error)
+
+    lowered = fetch_result.body.lower()
+    final = (fetch_result.final_url or "").lower()
+    status_code = int(fetch_result.status_code)
+
+    block_reason = classify_block_or_captcha_reason(
+        status_code=status_code,
+        final_url=final,
+        body_lowered=lowered,
+    )
+    if block_reason is not None:
+        return ParseState.BLOCKED_OR_CAPTCHA, block_reason
+
+    if not publications and any(keyword in visible_text for keyword in NO_RESULTS_KEYWORDS):
+        return ParseState.NO_RESULTS, "no_results_keyword_detected"
+
+    layout_warning = _first_layout_warning(warnings)
+    if layout_warning is not None:
+        return ParseState.LAYOUT_CHANGED, layout_warning
+
+    if _has_layout_row_failure(marker_counts, warnings):
+        return ParseState.LAYOUT_CHANGED, "layout_publication_rows_unparseable"
+
+    if has_show_more_button_flag and not articles_range:
+        return ParseState.LAYOUT_CHANGED, "layout_show_more_without_articles_range"
+
+    if not publications:
+        has_profile_markers = marker_counts.get("gsc_prf_in", 0) > 0
+        has_table_markers = marker_counts.get("gsc_a_tr", 0) > 0 or marker_counts.get("gsc_a_at", 0) > 0
+        if not has_profile_markers and not has_table_markers:
+            return ParseState.LAYOUT_CHANGED, "layout_markers_missing"
+        return ParseState.OK, "no_rows_with_known_markers"
+
+    return ParseState.OK, "publications_extracted"
+
+
+def detect_author_search_state(
+    fetch_result: FetchResult,
+    candidates: list[ScholarSearchCandidate],
+    marker_counts: dict[str, int],
+    *,
+    visible_text: str,
+) -> tuple[ParseState, str]:
+    if fetch_result.status_code is None:
+        return ParseState.NETWORK_ERROR, classify_network_error_reason(fetch_result.error)
+
+    lowered = fetch_result.body.lower()
+    final = (fetch_result.final_url or "").lower()
+    status_code = int(fetch_result.status_code)
+
+    block_reason = classify_block_or_captcha_reason(
+        status_code=status_code,
+        final_url=final,
+        body_lowered=lowered,
+    )
+    if block_reason is not None:
+        return ParseState.BLOCKED_OR_CAPTCHA, block_reason
+
+    if not candidates and any(keyword in visible_text for keyword in NO_AUTHOR_RESULTS_KEYWORDS):
+        return ParseState.NO_RESULTS, "no_results_keyword_detected"
+
+    if not candidates:
+        has_search_markers = marker_counts.get("gsc_1usr", 0) > 0 or marker_counts.get("gs_ai_name", 0) > 0
+        if not has_search_markers:
+            return ParseState.NO_RESULTS, "no_search_candidates_detected"
+        return ParseState.LAYOUT_CHANGED, "layout_author_candidates_unparseable"
+
+    return ParseState.OK, "author_candidates_extracted"