modularize service layer, add mobile nav drawer, and sync docs
This commit is contained in:
parent
7f7a8ce2b0
commit
6b6ed91b92
72 changed files with 8122 additions and 7501 deletions
|
|
@ -1,886 +1,3 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from enum import StrEnum
|
||||
from html import unescape
|
||||
from html.parser import HTMLParser
|
||||
from typing import Any
|
||||
from urllib.parse import parse_qs, urljoin, urlparse
|
||||
|
||||
from app.services.scholar_source import FetchResult
|
||||
|
||||
BLOCKED_KEYWORDS = [
|
||||
"unusual traffic",
|
||||
"sorry/index",
|
||||
"not a robot",
|
||||
"our systems have detected",
|
||||
"automated queries",
|
||||
"recaptcha",
|
||||
"captcha",
|
||||
]
|
||||
|
||||
NO_RESULTS_KEYWORDS = [
|
||||
"didn't match any articles",
|
||||
"did not match any articles",
|
||||
"no articles",
|
||||
"no documents",
|
||||
]
|
||||
|
||||
NO_AUTHOR_RESULTS_KEYWORDS = [
|
||||
"didn't match any user profiles",
|
||||
"did not match any user profiles",
|
||||
"didn't match any scholars",
|
||||
"did not match any scholars",
|
||||
"no user profiles",
|
||||
]
|
||||
|
||||
MARKER_KEYS = [
|
||||
"gsc_a_tr",
|
||||
"gsc_a_at",
|
||||
"gsc_a_ac",
|
||||
"gsc_a_h",
|
||||
"gsc_a_y",
|
||||
"gs_gray",
|
||||
"gsc_prf_in",
|
||||
"gsc_rsb_st",
|
||||
]
|
||||
|
||||
AUTHOR_SEARCH_MARKER_KEYS = [
|
||||
"gsc_1usr",
|
||||
"gs_ai_name",
|
||||
"gs_ai_aff",
|
||||
"gs_ai_eml",
|
||||
"gs_ai_cby",
|
||||
"gs_ai_one_int",
|
||||
]
|
||||
|
||||
NETWORK_DNS_ERROR_KEYWORDS = [
|
||||
"temporary failure in name resolution",
|
||||
"name or service not known",
|
||||
"nodename nor servname provided",
|
||||
"getaddrinfo failed",
|
||||
]
|
||||
|
||||
NETWORK_TIMEOUT_KEYWORDS = [
|
||||
"timed out",
|
||||
"timeout",
|
||||
]
|
||||
|
||||
NETWORK_TLS_ERROR_KEYWORDS = [
|
||||
"ssl",
|
||||
"tls",
|
||||
"certificate verify failed",
|
||||
]
|
||||
|
||||
TAG_RE = re.compile(r"<[^>]+>", re.S)
|
||||
SCRIPT_STYLE_RE = re.compile(r"<(script|style)\b[^>]*>.*?</\1>", re.I | re.S)
|
||||
SHOW_MORE_BUTTON_RE = re.compile(
|
||||
r"<button\b[^>]*\bid\s*=\s*['\"]gsc_bpf_more['\"][^>]*>",
|
||||
re.I | re.S,
|
||||
)
|
||||
PROFILE_ROW_PARSER_DIRECT_MARKERS = (
|
||||
"gs_ggs",
|
||||
"gs_ggsd",
|
||||
"gs_ggsa",
|
||||
"gs_or_ggsm",
|
||||
)
|
||||
PROFILE_ROW_DIRECT_LABEL_TOKENS = (
|
||||
"pdf",
|
||||
"[pdf]",
|
||||
"full text",
|
||||
"download",
|
||||
)
|
||||
|
||||
|
||||
class ParseState(StrEnum):
|
||||
OK = "ok"
|
||||
NO_RESULTS = "no_results"
|
||||
BLOCKED_OR_CAPTCHA = "blocked_or_captcha"
|
||||
LAYOUT_CHANGED = "layout_changed"
|
||||
NETWORK_ERROR = "network_error"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PublicationCandidate:
|
||||
title: str
|
||||
title_url: str | None
|
||||
cluster_id: str | None
|
||||
year: int | None
|
||||
citation_count: int | None
|
||||
authors_text: str | None
|
||||
venue_text: str | None
|
||||
pdf_url: str | None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ScholarSearchCandidate:
|
||||
scholar_id: str
|
||||
display_name: str
|
||||
affiliation: str | None
|
||||
email_domain: str | None
|
||||
cited_by_count: int | None
|
||||
interests: list[str]
|
||||
profile_url: str
|
||||
profile_image_url: str | None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ParsedProfilePage:
|
||||
state: ParseState
|
||||
state_reason: str
|
||||
profile_name: str | None
|
||||
profile_image_url: str | None
|
||||
publications: list[PublicationCandidate]
|
||||
marker_counts: dict[str, int]
|
||||
warnings: list[str]
|
||||
has_show_more_button: bool
|
||||
has_operation_error_banner: bool
|
||||
articles_range: str | None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ParsedAuthorSearchPage:
|
||||
state: ParseState
|
||||
state_reason: str
|
||||
candidates: list[ScholarSearchCandidate]
|
||||
marker_counts: dict[str, int]
|
||||
warnings: list[str]
|
||||
|
||||
|
||||
def normalize_space(value: str) -> str:
|
||||
return " ".join(unescape(value).split())
|
||||
|
||||
|
||||
def strip_tags(value: str) -> str:
|
||||
return normalize_space(TAG_RE.sub(" ", value))
|
||||
|
||||
|
||||
def attr_class(attrs: list[tuple[str, str | None]]) -> str:
|
||||
for name, raw_value in attrs:
|
||||
if name.lower() == "class":
|
||||
return raw_value or ""
|
||||
return ""
|
||||
|
||||
|
||||
def attr_href(attrs: list[tuple[str, str | None]]) -> str | None:
|
||||
for name, raw_value in attrs:
|
||||
if name.lower() == "href":
|
||||
return raw_value
|
||||
return None
|
||||
|
||||
|
||||
def attr_src(attrs: list[tuple[str, str | None]]) -> str | None:
|
||||
for name, raw_value in attrs:
|
||||
if name.lower() == "src":
|
||||
return raw_value
|
||||
return None
|
||||
|
||||
|
||||
def build_absolute_scholar_url(path_or_url: str | None) -> str | None:
|
||||
if not path_or_url:
|
||||
return None
|
||||
return urljoin("https://scholar.google.com", path_or_url)
|
||||
|
||||
|
||||
class ScholarRowParser(HTMLParser):
|
||||
def __init__(self) -> None:
|
||||
super().__init__(convert_charrefs=True)
|
||||
self.title_href: str | None = None
|
||||
self.direct_download_href: str | None = None
|
||||
self.title_parts: list[str] = []
|
||||
self.citation_parts: list[str] = []
|
||||
self.year_parts: list[str] = []
|
||||
self.gray_texts: list[str] = []
|
||||
|
||||
self._title_depth = 0
|
||||
self._citation_depth = 0
|
||||
self._year_depth = 0
|
||||
self._gray_stack: list[dict[str, Any]] = []
|
||||
self._direct_marker_depth = 0
|
||||
self._aux_link_stack: list[dict[str, Any]] = []
|
||||
|
||||
@staticmethod
|
||||
def _contains_direct_marker(classes: str) -> bool:
|
||||
lowered = classes.lower()
|
||||
return any(marker in lowered for marker in PROFILE_ROW_PARSER_DIRECT_MARKERS)
|
||||
|
||||
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
||||
if self._title_depth > 0:
|
||||
self._title_depth += 1
|
||||
if self._citation_depth > 0:
|
||||
self._citation_depth += 1
|
||||
if self._year_depth > 0:
|
||||
self._year_depth += 1
|
||||
if self._gray_stack:
|
||||
self._gray_stack[-1]["depth"] += 1
|
||||
if self._direct_marker_depth > 0:
|
||||
self._direct_marker_depth += 1
|
||||
if self._aux_link_stack:
|
||||
self._aux_link_stack[-1]["depth"] += 1
|
||||
|
||||
classes = attr_class(attrs)
|
||||
if tag in {"div", "span"} and self._contains_direct_marker(classes):
|
||||
self._direct_marker_depth = 1
|
||||
|
||||
if tag == "a" and "gsc_a_at" in classes:
|
||||
self._title_depth = 1
|
||||
self.title_href = attr_href(attrs)
|
||||
return
|
||||
|
||||
if tag == "a" and "gsc_a_ac" in classes:
|
||||
self._citation_depth = 1
|
||||
return
|
||||
|
||||
if tag in {"span", "a"} and ("gsc_a_h" in classes or "gsc_a_y" in classes):
|
||||
self._year_depth = 1
|
||||
return
|
||||
|
||||
if tag == "div" and "gs_gray" in classes:
|
||||
self._gray_stack.append({"depth": 1, "parts": []})
|
||||
return
|
||||
|
||||
if tag == "a":
|
||||
self._aux_link_stack.append(
|
||||
{
|
||||
"depth": 1,
|
||||
"href": attr_href(attrs),
|
||||
"classes": classes,
|
||||
"parts": [],
|
||||
}
|
||||
)
|
||||
|
||||
def handle_data(self, data: str) -> None:
|
||||
if self._title_depth > 0:
|
||||
self.title_parts.append(data)
|
||||
if self._citation_depth > 0:
|
||||
self.citation_parts.append(data)
|
||||
if self._year_depth > 0:
|
||||
self.year_parts.append(data)
|
||||
if self._gray_stack:
|
||||
self._gray_stack[-1]["parts"].append(data)
|
||||
if self._aux_link_stack:
|
||||
self._aux_link_stack[-1]["parts"].append(data)
|
||||
|
||||
def _capture_direct_download_href(self, link: dict[str, Any]) -> None:
|
||||
if self.direct_download_href:
|
||||
return
|
||||
href = link.get("href")
|
||||
if not isinstance(href, str) or not href.strip():
|
||||
return
|
||||
label = normalize_space("".join(link.get("parts", []))).lower()
|
||||
classes = str(link.get("classes", "")).lower()
|
||||
label_match = any(token in label for token in PROFILE_ROW_DIRECT_LABEL_TOKENS)
|
||||
marker_match = self._contains_direct_marker(classes) or self._direct_marker_depth > 0
|
||||
if label_match or marker_match:
|
||||
self.direct_download_href = href.strip()
|
||||
|
||||
def handle_endtag(self, _tag: str) -> None:
|
||||
if self._title_depth > 0:
|
||||
self._title_depth -= 1
|
||||
if self._citation_depth > 0:
|
||||
self._citation_depth -= 1
|
||||
if self._year_depth > 0:
|
||||
self._year_depth -= 1
|
||||
if self._gray_stack:
|
||||
self._gray_stack[-1]["depth"] -= 1
|
||||
if self._gray_stack[-1]["depth"] == 0:
|
||||
text = normalize_space("".join(self._gray_stack[-1]["parts"]))
|
||||
if text:
|
||||
self.gray_texts.append(text)
|
||||
self._gray_stack.pop()
|
||||
if self._aux_link_stack:
|
||||
self._aux_link_stack[-1]["depth"] -= 1
|
||||
if self._aux_link_stack[-1]["depth"] == 0:
|
||||
self._capture_direct_download_href(self._aux_link_stack[-1])
|
||||
self._aux_link_stack.pop()
|
||||
if self._direct_marker_depth > 0:
|
||||
self._direct_marker_depth -= 1
|
||||
|
||||
|
||||
def extract_rows(html: str) -> list[str]:
|
||||
pattern = re.compile(
|
||||
r"<tr\b(?=[^>]*\bclass\s*=\s*['\"][^'\"]*\bgsc_a_tr\b[^'\"]*['\"])[^>]*>(.*?)</tr>",
|
||||
re.I | re.S,
|
||||
)
|
||||
return [match.group(1) for match in pattern.finditer(html)]
|
||||
|
||||
|
||||
def parse_cluster_id_from_href(href: str | None) -> str | None:
|
||||
if not href:
|
||||
return None
|
||||
parsed = urlparse(href)
|
||||
query = parse_qs(parsed.query)
|
||||
|
||||
citation_for_view = query.get("citation_for_view")
|
||||
if citation_for_view:
|
||||
token = citation_for_view[0].strip()
|
||||
if token:
|
||||
if ":" in token:
|
||||
return token.rsplit(":", 1)[-1] or None
|
||||
return token
|
||||
|
||||
cluster = query.get("cluster")
|
||||
if cluster:
|
||||
token = cluster[0].strip()
|
||||
if token:
|
||||
return token
|
||||
return None
|
||||
|
||||
|
||||
def parse_scholar_id_from_href(href: str | None) -> str | None:
|
||||
if not href:
|
||||
return None
|
||||
parsed = urlparse(href)
|
||||
query = parse_qs(parsed.query)
|
||||
user_values = query.get("user")
|
||||
if not user_values:
|
||||
return None
|
||||
candidate = user_values[0].strip()
|
||||
return candidate or None
|
||||
|
||||
|
||||
def parse_year(parts: list[str]) -> int | None:
|
||||
text = normalize_space(" ".join(parts))
|
||||
match = re.search(r"\b(19|20)\d{2}\b", text)
|
||||
if not match:
|
||||
return None
|
||||
try:
|
||||
return int(match.group(0))
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def parse_citation_count(parts: list[str]) -> int | None:
|
||||
text = normalize_space(" ".join(parts))
|
||||
if not text:
|
||||
return 0
|
||||
match = re.search(r"\d+", text)
|
||||
if not match:
|
||||
return None
|
||||
return int(match.group(0))
|
||||
|
||||
|
||||
def _parse_publication_row(row_html: str) -> tuple[PublicationCandidate | None, list[str]]:
|
||||
parser = ScholarRowParser()
|
||||
parser.feed(row_html)
|
||||
warnings: list[str] = []
|
||||
title = normalize_space("".join(parser.title_parts))
|
||||
if not title:
|
||||
warnings.append("row_missing_title")
|
||||
return None, warnings
|
||||
if not parser.title_href:
|
||||
warnings.append("row_missing_title_href")
|
||||
|
||||
citation_text = normalize_space(" ".join(parser.citation_parts))
|
||||
citation_count = parse_citation_count(parser.citation_parts)
|
||||
if citation_text and citation_count is None:
|
||||
warnings.append("layout_row_citation_unparseable")
|
||||
|
||||
year_text = normalize_space(" ".join(parser.year_parts))
|
||||
year = parse_year(parser.year_parts)
|
||||
if year_text and year is None:
|
||||
warnings.append("layout_row_year_unparseable")
|
||||
|
||||
authors_text = parser.gray_texts[0] if len(parser.gray_texts) > 0 else None
|
||||
venue_text = parser.gray_texts[1] if len(parser.gray_texts) > 1 else None
|
||||
return (
|
||||
PublicationCandidate(
|
||||
title=title,
|
||||
title_url=parser.title_href,
|
||||
cluster_id=parse_cluster_id_from_href(parser.title_href),
|
||||
year=year,
|
||||
citation_count=citation_count,
|
||||
authors_text=authors_text,
|
||||
venue_text=venue_text,
|
||||
pdf_url=build_absolute_scholar_url(parser.direct_download_href),
|
||||
),
|
||||
warnings,
|
||||
)
|
||||
|
||||
|
||||
def parse_publications(html: str) -> tuple[list[PublicationCandidate], list[str]]:
|
||||
rows = extract_rows(html)
|
||||
warnings: list[str] = []
|
||||
publications: list[PublicationCandidate] = []
|
||||
|
||||
for row_html in rows:
|
||||
publication, row_warnings = _parse_publication_row(row_html)
|
||||
warnings.extend(row_warnings)
|
||||
if publication is None:
|
||||
continue
|
||||
publications.append(publication)
|
||||
|
||||
if not rows:
|
||||
warnings.append("no_rows_detected")
|
||||
if rows and not publications:
|
||||
warnings.append("layout_all_rows_unparseable")
|
||||
|
||||
return publications, sorted(set(warnings))
|
||||
|
||||
|
||||
def extract_profile_name(html: str) -> str | None:
|
||||
pattern = re.compile(
|
||||
r"<[^>]*\bid\s*=\s*['\"]gsc_prf_in['\"][^>]*>(.*?)</[^>]+>",
|
||||
re.I | re.S,
|
||||
)
|
||||
match = pattern.search(html)
|
||||
if not match:
|
||||
return None
|
||||
value = strip_tags(match.group(1))
|
||||
return value or None
|
||||
|
||||
|
||||
def extract_profile_image_url(html: str) -> str | None:
|
||||
og_image_pattern = re.compile(
|
||||
r"<meta[^>]+property=['\"]og:image['\"][^>]+content=['\"]([^'\"]+)['\"][^>]*>",
|
||||
re.I | re.S,
|
||||
)
|
||||
og_match = og_image_pattern.search(html)
|
||||
if og_match:
|
||||
value = normalize_space(og_match.group(1))
|
||||
absolute = build_absolute_scholar_url(value)
|
||||
if absolute:
|
||||
return absolute
|
||||
|
||||
image_pattern = re.compile(
|
||||
r"<img[^>]*\bid=['\"]gsc_prf_pup-img['\"][^>]*\bsrc=['\"]([^'\"]+)['\"][^>]*>",
|
||||
re.I | re.S,
|
||||
)
|
||||
image_match = image_pattern.search(html)
|
||||
if not image_match:
|
||||
return None
|
||||
|
||||
value = normalize_space(image_match.group(1))
|
||||
return build_absolute_scholar_url(value)
|
||||
|
||||
|
||||
def extract_articles_range(html: str) -> str | None:
|
||||
pattern = re.compile(
|
||||
r"<[^>]*\bid\s*=\s*['\"]gsc_a_nn['\"][^>]*>(.*?)</[^>]+>",
|
||||
re.I | re.S,
|
||||
)
|
||||
match = pattern.search(html)
|
||||
if not match:
|
||||
return None
|
||||
value = strip_tags(match.group(1))
|
||||
return value or None
|
||||
|
||||
|
||||
def has_show_more_button(html: str) -> bool:
|
||||
match = SHOW_MORE_BUTTON_RE.search(html)
|
||||
if match is None:
|
||||
return False
|
||||
|
||||
button_tag = match.group(0).lower()
|
||||
if "disabled" in button_tag:
|
||||
return False
|
||||
if 'aria-disabled="true"' in button_tag or "aria-disabled='true'" in button_tag:
|
||||
return False
|
||||
if "gs_dis" in button_tag:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def has_operation_error_banner(html: str) -> bool:
|
||||
lowered = html.lower()
|
||||
if "id=\"gsc_a_err\"" not in lowered and "id='gsc_a_err'" not in lowered:
|
||||
return False
|
||||
return "can't perform the operation now" in lowered or "cannot perform the operation now" in lowered
|
||||
|
||||
|
||||
def count_markers(html: str) -> dict[str, int]:
|
||||
lowered = html.lower()
|
||||
return {key: lowered.count(key.lower()) for key in MARKER_KEYS}
|
||||
|
||||
|
||||
def count_author_search_markers(html: str) -> dict[str, int]:
|
||||
lowered = html.lower()
|
||||
return {key: lowered.count(key.lower()) for key in AUTHOR_SEARCH_MARKER_KEYS}
|
||||
|
||||
|
||||
def _extract_verified_email_domain(value: str | None) -> str | None:
|
||||
if not value:
|
||||
return None
|
||||
match = re.search(r"verified email at\s+(.+)$", value.strip(), re.I)
|
||||
if not match:
|
||||
return None
|
||||
domain = normalize_space(match.group(1))
|
||||
return domain or None
|
||||
|
||||
|
||||
class ScholarAuthorSearchParser(HTMLParser):
|
||||
def __init__(self) -> None:
|
||||
super().__init__(convert_charrefs=True)
|
||||
self.candidates: list[ScholarSearchCandidate] = []
|
||||
self._candidate: dict[str, Any] | None = None
|
||||
|
||||
def _begin_candidate(self) -> None:
|
||||
self._candidate = {
|
||||
"depth": 1,
|
||||
"name_href": None,
|
||||
"name_parts": [],
|
||||
"aff_depth": 0,
|
||||
"aff_parts": [],
|
||||
"name_depth": 0,
|
||||
"eml_depth": 0,
|
||||
"eml_parts": [],
|
||||
"cby_depth": 0,
|
||||
"cby_parts": [],
|
||||
"interest_depth": 0,
|
||||
"interest_parts": [],
|
||||
"interests": [],
|
||||
"image_src": None,
|
||||
}
|
||||
|
||||
def _increment_capture_depths(self) -> None:
|
||||
if self._candidate is None:
|
||||
return
|
||||
for key in ("name_depth", "aff_depth", "eml_depth", "cby_depth", "interest_depth"):
|
||||
if self._candidate[key] > 0:
|
||||
self._candidate[key] += 1
|
||||
|
||||
def _finalize_candidate(self) -> None:
|
||||
if self._candidate is None:
|
||||
return
|
||||
|
||||
name = normalize_space("".join(self._candidate["name_parts"]))
|
||||
scholar_id = parse_scholar_id_from_href(self._candidate["name_href"])
|
||||
if not name or not scholar_id:
|
||||
return
|
||||
|
||||
affiliation = normalize_space("".join(self._candidate["aff_parts"])) or None
|
||||
email_domain = _extract_verified_email_domain(
|
||||
normalize_space("".join(self._candidate["eml_parts"])) or None
|
||||
)
|
||||
cited_by_text = normalize_space("".join(self._candidate["cby_parts"]))
|
||||
cited_by_match = re.search(r"\d+", cited_by_text)
|
||||
cited_by_count = int(cited_by_match.group(0)) if cited_by_match else None
|
||||
|
||||
seen_interests: set[str] = set()
|
||||
interests: list[str] = []
|
||||
for interest in self._candidate["interests"]:
|
||||
normalized = normalize_space(interest)
|
||||
if not normalized or normalized in seen_interests:
|
||||
continue
|
||||
seen_interests.add(normalized)
|
||||
interests.append(normalized)
|
||||
|
||||
profile_url = build_absolute_scholar_url(self._candidate["name_href"])
|
||||
if not profile_url:
|
||||
profile_url = (
|
||||
"https://scholar.google.com/citations"
|
||||
f"?hl=en&user={scholar_id}"
|
||||
)
|
||||
|
||||
self.candidates.append(
|
||||
ScholarSearchCandidate(
|
||||
scholar_id=scholar_id,
|
||||
display_name=name,
|
||||
affiliation=affiliation,
|
||||
email_domain=email_domain,
|
||||
cited_by_count=cited_by_count,
|
||||
interests=interests,
|
||||
profile_url=profile_url,
|
||||
profile_image_url=build_absolute_scholar_url(self._candidate["image_src"]),
|
||||
)
|
||||
)
|
||||
|
||||
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
||||
classes = attr_class(attrs)
|
||||
|
||||
if self._candidate is None:
|
||||
if tag == "div" and "gsc_1usr" in classes:
|
||||
self._begin_candidate()
|
||||
return
|
||||
|
||||
self._candidate["depth"] += 1
|
||||
self._increment_capture_depths()
|
||||
|
||||
if tag == "a" and "gs_ai_name" in classes:
|
||||
self._candidate["name_depth"] = 1
|
||||
self._candidate["name_href"] = attr_href(attrs)
|
||||
return
|
||||
|
||||
if tag == "div" and "gs_ai_aff" in classes:
|
||||
self._candidate["aff_depth"] = 1
|
||||
return
|
||||
|
||||
if tag == "div" and "gs_ai_eml" in classes:
|
||||
self._candidate["eml_depth"] = 1
|
||||
return
|
||||
|
||||
if tag == "div" and "gs_ai_cby" in classes:
|
||||
self._candidate["cby_depth"] = 1
|
||||
return
|
||||
|
||||
if tag == "a" and "gs_ai_one_int" in classes:
|
||||
self._candidate["interest_depth"] = 1
|
||||
self._candidate["interest_parts"] = []
|
||||
return
|
||||
|
||||
if tag == "img" and self._candidate["image_src"] is None:
|
||||
self._candidate["image_src"] = attr_src(attrs)
|
||||
|
||||
def handle_data(self, data: str) -> None:
|
||||
if self._candidate is None:
|
||||
return
|
||||
if self._candidate["name_depth"] > 0:
|
||||
self._candidate["name_parts"].append(data)
|
||||
if self._candidate["aff_depth"] > 0:
|
||||
self._candidate["aff_parts"].append(data)
|
||||
if self._candidate["eml_depth"] > 0:
|
||||
self._candidate["eml_parts"].append(data)
|
||||
if self._candidate["cby_depth"] > 0:
|
||||
self._candidate["cby_parts"].append(data)
|
||||
if self._candidate["interest_depth"] > 0:
|
||||
self._candidate["interest_parts"].append(data)
|
||||
|
||||
def _decrement_capture_depth(self, key: str) -> bool:
|
||||
if self._candidate is None:
|
||||
return False
|
||||
if self._candidate[key] <= 0:
|
||||
return False
|
||||
self._candidate[key] -= 1
|
||||
return self._candidate[key] == 0
|
||||
|
||||
def handle_endtag(self, _tag: str) -> None:
|
||||
if self._candidate is None:
|
||||
return
|
||||
|
||||
interest_closed = self._decrement_capture_depth("interest_depth")
|
||||
self._decrement_capture_depth("name_depth")
|
||||
self._decrement_capture_depth("aff_depth")
|
||||
self._decrement_capture_depth("eml_depth")
|
||||
self._decrement_capture_depth("cby_depth")
|
||||
|
||||
if interest_closed:
|
||||
interest_text = normalize_space("".join(self._candidate["interest_parts"]))
|
||||
if interest_text:
|
||||
self._candidate["interests"].append(interest_text)
|
||||
self._candidate["interest_parts"] = []
|
||||
|
||||
self._candidate["depth"] -= 1
|
||||
if self._candidate["depth"] > 0:
|
||||
return
|
||||
|
||||
self._finalize_candidate()
|
||||
self._candidate = None
|
||||
|
||||
|
||||
def classify_network_error_reason(fetch_error: str | None) -> str:
|
||||
lowered = (fetch_error or "").lower()
|
||||
if lowered:
|
||||
if any(keyword in lowered for keyword in NETWORK_DNS_ERROR_KEYWORDS):
|
||||
return "network_dns_resolution_failed"
|
||||
if any(keyword in lowered for keyword in NETWORK_TIMEOUT_KEYWORDS):
|
||||
return "network_timeout"
|
||||
if any(keyword in lowered for keyword in NETWORK_TLS_ERROR_KEYWORDS):
|
||||
return "network_tls_error"
|
||||
if "connection reset" in lowered:
|
||||
return "network_connection_reset"
|
||||
if "connection refused" in lowered:
|
||||
return "network_connection_refused"
|
||||
if "network is unreachable" in lowered:
|
||||
return "network_unreachable"
|
||||
return "network_error_missing_status_code"
|
||||
|
||||
|
||||
def classify_block_or_captcha_reason(
|
||||
*,
|
||||
status_code: int,
|
||||
final_url: str,
|
||||
body_lowered: str,
|
||||
) -> str | None:
|
||||
if "accounts.google.com" in final_url and ("signin" in final_url or "servicelogin" in final_url):
|
||||
return "blocked_accounts_redirect"
|
||||
if status_code == 429:
|
||||
return "blocked_http_429_rate_limited"
|
||||
if status_code == 403:
|
||||
if "recaptcha" in body_lowered or "captcha" in body_lowered or "sorry/index" in final_url:
|
||||
return "blocked_http_403_captcha_challenge"
|
||||
return "blocked_http_403_forbidden"
|
||||
if "sorry/index" in final_url or "sorry/index" in body_lowered:
|
||||
return "blocked_google_sorry_challenge"
|
||||
if "our systems have detected" in body_lowered or "unusual traffic" in body_lowered:
|
||||
return "blocked_unusual_traffic_detected"
|
||||
if "automated queries" in body_lowered:
|
||||
return "blocked_automated_queries_detected"
|
||||
if "not a robot" in body_lowered:
|
||||
return "blocked_not_a_robot_challenge"
|
||||
if "recaptcha" in body_lowered:
|
||||
return "blocked_recaptcha_challenge"
|
||||
if "captcha" in body_lowered:
|
||||
return "blocked_captcha_challenge"
|
||||
if any(keyword in body_lowered for keyword in BLOCKED_KEYWORDS):
|
||||
return "blocked_keyword_detected"
|
||||
return None
|
||||
|
||||
|
||||
def _warnings_contain(warnings: list[str], code: str) -> bool:
|
||||
return any(item == code for item in warnings)
|
||||
|
||||
|
||||
def _has_layout_row_failure(marker_counts: dict[str, int], warnings: list[str]) -> bool:
|
||||
if _warnings_contain(warnings, "layout_all_rows_unparseable"):
|
||||
return True
|
||||
if marker_counts.get("gsc_a_tr", 0) <= 0:
|
||||
return False
|
||||
if _warnings_contain(warnings, "row_missing_title"):
|
||||
return True
|
||||
return marker_counts.get("gsc_a_at", 0) <= 0
|
||||
|
||||
|
||||
def _first_layout_warning(warnings: list[str]) -> str | None:
|
||||
for warning in warnings:
|
||||
if warning.startswith("layout_"):
|
||||
return warning
|
||||
return None
|
||||
|
||||
|
||||
def detect_state(
|
||||
fetch_result: FetchResult,
|
||||
publications: list[PublicationCandidate],
|
||||
marker_counts: dict[str, int],
|
||||
*,
|
||||
warnings: list[str],
|
||||
has_show_more_button_flag: bool,
|
||||
articles_range: str | None,
|
||||
visible_text: str,
|
||||
) -> tuple[ParseState, str]:
|
||||
if fetch_result.status_code is None:
|
||||
return ParseState.NETWORK_ERROR, classify_network_error_reason(fetch_result.error)
|
||||
|
||||
lowered = fetch_result.body.lower()
|
||||
final = (fetch_result.final_url or "").lower()
|
||||
status_code = int(fetch_result.status_code)
|
||||
|
||||
block_reason = classify_block_or_captcha_reason(
|
||||
status_code=status_code,
|
||||
final_url=final,
|
||||
body_lowered=lowered,
|
||||
)
|
||||
if block_reason is not None:
|
||||
return ParseState.BLOCKED_OR_CAPTCHA, block_reason
|
||||
|
||||
if not publications and any(keyword in visible_text for keyword in NO_RESULTS_KEYWORDS):
|
||||
return ParseState.NO_RESULTS, "no_results_keyword_detected"
|
||||
|
||||
layout_warning = _first_layout_warning(warnings)
|
||||
if layout_warning is not None:
|
||||
return ParseState.LAYOUT_CHANGED, layout_warning
|
||||
|
||||
if _has_layout_row_failure(marker_counts, warnings):
|
||||
return ParseState.LAYOUT_CHANGED, "layout_publication_rows_unparseable"
|
||||
|
||||
if has_show_more_button_flag and not articles_range:
|
||||
return ParseState.LAYOUT_CHANGED, "layout_show_more_without_articles_range"
|
||||
|
||||
if not publications:
|
||||
has_profile_markers = marker_counts.get("gsc_prf_in", 0) > 0
|
||||
has_table_markers = marker_counts.get("gsc_a_tr", 0) > 0 or marker_counts.get("gsc_a_at", 0) > 0
|
||||
if not has_profile_markers and not has_table_markers:
|
||||
return ParseState.LAYOUT_CHANGED, "layout_markers_missing"
|
||||
return ParseState.OK, "no_rows_with_known_markers"
|
||||
|
||||
return ParseState.OK, "publications_extracted"
|
||||
|
||||
|
||||
def detect_author_search_state(
|
||||
fetch_result: FetchResult,
|
||||
candidates: list[ScholarSearchCandidate],
|
||||
marker_counts: dict[str, int],
|
||||
*,
|
||||
visible_text: str,
|
||||
) -> tuple[ParseState, str]:
|
||||
if fetch_result.status_code is None:
|
||||
return ParseState.NETWORK_ERROR, classify_network_error_reason(fetch_result.error)
|
||||
|
||||
lowered = fetch_result.body.lower()
|
||||
final = (fetch_result.final_url or "").lower()
|
||||
status_code = int(fetch_result.status_code)
|
||||
|
||||
block_reason = classify_block_or_captcha_reason(
|
||||
status_code=status_code,
|
||||
final_url=final,
|
||||
body_lowered=lowered,
|
||||
)
|
||||
if block_reason is not None:
|
||||
return ParseState.BLOCKED_OR_CAPTCHA, block_reason
|
||||
|
||||
if not candidates and any(keyword in visible_text for keyword in NO_AUTHOR_RESULTS_KEYWORDS):
|
||||
return ParseState.NO_RESULTS, "no_results_keyword_detected"
|
||||
|
||||
if not candidates:
|
||||
has_search_markers = marker_counts.get("gsc_1usr", 0) > 0 or marker_counts.get("gs_ai_name", 0) > 0
|
||||
if not has_search_markers:
|
||||
return ParseState.NO_RESULTS, "no_search_candidates_detected"
|
||||
return ParseState.LAYOUT_CHANGED, "layout_author_candidates_unparseable"
|
||||
|
||||
return ParseState.OK, "author_candidates_extracted"
|
||||
|
||||
|
||||
def parse_profile_page(fetch_result: FetchResult) -> ParsedProfilePage:
|
||||
publications, warnings = parse_publications(fetch_result.body)
|
||||
marker_counts = count_markers(fetch_result.body)
|
||||
visible_text = strip_tags(SCRIPT_STYLE_RE.sub(" ", fetch_result.body)).lower()
|
||||
|
||||
show_more = has_show_more_button(fetch_result.body)
|
||||
operation_error_banner = has_operation_error_banner(fetch_result.body)
|
||||
articles_range = extract_articles_range(fetch_result.body)
|
||||
|
||||
if show_more:
|
||||
warnings.append("possible_partial_page_show_more_present")
|
||||
if operation_error_banner:
|
||||
warnings.append("operation_error_banner_present")
|
||||
|
||||
warnings = sorted(set(warnings))
|
||||
|
||||
state, state_reason = detect_state(
|
||||
fetch_result,
|
||||
publications,
|
||||
marker_counts,
|
||||
warnings=warnings,
|
||||
has_show_more_button_flag=show_more,
|
||||
articles_range=articles_range,
|
||||
visible_text=visible_text,
|
||||
)
|
||||
|
||||
return ParsedProfilePage(
|
||||
state=state,
|
||||
state_reason=state_reason,
|
||||
profile_name=extract_profile_name(fetch_result.body),
|
||||
profile_image_url=extract_profile_image_url(fetch_result.body),
|
||||
publications=publications,
|
||||
marker_counts=marker_counts,
|
||||
warnings=warnings,
|
||||
has_show_more_button=show_more,
|
||||
has_operation_error_banner=operation_error_banner,
|
||||
articles_range=articles_range,
|
||||
)
|
||||
|
||||
|
||||
def parse_author_search_page(fetch_result: FetchResult) -> ParsedAuthorSearchPage:
|
||||
parser = ScholarAuthorSearchParser()
|
||||
parser.feed(fetch_result.body)
|
||||
|
||||
marker_counts = count_author_search_markers(fetch_result.body)
|
||||
visible_text = strip_tags(SCRIPT_STYLE_RE.sub(" ", fetch_result.body)).lower()
|
||||
warnings: list[str] = []
|
||||
if not parser.candidates:
|
||||
warnings.append("no_author_candidates_detected")
|
||||
|
||||
state, state_reason = detect_author_search_state(
|
||||
fetch_result,
|
||||
parser.candidates,
|
||||
marker_counts,
|
||||
visible_text=visible_text,
|
||||
)
|
||||
|
||||
return ParsedAuthorSearchPage(
|
||||
state=state,
|
||||
state_reason=state_reason,
|
||||
candidates=parser.candidates,
|
||||
marker_counts=marker_counts,
|
||||
warnings=warnings,
|
||||
)
|
||||
from app.services.domains.scholar.parser import *
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue