justinzeus/scholarr
1
0
Fork 0
Code Issues 1 Pull requests 18 Projects Releases Packages Wiki Activity Actions

harden domain architecture and add lazy Crossref+Unpaywall PDF enrichment with publications workspace refactor

Browse source
This commit is contained in:
Justin Visser 2026-02-20 22:56:52 +01:00
parent a527e7a535
commit 01454162bb
62 changed files with 2562 additions and 688 deletions
Download patch file Download diff file Expand all files Collapse all files

65
app/services/domains/scholar/parser.py
View file

@ -11,14 +11,12 @@ from app.services.domains.scholar.parser_types import (
ParsedAuthorSearchPage,
ParsedProfilePage,
PublicationCandidate,
ScholarDomInvariantError,
ScholarMalformedDataError,
ScholarParserError,
ScholarSearchCandidate,
)
from app.services.domains.scholar.parser_utils import (
attr_class,
attr_href,
attr_src,
build_absolute_scholar_url,
normalize_space,
strip_tags,
)
from app.services.domains.scholar.profile_rows import (
@ -44,6 +42,55 @@ from app.services.domains.scholar.state_detection import (
)
def _raise_dom_error(code: str, message: str) -> None:
raise ScholarDomInvariantError(code=code, message=message)
def _assert_profile_dom_invariants(
*,
fetch_result: FetchResult,
marker_counts: dict[str, int],
publications: list[PublicationCandidate],
warnings: list[str],
has_show_more_button_flag: bool,
articles_range: str | None,
) -> None:
if fetch_result.status_code is None:
return
final_url = (fetch_result.final_url or "").lower()
if "accounts.google.com" in final_url or "sorry/index" in final_url:
return
if any(code.startswith("layout_") for code in warnings):
reason = next(code for code in warnings if code.startswith("layout_"))
_raise_dom_error(reason, f"Detected layout warning: {reason}")
if has_show_more_button_flag and not articles_range:
_raise_dom_error(
"layout_show_more_without_articles_range",
"Show-more control exists without an articles range marker.",
)
if marker_counts.get("gsc_a_tr", 0) > 0 and marker_counts.get("gsc_a_at", 0) <= 0:
_raise_dom_error(
"layout_missing_publication_title_anchor",
"Publication rows were present but title anchors were absent.",
)
if not publications:
has_profile_markers = marker_counts.get("gsc_prf_in", 0) > 0
has_table_markers = marker_counts.get("gsc_a_tr", 0) > 0 or marker_counts.get("gsc_a_at", 0) > 0
if not has_profile_markers and not has_table_markers:
_raise_dom_error("layout_markers_missing", "Expected scholar profile markers were absent.")
for publication in publications:
if not publication.title.strip():
raise ScholarMalformedDataError(
code="malformed_publication_title",
message="Encountered a publication candidate with an empty title.",
)
if publication.citation_count is not None and int(publication.citation_count) < 0:
raise ScholarMalformedDataError(
code="malformed_publication_negative_citations",
message="Encountered a publication candidate with negative citations.",
)
def parse_profile_page(fetch_result: FetchResult) -> ParsedProfilePage:
publications, warnings = parse_publications(fetch_result.body)
marker_counts = count_markers(fetch_result.body)
@ -59,6 +106,14 @@ def parse_profile_page(fetch_result: FetchResult) -> ParsedProfilePage:
warnings.append("operation_error_banner_present")
warnings = sorted(set(warnings))
_assert_profile_dom_invariants(
fetch_result=fetch_result,
marker_counts=marker_counts,
publications=publications,
warnings=warnings,
has_show_more_button_flag=show_more,
articles_range=articles_range,
)
state, state_reason = detect_state(
fetch_result,

16
app/services/domains/scholar/parser_types.py
View file

@ -12,6 +12,22 @@ class ParseState(StrEnum):
NETWORK_ERROR = "network_error"
class ScholarParserError(RuntimeError):
code: str
def __init__(self, *, code: str, message: str) -> None:
super().__init__(message)
self.code = code
class ScholarDomInvariantError(ScholarParserError):
pass
class ScholarMalformedDataError(ScholarParserError):
pass
@dataclass(frozen=True)
class PublicationCandidate:
title: str

56
app/services/domains/scholar/profile_rows.py
View file

@ -2,13 +2,10 @@ from __future__ import annotations
import re
from html.parser import HTMLParser
from typing import Any
from urllib.parse import parse_qs, urlparse
from app.services.domains.scholar.parser_constants import (
MARKER_KEYS,
PROFILE_ROW_DIRECT_LABEL_TOKENS,
PROFILE_ROW_PARSER_DIRECT_MARKERS,
SHOW_MORE_BUTTON_RE,
)
from app.services.domains.scholar.parser_types import PublicationCandidate
@ -26,7 +23,6 @@ class ScholarRowParser(HTMLParser):
def __init__(self) -> None:
super().__init__(convert_charrefs=True)
self.title_href: str | None = None
self.direct_download_href: str | None = None
self.title_parts: list[str] = []
self.citation_parts: list[str] = []
self.year_parts: list[str] = []
@ -35,14 +31,7 @@ class ScholarRowParser(HTMLParser):
self._title_depth = 0
self._citation_depth = 0
self._year_depth = 0
self._gray_stack: list[dict[str, Any]] = []
self._direct_marker_depth = 0
self._aux_link_stack: list[dict[str, Any]] = []
@staticmethod
def _contains_direct_marker(classes: str) -> bool:
lowered = classes.lower()
return any(marker in lowered for marker in PROFILE_ROW_PARSER_DIRECT_MARKERS)
self._gray_stack: list[dict[str, object]] = []
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
if self._title_depth > 0:
@ -53,14 +42,8 @@ class ScholarRowParser(HTMLParser):
self._year_depth += 1
if self._gray_stack:
self._gray_stack[-1]["depth"] += 1
if self._direct_marker_depth > 0:
self._direct_marker_depth += 1
if self._aux_link_stack:
self._aux_link_stack[-1]["depth"] += 1
classes = attr_class(attrs)
if tag in {"div", "span"} and self._contains_direct_marker(classes):
self._direct_marker_depth = 1
if tag == "a" and "gsc_a_at" in classes:
self._title_depth = 1
@ -79,17 +62,6 @@ class ScholarRowParser(HTMLParser):
self._gray_stack.append({"depth": 1, "parts": []})
return
if tag == "a":
href = attr_href(attrs)
self._aux_link_stack.append(
{
"depth": 1,
"classes": classes,
"href": href,
"parts": [],
}
)
def handle_data(self, data: str) -> None:
if self._title_depth > 0:
self.title_parts.append(data)
@ -99,23 +71,6 @@ class ScholarRowParser(HTMLParser):
self.year_parts.append(data)
if self._gray_stack:
self._gray_stack[-1]["parts"].append(data)
if self._aux_link_stack:
self._aux_link_stack[-1]["parts"].append(data)
def _capture_direct_download_href(self, link_info: dict[str, Any]) -> None:
if self.direct_download_href:
return
href = link_info.get("href")
if not href:
return
classes = str(link_info.get("classes") or "")
label = normalize_space("".join(link_info.get("parts") or [])).lower()
if "gs_ggsd" in classes or "gs_ggs" in classes or "gs_ggsa" in classes:
self.direct_download_href = href
return
label_match = any(token in label for token in PROFILE_ROW_DIRECT_LABEL_TOKENS)
if self._direct_marker_depth > 0 and label_match:
self.direct_download_href = href
def handle_endtag(self, tag: str) -> None:
if self._title_depth > 0:
@ -131,13 +86,6 @@ class ScholarRowParser(HTMLParser):
if text_value:
self.gray_texts.append(text_value)
self._gray_stack.pop()
if self._aux_link_stack:
self._aux_link_stack[-1]["depth"] -= 1
if self._aux_link_stack[-1]["depth"] == 0:
link_info = self._aux_link_stack.pop()
self._capture_direct_download_href(link_info)
if self._direct_marker_depth > 0:
self._direct_marker_depth -= 1
def extract_rows(html: str) -> list[str]:
@ -223,7 +171,7 @@ def _parse_publication_row(row_html: str) -> tuple[PublicationCandidate | None,
citation_count=citation_count,
authors_text=authors_text,
venue_text=venue_text,
pdf_url=build_absolute_scholar_url(parser.direct_download_href),
pdf_url=None,
),
warnings,
)