777 lines
25 KiB
Python
Executable file
777 lines
25 KiB
Python
Executable file
#!/usr/bin/env python3
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import random
|
|
import re
|
|
import time
|
|
from dataclasses import dataclass, asdict
|
|
from datetime import datetime, timezone
|
|
from html import unescape
|
|
from html.parser import HTMLParser
|
|
from pathlib import Path
|
|
from typing import Any
|
|
from urllib.error import HTTPError, URLError
|
|
from urllib.parse import parse_qs, urlencode, urlparse
|
|
from urllib.request import Request, urlopen
|
|
|
|
ROBOTS_URL = "https://scholar.google.com/robots.txt"
|
|
PROFILE_URL = "https://scholar.google.com/citations"
|
|
|
|
DEFAULT_USER_AGENTS = [
|
|
(
|
|
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
|
|
"(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
|
|
),
|
|
(
|
|
"Mozilla/5.0 (Macintosh; Intel Mac OS X 14_7) AppleWebKit/605.1.15 "
|
|
"(KHTML, like Gecko) Version/18.1 Safari/605.1.15"
|
|
),
|
|
(
|
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:131.0) "
|
|
"Gecko/20100101 Firefox/131.0"
|
|
),
|
|
]
|
|
|
|
BLOCKED_KEYWORDS = [
|
|
"unusual traffic",
|
|
"sorry/index",
|
|
"not a robot",
|
|
"our systems have detected",
|
|
"automated queries",
|
|
]
|
|
|
|
NO_RESULTS_KEYWORDS = [
|
|
"didn't match any articles",
|
|
"did not match any articles",
|
|
"no articles",
|
|
"no documents",
|
|
]
|
|
|
|
MARKER_KEYS = [
|
|
"gsc_a_tr",
|
|
"gsc_a_at",
|
|
"gsc_a_ac",
|
|
"gsc_a_h",
|
|
"gsc_a_y",
|
|
"gs_gray",
|
|
"gsc_prf_in",
|
|
"gsc_rsb_st",
|
|
]
|
|
|
|
|
|
@dataclass
|
|
class FetchRecord:
|
|
source: str
|
|
url: str
|
|
file_name: str
|
|
status_code: int | None
|
|
fetched_at_utc: str
|
|
elapsed_seconds: float
|
|
final_url: str | None
|
|
error: str | None
|
|
|
|
|
|
@dataclass
|
|
class PublicationCandidate:
|
|
title: str
|
|
title_url: str | None
|
|
cluster_id: str | None
|
|
year: int | None
|
|
citation_count: int | None
|
|
authors_text: str | None
|
|
venue_text: str | None
|
|
|
|
|
|
@dataclass
|
|
class PageAnalysis:
|
|
source: str
|
|
status: str
|
|
profile_name: str | None
|
|
publication_count: int
|
|
articles_range: str | None
|
|
has_show_more_button: bool
|
|
has_operation_error_banner: bool
|
|
marker_counts: dict[str, int]
|
|
field_presence: dict[str, int]
|
|
parse_warnings: list[str]
|
|
|
|
|
|
def normalize_space(value: str) -> str:
|
|
return " ".join(unescape(value).split())
|
|
|
|
|
|
TAG_RE = re.compile(r"<[^>]+>", re.S)
|
|
|
|
|
|
def strip_tags(value: str) -> str:
|
|
return normalize_space(TAG_RE.sub(" ", value))
|
|
|
|
|
|
def attr_class(attrs: list[tuple[str, str | None]]) -> str:
|
|
for name, raw_value in attrs:
|
|
if name.lower() == "class":
|
|
return raw_value or ""
|
|
return ""
|
|
|
|
|
|
def attr_href(attrs: list[tuple[str, str | None]]) -> str | None:
|
|
for name, raw_value in attrs:
|
|
if name.lower() == "href":
|
|
return raw_value
|
|
return None
|
|
|
|
|
|
class ScholarRowParser(HTMLParser):
|
|
def __init__(self) -> None:
|
|
super().__init__(convert_charrefs=True)
|
|
self.title_href: str | None = None
|
|
self.title_parts: list[str] = []
|
|
self.citation_parts: list[str] = []
|
|
self.year_parts: list[str] = []
|
|
self.gray_texts: list[str] = []
|
|
|
|
self._title_depth = 0
|
|
self._citation_depth = 0
|
|
self._year_depth = 0
|
|
self._gray_stack: list[dict[str, Any]] = []
|
|
|
|
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
|
if self._title_depth > 0:
|
|
self._title_depth += 1
|
|
if self._citation_depth > 0:
|
|
self._citation_depth += 1
|
|
if self._year_depth > 0:
|
|
self._year_depth += 1
|
|
if self._gray_stack:
|
|
self._gray_stack[-1]["depth"] += 1
|
|
|
|
classes = attr_class(attrs)
|
|
|
|
if tag == "a" and "gsc_a_at" in classes:
|
|
self._title_depth = 1
|
|
self.title_href = attr_href(attrs)
|
|
return
|
|
|
|
if tag == "a" and "gsc_a_ac" in classes:
|
|
self._citation_depth = 1
|
|
return
|
|
|
|
if tag in {"span", "a"} and ("gsc_a_h" in classes or "gsc_a_y" in classes):
|
|
self._year_depth = 1
|
|
return
|
|
|
|
if tag == "div" and "gs_gray" in classes:
|
|
self._gray_stack.append({"depth": 1, "parts": []})
|
|
return
|
|
|
|
def handle_data(self, data: str) -> None:
|
|
if self._title_depth > 0:
|
|
self.title_parts.append(data)
|
|
if self._citation_depth > 0:
|
|
self.citation_parts.append(data)
|
|
if self._year_depth > 0:
|
|
self.year_parts.append(data)
|
|
if self._gray_stack:
|
|
self._gray_stack[-1]["parts"].append(data)
|
|
|
|
def handle_endtag(self, _tag: str) -> None:
|
|
if self._title_depth > 0:
|
|
self._title_depth -= 1
|
|
if self._citation_depth > 0:
|
|
self._citation_depth -= 1
|
|
if self._year_depth > 0:
|
|
self._year_depth -= 1
|
|
if self._gray_stack:
|
|
self._gray_stack[-1]["depth"] -= 1
|
|
if self._gray_stack[-1]["depth"] == 0:
|
|
text = normalize_space("".join(self._gray_stack[-1]["parts"]))
|
|
if text:
|
|
self.gray_texts.append(text)
|
|
self._gray_stack.pop()
|
|
|
|
|
|
def extract_rows(html: str) -> list[str]:
|
|
pattern = re.compile(
|
|
r"<tr\b(?=[^>]*\bclass\s*=\s*['\"][^'\"]*\bgsc_a_tr\b[^'\"]*['\"])[^>]*>(.*?)</tr>",
|
|
re.I | re.S,
|
|
)
|
|
return [match.group(1) for match in pattern.finditer(html)]
|
|
|
|
|
|
def parse_cluster_id_from_href(href: str | None) -> str | None:
|
|
if not href:
|
|
return None
|
|
parsed = urlparse(href)
|
|
query = parse_qs(parsed.query)
|
|
|
|
citation_for_view = query.get("citation_for_view")
|
|
if citation_for_view:
|
|
token = citation_for_view[0].strip()
|
|
if token:
|
|
if ":" in token:
|
|
return token.rsplit(":", 1)[-1] or None
|
|
return token
|
|
|
|
cluster = query.get("cluster")
|
|
if cluster:
|
|
token = cluster[0].strip()
|
|
if token:
|
|
return token
|
|
return None
|
|
|
|
|
|
def parse_year(parts: list[str]) -> int | None:
|
|
text = normalize_space(" ".join(parts))
|
|
match = re.search(r"\b(19|20)\d{2}\b", text)
|
|
if not match:
|
|
return None
|
|
try:
|
|
return int(match.group(0))
|
|
except ValueError:
|
|
return None
|
|
|
|
|
|
def parse_citation_count(parts: list[str]) -> int | None:
|
|
text = normalize_space(" ".join(parts))
|
|
if not text:
|
|
return 0
|
|
match = re.search(r"\d+", text)
|
|
if not match:
|
|
return None
|
|
return int(match.group(0))
|
|
|
|
|
|
def parse_publications(html: str) -> tuple[list[PublicationCandidate], list[str]]:
|
|
rows = extract_rows(html)
|
|
warnings: list[str] = []
|
|
publications: list[PublicationCandidate] = []
|
|
|
|
for row_html in rows:
|
|
parser = ScholarRowParser()
|
|
parser.feed(row_html)
|
|
|
|
title = normalize_space("".join(parser.title_parts))
|
|
if not title:
|
|
warnings.append("row_missing_title")
|
|
continue
|
|
|
|
authors_text = parser.gray_texts[0] if len(parser.gray_texts) > 0 else None
|
|
venue_text = parser.gray_texts[1] if len(parser.gray_texts) > 1 else None
|
|
|
|
publications.append(
|
|
PublicationCandidate(
|
|
title=title,
|
|
title_url=parser.title_href,
|
|
cluster_id=parse_cluster_id_from_href(parser.title_href),
|
|
year=parse_year(parser.year_parts),
|
|
citation_count=parse_citation_count(parser.citation_parts),
|
|
authors_text=authors_text,
|
|
venue_text=venue_text,
|
|
)
|
|
)
|
|
|
|
if not rows:
|
|
warnings.append("no_rows_detected")
|
|
|
|
return publications, sorted(set(warnings))
|
|
|
|
|
|
def extract_profile_name(html: str) -> str | None:
|
|
pattern = re.compile(
|
|
r"<[^>]*\bid\s*=\s*['\"]gsc_prf_in['\"][^>]*>(.*?)</[^>]+>",
|
|
re.I | re.S,
|
|
)
|
|
match = pattern.search(html)
|
|
if not match:
|
|
return None
|
|
value = strip_tags(match.group(1))
|
|
return value or None
|
|
|
|
|
|
def extract_articles_range(html: str) -> str | None:
|
|
pattern = re.compile(
|
|
r"<[^>]*\bid\s*=\s*['\"]gsc_a_nn['\"][^>]*>(.*?)</[^>]+>",
|
|
re.I | re.S,
|
|
)
|
|
match = pattern.search(html)
|
|
if not match:
|
|
return None
|
|
value = strip_tags(match.group(1))
|
|
return value or None
|
|
|
|
|
|
def has_show_more_button(html: str) -> bool:
|
|
lowered = html.lower()
|
|
return "id=\"gsc_bpf_more\"" in lowered or "id='gsc_bpf_more'" in lowered
|
|
|
|
|
|
def has_operation_error_banner(html: str) -> bool:
|
|
lowered = html.lower()
|
|
if "id=\"gsc_a_err\"" not in lowered and "id='gsc_a_err'" not in lowered:
|
|
return False
|
|
return "can't perform the operation now" in lowered or "cannot perform the operation now" in lowered
|
|
|
|
|
|
def count_markers(html: str) -> dict[str, int]:
|
|
lowered = html.lower()
|
|
return {key: lowered.count(key.lower()) for key in MARKER_KEYS}
|
|
|
|
|
|
def detect_status(
|
|
*,
|
|
status_code: int | None,
|
|
final_url: str | None,
|
|
html: str,
|
|
publication_count: int,
|
|
marker_counts: dict[str, int],
|
|
) -> str:
|
|
if status_code is None:
|
|
return "network_error"
|
|
|
|
lowered = html.lower()
|
|
final = (final_url or "").lower()
|
|
|
|
if "accounts.google.com" in final and ("signin" in final or "servicelogin" in final):
|
|
return "blocked_or_captcha"
|
|
|
|
if any(keyword in lowered for keyword in BLOCKED_KEYWORDS) or "sorry/index" in final:
|
|
return "blocked_or_captcha"
|
|
|
|
if publication_count == 0 and any(keyword in lowered for keyword in NO_RESULTS_KEYWORDS):
|
|
return "no_results"
|
|
|
|
if publication_count == 0:
|
|
has_profile_markers = marker_counts.get("gsc_prf_in", 0) > 0
|
|
has_table_markers = marker_counts.get("gsc_a_tr", 0) > 0 or marker_counts.get("gsc_a_at", 0) > 0
|
|
if not has_profile_markers and not has_table_markers:
|
|
return "layout_changed"
|
|
|
|
return "ok"
|
|
|
|
|
|
def load_scholar_ids(id_file: Path | None, cli_ids: list[str]) -> list[str]:
|
|
seen: set[str] = set()
|
|
collected: list[str] = []
|
|
|
|
def maybe_add(raw: str) -> None:
|
|
item = raw.strip()
|
|
if not item:
|
|
return
|
|
if item.startswith("#"):
|
|
return
|
|
if item in seen:
|
|
return
|
|
seen.add(item)
|
|
collected.append(item)
|
|
|
|
for item in cli_ids:
|
|
maybe_add(item)
|
|
|
|
if id_file and id_file.exists():
|
|
for line in id_file.read_text(encoding="utf-8").splitlines():
|
|
maybe_add(line)
|
|
|
|
return collected
|
|
|
|
|
|
def fetch_url(url: str, user_agent: str, timeout_seconds: float) -> tuple[int | None, str | None, str, str | None]:
|
|
request = Request(
|
|
url,
|
|
headers={
|
|
"User-Agent": user_agent,
|
|
"Accept": "text/html,application/xhtml+xml",
|
|
"Accept-Language": "en-US,en;q=0.9",
|
|
"Connection": "close",
|
|
},
|
|
)
|
|
start = time.perf_counter()
|
|
|
|
try:
|
|
with urlopen(request, timeout=timeout_seconds) as response:
|
|
body = response.read().decode("utf-8", errors="replace")
|
|
elapsed = time.perf_counter() - start
|
|
status = getattr(response, "status", 200)
|
|
final_url = response.geturl()
|
|
return status, final_url, body, None
|
|
except HTTPError as exc:
|
|
body = ""
|
|
try:
|
|
body = exc.read().decode("utf-8", errors="replace")
|
|
except Exception:
|
|
body = ""
|
|
elapsed = time.perf_counter() - start
|
|
_ = elapsed
|
|
return exc.code, exc.geturl(), body, str(exc)
|
|
except URLError as exc:
|
|
elapsed = time.perf_counter() - start
|
|
_ = elapsed
|
|
return None, None, "", str(exc)
|
|
|
|
|
|
def write_json(path: Path, payload: Any) -> None:
|
|
path.write_text(json.dumps(payload, indent=2, sort_keys=True), encoding="utf-8")
|
|
|
|
|
|
def render_markdown(
|
|
*,
|
|
generated_at: str,
|
|
run_dir: Path,
|
|
fetch_records: list[FetchRecord],
|
|
page_analyses: list[PageAnalysis],
|
|
publications_by_source: dict[str, list[PublicationCandidate]],
|
|
robots_excerpt: str,
|
|
) -> str:
|
|
lines: list[str] = []
|
|
lines.append("# Scholar Scrape Probe Report")
|
|
lines.append("")
|
|
lines.append(f"Generated UTC: `{generated_at}`")
|
|
lines.append(f"Run fixtures dir: `{run_dir.as_posix()}`")
|
|
lines.append("")
|
|
|
|
lines.append("## Robots Snapshot")
|
|
lines.append("")
|
|
lines.append("```text")
|
|
lines.append(robots_excerpt.rstrip() or "(robots fetch unavailable)")
|
|
lines.append("```")
|
|
lines.append("")
|
|
|
|
lines.append("## Fetch Summary")
|
|
lines.append("")
|
|
lines.append("| Source | Status Code | Status/Error | Final URL |")
|
|
lines.append("| --- | --- | --- | --- |")
|
|
for record in fetch_records:
|
|
status = str(record.status_code) if record.status_code is not None else "-"
|
|
state = record.error or "ok"
|
|
final_url = record.final_url or "-"
|
|
lines.append(f"| `{record.source}` | {status} | {state} | `{final_url}` |")
|
|
lines.append("")
|
|
|
|
lines.append("## Parse Summary")
|
|
lines.append("")
|
|
lines.append("| Source | Parse Status | Profile | Publications | Articles Range | Show More | Warnings |")
|
|
lines.append("| --- | --- | --- | --- | --- | --- | --- |")
|
|
for analysis in page_analyses:
|
|
warnings = ", ".join(analysis.parse_warnings) if analysis.parse_warnings else "-"
|
|
profile = analysis.profile_name or "-"
|
|
articles_range = analysis.articles_range or "-"
|
|
show_more = "yes" if analysis.has_show_more_button else "no"
|
|
lines.append(
|
|
f"| `{analysis.source}` | `{analysis.status}` | {profile} | {analysis.publication_count} | {articles_range} | {show_more} | {warnings} |"
|
|
)
|
|
lines.append("")
|
|
|
|
all_publications = [
|
|
publication
|
|
for publications in publications_by_source.values()
|
|
for publication in publications
|
|
]
|
|
|
|
if all_publications:
|
|
total = len(all_publications)
|
|
def pct(count: int) -> str:
|
|
return f"{(count / total) * 100:.1f}%"
|
|
|
|
title_count = sum(1 for item in all_publications if item.title)
|
|
cluster_count = sum(1 for item in all_publications if item.cluster_id)
|
|
year_count = sum(1 for item in all_publications if item.year is not None)
|
|
citation_count = sum(1 for item in all_publications if item.citation_count is not None)
|
|
author_count = sum(1 for item in all_publications if item.authors_text)
|
|
venue_count = sum(1 for item in all_publications if item.venue_text)
|
|
|
|
lines.append("## Field Coverage")
|
|
lines.append("")
|
|
lines.append(f"Total parsed publication rows: **{total}**")
|
|
lines.append("")
|
|
lines.append("| Field | Present | Coverage |")
|
|
lines.append("| --- | --- | --- |")
|
|
lines.append(f"| `title` | {title_count} | {pct(title_count)} |")
|
|
lines.append(f"| `cluster_id` | {cluster_count} | {pct(cluster_count)} |")
|
|
lines.append(f"| `year` | {year_count} | {pct(year_count)} |")
|
|
lines.append(f"| `citation_count` | {citation_count} | {pct(citation_count)} |")
|
|
lines.append(f"| `authors_text` | {author_count} | {pct(author_count)} |")
|
|
lines.append(f"| `venue_text` | {venue_count} | {pct(venue_count)} |")
|
|
lines.append("")
|
|
|
|
lines.append("## Parser Contract Recommendation")
|
|
lines.append("")
|
|
lines.append("- Primary row marker: `tr.gsc_a_tr`.")
|
|
lines.append("- Title anchor marker: `a.gsc_a_at`; derive `cluster_id` from `citation_for_view` query token.")
|
|
lines.append("- Metadata text markers: first/second `div.gs_gray` per row for authors and venue.")
|
|
lines.append("- Year marker fallback: classes containing `gsc_a_h` or `gsc_a_y` and 4-digit year regex.")
|
|
lines.append("- Failure states to persist: `ok`, `no_results`, `blocked_or_captcha`, `layout_changed`, `network_error`.")
|
|
lines.append("")
|
|
|
|
lines.append("## Future-Proofing Notes")
|
|
lines.append("")
|
|
lines.append("- Keep raw HTML fixture snapshots and update parser tests on DOM drift.")
|
|
lines.append("- Treat blocked pages as retriable with backoff, not parser errors.")
|
|
lines.append("- If `Show more` is present, treat first-page-only results as partial and surface that in run status/UI.")
|
|
lines.append("- Track robots policy changes because `/citations?*cstart=` is currently disallowed.")
|
|
lines.append("- Add marker-count assertions in CI to catch silent layout shifts early.")
|
|
lines.append("- Use explicit parse status per run/scholar so automation can degrade gracefully.")
|
|
|
|
lines.append("")
|
|
return "\n".join(lines)
|
|
|
|
|
|
def build_parser() -> argparse.ArgumentParser:
|
|
parser = argparse.ArgumentParser(
|
|
description="Temporary probe: capture and analyze Google Scholar profile HTML for parser planning."
|
|
)
|
|
parser.add_argument(
|
|
"--output-root",
|
|
default="planning/scholar_probe_tmp",
|
|
help="Root directory of temporary probe workspace.",
|
|
)
|
|
parser.add_argument(
|
|
"--scholar-id",
|
|
action="append",
|
|
default=[],
|
|
help="Scholar profile id (can be passed multiple times).",
|
|
)
|
|
parser.add_argument(
|
|
"--id-file",
|
|
default="planning/scholar_probe_tmp/notes/seed_scholar_ids.txt",
|
|
help="Path to newline-delimited scholar ids.",
|
|
)
|
|
parser.add_argument(
|
|
"--max-profiles",
|
|
type=int,
|
|
default=5,
|
|
help="Maximum number of scholar profiles to fetch in this probe run.",
|
|
)
|
|
parser.add_argument(
|
|
"--request-delay-seconds",
|
|
type=float,
|
|
default=8.0,
|
|
help="Base delay between live requests.",
|
|
)
|
|
parser.add_argument(
|
|
"--request-jitter-seconds",
|
|
type=float,
|
|
default=2.0,
|
|
help="Randomized additional delay in seconds.",
|
|
)
|
|
parser.add_argument(
|
|
"--timeout-seconds",
|
|
type=float,
|
|
default=25.0,
|
|
help="HTTP timeout seconds.",
|
|
)
|
|
parser.add_argument(
|
|
"--allow-live-fetch",
|
|
action="store_true",
|
|
help="Enable outbound fetch for robots and scholar profile pages.",
|
|
)
|
|
parser.add_argument(
|
|
"--analyze-existing-fixtures",
|
|
action="store_true",
|
|
help="Analyze existing fixtures in output-root/fixtures even if live fetch is disabled.",
|
|
)
|
|
return parser
|
|
|
|
|
|
def main() -> int:
|
|
args = build_parser().parse_args()
|
|
|
|
output_root = Path(args.output_root)
|
|
fixtures_root = output_root / "fixtures"
|
|
notes_root = output_root / "notes"
|
|
fixtures_root.mkdir(parents=True, exist_ok=True)
|
|
notes_root.mkdir(parents=True, exist_ok=True)
|
|
|
|
now = datetime.now(timezone.utc)
|
|
run_id = now.strftime("run_%Y%m%dT%H%M%SZ")
|
|
run_dir = fixtures_root / run_id
|
|
run_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
id_file = Path(args.id_file)
|
|
scholar_ids = load_scholar_ids(id_file, args.scholar_id)
|
|
if args.max_profiles > 0:
|
|
scholar_ids = scholar_ids[: args.max_profiles]
|
|
|
|
fetch_records: list[FetchRecord] = []
|
|
robots_excerpt = ""
|
|
|
|
if args.allow_live_fetch:
|
|
robots_start = time.perf_counter()
|
|
robots_status, robots_final_url, robots_body, robots_error = fetch_url(
|
|
ROBOTS_URL,
|
|
user_agent=random.choice(DEFAULT_USER_AGENTS),
|
|
timeout_seconds=args.timeout_seconds,
|
|
)
|
|
robots_elapsed = time.perf_counter() - robots_start
|
|
|
|
robots_path = run_dir / "robots.txt"
|
|
robots_path.write_text(robots_body or "", encoding="utf-8")
|
|
robots_excerpt = "\n".join((robots_body or "").splitlines()[:50])
|
|
|
|
fetch_records.append(
|
|
FetchRecord(
|
|
source="robots",
|
|
url=ROBOTS_URL,
|
|
file_name=robots_path.name,
|
|
status_code=robots_status,
|
|
fetched_at_utc=datetime.now(timezone.utc).isoformat(),
|
|
elapsed_seconds=round(robots_elapsed, 3),
|
|
final_url=robots_final_url,
|
|
error=robots_error,
|
|
)
|
|
)
|
|
|
|
for index, scholar_id in enumerate(scholar_ids):
|
|
params = {"hl": "en", "user": scholar_id}
|
|
url = f"{PROFILE_URL}?{urlencode(params)}"
|
|
|
|
user_agent = DEFAULT_USER_AGENTS[index % len(DEFAULT_USER_AGENTS)]
|
|
start = time.perf_counter()
|
|
status_code, final_url, body, error = fetch_url(
|
|
url,
|
|
user_agent=user_agent,
|
|
timeout_seconds=args.timeout_seconds,
|
|
)
|
|
elapsed = time.perf_counter() - start
|
|
|
|
safe_source = f"profile_{scholar_id}"
|
|
html_name = f"{safe_source}.html"
|
|
html_path = run_dir / html_name
|
|
html_path.write_text(body or "", encoding="utf-8")
|
|
|
|
fetch_records.append(
|
|
FetchRecord(
|
|
source=safe_source,
|
|
url=url,
|
|
file_name=html_name,
|
|
status_code=status_code,
|
|
fetched_at_utc=datetime.now(timezone.utc).isoformat(),
|
|
elapsed_seconds=round(elapsed, 3),
|
|
final_url=final_url,
|
|
error=error,
|
|
)
|
|
)
|
|
|
|
if index < len(scholar_ids) - 1:
|
|
delay = max(0.0, args.request_delay_seconds) + random.uniform(
|
|
0.0,
|
|
max(0.0, args.request_jitter_seconds),
|
|
)
|
|
time.sleep(delay)
|
|
|
|
if not robots_excerpt:
|
|
robots_candidates = sorted(fixtures_root.glob("run_*/robots.txt"), reverse=True)
|
|
if robots_candidates:
|
|
robots_excerpt = "\n".join(
|
|
robots_candidates[0].read_text(encoding="utf-8").splitlines()[:50]
|
|
)
|
|
|
|
html_files: list[Path] = []
|
|
html_files.extend(run_dir.glob("profile_*.html"))
|
|
|
|
if args.analyze_existing_fixtures:
|
|
latest_runs = sorted(fixtures_root.glob("run_*"), reverse=True)
|
|
for existing_run in latest_runs:
|
|
if existing_run == run_dir:
|
|
continue
|
|
html_files.extend(existing_run.glob("profile_*.html"))
|
|
|
|
by_source: dict[str, Path] = {}
|
|
for candidate in sorted(html_files, reverse=True):
|
|
source = candidate.stem
|
|
if source in by_source:
|
|
continue
|
|
by_source[source] = candidate
|
|
deduped_files = [by_source[source] for source in sorted(by_source)]
|
|
|
|
page_analyses: list[PageAnalysis] = []
|
|
publications_by_source: dict[str, list[PublicationCandidate]] = {}
|
|
|
|
fetch_record_by_source = {record.source: record for record in fetch_records}
|
|
|
|
for html_path in deduped_files:
|
|
source = html_path.stem
|
|
html = html_path.read_text(encoding="utf-8")
|
|
publications, warnings = parse_publications(html)
|
|
markers = count_markers(html)
|
|
articles_range = extract_articles_range(html)
|
|
show_more = has_show_more_button(html)
|
|
operation_error_banner = has_operation_error_banner(html)
|
|
|
|
if show_more:
|
|
warnings.append("possible_partial_page_show_more_present")
|
|
if operation_error_banner:
|
|
warnings.append("operation_error_banner_present")
|
|
warnings = sorted(set(warnings))
|
|
|
|
record = fetch_record_by_source.get(source)
|
|
status = detect_status(
|
|
status_code=record.status_code if record else 200,
|
|
final_url=record.final_url if record else None,
|
|
html=html,
|
|
publication_count=len(publications),
|
|
marker_counts=markers,
|
|
)
|
|
|
|
field_presence = {
|
|
"title": sum(1 for item in publications if item.title),
|
|
"cluster_id": sum(1 for item in publications if item.cluster_id),
|
|
"year": sum(1 for item in publications if item.year is not None),
|
|
"citation_count": sum(1 for item in publications if item.citation_count is not None),
|
|
"authors_text": sum(1 for item in publications if item.authors_text),
|
|
"venue_text": sum(1 for item in publications if item.venue_text),
|
|
}
|
|
|
|
analysis = PageAnalysis(
|
|
source=source,
|
|
status=status,
|
|
profile_name=extract_profile_name(html),
|
|
publication_count=len(publications),
|
|
articles_range=articles_range,
|
|
has_show_more_button=show_more,
|
|
has_operation_error_banner=operation_error_banner,
|
|
marker_counts=markers,
|
|
field_presence=field_presence,
|
|
parse_warnings=warnings,
|
|
)
|
|
page_analyses.append(analysis)
|
|
publications_by_source[source] = publications
|
|
|
|
report_payload = {
|
|
"generated_at_utc": datetime.now(timezone.utc).isoformat(),
|
|
"run_dir": run_dir.as_posix(),
|
|
"fetch_records": [asdict(record) for record in fetch_records],
|
|
"page_analyses": [asdict(analysis) for analysis in page_analyses],
|
|
"publications_by_source": {
|
|
source: [asdict(item) for item in publications]
|
|
for source, publications in publications_by_source.items()
|
|
},
|
|
}
|
|
|
|
json_report_path = notes_root / f"probe_report_{run_id}.json"
|
|
write_json(json_report_path, report_payload)
|
|
|
|
markdown_report = render_markdown(
|
|
generated_at=report_payload["generated_at_utc"],
|
|
run_dir=run_dir,
|
|
fetch_records=fetch_records,
|
|
page_analyses=page_analyses,
|
|
publications_by_source=publications_by_source,
|
|
robots_excerpt=robots_excerpt,
|
|
)
|
|
markdown_report_path = notes_root / f"probe_report_{run_id}.md"
|
|
markdown_report_path.write_text(markdown_report, encoding="utf-8")
|
|
|
|
summary = {
|
|
"run_id": run_id,
|
|
"run_dir": run_dir.as_posix(),
|
|
"json_report": json_report_path.as_posix(),
|
|
"markdown_report": markdown_report_path.as_posix(),
|
|
"profiles_analyzed": len(page_analyses),
|
|
}
|
|
print(json.dumps(summary, indent=2))
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|