First product

planning/scholar_probe_tmp/scripts/scholar_probe.py

@@ -0,0 +1,777 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import json
+import random
+import re
+import time
+from dataclasses import dataclass, asdict
+from datetime import datetime, timezone
+from html import unescape
+from html.parser import HTMLParser
+from pathlib import Path
+from typing import Any
+from urllib.error import HTTPError, URLError
+from urllib.parse import parse_qs, urlencode, urlparse
+from urllib.request import Request, urlopen
+
+ROBOTS_URL = "https://scholar.google.com/robots.txt"
+PROFILE_URL = "https://scholar.google.com/citations"
+
+DEFAULT_USER_AGENTS = [
+    (
+        "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
+        "(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
+    ),
+    (
+        "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_7) AppleWebKit/605.1.15 "
+        "(KHTML, like Gecko) Version/18.1 Safari/605.1.15"
+    ),
+    (
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:131.0) "
+        "Gecko/20100101 Firefox/131.0"
+    ),
+]
+
+BLOCKED_KEYWORDS = [
+    "unusual traffic",
+    "sorry/index",
+    "not a robot",
+    "our systems have detected",
+    "automated queries",
+]
+
+NO_RESULTS_KEYWORDS = [
+    "didn't match any articles",
+    "did not match any articles",
+    "no articles",
+    "no documents",
+]
+
+MARKER_KEYS = [
+    "gsc_a_tr",
+    "gsc_a_at",
+    "gsc_a_ac",
+    "gsc_a_h",
+    "gsc_a_y",
+    "gs_gray",
+    "gsc_prf_in",
+    "gsc_rsb_st",
+]
+
+
+@dataclass
+class FetchRecord:
+    source: str
+    url: str
+    file_name: str
+    status_code: int | None
+    fetched_at_utc: str
+    elapsed_seconds: float
+    final_url: str | None
+    error: str | None
+
+
+@dataclass
+class PublicationCandidate:
+    title: str
+    title_url: str | None
+    cluster_id: str | None
+    year: int | None
+    citation_count: int | None
+    authors_text: str | None
+    venue_text: str | None
+
+
+@dataclass
+class PageAnalysis:
+    source: str
+    status: str
+    profile_name: str | None
+    publication_count: int
+    articles_range: str | None
+    has_show_more_button: bool
+    has_operation_error_banner: bool
+    marker_counts: dict[str, int]
+    field_presence: dict[str, int]
+    parse_warnings: list[str]
+
+
+def normalize_space(value: str) -> str:
+    return " ".join(unescape(value).split())
+
+
+TAG_RE = re.compile(r"<[^>]+>", re.S)
+
+
+def strip_tags(value: str) -> str:
+    return normalize_space(TAG_RE.sub(" ", value))
+
+
+def attr_class(attrs: list[tuple[str, str | None]]) -> str:
+    for name, raw_value in attrs:
+        if name.lower() == "class":
+            return raw_value or ""
+    return ""
+
+
+def attr_href(attrs: list[tuple[str, str | None]]) -> str | None:
+    for name, raw_value in attrs:
+        if name.lower() == "href":
+            return raw_value
+    return None
+
+
+class ScholarRowParser(HTMLParser):
+    def __init__(self) -> None:
+        super().__init__(convert_charrefs=True)
+        self.title_href: str | None = None
+        self.title_parts: list[str] = []
+        self.citation_parts: list[str] = []
+        self.year_parts: list[str] = []
+        self.gray_texts: list[str] = []
+
+        self._title_depth = 0
+        self._citation_depth = 0
+        self._year_depth = 0
+        self._gray_stack: list[dict[str, Any]] = []
+
+    def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
+        if self._title_depth > 0:
+            self._title_depth += 1
+        if self._citation_depth > 0:
+            self._citation_depth += 1
+        if self._year_depth > 0:
+            self._year_depth += 1
+        if self._gray_stack:
+            self._gray_stack[-1]["depth"] += 1
+
+        classes = attr_class(attrs)
+
+        if tag == "a" and "gsc_a_at" in classes:
+            self._title_depth = 1
+            self.title_href = attr_href(attrs)
+            return
+
+        if tag == "a" and "gsc_a_ac" in classes:
+            self._citation_depth = 1
+            return
+
+        if tag in {"span", "a"} and ("gsc_a_h" in classes or "gsc_a_y" in classes):
+            self._year_depth = 1
+            return
+
+        if tag == "div" and "gs_gray" in classes:
+            self._gray_stack.append({"depth": 1, "parts": []})
+            return
+
+    def handle_data(self, data: str) -> None:
+        if self._title_depth > 0:
+            self.title_parts.append(data)
+        if self._citation_depth > 0:
+            self.citation_parts.append(data)
+        if self._year_depth > 0:
+            self.year_parts.append(data)
+        if self._gray_stack:
+            self._gray_stack[-1]["parts"].append(data)
+
+    def handle_endtag(self, _tag: str) -> None:
+        if self._title_depth > 0:
+            self._title_depth -= 1
+        if self._citation_depth > 0:
+            self._citation_depth -= 1
+        if self._year_depth > 0:
+            self._year_depth -= 1
+        if self._gray_stack:
+            self._gray_stack[-1]["depth"] -= 1
+            if self._gray_stack[-1]["depth"] == 0:
+                text = normalize_space("".join(self._gray_stack[-1]["parts"]))
+                if text:
+                    self.gray_texts.append(text)
+                self._gray_stack.pop()
+
+
+def extract_rows(html: str) -> list[str]:
+    pattern = re.compile(
+        r"<tr\b(?=[^>]*\bclass\s*=\s*['\"][^'\"]*\bgsc_a_tr\b[^'\"]*['\"])[^>]*>(.*?)</tr>",
+        re.I | re.S,
+    )
+    return [match.group(1) for match in pattern.finditer(html)]
+
+
+def parse_cluster_id_from_href(href: str | None) -> str | None:
+    if not href:
+        return None
+    parsed = urlparse(href)
+    query = parse_qs(parsed.query)
+
+    citation_for_view = query.get("citation_for_view")
+    if citation_for_view:
+        token = citation_for_view[0].strip()
+        if token:
+            if ":" in token:
+                return token.rsplit(":", 1)[-1] or None
+            return token
+
+    cluster = query.get("cluster")
+    if cluster:
+        token = cluster[0].strip()
+        if token:
+            return token
+    return None
+
+
+def parse_year(parts: list[str]) -> int | None:
+    text = normalize_space(" ".join(parts))
+    match = re.search(r"\b(19|20)\d{2}\b", text)
+    if not match:
+        return None
+    try:
+        return int(match.group(0))
+    except ValueError:
+        return None
+
+
+def parse_citation_count(parts: list[str]) -> int | None:
+    text = normalize_space(" ".join(parts))
+    if not text:
+        return 0
+    match = re.search(r"\d+", text)
+    if not match:
+        return None
+    return int(match.group(0))
+
+
+def parse_publications(html: str) -> tuple[list[PublicationCandidate], list[str]]:
+    rows = extract_rows(html)
+    warnings: list[str] = []
+    publications: list[PublicationCandidate] = []
+
+    for row_html in rows:
+        parser = ScholarRowParser()
+        parser.feed(row_html)
+
+        title = normalize_space("".join(parser.title_parts))
+        if not title:
+            warnings.append("row_missing_title")
+            continue
+
+        authors_text = parser.gray_texts[0] if len(parser.gray_texts) > 0 else None
+        venue_text = parser.gray_texts[1] if len(parser.gray_texts) > 1 else None
+
+        publications.append(
+            PublicationCandidate(
+                title=title,
+                title_url=parser.title_href,
+                cluster_id=parse_cluster_id_from_href(parser.title_href),
+                year=parse_year(parser.year_parts),
+                citation_count=parse_citation_count(parser.citation_parts),
+                authors_text=authors_text,
+                venue_text=venue_text,
+            )
+        )
+
+    if not rows:
+        warnings.append("no_rows_detected")
+
+    return publications, sorted(set(warnings))
+
+
+def extract_profile_name(html: str) -> str | None:
+    pattern = re.compile(
+        r"<[^>]*\bid\s*=\s*['\"]gsc_prf_in['\"][^>]*>(.*?)</[^>]+>",
+        re.I | re.S,
+    )
+    match = pattern.search(html)
+    if not match:
+        return None
+    value = strip_tags(match.group(1))
+    return value or None
+
+
+def extract_articles_range(html: str) -> str | None:
+    pattern = re.compile(
+        r"<[^>]*\bid\s*=\s*['\"]gsc_a_nn['\"][^>]*>(.*?)</[^>]+>",
+        re.I | re.S,
+    )
+    match = pattern.search(html)
+    if not match:
+        return None
+    value = strip_tags(match.group(1))
+    return value or None
+
+
+def has_show_more_button(html: str) -> bool:
+    lowered = html.lower()
+    return "id=\"gsc_bpf_more\"" in lowered or "id='gsc_bpf_more'" in lowered
+
+
+def has_operation_error_banner(html: str) -> bool:
+    lowered = html.lower()
+    if "id=\"gsc_a_err\"" not in lowered and "id='gsc_a_err'" not in lowered:
+        return False
+    return "can't perform the operation now" in lowered or "cannot perform the operation now" in lowered
+
+
+def count_markers(html: str) -> dict[str, int]:
+    lowered = html.lower()
+    return {key: lowered.count(key.lower()) for key in MARKER_KEYS}
+
+
+def detect_status(
+    *,
+    status_code: int | None,
+    final_url: str | None,
+    html: str,
+    publication_count: int,
+    marker_counts: dict[str, int],
+) -> str:
+    if status_code is None:
+        return "network_error"
+
+    lowered = html.lower()
+    final = (final_url or "").lower()
+
+    if "accounts.google.com" in final and ("signin" in final or "servicelogin" in final):
+        return "blocked_or_captcha"
+
+    if any(keyword in lowered for keyword in BLOCKED_KEYWORDS) or "sorry/index" in final:
+        return "blocked_or_captcha"
+
+    if publication_count == 0 and any(keyword in lowered for keyword in NO_RESULTS_KEYWORDS):
+        return "no_results"
+
+    if publication_count == 0:
+        has_profile_markers = marker_counts.get("gsc_prf_in", 0) > 0
+        has_table_markers = marker_counts.get("gsc_a_tr", 0) > 0 or marker_counts.get("gsc_a_at", 0) > 0
+        if not has_profile_markers and not has_table_markers:
+            return "layout_changed"
+
+    return "ok"
+
+
+def load_scholar_ids(id_file: Path | None, cli_ids: list[str]) -> list[str]:
+    seen: set[str] = set()
+    collected: list[str] = []
+
+    def maybe_add(raw: str) -> None:
+        item = raw.strip()
+        if not item:
+            return
+        if item.startswith("#"):
+            return
+        if item in seen:
+            return
+        seen.add(item)
+        collected.append(item)
+
+    for item in cli_ids:
+        maybe_add(item)
+
+    if id_file and id_file.exists():
+        for line in id_file.read_text(encoding="utf-8").splitlines():
+            maybe_add(line)
+
+    return collected
+
+
+def fetch_url(url: str, user_agent: str, timeout_seconds: float) -> tuple[int | None, str | None, str, str | None]:
+    request = Request(
+        url,
+        headers={
+            "User-Agent": user_agent,
+            "Accept": "text/html,application/xhtml+xml",
+            "Accept-Language": "en-US,en;q=0.9",
+            "Connection": "close",
+        },
+    )
+    start = time.perf_counter()
+
+    try:
+        with urlopen(request, timeout=timeout_seconds) as response:
+            body = response.read().decode("utf-8", errors="replace")
+            elapsed = time.perf_counter() - start
+            status = getattr(response, "status", 200)
+            final_url = response.geturl()
+            return status, final_url, body, None
+    except HTTPError as exc:
+        body = ""
+        try:
+            body = exc.read().decode("utf-8", errors="replace")
+        except Exception:
+            body = ""
+        elapsed = time.perf_counter() - start
+        _ = elapsed
+        return exc.code, exc.geturl(), body, str(exc)
+    except URLError as exc:
+        elapsed = time.perf_counter() - start
+        _ = elapsed
+        return None, None, "", str(exc)
+
+
+def write_json(path: Path, payload: Any) -> None:
+    path.write_text(json.dumps(payload, indent=2, sort_keys=True), encoding="utf-8")
+
+
+def render_markdown(
+    *,
+    generated_at: str,
+    run_dir: Path,
+    fetch_records: list[FetchRecord],
+    page_analyses: list[PageAnalysis],
+    publications_by_source: dict[str, list[PublicationCandidate]],
+    robots_excerpt: str,
+) -> str:
+    lines: list[str] = []
+    lines.append("# Scholar Scrape Probe Report")
+    lines.append("")
+    lines.append(f"Generated UTC: `{generated_at}`")
+    lines.append(f"Run fixtures dir: `{run_dir.as_posix()}`")
+    lines.append("")
+
+    lines.append("## Robots Snapshot")
+    lines.append("")
+    lines.append("```text")
+    lines.append(robots_excerpt.rstrip() or "(robots fetch unavailable)")
+    lines.append("```")
+    lines.append("")
+
+    lines.append("## Fetch Summary")
+    lines.append("")
+    lines.append("| Source | Status Code | Status/Error | Final URL |")
+    lines.append("| --- | --- | --- | --- |")
+    for record in fetch_records:
+        status = str(record.status_code) if record.status_code is not None else "-"
+        state = record.error or "ok"
+        final_url = record.final_url or "-"
+        lines.append(f"| `{record.source}` | {status} | {state} | `{final_url}` |")
+    lines.append("")
+
+    lines.append("## Parse Summary")
+    lines.append("")
+    lines.append("| Source | Parse Status | Profile | Publications | Articles Range | Show More | Warnings |")
+    lines.append("| --- | --- | --- | --- | --- | --- | --- |")
+    for analysis in page_analyses:
+        warnings = ", ".join(analysis.parse_warnings) if analysis.parse_warnings else "-"
+        profile = analysis.profile_name or "-"
+        articles_range = analysis.articles_range or "-"
+        show_more = "yes" if analysis.has_show_more_button else "no"
+        lines.append(
+            f"| `{analysis.source}` | `{analysis.status}` | {profile} | {analysis.publication_count} | {articles_range} | {show_more} | {warnings} |"
+        )
+    lines.append("")
+
+    all_publications = [
+        publication
+        for publications in publications_by_source.values()
+        for publication in publications
+    ]
+
+    if all_publications:
+        total = len(all_publications)
+        def pct(count: int) -> str:
+            return f"{(count / total) * 100:.1f}%"
+
+        title_count = sum(1 for item in all_publications if item.title)
+        cluster_count = sum(1 for item in all_publications if item.cluster_id)
+        year_count = sum(1 for item in all_publications if item.year is not None)
+        citation_count = sum(1 for item in all_publications if item.citation_count is not None)
+        author_count = sum(1 for item in all_publications if item.authors_text)
+        venue_count = sum(1 for item in all_publications if item.venue_text)
+
+        lines.append("## Field Coverage")
+        lines.append("")
+        lines.append(f"Total parsed publication rows: **{total}**")
+        lines.append("")
+        lines.append("| Field | Present | Coverage |")
+        lines.append("| --- | --- | --- |")
+        lines.append(f"| `title` | {title_count} | {pct(title_count)} |")
+        lines.append(f"| `cluster_id` | {cluster_count} | {pct(cluster_count)} |")
+        lines.append(f"| `year` | {year_count} | {pct(year_count)} |")
+        lines.append(f"| `citation_count` | {citation_count} | {pct(citation_count)} |")
+        lines.append(f"| `authors_text` | {author_count} | {pct(author_count)} |")
+        lines.append(f"| `venue_text` | {venue_count} | {pct(venue_count)} |")
+        lines.append("")
+
+    lines.append("## Parser Contract Recommendation")
+    lines.append("")
+    lines.append("- Primary row marker: `tr.gsc_a_tr`.")
+    lines.append("- Title anchor marker: `a.gsc_a_at`; derive `cluster_id` from `citation_for_view` query token.")
+    lines.append("- Metadata text markers: first/second `div.gs_gray` per row for authors and venue.")
+    lines.append("- Year marker fallback: classes containing `gsc_a_h` or `gsc_a_y` and 4-digit year regex.")
+    lines.append("- Failure states to persist: `ok`, `no_results`, `blocked_or_captcha`, `layout_changed`, `network_error`.")
+    lines.append("")
+
+    lines.append("## Future-Proofing Notes")
+    lines.append("")
+    lines.append("- Keep raw HTML fixture snapshots and update parser tests on DOM drift.")
+    lines.append("- Treat blocked pages as retriable with backoff, not parser errors.")
+    lines.append("- If `Show more` is present, treat first-page-only results as partial and surface that in run status/UI.")
+    lines.append("- Track robots policy changes because `/citations?*cstart=` is currently disallowed.")
+    lines.append("- Add marker-count assertions in CI to catch silent layout shifts early.")
+    lines.append("- Use explicit parse status per run/scholar so automation can degrade gracefully.")
+
+    lines.append("")
+    return "\n".join(lines)
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        description="Temporary probe: capture and analyze Google Scholar profile HTML for parser planning."
+    )
+    parser.add_argument(
+        "--output-root",
+        default="planning/scholar_probe_tmp",
+        help="Root directory of temporary probe workspace.",
+    )
+    parser.add_argument(
+        "--scholar-id",
+        action="append",
+        default=[],
+        help="Scholar profile id (can be passed multiple times).",
+    )
+    parser.add_argument(
+        "--id-file",
+        default="planning/scholar_probe_tmp/notes/seed_scholar_ids.txt",
+        help="Path to newline-delimited scholar ids.",
+    )
+    parser.add_argument(
+        "--max-profiles",
+        type=int,
+        default=5,
+        help="Maximum number of scholar profiles to fetch in this probe run.",
+    )
+    parser.add_argument(
+        "--request-delay-seconds",
+        type=float,
+        default=8.0,
+        help="Base delay between live requests.",
+    )
+    parser.add_argument(
+        "--request-jitter-seconds",
+        type=float,
+        default=2.0,
+        help="Randomized additional delay in seconds.",
+    )
+    parser.add_argument(
+        "--timeout-seconds",
+        type=float,
+        default=25.0,
+        help="HTTP timeout seconds.",
+    )
+    parser.add_argument(
+        "--allow-live-fetch",
+        action="store_true",
+        help="Enable outbound fetch for robots and scholar profile pages.",
+    )
+    parser.add_argument(
+        "--analyze-existing-fixtures",
+        action="store_true",
+        help="Analyze existing fixtures in output-root/fixtures even if live fetch is disabled.",
+    )
+    return parser
+
+
+def main() -> int:
+    args = build_parser().parse_args()
+
+    output_root = Path(args.output_root)
+    fixtures_root = output_root / "fixtures"
+    notes_root = output_root / "notes"
+    fixtures_root.mkdir(parents=True, exist_ok=True)
+    notes_root.mkdir(parents=True, exist_ok=True)
+
+    now = datetime.now(timezone.utc)
+    run_id = now.strftime("run_%Y%m%dT%H%M%SZ")
+    run_dir = fixtures_root / run_id
+    run_dir.mkdir(parents=True, exist_ok=True)
+
+    id_file = Path(args.id_file)
+    scholar_ids = load_scholar_ids(id_file, args.scholar_id)
+    if args.max_profiles > 0:
+        scholar_ids = scholar_ids[: args.max_profiles]
+
+    fetch_records: list[FetchRecord] = []
+    robots_excerpt = ""
+
+    if args.allow_live_fetch:
+        robots_start = time.perf_counter()
+        robots_status, robots_final_url, robots_body, robots_error = fetch_url(
+            ROBOTS_URL,
+            user_agent=random.choice(DEFAULT_USER_AGENTS),
+            timeout_seconds=args.timeout_seconds,
+        )
+        robots_elapsed = time.perf_counter() - robots_start
+
+        robots_path = run_dir / "robots.txt"
+        robots_path.write_text(robots_body or "", encoding="utf-8")
+        robots_excerpt = "\n".join((robots_body or "").splitlines()[:50])
+
+        fetch_records.append(
+            FetchRecord(
+                source="robots",
+                url=ROBOTS_URL,
+                file_name=robots_path.name,
+                status_code=robots_status,
+                fetched_at_utc=datetime.now(timezone.utc).isoformat(),
+                elapsed_seconds=round(robots_elapsed, 3),
+                final_url=robots_final_url,
+                error=robots_error,
+            )
+        )
+
+        for index, scholar_id in enumerate(scholar_ids):
+            params = {"hl": "en", "user": scholar_id}
+            url = f"{PROFILE_URL}?{urlencode(params)}"
+
+            user_agent = DEFAULT_USER_AGENTS[index % len(DEFAULT_USER_AGENTS)]
+            start = time.perf_counter()
+            status_code, final_url, body, error = fetch_url(
+                url,
+                user_agent=user_agent,
+                timeout_seconds=args.timeout_seconds,
+            )
+            elapsed = time.perf_counter() - start
+
+            safe_source = f"profile_{scholar_id}"
+            html_name = f"{safe_source}.html"
+            html_path = run_dir / html_name
+            html_path.write_text(body or "", encoding="utf-8")
+
+            fetch_records.append(
+                FetchRecord(
+                    source=safe_source,
+                    url=url,
+                    file_name=html_name,
+                    status_code=status_code,
+                    fetched_at_utc=datetime.now(timezone.utc).isoformat(),
+                    elapsed_seconds=round(elapsed, 3),
+                    final_url=final_url,
+                    error=error,
+                )
+            )
+
+            if index < len(scholar_ids) - 1:
+                delay = max(0.0, args.request_delay_seconds) + random.uniform(
+                    0.0,
+                    max(0.0, args.request_jitter_seconds),
+                )
+                time.sleep(delay)
+
+    if not robots_excerpt:
+        robots_candidates = sorted(fixtures_root.glob("run_*/robots.txt"), reverse=True)
+        if robots_candidates:
+            robots_excerpt = "\n".join(
+                robots_candidates[0].read_text(encoding="utf-8").splitlines()[:50]
+            )
+
+    html_files: list[Path] = []
+    html_files.extend(run_dir.glob("profile_*.html"))
+
+    if args.analyze_existing_fixtures:
+        latest_runs = sorted(fixtures_root.glob("run_*"), reverse=True)
+        for existing_run in latest_runs:
+            if existing_run == run_dir:
+                continue
+            html_files.extend(existing_run.glob("profile_*.html"))
+
+    by_source: dict[str, Path] = {}
+    for candidate in sorted(html_files, reverse=True):
+        source = candidate.stem
+        if source in by_source:
+            continue
+        by_source[source] = candidate
+    deduped_files = [by_source[source] for source in sorted(by_source)]
+
+    page_analyses: list[PageAnalysis] = []
+    publications_by_source: dict[str, list[PublicationCandidate]] = {}
+
+    fetch_record_by_source = {record.source: record for record in fetch_records}
+
+    for html_path in deduped_files:
+        source = html_path.stem
+        html = html_path.read_text(encoding="utf-8")
+        publications, warnings = parse_publications(html)
+        markers = count_markers(html)
+        articles_range = extract_articles_range(html)
+        show_more = has_show_more_button(html)
+        operation_error_banner = has_operation_error_banner(html)
+
+        if show_more:
+            warnings.append("possible_partial_page_show_more_present")
+        if operation_error_banner:
+            warnings.append("operation_error_banner_present")
+        warnings = sorted(set(warnings))
+
+        record = fetch_record_by_source.get(source)
+        status = detect_status(
+            status_code=record.status_code if record else 200,
+            final_url=record.final_url if record else None,
+            html=html,
+            publication_count=len(publications),
+            marker_counts=markers,
+        )
+
+        field_presence = {
+            "title": sum(1 for item in publications if item.title),
+            "cluster_id": sum(1 for item in publications if item.cluster_id),
+            "year": sum(1 for item in publications if item.year is not None),
+            "citation_count": sum(1 for item in publications if item.citation_count is not None),
+            "authors_text": sum(1 for item in publications if item.authors_text),
+            "venue_text": sum(1 for item in publications if item.venue_text),
+        }
+
+        analysis = PageAnalysis(
+            source=source,
+            status=status,
+            profile_name=extract_profile_name(html),
+            publication_count=len(publications),
+            articles_range=articles_range,
+            has_show_more_button=show_more,
+            has_operation_error_banner=operation_error_banner,
+            marker_counts=markers,
+            field_presence=field_presence,
+            parse_warnings=warnings,
+        )
+        page_analyses.append(analysis)
+        publications_by_source[source] = publications
+
+    report_payload = {
+        "generated_at_utc": datetime.now(timezone.utc).isoformat(),
+        "run_dir": run_dir.as_posix(),
+        "fetch_records": [asdict(record) for record in fetch_records],
+        "page_analyses": [asdict(analysis) for analysis in page_analyses],
+        "publications_by_source": {
+            source: [asdict(item) for item in publications]
+            for source, publications in publications_by_source.items()
+        },
+    }
+
+    json_report_path = notes_root / f"probe_report_{run_id}.json"
+    write_json(json_report_path, report_payload)
+
+    markdown_report = render_markdown(
+        generated_at=report_payload["generated_at_utc"],
+        run_dir=run_dir,
+        fetch_records=fetch_records,
+        page_analyses=page_analyses,
+        publications_by_source=publications_by_source,
+        robots_excerpt=robots_excerpt,
+    )
+    markdown_report_path = notes_root / f"probe_report_{run_id}.md"
+    markdown_report_path.write_text(markdown_report, encoding="utf-8")
+
+    summary = {
+        "run_id": run_id,
+        "run_dir": run_dir.as_posix(),
+        "json_report": json_report_path.as_posix(),
+        "markdown_report": markdown_report_path.as_posix(),
+        "profiles_analyzed": len(page_analyses),
+    }
+    print(json.dumps(summary, indent=2))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())