test
This commit is contained in:
parent
efd21a7297
commit
441676be27
97 changed files with 10764 additions and 223 deletions
|
|
@ -181,6 +181,23 @@ async def increment_attempt_count(
|
|||
return item
|
||||
|
||||
|
||||
async def reset_attempt_count(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
job_id: int,
|
||||
) -> IngestionQueueItem | None:
|
||||
now = datetime.now(timezone.utc)
|
||||
result = await db_session.execute(
|
||||
select(IngestionQueueItem).where(IngestionQueueItem.id == job_id)
|
||||
)
|
||||
item = result.scalar_one_or_none()
|
||||
if item is None:
|
||||
return None
|
||||
item.attempt_count = 0
|
||||
item.updated_at = now
|
||||
return item
|
||||
|
||||
|
||||
async def reschedule_job(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import asyncio
|
|||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from typing import Any
|
||||
|
|
@ -45,6 +46,7 @@ RESUMABLE_PARTIAL_REASONS = {
|
|||
"pagination_cursor_stalled",
|
||||
}
|
||||
RESUMABLE_PARTIAL_REASON_PREFIXES = ("page_state_network_error",)
|
||||
INITIAL_PAGE_FINGERPRINT_MAX_PUBLICATIONS = 30
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
|
|
@ -63,6 +65,9 @@ class RunExecutionSummary:
|
|||
class PagedParseResult:
|
||||
fetch_result: FetchResult
|
||||
parsed_page: ParsedProfilePage
|
||||
first_page_fetch_result: FetchResult
|
||||
first_page_parsed_page: ParsedProfilePage
|
||||
first_page_fingerprint_sha256: str | None
|
||||
publications: list[PublicationCandidate]
|
||||
attempt_log: list[dict[str, Any]]
|
||||
page_logs: list[dict[str, Any]]
|
||||
|
|
@ -71,6 +76,7 @@ class PagedParseResult:
|
|||
has_more_remaining: bool
|
||||
pagination_truncated_reason: str | None
|
||||
continuation_cstart: int | None
|
||||
skipped_no_change: bool
|
||||
|
||||
|
||||
class RunAlreadyInProgressError(RuntimeError):
|
||||
|
|
@ -96,6 +102,7 @@ class ScholarIngestionService:
|
|||
start_cstart_by_scholar_id: dict[int, int] | None = None,
|
||||
auto_queue_continuations: bool = True,
|
||||
queue_delay_seconds: int = 60,
|
||||
idempotency_key: str | None = None,
|
||||
) -> RunExecutionSummary:
|
||||
lock_acquired = await self._try_acquire_user_lock(
|
||||
db_session,
|
||||
|
|
@ -153,6 +160,7 @@ class ScholarIngestionService:
|
|||
"retry_backoff_seconds": retry_backoff_seconds,
|
||||
"max_pages_per_scholar": max_pages_per_scholar,
|
||||
"page_size": page_size,
|
||||
"idempotency_key": idempotency_key,
|
||||
},
|
||||
)
|
||||
|
||||
|
|
@ -162,6 +170,7 @@ class ScholarIngestionService:
|
|||
status=RunStatus.RUNNING,
|
||||
scholar_count=len(scholars),
|
||||
new_pub_count=0,
|
||||
idempotency_key=idempotency_key,
|
||||
error_log={},
|
||||
)
|
||||
db_session.add(run)
|
||||
|
|
@ -187,6 +196,7 @@ class ScholarIngestionService:
|
|||
retry_backoff_seconds=retry_backoff_seconds,
|
||||
max_pages=max_pages_per_scholar,
|
||||
page_size=page_size,
|
||||
previous_initial_page_fingerprint_sha256=scholar.last_initial_page_fingerprint_sha256,
|
||||
)
|
||||
fetch_result = paged_parse_result.fetch_result
|
||||
parsed_page = paged_parse_result.parsed_page
|
||||
|
|
@ -194,6 +204,17 @@ class ScholarIngestionService:
|
|||
attempt_log = paged_parse_result.attempt_log
|
||||
page_logs = paged_parse_result.page_logs
|
||||
|
||||
first_page = paged_parse_result.first_page_parsed_page
|
||||
if first_page.profile_name and not (scholar.display_name or "").strip():
|
||||
scholar.display_name = first_page.profile_name
|
||||
if first_page.profile_image_url:
|
||||
scholar.profile_image_url = first_page.profile_image_url
|
||||
if paged_parse_result.first_page_fingerprint_sha256:
|
||||
scholar.last_initial_page_fingerprint_sha256 = (
|
||||
paged_parse_result.first_page_fingerprint_sha256
|
||||
)
|
||||
scholar.last_initial_page_checked_at = run_dt
|
||||
|
||||
logger.info(
|
||||
"ingestion.scholar_parsed",
|
||||
extra={
|
||||
|
|
@ -210,6 +231,7 @@ class ScholarIngestionService:
|
|||
"has_more_remaining": paged_parse_result.has_more_remaining,
|
||||
"pagination_truncated_reason": paged_parse_result.pagination_truncated_reason,
|
||||
"warning_count": len(parsed_page.warnings),
|
||||
"skipped_no_change": paged_parse_result.skipped_no_change,
|
||||
},
|
||||
)
|
||||
|
||||
|
|
@ -230,7 +252,55 @@ class ScholarIngestionService:
|
|||
"has_more_remaining": paged_parse_result.has_more_remaining,
|
||||
"pagination_truncated_reason": paged_parse_result.pagination_truncated_reason,
|
||||
"continuation_cstart": paged_parse_result.continuation_cstart,
|
||||
"skipped_no_change": paged_parse_result.skipped_no_change,
|
||||
"initial_page_fingerprint_sha256": paged_parse_result.first_page_fingerprint_sha256,
|
||||
}
|
||||
if paged_parse_result.skipped_no_change:
|
||||
scholar.last_run_status = RunStatus.SUCCESS
|
||||
scholar.last_run_dt = run_dt
|
||||
succeeded_count += 1
|
||||
result_entry["state"] = first_page.state.value
|
||||
result_entry["state_reason"] = "no_change_initial_page_signature"
|
||||
result_entry["outcome"] = "success"
|
||||
result_entry["publication_count"] = 0
|
||||
result_entry["warnings"] = first_page.warnings
|
||||
result_entry["debug"] = {
|
||||
"state_reason": "no_change_initial_page_signature",
|
||||
"first_page_fingerprint_sha256": paged_parse_result.first_page_fingerprint_sha256,
|
||||
"attempt_log": attempt_log,
|
||||
"page_logs": page_logs,
|
||||
}
|
||||
scholar_results.append(result_entry)
|
||||
queue_reason, queue_cstart = self._resolve_continuation_queue_target(
|
||||
outcome=str(result_entry.get("outcome", "")),
|
||||
state=str(result_entry.get("state", "")),
|
||||
pagination_truncated_reason=paged_parse_result.pagination_truncated_reason,
|
||||
continuation_cstart=paged_parse_result.continuation_cstart,
|
||||
fallback_cstart=start_cstart,
|
||||
)
|
||||
if auto_queue_continuations and queue_reason is not None:
|
||||
await queue_service.upsert_job(
|
||||
db_session,
|
||||
user_id=user_id,
|
||||
scholar_profile_id=scholar.id,
|
||||
resume_cstart=queue_cstart,
|
||||
reason=queue_reason,
|
||||
run_id=run.id,
|
||||
delay_seconds=queue_delay_seconds,
|
||||
)
|
||||
result_entry["continuation_enqueued"] = True
|
||||
result_entry["continuation_reason"] = queue_reason
|
||||
result_entry["continuation_cstart"] = queue_cstart
|
||||
else:
|
||||
cleared = await queue_service.clear_job_for_scholar(
|
||||
db_session,
|
||||
user_id=user_id,
|
||||
scholar_profile_id=scholar.id,
|
||||
)
|
||||
if cleared:
|
||||
result_entry["continuation_cleared"] = True
|
||||
continue
|
||||
|
||||
had_page_failure = parsed_page.state not in {ParseState.OK, ParseState.NO_RESULTS}
|
||||
has_partial_publication_set = len(publications) > 0 and had_page_failure
|
||||
is_partial_due_to_pagination = (
|
||||
|
|
@ -373,6 +443,11 @@ class ScholarIngestionService:
|
|||
"failed_state_counts": failed_state_counts,
|
||||
"failed_reason_counts": failed_reason_counts,
|
||||
},
|
||||
"meta": {
|
||||
"idempotency_key": idempotency_key,
|
||||
}
|
||||
if idempotency_key
|
||||
else {},
|
||||
}
|
||||
|
||||
await db_session.commit()
|
||||
|
|
@ -519,6 +594,7 @@ class ScholarIngestionService:
|
|||
retry_backoff_seconds: float,
|
||||
max_pages: int,
|
||||
page_size: int,
|
||||
previous_initial_page_fingerprint_sha256: str | None = None,
|
||||
) -> PagedParseResult:
|
||||
bounded_max_pages = max(1, int(max_pages))
|
||||
bounded_page_size = max(1, int(page_size))
|
||||
|
|
@ -534,6 +610,9 @@ class ScholarIngestionService:
|
|||
network_error_retries=network_error_retries,
|
||||
retry_backoff_seconds=retry_backoff_seconds,
|
||||
)
|
||||
first_page_fetch_result = fetch_result
|
||||
first_page_parsed_page = parsed_page
|
||||
first_page_fingerprint_sha256 = build_initial_page_fingerprint(parsed_page)
|
||||
|
||||
attempt_log: list[dict[str, Any]] = list(first_attempt_log)
|
||||
page_logs: list[dict[str, Any]] = []
|
||||
|
|
@ -553,11 +632,39 @@ class ScholarIngestionService:
|
|||
)
|
||||
pages_attempted = 1
|
||||
|
||||
should_skip_no_change = (
|
||||
start_cstart <= 0
|
||||
and first_page_fingerprint_sha256 is not None
|
||||
and previous_initial_page_fingerprint_sha256 is not None
|
||||
and first_page_fingerprint_sha256 == previous_initial_page_fingerprint_sha256
|
||||
and parsed_page.state in {ParseState.OK, ParseState.NO_RESULTS}
|
||||
)
|
||||
if should_skip_no_change:
|
||||
return PagedParseResult(
|
||||
fetch_result=fetch_result,
|
||||
parsed_page=parsed_page,
|
||||
first_page_fetch_result=first_page_fetch_result,
|
||||
first_page_parsed_page=first_page_parsed_page,
|
||||
first_page_fingerprint_sha256=first_page_fingerprint_sha256,
|
||||
publications=[],
|
||||
attempt_log=attempt_log,
|
||||
page_logs=page_logs,
|
||||
pages_fetched=1,
|
||||
pages_attempted=pages_attempted,
|
||||
has_more_remaining=False,
|
||||
pagination_truncated_reason=None,
|
||||
continuation_cstart=None,
|
||||
skipped_no_change=True,
|
||||
)
|
||||
|
||||
# Immediate hard failure: nothing to salvage.
|
||||
if parsed_page.state not in {ParseState.OK, ParseState.NO_RESULTS}:
|
||||
return PagedParseResult(
|
||||
fetch_result=fetch_result,
|
||||
parsed_page=parsed_page,
|
||||
first_page_fetch_result=first_page_fetch_result,
|
||||
first_page_parsed_page=first_page_parsed_page,
|
||||
first_page_fingerprint_sha256=first_page_fingerprint_sha256,
|
||||
publications=[],
|
||||
attempt_log=attempt_log,
|
||||
page_logs=page_logs,
|
||||
|
|
@ -568,6 +675,7 @@ class ScholarIngestionService:
|
|||
continuation_cstart=(
|
||||
start_cstart if parsed_page.state == ParseState.NETWORK_ERROR else None
|
||||
),
|
||||
skipped_no_change=False,
|
||||
)
|
||||
|
||||
publications = list(parsed_page.publications)
|
||||
|
|
@ -649,6 +757,9 @@ class ScholarIngestionService:
|
|||
return PagedParseResult(
|
||||
fetch_result=fetch_result,
|
||||
parsed_page=parsed_page,
|
||||
first_page_fetch_result=first_page_fetch_result,
|
||||
first_page_parsed_page=first_page_parsed_page,
|
||||
first_page_fingerprint_sha256=first_page_fingerprint_sha256,
|
||||
publications=_dedupe_publication_candidates(publications),
|
||||
attempt_log=attempt_log,
|
||||
page_logs=page_logs,
|
||||
|
|
@ -657,6 +768,7 @@ class ScholarIngestionService:
|
|||
has_more_remaining=has_more_remaining,
|
||||
pagination_truncated_reason=pagination_truncated_reason,
|
||||
continuation_cstart=continuation_cstart,
|
||||
skipped_no_change=False,
|
||||
)
|
||||
|
||||
async def _upsert_profile_publications(
|
||||
|
|
@ -838,6 +950,7 @@ class ScholarIngestionService:
|
|||
"fetch_error": fetch_result.error,
|
||||
"state_reason": parsed_page.state_reason,
|
||||
"profile_name": parsed_page.profile_name,
|
||||
"profile_image_url": parsed_page.profile_image_url,
|
||||
"articles_range": parsed_page.articles_range,
|
||||
"has_show_more_button": parsed_page.has_show_more_button,
|
||||
"has_operation_error_banner": parsed_page.has_operation_error_banner,
|
||||
|
|
@ -914,6 +1027,37 @@ def build_publication_fingerprint(candidate: PublicationCandidate) -> str:
|
|||
return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def build_initial_page_fingerprint(parsed_page: ParsedProfilePage) -> str | None:
|
||||
if parsed_page.state not in {ParseState.OK, ParseState.NO_RESULTS}:
|
||||
return None
|
||||
|
||||
normalized_rows: list[dict[str, Any]] = []
|
||||
for publication in parsed_page.publications[:INITIAL_PAGE_FINGERPRINT_MAX_PUBLICATIONS]:
|
||||
normalized_rows.append(
|
||||
{
|
||||
"cluster_id": publication.cluster_id or "",
|
||||
"title_normalized": normalize_title(publication.title),
|
||||
"year": publication.year,
|
||||
"citation_count": publication.citation_count,
|
||||
}
|
||||
)
|
||||
|
||||
payload = {
|
||||
"state": parsed_page.state.value,
|
||||
"articles_range": parsed_page.articles_range or "",
|
||||
"has_show_more_button": parsed_page.has_show_more_button,
|
||||
"profile_name": parsed_page.profile_name or "",
|
||||
"publications": normalized_rows,
|
||||
}
|
||||
canonical = json.dumps(
|
||||
payload,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
ensure_ascii=True,
|
||||
)
|
||||
return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def build_publication_url(path_or_url: str | None) -> str | None:
|
||||
if not path_or_url:
|
||||
return None
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ from __future__ import annotations
|
|||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import Select, func, select, update
|
||||
from sqlalchemy import Select, func, select, tuple_, update
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.models import (
|
||||
|
|
@ -295,3 +295,39 @@ async def mark_all_unread_as_read_for_user(
|
|||
|
||||
rowcount = result.rowcount
|
||||
return int(rowcount or 0)
|
||||
|
||||
|
||||
async def mark_selected_as_read_for_user(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
selections: list[tuple[int, int]],
|
||||
) -> int:
|
||||
normalized_pairs = {
|
||||
(int(scholar_profile_id), int(publication_id))
|
||||
for scholar_profile_id, publication_id in selections
|
||||
if int(scholar_profile_id) > 0 and int(publication_id) > 0
|
||||
}
|
||||
if not normalized_pairs:
|
||||
return 0
|
||||
|
||||
scholar_ids = (
|
||||
select(ScholarProfile.id)
|
||||
.where(ScholarProfile.user_id == user_id)
|
||||
.scalar_subquery()
|
||||
)
|
||||
stmt = (
|
||||
update(ScholarPublication)
|
||||
.where(
|
||||
ScholarPublication.scholar_profile_id.in_(scholar_ids),
|
||||
tuple_(
|
||||
ScholarPublication.scholar_profile_id,
|
||||
ScholarPublication.publication_id,
|
||||
).in_(list(normalized_pairs)),
|
||||
ScholarPublication.is_read.is_(False),
|
||||
)
|
||||
.values(is_read=True)
|
||||
)
|
||||
result = await db_session.execute(stmt)
|
||||
await db_session.commit()
|
||||
return int(result.rowcount or 0)
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ from dataclasses import dataclass
|
|||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import and_, case, func, select
|
||||
from sqlalchemy import and_, case, func, or_, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.models import (
|
||||
|
|
@ -152,7 +152,10 @@ async def get_manual_run_by_idempotency_key(
|
|||
.where(
|
||||
CrawlRun.user_id == user_id,
|
||||
CrawlRun.trigger_type == RunTriggerType.MANUAL,
|
||||
CrawlRun.error_log["meta"]["idempotency_key"].astext == idempotency_key,
|
||||
or_(
|
||||
CrawlRun.idempotency_key == idempotency_key,
|
||||
CrawlRun.error_log["meta"]["idempotency_key"].astext == idempotency_key,
|
||||
),
|
||||
)
|
||||
.order_by(CrawlRun.start_dt.desc(), CrawlRun.id.desc())
|
||||
.limit(1)
|
||||
|
|
@ -160,33 +163,6 @@ async def get_manual_run_by_idempotency_key(
|
|||
return result.scalar_one_or_none()
|
||||
|
||||
|
||||
async def set_manual_run_idempotency_key(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
run_id: int,
|
||||
idempotency_key: str,
|
||||
) -> bool:
|
||||
run = await get_run_for_user(
|
||||
db_session,
|
||||
user_id=user_id,
|
||||
run_id=run_id,
|
||||
)
|
||||
if run is None:
|
||||
return False
|
||||
if run.trigger_type != RunTriggerType.MANUAL:
|
||||
return False
|
||||
|
||||
payload = dict(run.error_log) if isinstance(run.error_log, dict) else {}
|
||||
meta = payload.get("meta")
|
||||
meta_dict = dict(meta) if isinstance(meta, dict) else {}
|
||||
meta_dict["idempotency_key"] = idempotency_key
|
||||
payload["meta"] = meta_dict
|
||||
run.error_log = payload
|
||||
await db_session.commit()
|
||||
return True
|
||||
|
||||
|
||||
async def list_queue_items_for_user(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
|
|
|
|||
|
|
@ -399,6 +399,39 @@ class SchedulerService:
|
|||
return
|
||||
|
||||
async with session_factory() as session:
|
||||
# Failed-attempt budget should advance only when continuation execution fails.
|
||||
if int(run_summary.failed_count) <= 0:
|
||||
queue_item = await queue_service.reset_attempt_count(
|
||||
session,
|
||||
job_id=job.id,
|
||||
)
|
||||
if queue_item is None:
|
||||
await session.commit()
|
||||
logger.info(
|
||||
"scheduler.queue_item_resolved",
|
||||
extra={
|
||||
"event": "scheduler.queue_item_resolved",
|
||||
"queue_item_id": job.id,
|
||||
"user_id": job.user_id,
|
||||
"run_id": run_summary.crawl_run_id,
|
||||
"status": run_summary.status.value,
|
||||
},
|
||||
)
|
||||
return
|
||||
await session.commit()
|
||||
logger.info(
|
||||
"scheduler.queue_item_progressed",
|
||||
extra={
|
||||
"event": "scheduler.queue_item_progressed",
|
||||
"queue_item_id": job.id,
|
||||
"user_id": job.user_id,
|
||||
"attempt_count": int(queue_item.attempt_count),
|
||||
"run_id": run_summary.crawl_run_id,
|
||||
"status": run_summary.status.value,
|
||||
},
|
||||
)
|
||||
return
|
||||
|
||||
queue_item = await queue_service.increment_attempt_count(
|
||||
session,
|
||||
job_id=job.id,
|
||||
|
|
@ -451,9 +484,9 @@ class SchedulerService:
|
|||
)
|
||||
await session.commit()
|
||||
logger.info(
|
||||
"scheduler.queue_item_rescheduled",
|
||||
"scheduler.queue_item_rescheduled_failed",
|
||||
extra={
|
||||
"event": "scheduler.queue_item_rescheduled",
|
||||
"event": "scheduler.queue_item_rescheduled_failed",
|
||||
"queue_item_id": job.id,
|
||||
"user_id": job.user_id,
|
||||
"attempt_count": queue_item.attempt_count,
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ from enum import StrEnum
|
|||
from html import unescape
|
||||
from html.parser import HTMLParser
|
||||
from typing import Any
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
from urllib.parse import parse_qs, urljoin, urlparse
|
||||
|
||||
from app.services.scholar_source import FetchResult
|
||||
|
||||
|
|
@ -16,6 +16,8 @@ BLOCKED_KEYWORDS = [
|
|||
"not a robot",
|
||||
"our systems have detected",
|
||||
"automated queries",
|
||||
"recaptcha",
|
||||
"captcha",
|
||||
]
|
||||
|
||||
NO_RESULTS_KEYWORDS = [
|
||||
|
|
@ -25,6 +27,14 @@ NO_RESULTS_KEYWORDS = [
|
|||
"no documents",
|
||||
]
|
||||
|
||||
NO_AUTHOR_RESULTS_KEYWORDS = [
|
||||
"didn't match any user profiles",
|
||||
"did not match any user profiles",
|
||||
"didn't match any scholars",
|
||||
"did not match any scholars",
|
||||
"no user profiles",
|
||||
]
|
||||
|
||||
MARKER_KEYS = [
|
||||
"gsc_a_tr",
|
||||
"gsc_a_at",
|
||||
|
|
@ -36,6 +46,33 @@ MARKER_KEYS = [
|
|||
"gsc_rsb_st",
|
||||
]
|
||||
|
||||
AUTHOR_SEARCH_MARKER_KEYS = [
|
||||
"gsc_1usr",
|
||||
"gs_ai_name",
|
||||
"gs_ai_aff",
|
||||
"gs_ai_eml",
|
||||
"gs_ai_cby",
|
||||
"gs_ai_one_int",
|
||||
]
|
||||
|
||||
NETWORK_DNS_ERROR_KEYWORDS = [
|
||||
"temporary failure in name resolution",
|
||||
"name or service not known",
|
||||
"nodename nor servname provided",
|
||||
"getaddrinfo failed",
|
||||
]
|
||||
|
||||
NETWORK_TIMEOUT_KEYWORDS = [
|
||||
"timed out",
|
||||
"timeout",
|
||||
]
|
||||
|
||||
NETWORK_TLS_ERROR_KEYWORDS = [
|
||||
"ssl",
|
||||
"tls",
|
||||
"certificate verify failed",
|
||||
]
|
||||
|
||||
TAG_RE = re.compile(r"<[^>]+>", re.S)
|
||||
SCRIPT_STYLE_RE = re.compile(r"<(script|style)\b[^>]*>.*?</\1>", re.I | re.S)
|
||||
SHOW_MORE_BUTTON_RE = re.compile(
|
||||
|
|
@ -63,11 +100,24 @@ class PublicationCandidate:
|
|||
venue_text: str | None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ScholarSearchCandidate:
|
||||
scholar_id: str
|
||||
display_name: str
|
||||
affiliation: str | None
|
||||
email_domain: str | None
|
||||
cited_by_count: int | None
|
||||
interests: list[str]
|
||||
profile_url: str
|
||||
profile_image_url: str | None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ParsedProfilePage:
|
||||
state: ParseState
|
||||
state_reason: str
|
||||
profile_name: str | None
|
||||
profile_image_url: str | None
|
||||
publications: list[PublicationCandidate]
|
||||
marker_counts: dict[str, int]
|
||||
warnings: list[str]
|
||||
|
|
@ -76,6 +126,15 @@ class ParsedProfilePage:
|
|||
articles_range: str | None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ParsedAuthorSearchPage:
|
||||
state: ParseState
|
||||
state_reason: str
|
||||
candidates: list[ScholarSearchCandidate]
|
||||
marker_counts: dict[str, int]
|
||||
warnings: list[str]
|
||||
|
||||
|
||||
def normalize_space(value: str) -> str:
|
||||
return " ".join(unescape(value).split())
|
||||
|
||||
|
|
@ -98,6 +157,19 @@ def attr_href(attrs: list[tuple[str, str | None]]) -> str | None:
|
|||
return None
|
||||
|
||||
|
||||
def attr_src(attrs: list[tuple[str, str | None]]) -> str | None:
|
||||
for name, raw_value in attrs:
|
||||
if name.lower() == "src":
|
||||
return raw_value
|
||||
return None
|
||||
|
||||
|
||||
def build_absolute_scholar_url(path_or_url: str | None) -> str | None:
|
||||
if not path_or_url:
|
||||
return None
|
||||
return urljoin("https://scholar.google.com", path_or_url)
|
||||
|
||||
|
||||
class ScholarRowParser(HTMLParser):
|
||||
def __init__(self) -> None:
|
||||
super().__init__(convert_charrefs=True)
|
||||
|
|
@ -197,6 +269,18 @@ def parse_cluster_id_from_href(href: str | None) -> str | None:
|
|||
return None
|
||||
|
||||
|
||||
def parse_scholar_id_from_href(href: str | None) -> str | None:
|
||||
if not href:
|
||||
return None
|
||||
parsed = urlparse(href)
|
||||
query = parse_qs(parsed.query)
|
||||
user_values = query.get("user")
|
||||
if not user_values:
|
||||
return None
|
||||
candidate = user_values[0].strip()
|
||||
return candidate or None
|
||||
|
||||
|
||||
def parse_year(parts: list[str]) -> int | None:
|
||||
text = normalize_space(" ".join(parts))
|
||||
match = re.search(r"\b(19|20)\d{2}\b", text)
|
||||
|
|
@ -265,6 +349,30 @@ def extract_profile_name(html: str) -> str | None:
|
|||
return value or None
|
||||
|
||||
|
||||
def extract_profile_image_url(html: str) -> str | None:
|
||||
og_image_pattern = re.compile(
|
||||
r"<meta[^>]+property=['\"]og:image['\"][^>]+content=['\"]([^'\"]+)['\"][^>]*>",
|
||||
re.I | re.S,
|
||||
)
|
||||
og_match = og_image_pattern.search(html)
|
||||
if og_match:
|
||||
value = normalize_space(og_match.group(1))
|
||||
absolute = build_absolute_scholar_url(value)
|
||||
if absolute:
|
||||
return absolute
|
||||
|
||||
image_pattern = re.compile(
|
||||
r"<img[^>]*\bid=['\"]gsc_prf_pup-img['\"][^>]*\bsrc=['\"]([^'\"]+)['\"][^>]*>",
|
||||
re.I | re.S,
|
||||
)
|
||||
image_match = image_pattern.search(html)
|
||||
if not image_match:
|
||||
return None
|
||||
|
||||
value = normalize_space(image_match.group(1))
|
||||
return build_absolute_scholar_url(value)
|
||||
|
||||
|
||||
def extract_articles_range(html: str) -> str | None:
|
||||
pattern = re.compile(
|
||||
r"<[^>]*\bid\s*=\s*['\"]gsc_a_nn['\"][^>]*>(.*?)</[^>]+>",
|
||||
|
|
@ -304,6 +412,229 @@ def count_markers(html: str) -> dict[str, int]:
|
|||
return {key: lowered.count(key.lower()) for key in MARKER_KEYS}
|
||||
|
||||
|
||||
def count_author_search_markers(html: str) -> dict[str, int]:
|
||||
lowered = html.lower()
|
||||
return {key: lowered.count(key.lower()) for key in AUTHOR_SEARCH_MARKER_KEYS}
|
||||
|
||||
|
||||
def _extract_verified_email_domain(value: str | None) -> str | None:
|
||||
if not value:
|
||||
return None
|
||||
match = re.search(r"verified email at\s+(.+)$", value.strip(), re.I)
|
||||
if not match:
|
||||
return None
|
||||
domain = normalize_space(match.group(1))
|
||||
return domain or None
|
||||
|
||||
|
||||
class ScholarAuthorSearchParser(HTMLParser):
|
||||
def __init__(self) -> None:
|
||||
super().__init__(convert_charrefs=True)
|
||||
self.candidates: list[ScholarSearchCandidate] = []
|
||||
self._candidate: dict[str, Any] | None = None
|
||||
|
||||
def _begin_candidate(self) -> None:
|
||||
self._candidate = {
|
||||
"depth": 1,
|
||||
"name_href": None,
|
||||
"name_parts": [],
|
||||
"aff_depth": 0,
|
||||
"aff_parts": [],
|
||||
"name_depth": 0,
|
||||
"eml_depth": 0,
|
||||
"eml_parts": [],
|
||||
"cby_depth": 0,
|
||||
"cby_parts": [],
|
||||
"interest_depth": 0,
|
||||
"interest_parts": [],
|
||||
"interests": [],
|
||||
"image_src": None,
|
||||
}
|
||||
|
||||
def _increment_capture_depths(self) -> None:
|
||||
if self._candidate is None:
|
||||
return
|
||||
for key in ("name_depth", "aff_depth", "eml_depth", "cby_depth", "interest_depth"):
|
||||
if self._candidate[key] > 0:
|
||||
self._candidate[key] += 1
|
||||
|
||||
def _finalize_candidate(self) -> None:
|
||||
if self._candidate is None:
|
||||
return
|
||||
|
||||
name = normalize_space("".join(self._candidate["name_parts"]))
|
||||
scholar_id = parse_scholar_id_from_href(self._candidate["name_href"])
|
||||
if not name or not scholar_id:
|
||||
return
|
||||
|
||||
affiliation = normalize_space("".join(self._candidate["aff_parts"])) or None
|
||||
email_domain = _extract_verified_email_domain(
|
||||
normalize_space("".join(self._candidate["eml_parts"])) or None
|
||||
)
|
||||
cited_by_text = normalize_space("".join(self._candidate["cby_parts"]))
|
||||
cited_by_match = re.search(r"\d+", cited_by_text)
|
||||
cited_by_count = int(cited_by_match.group(0)) if cited_by_match else None
|
||||
|
||||
seen_interests: set[str] = set()
|
||||
interests: list[str] = []
|
||||
for interest in self._candidate["interests"]:
|
||||
normalized = normalize_space(interest)
|
||||
if not normalized or normalized in seen_interests:
|
||||
continue
|
||||
seen_interests.add(normalized)
|
||||
interests.append(normalized)
|
||||
|
||||
profile_url = build_absolute_scholar_url(self._candidate["name_href"])
|
||||
if not profile_url:
|
||||
profile_url = (
|
||||
"https://scholar.google.com/citations"
|
||||
f"?hl=en&user={scholar_id}"
|
||||
)
|
||||
|
||||
self.candidates.append(
|
||||
ScholarSearchCandidate(
|
||||
scholar_id=scholar_id,
|
||||
display_name=name,
|
||||
affiliation=affiliation,
|
||||
email_domain=email_domain,
|
||||
cited_by_count=cited_by_count,
|
||||
interests=interests,
|
||||
profile_url=profile_url,
|
||||
profile_image_url=build_absolute_scholar_url(self._candidate["image_src"]),
|
||||
)
|
||||
)
|
||||
|
||||
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
||||
classes = attr_class(attrs)
|
||||
|
||||
if self._candidate is None:
|
||||
if tag == "div" and "gsc_1usr" in classes:
|
||||
self._begin_candidate()
|
||||
return
|
||||
|
||||
self._candidate["depth"] += 1
|
||||
self._increment_capture_depths()
|
||||
|
||||
if tag == "a" and "gs_ai_name" in classes:
|
||||
self._candidate["name_depth"] = 1
|
||||
self._candidate["name_href"] = attr_href(attrs)
|
||||
return
|
||||
|
||||
if tag == "div" and "gs_ai_aff" in classes:
|
||||
self._candidate["aff_depth"] = 1
|
||||
return
|
||||
|
||||
if tag == "div" and "gs_ai_eml" in classes:
|
||||
self._candidate["eml_depth"] = 1
|
||||
return
|
||||
|
||||
if tag == "div" and "gs_ai_cby" in classes:
|
||||
self._candidate["cby_depth"] = 1
|
||||
return
|
||||
|
||||
if tag == "a" and "gs_ai_one_int" in classes:
|
||||
self._candidate["interest_depth"] = 1
|
||||
self._candidate["interest_parts"] = []
|
||||
return
|
||||
|
||||
if tag == "img" and self._candidate["image_src"] is None:
|
||||
self._candidate["image_src"] = attr_src(attrs)
|
||||
|
||||
def handle_data(self, data: str) -> None:
|
||||
if self._candidate is None:
|
||||
return
|
||||
if self._candidate["name_depth"] > 0:
|
||||
self._candidate["name_parts"].append(data)
|
||||
if self._candidate["aff_depth"] > 0:
|
||||
self._candidate["aff_parts"].append(data)
|
||||
if self._candidate["eml_depth"] > 0:
|
||||
self._candidate["eml_parts"].append(data)
|
||||
if self._candidate["cby_depth"] > 0:
|
||||
self._candidate["cby_parts"].append(data)
|
||||
if self._candidate["interest_depth"] > 0:
|
||||
self._candidate["interest_parts"].append(data)
|
||||
|
||||
def _decrement_capture_depth(self, key: str) -> bool:
|
||||
if self._candidate is None:
|
||||
return False
|
||||
if self._candidate[key] <= 0:
|
||||
return False
|
||||
self._candidate[key] -= 1
|
||||
return self._candidate[key] == 0
|
||||
|
||||
def handle_endtag(self, _tag: str) -> None:
|
||||
if self._candidate is None:
|
||||
return
|
||||
|
||||
interest_closed = self._decrement_capture_depth("interest_depth")
|
||||
self._decrement_capture_depth("name_depth")
|
||||
self._decrement_capture_depth("aff_depth")
|
||||
self._decrement_capture_depth("eml_depth")
|
||||
self._decrement_capture_depth("cby_depth")
|
||||
|
||||
if interest_closed:
|
||||
interest_text = normalize_space("".join(self._candidate["interest_parts"]))
|
||||
if interest_text:
|
||||
self._candidate["interests"].append(interest_text)
|
||||
self._candidate["interest_parts"] = []
|
||||
|
||||
self._candidate["depth"] -= 1
|
||||
if self._candidate["depth"] > 0:
|
||||
return
|
||||
|
||||
self._finalize_candidate()
|
||||
self._candidate = None
|
||||
|
||||
|
||||
def classify_network_error_reason(fetch_error: str | None) -> str:
|
||||
lowered = (fetch_error or "").lower()
|
||||
if lowered:
|
||||
if any(keyword in lowered for keyword in NETWORK_DNS_ERROR_KEYWORDS):
|
||||
return "network_dns_resolution_failed"
|
||||
if any(keyword in lowered for keyword in NETWORK_TIMEOUT_KEYWORDS):
|
||||
return "network_timeout"
|
||||
if any(keyword in lowered for keyword in NETWORK_TLS_ERROR_KEYWORDS):
|
||||
return "network_tls_error"
|
||||
if "connection reset" in lowered:
|
||||
return "network_connection_reset"
|
||||
if "connection refused" in lowered:
|
||||
return "network_connection_refused"
|
||||
if "network is unreachable" in lowered:
|
||||
return "network_unreachable"
|
||||
return "network_error_missing_status_code"
|
||||
|
||||
|
||||
def classify_block_or_captcha_reason(
|
||||
*,
|
||||
status_code: int,
|
||||
final_url: str,
|
||||
body_lowered: str,
|
||||
) -> str | None:
|
||||
if "accounts.google.com" in final_url and ("signin" in final_url or "servicelogin" in final_url):
|
||||
return "blocked_accounts_redirect"
|
||||
if status_code == 429:
|
||||
return "blocked_http_429_rate_limited"
|
||||
if status_code == 403:
|
||||
if "recaptcha" in body_lowered or "captcha" in body_lowered or "sorry/index" in final_url:
|
||||
return "blocked_http_403_captcha_challenge"
|
||||
return "blocked_http_403_forbidden"
|
||||
if "sorry/index" in final_url or "sorry/index" in body_lowered:
|
||||
return "blocked_google_sorry_challenge"
|
||||
if "our systems have detected" in body_lowered or "unusual traffic" in body_lowered:
|
||||
return "blocked_unusual_traffic_detected"
|
||||
if "automated queries" in body_lowered:
|
||||
return "blocked_automated_queries_detected"
|
||||
if "not a robot" in body_lowered:
|
||||
return "blocked_not_a_robot_challenge"
|
||||
if "recaptcha" in body_lowered:
|
||||
return "blocked_recaptcha_challenge"
|
||||
if "captcha" in body_lowered:
|
||||
return "blocked_captcha_challenge"
|
||||
if any(keyword in body_lowered for keyword in BLOCKED_KEYWORDS):
|
||||
return "blocked_keyword_detected"
|
||||
return None
|
||||
|
||||
|
||||
def detect_state(
|
||||
fetch_result: FetchResult,
|
||||
publications: list[PublicationCandidate],
|
||||
|
|
@ -312,15 +643,19 @@ def detect_state(
|
|||
visible_text: str,
|
||||
) -> tuple[ParseState, str]:
|
||||
if fetch_result.status_code is None:
|
||||
return ParseState.NETWORK_ERROR, "network_error_missing_status_code"
|
||||
return ParseState.NETWORK_ERROR, classify_network_error_reason(fetch_result.error)
|
||||
|
||||
lowered = fetch_result.body.lower()
|
||||
final = (fetch_result.final_url or "").lower()
|
||||
status_code = int(fetch_result.status_code)
|
||||
|
||||
if "accounts.google.com" in final and ("signin" in final or "servicelogin" in final):
|
||||
return ParseState.BLOCKED_OR_CAPTCHA, "blocked_accounts_redirect"
|
||||
if any(keyword in lowered for keyword in BLOCKED_KEYWORDS) or "sorry/index" in final:
|
||||
return ParseState.BLOCKED_OR_CAPTCHA, "blocked_keyword_detected"
|
||||
block_reason = classify_block_or_captcha_reason(
|
||||
status_code=status_code,
|
||||
final_url=final,
|
||||
body_lowered=lowered,
|
||||
)
|
||||
if block_reason is not None:
|
||||
return ParseState.BLOCKED_OR_CAPTCHA, block_reason
|
||||
|
||||
if not publications and any(keyword in visible_text for keyword in NO_RESULTS_KEYWORDS):
|
||||
return ParseState.NO_RESULTS, "no_results_keyword_detected"
|
||||
|
|
@ -335,6 +670,40 @@ def detect_state(
|
|||
return ParseState.OK, "publications_extracted"
|
||||
|
||||
|
||||
def detect_author_search_state(
|
||||
fetch_result: FetchResult,
|
||||
candidates: list[ScholarSearchCandidate],
|
||||
marker_counts: dict[str, int],
|
||||
*,
|
||||
visible_text: str,
|
||||
) -> tuple[ParseState, str]:
|
||||
if fetch_result.status_code is None:
|
||||
return ParseState.NETWORK_ERROR, classify_network_error_reason(fetch_result.error)
|
||||
|
||||
lowered = fetch_result.body.lower()
|
||||
final = (fetch_result.final_url or "").lower()
|
||||
status_code = int(fetch_result.status_code)
|
||||
|
||||
block_reason = classify_block_or_captcha_reason(
|
||||
status_code=status_code,
|
||||
final_url=final,
|
||||
body_lowered=lowered,
|
||||
)
|
||||
if block_reason is not None:
|
||||
return ParseState.BLOCKED_OR_CAPTCHA, block_reason
|
||||
|
||||
if not candidates and any(keyword in visible_text for keyword in NO_AUTHOR_RESULTS_KEYWORDS):
|
||||
return ParseState.NO_RESULTS, "no_results_keyword_detected"
|
||||
|
||||
if not candidates:
|
||||
has_search_markers = marker_counts.get("gsc_1usr", 0) > 0 or marker_counts.get("gs_ai_name", 0) > 0
|
||||
if not has_search_markers:
|
||||
return ParseState.NO_RESULTS, "no_search_candidates_detected"
|
||||
return ParseState.OK, "search_markers_present_with_empty_results"
|
||||
|
||||
return ParseState.OK, "author_candidates_extracted"
|
||||
|
||||
|
||||
def parse_profile_page(fetch_result: FetchResult) -> ParsedProfilePage:
|
||||
publications, warnings = parse_publications(fetch_result.body)
|
||||
marker_counts = count_markers(fetch_result.body)
|
||||
|
|
@ -361,6 +730,7 @@ def parse_profile_page(fetch_result: FetchResult) -> ParsedProfilePage:
|
|||
state=state,
|
||||
state_reason=state_reason,
|
||||
profile_name=extract_profile_name(fetch_result.body),
|
||||
profile_image_url=extract_profile_image_url(fetch_result.body),
|
||||
publications=publications,
|
||||
marker_counts=marker_counts,
|
||||
warnings=warnings,
|
||||
|
|
@ -368,3 +738,29 @@ def parse_profile_page(fetch_result: FetchResult) -> ParsedProfilePage:
|
|||
has_operation_error_banner=operation_error_banner,
|
||||
articles_range=extract_articles_range(fetch_result.body),
|
||||
)
|
||||
|
||||
|
||||
def parse_author_search_page(fetch_result: FetchResult) -> ParsedAuthorSearchPage:
|
||||
parser = ScholarAuthorSearchParser()
|
||||
parser.feed(fetch_result.body)
|
||||
|
||||
marker_counts = count_author_search_markers(fetch_result.body)
|
||||
visible_text = strip_tags(SCRIPT_STYLE_RE.sub(" ", fetch_result.body)).lower()
|
||||
warnings: list[str] = []
|
||||
if not parser.candidates:
|
||||
warnings.append("no_author_candidates_detected")
|
||||
|
||||
state, state_reason = detect_author_search_state(
|
||||
fetch_result,
|
||||
parser.candidates,
|
||||
marker_counts,
|
||||
visible_text=visible_text,
|
||||
)
|
||||
|
||||
return ParsedAuthorSearchPage(
|
||||
state=state,
|
||||
state_reason=state_reason,
|
||||
candidates=parser.candidates,
|
||||
marker_counts=marker_counts,
|
||||
warnings=warnings,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -52,6 +52,14 @@ class ScholarSource(Protocol):
|
|||
) -> FetchResult:
|
||||
...
|
||||
|
||||
async def fetch_author_search_html(
|
||||
self,
|
||||
query: str,
|
||||
*,
|
||||
start: int,
|
||||
) -> FetchResult:
|
||||
...
|
||||
|
||||
|
||||
class LiveScholarSource:
|
||||
def __init__(
|
||||
|
|
@ -94,6 +102,27 @@ class LiveScholarSource:
|
|||
)
|
||||
return await asyncio.to_thread(self._fetch_sync, requested_url)
|
||||
|
||||
async def fetch_author_search_html(
|
||||
self,
|
||||
query: str,
|
||||
*,
|
||||
start: int = 0,
|
||||
) -> FetchResult:
|
||||
requested_url = _build_author_search_url(
|
||||
query=query,
|
||||
start=start,
|
||||
)
|
||||
logger.debug(
|
||||
"scholar_source.search_fetch_started",
|
||||
extra={
|
||||
"event": "scholar_source.search_fetch_started",
|
||||
"query": query,
|
||||
"requested_url": requested_url,
|
||||
"start": start,
|
||||
},
|
||||
)
|
||||
return await asyncio.to_thread(self._fetch_sync, requested_url)
|
||||
|
||||
def _fetch_sync(self, requested_url: str) -> FetchResult:
|
||||
request = Request(
|
||||
requested_url,
|
||||
|
|
@ -169,3 +198,14 @@ def _build_profile_url(*, scholar_id: str, cstart: int, pagesize: int) -> str:
|
|||
if pagesize > 0 and cstart > 0:
|
||||
query["pagesize"] = int(pagesize)
|
||||
return f"{SCHOLAR_PROFILE_URL}?{urlencode(query)}"
|
||||
|
||||
|
||||
def _build_author_search_url(*, query: str, start: int) -> str:
|
||||
params: dict[str, int | str] = {
|
||||
"hl": "en",
|
||||
"view_op": "search_authors",
|
||||
"mauthors": query,
|
||||
}
|
||||
if start > 0:
|
||||
params["astart"] = int(start)
|
||||
return f"{SCHOLAR_PROFILE_URL}?{urlencode(params)}"
|
||||
|
|
|
|||
|
|
@ -1,14 +1,106 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from collections import OrderedDict
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import replace
|
||||
from datetime import datetime
|
||||
from datetime import timedelta
|
||||
from datetime import timezone
|
||||
import logging
|
||||
import os
|
||||
import random
|
||||
import re
|
||||
import time
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlparse
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.models import ScholarProfile
|
||||
from app.services.scholar_parser import (
|
||||
ParseState,
|
||||
ParsedAuthorSearchPage,
|
||||
parse_author_search_page,
|
||||
parse_profile_page,
|
||||
)
|
||||
from app.services.scholar_source import ScholarSource
|
||||
|
||||
SCHOLAR_ID_PATTERN = re.compile(r"^[a-zA-Z0-9_-]{12}$")
|
||||
MAX_IMAGE_URL_LENGTH = 2048
|
||||
MAX_AUTHOR_SEARCH_LIMIT = 25
|
||||
DEFAULT_AUTHOR_SEARCH_CACHE_MAX_ENTRIES = 512
|
||||
DEFAULT_AUTHOR_SEARCH_BLOCKED_CACHE_TTL_SECONDS = 300
|
||||
DEFAULT_AUTHOR_SEARCH_COOLDOWN_BLOCK_THRESHOLD = 1
|
||||
DEFAULT_AUTHOR_SEARCH_COOLDOWN_SECONDS = 1800
|
||||
DEFAULT_AUTHOR_SEARCH_MIN_INTERVAL_SECONDS = 3.0
|
||||
DEFAULT_AUTHOR_SEARCH_INTERVAL_JITTER_SECONDS = 1.0
|
||||
ALLOWED_IMAGE_UPLOAD_CONTENT_TYPES = {
|
||||
"image/jpeg": ".jpg",
|
||||
"image/png": ".png",
|
||||
"image/webp": ".webp",
|
||||
"image/gif": ".gif",
|
||||
}
|
||||
SEARCH_DISABLED_REASON = "search_disabled_by_configuration"
|
||||
SEARCH_COOLDOWN_REASON = "search_temporarily_disabled_due_to_repeated_blocks"
|
||||
SEARCH_CACHED_BLOCK_REASON = "search_temporarily_disabled_from_cached_blocked_response"
|
||||
|
||||
STATE_REASON_HINTS: dict[str, str] = {
|
||||
SEARCH_DISABLED_REASON: (
|
||||
"Scholar name search is currently disabled by service policy. "
|
||||
"Add scholars by profile URL or Scholar ID."
|
||||
),
|
||||
SEARCH_COOLDOWN_REASON: (
|
||||
"Scholar name search is temporarily paused after repeated block responses. "
|
||||
"Use Scholar URL/ID adds until cooldown expires."
|
||||
),
|
||||
SEARCH_CACHED_BLOCK_REASON: (
|
||||
"A recent blocked response was cached to reduce traffic. "
|
||||
"Retry later or add by Scholar URL/ID."
|
||||
),
|
||||
"network_dns_resolution_failed": (
|
||||
"DNS resolution failed while reaching scholar.google.com. "
|
||||
"Verify container DNS/network and retry."
|
||||
),
|
||||
"network_timeout": (
|
||||
"Request timed out before Google Scholar responded. "
|
||||
"Increase delay/backoff and retry."
|
||||
),
|
||||
"network_tls_error": (
|
||||
"TLS handshake/certificate validation failed. "
|
||||
"Verify outbound TLS/network configuration."
|
||||
),
|
||||
"blocked_http_429_rate_limited": (
|
||||
"Google Scholar rate-limited the request. "
|
||||
"Slow request cadence and retry later."
|
||||
),
|
||||
"blocked_unusual_traffic_detected": (
|
||||
"Google Scholar flagged traffic as unusual. "
|
||||
"Increase delay/jitter and reduce concurrent scraping."
|
||||
),
|
||||
"blocked_accounts_redirect": (
|
||||
"Request was redirected to Google Account sign-in. "
|
||||
"Treat as access block and retry later."
|
||||
),
|
||||
}
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _AuthorSearchCacheEntry:
|
||||
parsed: ParsedAuthorSearchPage
|
||||
expires_at_monotonic: float
|
||||
cached_at_utc: datetime
|
||||
|
||||
|
||||
_AUTHOR_SEARCH_EXECUTION_LOCK = asyncio.Lock()
|
||||
_AUTHOR_SEARCH_CACHE: OrderedDict[str, _AuthorSearchCacheEntry] = OrderedDict()
|
||||
_AUTHOR_SEARCH_LAST_LIVE_REQUEST_MONOTONIC = 0.0
|
||||
_AUTHOR_SEARCH_COOLDOWN_UNTIL_UTC: datetime | None = None
|
||||
_AUTHOR_SEARCH_CONSECUTIVE_BLOCKED_COUNT = 0
|
||||
|
||||
|
||||
class ScholarServiceError(ValueError):
|
||||
|
|
@ -27,6 +119,174 @@ def normalize_display_name(value: str) -> str | None:
|
|||
return normalized if normalized else None
|
||||
|
||||
|
||||
def normalize_profile_image_url(value: str | None) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
|
||||
candidate = value.strip()
|
||||
if not candidate:
|
||||
return None
|
||||
|
||||
if len(candidate) > MAX_IMAGE_URL_LENGTH:
|
||||
raise ScholarServiceError(
|
||||
f"Image URL must be {MAX_IMAGE_URL_LENGTH} characters or fewer."
|
||||
)
|
||||
|
||||
parsed = urlparse(candidate)
|
||||
if parsed.scheme.lower() not in {"http", "https"} or not parsed.netloc:
|
||||
raise ScholarServiceError("Image URL must be an absolute http(s) URL.")
|
||||
|
||||
return candidate
|
||||
|
||||
|
||||
def _ensure_upload_root(upload_dir: str, *, create: bool) -> Path:
|
||||
root = Path(upload_dir).expanduser().resolve()
|
||||
if create:
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
return root
|
||||
|
||||
|
||||
def _resolve_upload_path(upload_root: Path, relative_path: str) -> Path:
|
||||
candidate = (upload_root / relative_path).resolve()
|
||||
if upload_root != candidate and upload_root not in candidate.parents:
|
||||
raise ScholarServiceError("Invalid scholar image path.")
|
||||
return candidate
|
||||
|
||||
|
||||
def _safe_remove_upload(upload_root: Path, relative_path: str | None) -> None:
|
||||
if not relative_path:
|
||||
return
|
||||
try:
|
||||
file_path = _resolve_upload_path(upload_root, relative_path)
|
||||
except ScholarServiceError:
|
||||
return
|
||||
|
||||
try:
|
||||
if file_path.exists() and file_path.is_file():
|
||||
file_path.unlink()
|
||||
except OSError:
|
||||
return
|
||||
|
||||
|
||||
def resolve_profile_image(
|
||||
profile: ScholarProfile,
|
||||
*,
|
||||
uploaded_image_url: str | None,
|
||||
) -> tuple[str | None, str]:
|
||||
if profile.profile_image_upload_path and uploaded_image_url:
|
||||
return uploaded_image_url, "upload"
|
||||
if profile.profile_image_override_url:
|
||||
return profile.profile_image_override_url, "override"
|
||||
if profile.profile_image_url:
|
||||
return profile.profile_image_url, "scraped"
|
||||
return None, "none"
|
||||
|
||||
|
||||
def resolve_upload_file_path(*, upload_dir: str, relative_path: str) -> Path:
|
||||
root = _ensure_upload_root(upload_dir, create=False)
|
||||
return _resolve_upload_path(root, relative_path)
|
||||
|
||||
|
||||
def scrape_state_hint(*, state: ParseState, state_reason: str) -> str | None:
|
||||
if state not in {ParseState.NETWORK_ERROR, ParseState.BLOCKED_OR_CAPTCHA}:
|
||||
return None
|
||||
return STATE_REASON_HINTS.get(state_reason)
|
||||
|
||||
|
||||
def _merge_warnings(base: list[str], extra: list[str]) -> list[str]:
|
||||
if not extra:
|
||||
return sorted(set(base))
|
||||
return sorted(set(base + extra))
|
||||
|
||||
|
||||
def _trim_author_search_result(
|
||||
parsed: ParsedAuthorSearchPage,
|
||||
*,
|
||||
limit: int,
|
||||
extra_warnings: list[str] | None = None,
|
||||
state_reason_override: str | None = None,
|
||||
) -> ParsedAuthorSearchPage:
|
||||
return ParsedAuthorSearchPage(
|
||||
state=parsed.state,
|
||||
state_reason=state_reason_override or parsed.state_reason,
|
||||
candidates=parsed.candidates[: max(1, min(int(limit), MAX_AUTHOR_SEARCH_LIMIT))],
|
||||
marker_counts=parsed.marker_counts,
|
||||
warnings=_merge_warnings(parsed.warnings, extra_warnings or []),
|
||||
)
|
||||
|
||||
|
||||
def _policy_blocked_author_search_result(
|
||||
*,
|
||||
reason: str,
|
||||
warning_codes: list[str],
|
||||
limit: int,
|
||||
) -> ParsedAuthorSearchPage:
|
||||
_ = limit
|
||||
return ParsedAuthorSearchPage(
|
||||
state=ParseState.BLOCKED_OR_CAPTCHA,
|
||||
state_reason=reason,
|
||||
candidates=[],
|
||||
marker_counts={},
|
||||
warnings=_merge_warnings([], warning_codes),
|
||||
)
|
||||
|
||||
|
||||
def _cache_get_author_search_result(query_key: str, now_monotonic: float) -> _AuthorSearchCacheEntry | None:
|
||||
entry = _AUTHOR_SEARCH_CACHE.get(query_key)
|
||||
if entry is None:
|
||||
return None
|
||||
if entry.expires_at_monotonic <= now_monotonic:
|
||||
_AUTHOR_SEARCH_CACHE.pop(query_key, None)
|
||||
return None
|
||||
_AUTHOR_SEARCH_CACHE.move_to_end(query_key)
|
||||
return entry
|
||||
|
||||
|
||||
def _cache_set_author_search_result(
|
||||
*,
|
||||
query_key: str,
|
||||
parsed: ParsedAuthorSearchPage,
|
||||
ttl_seconds: float,
|
||||
max_entries: int,
|
||||
) -> None:
|
||||
ttl = max(float(ttl_seconds), 0.0)
|
||||
if ttl <= 0.0:
|
||||
_AUTHOR_SEARCH_CACHE.pop(query_key, None)
|
||||
return
|
||||
|
||||
_AUTHOR_SEARCH_CACHE[query_key] = _AuthorSearchCacheEntry(
|
||||
parsed=parsed,
|
||||
expires_at_monotonic=time.monotonic() + ttl,
|
||||
cached_at_utc=datetime.now(timezone.utc),
|
||||
)
|
||||
_AUTHOR_SEARCH_CACHE.move_to_end(query_key)
|
||||
|
||||
bounded_max_entries = max(1, int(max_entries))
|
||||
while len(_AUTHOR_SEARCH_CACHE) > bounded_max_entries:
|
||||
_AUTHOR_SEARCH_CACHE.popitem(last=False)
|
||||
|
||||
|
||||
def _is_author_search_block_state(parsed: ParsedAuthorSearchPage) -> bool:
|
||||
return parsed.state == ParseState.BLOCKED_OR_CAPTCHA
|
||||
|
||||
|
||||
def _author_search_cooldown_remaining_seconds(now_utc: datetime) -> int:
|
||||
if _AUTHOR_SEARCH_COOLDOWN_UNTIL_UTC is None:
|
||||
return 0
|
||||
remaining_seconds = int((_AUTHOR_SEARCH_COOLDOWN_UNTIL_UTC - now_utc).total_seconds())
|
||||
return max(0, remaining_seconds)
|
||||
|
||||
|
||||
def _reset_author_search_runtime_state_for_tests() -> None:
|
||||
global _AUTHOR_SEARCH_LAST_LIVE_REQUEST_MONOTONIC
|
||||
global _AUTHOR_SEARCH_COOLDOWN_UNTIL_UTC
|
||||
global _AUTHOR_SEARCH_CONSECUTIVE_BLOCKED_COUNT
|
||||
_AUTHOR_SEARCH_CACHE.clear()
|
||||
_AUTHOR_SEARCH_LAST_LIVE_REQUEST_MONOTONIC = 0.0
|
||||
_AUTHOR_SEARCH_COOLDOWN_UNTIL_UTC = None
|
||||
_AUTHOR_SEARCH_CONSECUTIVE_BLOCKED_COUNT = 0
|
||||
|
||||
|
||||
async def list_scholars_for_user(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
|
|
@ -46,11 +306,13 @@ async def create_scholar_for_user(
|
|||
user_id: int,
|
||||
scholar_id: str,
|
||||
display_name: str,
|
||||
profile_image_url: str | None = None,
|
||||
) -> ScholarProfile:
|
||||
profile = ScholarProfile(
|
||||
user_id=user_id,
|
||||
scholar_id=validate_scholar_id(scholar_id),
|
||||
display_name=normalize_display_name(display_name),
|
||||
profile_image_url=normalize_profile_image_url(profile_image_url),
|
||||
)
|
||||
db_session.add(profile)
|
||||
try:
|
||||
|
|
@ -92,7 +354,293 @@ async def delete_scholar(
|
|||
db_session: AsyncSession,
|
||||
*,
|
||||
profile: ScholarProfile,
|
||||
upload_dir: str | None = None,
|
||||
) -> None:
|
||||
if upload_dir:
|
||||
upload_root = _ensure_upload_root(upload_dir, create=True)
|
||||
_safe_remove_upload(upload_root, profile.profile_image_upload_path)
|
||||
|
||||
await db_session.delete(profile)
|
||||
await db_session.commit()
|
||||
|
||||
|
||||
async def search_author_candidates(
|
||||
*,
|
||||
source: ScholarSource,
|
||||
query: str,
|
||||
limit: int,
|
||||
network_error_retries: int = 1,
|
||||
retry_backoff_seconds: float = 1.0,
|
||||
search_enabled: bool = True,
|
||||
cache_ttl_seconds: int = 21_600,
|
||||
blocked_cache_ttl_seconds: int = DEFAULT_AUTHOR_SEARCH_BLOCKED_CACHE_TTL_SECONDS,
|
||||
cache_max_entries: int = DEFAULT_AUTHOR_SEARCH_CACHE_MAX_ENTRIES,
|
||||
min_interval_seconds: float = DEFAULT_AUTHOR_SEARCH_MIN_INTERVAL_SECONDS,
|
||||
interval_jitter_seconds: float = DEFAULT_AUTHOR_SEARCH_INTERVAL_JITTER_SECONDS,
|
||||
cooldown_block_threshold: int = DEFAULT_AUTHOR_SEARCH_COOLDOWN_BLOCK_THRESHOLD,
|
||||
cooldown_seconds: int = DEFAULT_AUTHOR_SEARCH_COOLDOWN_SECONDS,
|
||||
) -> ParsedAuthorSearchPage:
|
||||
global _AUTHOR_SEARCH_LAST_LIVE_REQUEST_MONOTONIC
|
||||
global _AUTHOR_SEARCH_COOLDOWN_UNTIL_UTC
|
||||
global _AUTHOR_SEARCH_CONSECUTIVE_BLOCKED_COUNT
|
||||
|
||||
normalized_query = query.strip()
|
||||
if len(normalized_query) < 2:
|
||||
raise ScholarServiceError("Search query must be at least 2 characters.")
|
||||
bounded_limit = max(1, min(int(limit), MAX_AUTHOR_SEARCH_LIMIT))
|
||||
query_key = normalized_query.casefold()
|
||||
|
||||
if not search_enabled:
|
||||
logger.warning(
|
||||
"scholar_search.disabled_by_configuration",
|
||||
extra={
|
||||
"event": "scholar_search.disabled_by_configuration",
|
||||
"query": normalized_query,
|
||||
},
|
||||
)
|
||||
return _policy_blocked_author_search_result(
|
||||
reason=SEARCH_DISABLED_REASON,
|
||||
warning_codes=["author_search_disabled_by_configuration"],
|
||||
limit=bounded_limit,
|
||||
)
|
||||
|
||||
async with _AUTHOR_SEARCH_EXECUTION_LOCK:
|
||||
now_utc = datetime.now(timezone.utc)
|
||||
now_monotonic = time.monotonic()
|
||||
|
||||
cooldown_remaining_seconds = _author_search_cooldown_remaining_seconds(now_utc)
|
||||
if cooldown_remaining_seconds > 0:
|
||||
logger.warning(
|
||||
"scholar_search.cooldown_active",
|
||||
extra={
|
||||
"event": "scholar_search.cooldown_active",
|
||||
"query": normalized_query,
|
||||
"cooldown_remaining_seconds": cooldown_remaining_seconds,
|
||||
"cooldown_until_utc": _AUTHOR_SEARCH_COOLDOWN_UNTIL_UTC.isoformat()
|
||||
if _AUTHOR_SEARCH_COOLDOWN_UNTIL_UTC
|
||||
else None,
|
||||
},
|
||||
)
|
||||
return _policy_blocked_author_search_result(
|
||||
reason=SEARCH_COOLDOWN_REASON,
|
||||
warning_codes=[
|
||||
"author_search_cooldown_active",
|
||||
f"author_search_cooldown_remaining_{cooldown_remaining_seconds}s",
|
||||
],
|
||||
limit=bounded_limit,
|
||||
)
|
||||
|
||||
cached_entry = _cache_get_author_search_result(query_key, now_monotonic)
|
||||
if cached_entry is not None:
|
||||
cached = cached_entry.parsed
|
||||
state_reason_override = (
|
||||
SEARCH_CACHED_BLOCK_REASON if _is_author_search_block_state(cached) else None
|
||||
)
|
||||
logger.info(
|
||||
"scholar_search.cache_hit",
|
||||
extra={
|
||||
"event": "scholar_search.cache_hit",
|
||||
"query": normalized_query,
|
||||
"state": cached.state.value,
|
||||
"state_reason": cached.state_reason,
|
||||
},
|
||||
)
|
||||
return _trim_author_search_result(
|
||||
cached,
|
||||
limit=bounded_limit,
|
||||
extra_warnings=["author_search_served_from_cache"],
|
||||
state_reason_override=state_reason_override,
|
||||
)
|
||||
|
||||
enforced_wait_seconds = (
|
||||
(_AUTHOR_SEARCH_LAST_LIVE_REQUEST_MONOTONIC + max(float(min_interval_seconds), 0.0))
|
||||
- now_monotonic
|
||||
)
|
||||
jitter_seconds = random.uniform(0.0, max(float(interval_jitter_seconds), 0.0))
|
||||
sleep_seconds = max(0.0, enforced_wait_seconds) + jitter_seconds
|
||||
if sleep_seconds > 0.0:
|
||||
logger.info(
|
||||
"scholar_search.throttle_wait",
|
||||
extra={
|
||||
"event": "scholar_search.throttle_wait",
|
||||
"query": normalized_query,
|
||||
"sleep_seconds": round(sleep_seconds, 3),
|
||||
},
|
||||
)
|
||||
await asyncio.sleep(sleep_seconds)
|
||||
|
||||
max_attempts = max(1, int(network_error_retries) + 1)
|
||||
parsed: ParsedAuthorSearchPage | None = None
|
||||
retry_warnings: list[str] = []
|
||||
|
||||
for attempt_index in range(max_attempts):
|
||||
fetch_result = await source.fetch_author_search_html(normalized_query, start=0)
|
||||
parsed = parse_author_search_page(fetch_result)
|
||||
if parsed.state != ParseState.NETWORK_ERROR or attempt_index >= max_attempts - 1:
|
||||
break
|
||||
|
||||
retry_warnings.append("network_retry_scheduled_for_author_search")
|
||||
sleep_seconds = max(float(retry_backoff_seconds), 0.0) * (2**attempt_index)
|
||||
if sleep_seconds > 0:
|
||||
await asyncio.sleep(sleep_seconds)
|
||||
|
||||
_AUTHOR_SEARCH_LAST_LIVE_REQUEST_MONOTONIC = time.monotonic()
|
||||
|
||||
if parsed is None:
|
||||
raise ScholarServiceError("Unable to complete scholar author search.")
|
||||
|
||||
merged_parsed = replace(
|
||||
parsed,
|
||||
warnings=_merge_warnings(parsed.warnings, retry_warnings),
|
||||
)
|
||||
|
||||
if _is_author_search_block_state(merged_parsed):
|
||||
_AUTHOR_SEARCH_CONSECUTIVE_BLOCKED_COUNT += 1
|
||||
logger.warning(
|
||||
"scholar_search.block_detected",
|
||||
extra={
|
||||
"event": "scholar_search.block_detected",
|
||||
"query": normalized_query,
|
||||
"state_reason": merged_parsed.state_reason,
|
||||
"consecutive_blocked_count": _AUTHOR_SEARCH_CONSECUTIVE_BLOCKED_COUNT,
|
||||
},
|
||||
)
|
||||
if _AUTHOR_SEARCH_CONSECUTIVE_BLOCKED_COUNT >= max(1, int(cooldown_block_threshold)):
|
||||
_AUTHOR_SEARCH_COOLDOWN_UNTIL_UTC = datetime.now(timezone.utc) + timedelta(
|
||||
seconds=max(60, int(cooldown_seconds))
|
||||
)
|
||||
_AUTHOR_SEARCH_CONSECUTIVE_BLOCKED_COUNT = 0
|
||||
merged_parsed = replace(
|
||||
merged_parsed,
|
||||
warnings=_merge_warnings(
|
||||
merged_parsed.warnings,
|
||||
["author_search_circuit_breaker_armed"],
|
||||
),
|
||||
)
|
||||
logger.error(
|
||||
"scholar_search.cooldown_activated",
|
||||
extra={
|
||||
"event": "scholar_search.cooldown_activated",
|
||||
"query": normalized_query,
|
||||
"cooldown_until_utc": _AUTHOR_SEARCH_COOLDOWN_UNTIL_UTC.isoformat(),
|
||||
},
|
||||
)
|
||||
else:
|
||||
_AUTHOR_SEARCH_CONSECUTIVE_BLOCKED_COUNT = 0
|
||||
|
||||
ttl_seconds = (
|
||||
min(max(1, int(blocked_cache_ttl_seconds)), max(1, int(cache_ttl_seconds)))
|
||||
if _is_author_search_block_state(merged_parsed)
|
||||
else max(1, int(cache_ttl_seconds))
|
||||
)
|
||||
_cache_set_author_search_result(
|
||||
query_key=query_key,
|
||||
parsed=merged_parsed,
|
||||
ttl_seconds=float(ttl_seconds),
|
||||
max_entries=cache_max_entries,
|
||||
)
|
||||
|
||||
return _trim_author_search_result(
|
||||
merged_parsed,
|
||||
limit=bounded_limit,
|
||||
)
|
||||
|
||||
|
||||
async def hydrate_profile_metadata(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
profile: ScholarProfile,
|
||||
source: ScholarSource,
|
||||
) -> ScholarProfile:
|
||||
fetch_result = await source.fetch_profile_html(profile.scholar_id)
|
||||
parsed_page = parse_profile_page(fetch_result)
|
||||
|
||||
if parsed_page.profile_name and not (profile.display_name or "").strip():
|
||||
profile.display_name = parsed_page.profile_name
|
||||
if parsed_page.profile_image_url and not profile.profile_image_url:
|
||||
profile.profile_image_url = parsed_page.profile_image_url
|
||||
|
||||
await db_session.commit()
|
||||
await db_session.refresh(profile)
|
||||
return profile
|
||||
|
||||
|
||||
async def set_profile_image_override_url(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
profile: ScholarProfile,
|
||||
image_url: str | None,
|
||||
upload_dir: str,
|
||||
) -> ScholarProfile:
|
||||
upload_root = _ensure_upload_root(upload_dir, create=True)
|
||||
_safe_remove_upload(upload_root, profile.profile_image_upload_path)
|
||||
|
||||
profile.profile_image_upload_path = None
|
||||
profile.profile_image_override_url = normalize_profile_image_url(image_url)
|
||||
|
||||
await db_session.commit()
|
||||
await db_session.refresh(profile)
|
||||
return profile
|
||||
|
||||
|
||||
async def clear_profile_image_customization(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
profile: ScholarProfile,
|
||||
upload_dir: str,
|
||||
) -> ScholarProfile:
|
||||
upload_root = _ensure_upload_root(upload_dir, create=True)
|
||||
_safe_remove_upload(upload_root, profile.profile_image_upload_path)
|
||||
|
||||
profile.profile_image_upload_path = None
|
||||
profile.profile_image_override_url = None
|
||||
|
||||
await db_session.commit()
|
||||
await db_session.refresh(profile)
|
||||
return profile
|
||||
|
||||
|
||||
async def set_profile_image_upload(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
profile: ScholarProfile,
|
||||
content_type: str | None,
|
||||
image_bytes: bytes,
|
||||
upload_dir: str,
|
||||
max_upload_bytes: int,
|
||||
) -> ScholarProfile:
|
||||
normalized_content_type = (content_type or "").strip().lower()
|
||||
extension = ALLOWED_IMAGE_UPLOAD_CONTENT_TYPES.get(normalized_content_type)
|
||||
if extension is None:
|
||||
raise ScholarServiceError(
|
||||
"Unsupported image type. Use JPEG, PNG, WEBP, or GIF."
|
||||
)
|
||||
|
||||
if not image_bytes:
|
||||
raise ScholarServiceError("Uploaded image file is empty.")
|
||||
|
||||
if len(image_bytes) > max_upload_bytes:
|
||||
raise ScholarServiceError(
|
||||
f"Uploaded image exceeds {max_upload_bytes} bytes."
|
||||
)
|
||||
|
||||
upload_root = _ensure_upload_root(upload_dir, create=True)
|
||||
user_dir = upload_root / str(profile.user_id)
|
||||
user_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
filename = f"{profile.id}_{uuid4().hex}{extension}"
|
||||
relative_path = os.path.join(str(profile.user_id), filename)
|
||||
absolute_path = _resolve_upload_path(upload_root, relative_path)
|
||||
absolute_path.write_bytes(image_bytes)
|
||||
|
||||
old_path = profile.profile_image_upload_path
|
||||
profile.profile_image_upload_path = relative_path
|
||||
profile.profile_image_override_url = None
|
||||
|
||||
await db_session.commit()
|
||||
await db_session.refresh(profile)
|
||||
|
||||
if old_path and old_path != relative_path:
|
||||
_safe_remove_upload(upload_root, old_path)
|
||||
|
||||
return profile
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue