modularize service layer, add mobile nav drawer, and sync docs
This commit is contained in:
parent
7f7a8ce2b0
commit
6b6ed91b92
72 changed files with 8122 additions and 7501 deletions
|
|
@ -2,8 +2,8 @@ from __future__ import annotations
|
|||
|
||||
from fastapi import Depends
|
||||
|
||||
from app.services import ingestion as ingestion_service
|
||||
from app.services import scholar_source as scholar_source_service
|
||||
from app.services.domains.ingestion import application as ingestion_service
|
||||
from app.services.domains.scholar import source as scholar_source_service
|
||||
|
||||
|
||||
def get_scholar_source() -> scholar_source_service.ScholarSource:
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ from app.http.middleware import (
|
|||
)
|
||||
from app.logging_config import configure_logging, parse_redact_fields
|
||||
from app.security.csrf import CSRFMiddleware
|
||||
from app.services.scheduler import SchedulerService
|
||||
from app.services.domains.ingestion.scheduler import SchedulerService
|
||||
from app.settings import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
|
|
|||
|
|
@ -1,348 +1,3 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.models import IngestionQueueItem, QueueItemStatus
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ContinuationQueueJob:
|
||||
id: int
|
||||
user_id: int
|
||||
scholar_profile_id: int
|
||||
resume_cstart: int
|
||||
reason: str
|
||||
status: str
|
||||
attempt_count: int
|
||||
next_attempt_dt: datetime
|
||||
|
||||
|
||||
ACTIVE_QUEUE_STATUSES: tuple[str, ...] = (
|
||||
QueueItemStatus.QUEUED.value,
|
||||
QueueItemStatus.RETRYING.value,
|
||||
)
|
||||
|
||||
|
||||
def normalize_cstart(value: int | None) -> int:
|
||||
if value is None:
|
||||
return 0
|
||||
return max(0, int(value))
|
||||
|
||||
|
||||
def compute_backoff_seconds(*, base_seconds: int, attempt_count: int, max_seconds: int) -> int:
|
||||
base = max(1, int(base_seconds))
|
||||
attempts = max(1, int(attempt_count))
|
||||
maximum = max(base, int(max_seconds))
|
||||
seconds = base * (2 ** max(0, attempts - 1))
|
||||
return min(seconds, maximum)
|
||||
|
||||
|
||||
async def _get_item_for_user_scholar(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
scholar_profile_id: int,
|
||||
) -> IngestionQueueItem | None:
|
||||
result = await db_session.execute(
|
||||
select(IngestionQueueItem).where(
|
||||
IngestionQueueItem.user_id == user_id,
|
||||
IngestionQueueItem.scholar_profile_id == scholar_profile_id,
|
||||
)
|
||||
)
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
|
||||
def _build_queue_item(
|
||||
*,
|
||||
now: datetime,
|
||||
user_id: int,
|
||||
scholar_profile_id: int,
|
||||
normalized_cstart: int,
|
||||
reason: str,
|
||||
run_id: int | None,
|
||||
next_attempt_dt: datetime,
|
||||
) -> IngestionQueueItem:
|
||||
return IngestionQueueItem(
|
||||
user_id=user_id,
|
||||
scholar_profile_id=scholar_profile_id,
|
||||
resume_cstart=normalized_cstart,
|
||||
reason=reason,
|
||||
status=QueueItemStatus.QUEUED.value,
|
||||
attempt_count=0,
|
||||
next_attempt_dt=next_attempt_dt,
|
||||
last_run_id=run_id,
|
||||
last_error=None,
|
||||
dropped_reason=None,
|
||||
dropped_at=None,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
|
||||
|
||||
def _update_existing_queue_item(
|
||||
*,
|
||||
item: IngestionQueueItem,
|
||||
now: datetime,
|
||||
normalized_cstart: int,
|
||||
reason: str,
|
||||
run_id: int | None,
|
||||
next_attempt_dt: datetime,
|
||||
) -> None:
|
||||
item.resume_cstart = normalized_cstart
|
||||
item.reason = reason
|
||||
if item.status == QueueItemStatus.DROPPED.value:
|
||||
item.attempt_count = 0
|
||||
item.status = QueueItemStatus.QUEUED.value
|
||||
item.next_attempt_dt = next_attempt_dt
|
||||
item.last_run_id = run_id
|
||||
item.last_error = None
|
||||
item.dropped_reason = None
|
||||
item.dropped_at = None
|
||||
item.updated_at = now
|
||||
|
||||
|
||||
async def upsert_job(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
scholar_profile_id: int,
|
||||
resume_cstart: int,
|
||||
reason: str,
|
||||
run_id: int | None,
|
||||
delay_seconds: int,
|
||||
) -> IngestionQueueItem:
|
||||
now = datetime.now(timezone.utc)
|
||||
next_attempt_dt = now + timedelta(seconds=max(0, int(delay_seconds)))
|
||||
item = await _get_item_for_user_scholar(
|
||||
db_session,
|
||||
user_id=user_id,
|
||||
scholar_profile_id=scholar_profile_id,
|
||||
)
|
||||
normalized_cstart = normalize_cstart(resume_cstart)
|
||||
if item is None:
|
||||
item = _build_queue_item(
|
||||
now=now,
|
||||
user_id=user_id,
|
||||
scholar_profile_id=scholar_profile_id,
|
||||
normalized_cstart=normalized_cstart,
|
||||
reason=reason,
|
||||
run_id=run_id,
|
||||
next_attempt_dt=next_attempt_dt,
|
||||
)
|
||||
db_session.add(item)
|
||||
return item
|
||||
|
||||
_update_existing_queue_item(
|
||||
item=item,
|
||||
now=now,
|
||||
normalized_cstart=normalized_cstart,
|
||||
reason=reason,
|
||||
run_id=run_id,
|
||||
next_attempt_dt=next_attempt_dt,
|
||||
)
|
||||
return item
|
||||
|
||||
|
||||
async def clear_job_for_scholar(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
scholar_profile_id: int,
|
||||
) -> bool:
|
||||
result = await db_session.execute(
|
||||
select(IngestionQueueItem).where(
|
||||
IngestionQueueItem.user_id == user_id,
|
||||
IngestionQueueItem.scholar_profile_id == scholar_profile_id,
|
||||
)
|
||||
)
|
||||
item = result.scalar_one_or_none()
|
||||
if item is None:
|
||||
return False
|
||||
await db_session.delete(item)
|
||||
return True
|
||||
|
||||
|
||||
async def delete_job_by_id(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
job_id: int,
|
||||
) -> bool:
|
||||
result = await db_session.execute(
|
||||
select(IngestionQueueItem).where(IngestionQueueItem.id == job_id)
|
||||
)
|
||||
item = result.scalar_one_or_none()
|
||||
if item is None:
|
||||
return False
|
||||
await db_session.delete(item)
|
||||
return True
|
||||
|
||||
|
||||
async def list_due_jobs(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
now: datetime,
|
||||
limit: int,
|
||||
) -> list[ContinuationQueueJob]:
|
||||
result = await db_session.execute(
|
||||
select(IngestionQueueItem)
|
||||
.where(
|
||||
IngestionQueueItem.next_attempt_dt <= now,
|
||||
IngestionQueueItem.status.in_(ACTIVE_QUEUE_STATUSES),
|
||||
)
|
||||
.order_by(
|
||||
IngestionQueueItem.next_attempt_dt.asc(),
|
||||
IngestionQueueItem.id.asc(),
|
||||
)
|
||||
.limit(limit)
|
||||
)
|
||||
rows = list(result.scalars().all())
|
||||
jobs: list[ContinuationQueueJob] = []
|
||||
for row in rows:
|
||||
jobs.append(
|
||||
ContinuationQueueJob(
|
||||
id=int(row.id),
|
||||
user_id=int(row.user_id),
|
||||
scholar_profile_id=int(row.scholar_profile_id),
|
||||
resume_cstart=normalize_cstart(row.resume_cstart),
|
||||
reason=row.reason,
|
||||
status=row.status,
|
||||
attempt_count=int(row.attempt_count),
|
||||
next_attempt_dt=row.next_attempt_dt,
|
||||
)
|
||||
)
|
||||
return jobs
|
||||
|
||||
|
||||
async def increment_attempt_count(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
job_id: int,
|
||||
) -> IngestionQueueItem | None:
|
||||
now = datetime.now(timezone.utc)
|
||||
result = await db_session.execute(
|
||||
select(IngestionQueueItem).where(IngestionQueueItem.id == job_id)
|
||||
)
|
||||
item = result.scalar_one_or_none()
|
||||
if item is None:
|
||||
return None
|
||||
item.attempt_count = int(item.attempt_count or 0) + 1
|
||||
item.updated_at = now
|
||||
return item
|
||||
|
||||
|
||||
async def reset_attempt_count(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
job_id: int,
|
||||
) -> IngestionQueueItem | None:
|
||||
now = datetime.now(timezone.utc)
|
||||
result = await db_session.execute(
|
||||
select(IngestionQueueItem).where(IngestionQueueItem.id == job_id)
|
||||
)
|
||||
item = result.scalar_one_or_none()
|
||||
if item is None:
|
||||
return None
|
||||
item.attempt_count = 0
|
||||
item.updated_at = now
|
||||
return item
|
||||
|
||||
|
||||
async def reschedule_job(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
job_id: int,
|
||||
delay_seconds: int,
|
||||
reason: str,
|
||||
error: str | None = None,
|
||||
) -> IngestionQueueItem | None:
|
||||
now = datetime.now(timezone.utc)
|
||||
result = await db_session.execute(
|
||||
select(IngestionQueueItem).where(IngestionQueueItem.id == job_id)
|
||||
)
|
||||
item = result.scalar_one_or_none()
|
||||
if item is None:
|
||||
return None
|
||||
item.next_attempt_dt = now + timedelta(seconds=max(1, int(delay_seconds)))
|
||||
item.status = QueueItemStatus.QUEUED.value
|
||||
item.reason = reason
|
||||
item.last_error = error
|
||||
item.dropped_reason = None
|
||||
item.dropped_at = None
|
||||
item.updated_at = now
|
||||
return item
|
||||
|
||||
|
||||
async def mark_retrying(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
job_id: int,
|
||||
reason: str | None = None,
|
||||
) -> IngestionQueueItem | None:
|
||||
now = datetime.now(timezone.utc)
|
||||
result = await db_session.execute(
|
||||
select(IngestionQueueItem).where(IngestionQueueItem.id == job_id)
|
||||
)
|
||||
item = result.scalar_one_or_none()
|
||||
if item is None:
|
||||
return None
|
||||
if item.status == QueueItemStatus.DROPPED.value:
|
||||
return item
|
||||
item.status = QueueItemStatus.RETRYING.value
|
||||
if reason:
|
||||
item.reason = reason
|
||||
item.updated_at = now
|
||||
return item
|
||||
|
||||
|
||||
async def mark_dropped(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
job_id: int,
|
||||
reason: str,
|
||||
error: str | None = None,
|
||||
) -> IngestionQueueItem | None:
|
||||
now = datetime.now(timezone.utc)
|
||||
result = await db_session.execute(
|
||||
select(IngestionQueueItem).where(IngestionQueueItem.id == job_id)
|
||||
)
|
||||
item = result.scalar_one_or_none()
|
||||
if item is None:
|
||||
return None
|
||||
item.status = QueueItemStatus.DROPPED.value
|
||||
item.reason = "dropped"
|
||||
item.dropped_reason = reason
|
||||
item.dropped_at = now
|
||||
if error is not None:
|
||||
item.last_error = error
|
||||
item.updated_at = now
|
||||
return item
|
||||
|
||||
|
||||
async def mark_queued_now(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
job_id: int,
|
||||
reason: str,
|
||||
reset_attempt_count: bool = False,
|
||||
) -> IngestionQueueItem | None:
|
||||
now = datetime.now(timezone.utc)
|
||||
result = await db_session.execute(
|
||||
select(IngestionQueueItem).where(IngestionQueueItem.id == job_id)
|
||||
)
|
||||
item = result.scalar_one_or_none()
|
||||
if item is None:
|
||||
return None
|
||||
item.status = QueueItemStatus.QUEUED.value
|
||||
item.reason = reason
|
||||
item.next_attempt_dt = now
|
||||
if reset_attempt_count:
|
||||
item.attempt_count = 0
|
||||
item.last_error = None
|
||||
item.dropped_reason = None
|
||||
item.dropped_at = None
|
||||
item.updated_at = now
|
||||
return item
|
||||
from app.services.domains.ingestion.queue import *
|
||||
|
|
|
|||
1
app/services/domains/__init__.py
Normal file
1
app/services/domains/__init__.py
Normal file
|
|
@ -0,0 +1 @@
|
|||
from __future__ import annotations
|
||||
1
app/services/domains/ingestion/__init__.py
Normal file
1
app/services/domains/ingestion/__init__.py
Normal file
|
|
@ -0,0 +1 @@
|
|||
from __future__ import annotations
|
||||
2110
app/services/domains/ingestion/application.py
Normal file
2110
app/services/domains/ingestion/application.py
Normal file
File diff suppressed because it is too large
Load diff
31
app/services/domains/ingestion/constants.py
Normal file
31
app/services/domains/ingestion/constants.py
Normal file
|
|
@ -0,0 +1,31 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from app.services.domains.scholar.parser import ParseState
|
||||
|
||||
TITLE_ALNUM_RE = re.compile(r"[^a-z0-9]+")
|
||||
WORD_RE = re.compile(r"[a-z0-9]+")
|
||||
HTML_TAG_RE = re.compile(r"<[^>]+>", re.S)
|
||||
SPACE_RE = re.compile(r"\s+")
|
||||
|
||||
FAILED_STATES = {
|
||||
ParseState.BLOCKED_OR_CAPTCHA.value,
|
||||
ParseState.LAYOUT_CHANGED.value,
|
||||
ParseState.NETWORK_ERROR.value,
|
||||
"ingestion_error",
|
||||
}
|
||||
|
||||
FAILURE_BUCKET_BLOCKED = "blocked_or_captcha"
|
||||
FAILURE_BUCKET_NETWORK = "network_error"
|
||||
FAILURE_BUCKET_LAYOUT = "layout_changed"
|
||||
FAILURE_BUCKET_INGESTION = "ingestion_error"
|
||||
FAILURE_BUCKET_OTHER = "other_failure"
|
||||
|
||||
RUN_LOCK_NAMESPACE = 8217
|
||||
RESUMABLE_PARTIAL_REASONS = {
|
||||
"max_pages_reached",
|
||||
"pagination_cursor_stalled",
|
||||
}
|
||||
RESUMABLE_PARTIAL_REASON_PREFIXES = ("page_state_network_error",)
|
||||
INITIAL_PAGE_FINGERPRINT_MAX_PUBLICATIONS = 30
|
||||
136
app/services/domains/ingestion/fingerprints.py
Normal file
136
app/services/domains/ingestion/fingerprints.py
Normal file
|
|
@ -0,0 +1,136 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from typing import Any
|
||||
from urllib.parse import urljoin
|
||||
|
||||
from app.services.domains.ingestion.constants import (
|
||||
HTML_TAG_RE,
|
||||
INITIAL_PAGE_FINGERPRINT_MAX_PUBLICATIONS,
|
||||
SPACE_RE,
|
||||
TITLE_ALNUM_RE,
|
||||
WORD_RE,
|
||||
)
|
||||
from app.services.domains.scholar.parser import ParseState, ParsedProfilePage, PublicationCandidate
|
||||
|
||||
|
||||
def normalize_title(value: str) -> str:
|
||||
lowered = value.lower()
|
||||
return TITLE_ALNUM_RE.sub("", lowered)
|
||||
|
||||
|
||||
def _first_author_last_name(authors_text: str | None) -> str:
|
||||
if not authors_text:
|
||||
return ""
|
||||
first_author = authors_text.split(",", maxsplit=1)[0].strip().lower()
|
||||
words = WORD_RE.findall(first_author)
|
||||
if not words:
|
||||
return ""
|
||||
return words[-1]
|
||||
|
||||
|
||||
def _first_venue_word(venue_text: str | None) -> str:
|
||||
if not venue_text:
|
||||
return ""
|
||||
words = WORD_RE.findall(venue_text.lower())
|
||||
if not words:
|
||||
return ""
|
||||
return words[0]
|
||||
|
||||
|
||||
def build_publication_fingerprint(candidate: PublicationCandidate) -> str:
|
||||
canonical = "|".join(
|
||||
[
|
||||
normalize_title(candidate.title),
|
||||
str(candidate.year) if candidate.year is not None else "",
|
||||
_first_author_last_name(candidate.authors_text),
|
||||
_first_venue_word(candidate.venue_text),
|
||||
]
|
||||
)
|
||||
return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def build_initial_page_fingerprint(parsed_page: ParsedProfilePage) -> str | None:
|
||||
if parsed_page.state not in {ParseState.OK, ParseState.NO_RESULTS}:
|
||||
return None
|
||||
|
||||
normalized_rows: list[dict[str, Any]] = []
|
||||
for publication in parsed_page.publications[:INITIAL_PAGE_FINGERPRINT_MAX_PUBLICATIONS]:
|
||||
normalized_rows.append(
|
||||
{
|
||||
"cluster_id": publication.cluster_id or "",
|
||||
"title_normalized": normalize_title(publication.title),
|
||||
"year": publication.year,
|
||||
"citation_count": publication.citation_count,
|
||||
}
|
||||
)
|
||||
|
||||
payload = {
|
||||
"state": parsed_page.state.value,
|
||||
"articles_range": parsed_page.articles_range or "",
|
||||
"has_show_more_button": parsed_page.has_show_more_button,
|
||||
"profile_name": parsed_page.profile_name or "",
|
||||
"publications": normalized_rows,
|
||||
}
|
||||
canonical = json.dumps(
|
||||
payload,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
ensure_ascii=True,
|
||||
)
|
||||
return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def build_publication_url(path_or_url: str | None) -> str | None:
|
||||
if not path_or_url:
|
||||
return None
|
||||
return urljoin("https://scholar.google.com", path_or_url)
|
||||
|
||||
|
||||
def _next_cstart_value(*, articles_range: str | None, fallback: int) -> int:
|
||||
if articles_range:
|
||||
numbers = re.findall(r"\d+", articles_range)
|
||||
if len(numbers) >= 2:
|
||||
try:
|
||||
return int(numbers[1])
|
||||
except ValueError:
|
||||
pass
|
||||
return int(fallback)
|
||||
|
||||
|
||||
def _dedupe_publication_candidates(
|
||||
publications: list[PublicationCandidate],
|
||||
) -> list[PublicationCandidate]:
|
||||
deduped: list[PublicationCandidate] = []
|
||||
seen: set[str] = set()
|
||||
for publication in publications:
|
||||
if publication.cluster_id:
|
||||
identity = f"cluster:{publication.cluster_id}"
|
||||
else:
|
||||
identity = "|".join(
|
||||
[
|
||||
"fallback",
|
||||
normalize_title(publication.title),
|
||||
str(publication.year) if publication.year is not None else "",
|
||||
publication.authors_text or "",
|
||||
publication.venue_text or "",
|
||||
]
|
||||
)
|
||||
if identity in seen:
|
||||
continue
|
||||
seen.add(identity)
|
||||
deduped.append(publication)
|
||||
return deduped
|
||||
|
||||
|
||||
def _build_body_excerpt(body: str, *, max_chars: int = 220) -> str | None:
|
||||
if not body:
|
||||
return None
|
||||
flattened = SPACE_RE.sub(" ", HTML_TAG_RE.sub(" ", body)).strip()
|
||||
if not flattened:
|
||||
return None
|
||||
if len(flattened) <= max_chars:
|
||||
return flattened
|
||||
return f"{flattened[:max_chars - 1]}..."
|
||||
348
app/services/domains/ingestion/queue.py
Normal file
348
app/services/domains/ingestion/queue.py
Normal file
|
|
@ -0,0 +1,348 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.models import IngestionQueueItem, QueueItemStatus
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ContinuationQueueJob:
|
||||
id: int
|
||||
user_id: int
|
||||
scholar_profile_id: int
|
||||
resume_cstart: int
|
||||
reason: str
|
||||
status: str
|
||||
attempt_count: int
|
||||
next_attempt_dt: datetime
|
||||
|
||||
|
||||
ACTIVE_QUEUE_STATUSES: tuple[str, ...] = (
|
||||
QueueItemStatus.QUEUED.value,
|
||||
QueueItemStatus.RETRYING.value,
|
||||
)
|
||||
|
||||
|
||||
def normalize_cstart(value: int | None) -> int:
|
||||
if value is None:
|
||||
return 0
|
||||
return max(0, int(value))
|
||||
|
||||
|
||||
def compute_backoff_seconds(*, base_seconds: int, attempt_count: int, max_seconds: int) -> int:
|
||||
base = max(1, int(base_seconds))
|
||||
attempts = max(1, int(attempt_count))
|
||||
maximum = max(base, int(max_seconds))
|
||||
seconds = base * (2 ** max(0, attempts - 1))
|
||||
return min(seconds, maximum)
|
||||
|
||||
|
||||
async def _get_item_for_user_scholar(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
scholar_profile_id: int,
|
||||
) -> IngestionQueueItem | None:
|
||||
result = await db_session.execute(
|
||||
select(IngestionQueueItem).where(
|
||||
IngestionQueueItem.user_id == user_id,
|
||||
IngestionQueueItem.scholar_profile_id == scholar_profile_id,
|
||||
)
|
||||
)
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
|
||||
def _build_queue_item(
|
||||
*,
|
||||
now: datetime,
|
||||
user_id: int,
|
||||
scholar_profile_id: int,
|
||||
normalized_cstart: int,
|
||||
reason: str,
|
||||
run_id: int | None,
|
||||
next_attempt_dt: datetime,
|
||||
) -> IngestionQueueItem:
|
||||
return IngestionQueueItem(
|
||||
user_id=user_id,
|
||||
scholar_profile_id=scholar_profile_id,
|
||||
resume_cstart=normalized_cstart,
|
||||
reason=reason,
|
||||
status=QueueItemStatus.QUEUED.value,
|
||||
attempt_count=0,
|
||||
next_attempt_dt=next_attempt_dt,
|
||||
last_run_id=run_id,
|
||||
last_error=None,
|
||||
dropped_reason=None,
|
||||
dropped_at=None,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
|
||||
|
||||
def _update_existing_queue_item(
|
||||
*,
|
||||
item: IngestionQueueItem,
|
||||
now: datetime,
|
||||
normalized_cstart: int,
|
||||
reason: str,
|
||||
run_id: int | None,
|
||||
next_attempt_dt: datetime,
|
||||
) -> None:
|
||||
item.resume_cstart = normalized_cstart
|
||||
item.reason = reason
|
||||
if item.status == QueueItemStatus.DROPPED.value:
|
||||
item.attempt_count = 0
|
||||
item.status = QueueItemStatus.QUEUED.value
|
||||
item.next_attempt_dt = next_attempt_dt
|
||||
item.last_run_id = run_id
|
||||
item.last_error = None
|
||||
item.dropped_reason = None
|
||||
item.dropped_at = None
|
||||
item.updated_at = now
|
||||
|
||||
|
||||
async def upsert_job(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
scholar_profile_id: int,
|
||||
resume_cstart: int,
|
||||
reason: str,
|
||||
run_id: int | None,
|
||||
delay_seconds: int,
|
||||
) -> IngestionQueueItem:
|
||||
now = datetime.now(timezone.utc)
|
||||
next_attempt_dt = now + timedelta(seconds=max(0, int(delay_seconds)))
|
||||
item = await _get_item_for_user_scholar(
|
||||
db_session,
|
||||
user_id=user_id,
|
||||
scholar_profile_id=scholar_profile_id,
|
||||
)
|
||||
normalized_cstart = normalize_cstart(resume_cstart)
|
||||
if item is None:
|
||||
item = _build_queue_item(
|
||||
now=now,
|
||||
user_id=user_id,
|
||||
scholar_profile_id=scholar_profile_id,
|
||||
normalized_cstart=normalized_cstart,
|
||||
reason=reason,
|
||||
run_id=run_id,
|
||||
next_attempt_dt=next_attempt_dt,
|
||||
)
|
||||
db_session.add(item)
|
||||
return item
|
||||
|
||||
_update_existing_queue_item(
|
||||
item=item,
|
||||
now=now,
|
||||
normalized_cstart=normalized_cstart,
|
||||
reason=reason,
|
||||
run_id=run_id,
|
||||
next_attempt_dt=next_attempt_dt,
|
||||
)
|
||||
return item
|
||||
|
||||
|
||||
async def clear_job_for_scholar(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
scholar_profile_id: int,
|
||||
) -> bool:
|
||||
result = await db_session.execute(
|
||||
select(IngestionQueueItem).where(
|
||||
IngestionQueueItem.user_id == user_id,
|
||||
IngestionQueueItem.scholar_profile_id == scholar_profile_id,
|
||||
)
|
||||
)
|
||||
item = result.scalar_one_or_none()
|
||||
if item is None:
|
||||
return False
|
||||
await db_session.delete(item)
|
||||
return True
|
||||
|
||||
|
||||
async def delete_job_by_id(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
job_id: int,
|
||||
) -> bool:
|
||||
result = await db_session.execute(
|
||||
select(IngestionQueueItem).where(IngestionQueueItem.id == job_id)
|
||||
)
|
||||
item = result.scalar_one_or_none()
|
||||
if item is None:
|
||||
return False
|
||||
await db_session.delete(item)
|
||||
return True
|
||||
|
||||
|
||||
async def list_due_jobs(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
now: datetime,
|
||||
limit: int,
|
||||
) -> list[ContinuationQueueJob]:
|
||||
result = await db_session.execute(
|
||||
select(IngestionQueueItem)
|
||||
.where(
|
||||
IngestionQueueItem.next_attempt_dt <= now,
|
||||
IngestionQueueItem.status.in_(ACTIVE_QUEUE_STATUSES),
|
||||
)
|
||||
.order_by(
|
||||
IngestionQueueItem.next_attempt_dt.asc(),
|
||||
IngestionQueueItem.id.asc(),
|
||||
)
|
||||
.limit(limit)
|
||||
)
|
||||
rows = list(result.scalars().all())
|
||||
jobs: list[ContinuationQueueJob] = []
|
||||
for row in rows:
|
||||
jobs.append(
|
||||
ContinuationQueueJob(
|
||||
id=int(row.id),
|
||||
user_id=int(row.user_id),
|
||||
scholar_profile_id=int(row.scholar_profile_id),
|
||||
resume_cstart=normalize_cstart(row.resume_cstart),
|
||||
reason=row.reason,
|
||||
status=row.status,
|
||||
attempt_count=int(row.attempt_count),
|
||||
next_attempt_dt=row.next_attempt_dt,
|
||||
)
|
||||
)
|
||||
return jobs
|
||||
|
||||
|
||||
async def increment_attempt_count(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
job_id: int,
|
||||
) -> IngestionQueueItem | None:
|
||||
now = datetime.now(timezone.utc)
|
||||
result = await db_session.execute(
|
||||
select(IngestionQueueItem).where(IngestionQueueItem.id == job_id)
|
||||
)
|
||||
item = result.scalar_one_or_none()
|
||||
if item is None:
|
||||
return None
|
||||
item.attempt_count = int(item.attempt_count or 0) + 1
|
||||
item.updated_at = now
|
||||
return item
|
||||
|
||||
|
||||
async def reset_attempt_count(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
job_id: int,
|
||||
) -> IngestionQueueItem | None:
|
||||
now = datetime.now(timezone.utc)
|
||||
result = await db_session.execute(
|
||||
select(IngestionQueueItem).where(IngestionQueueItem.id == job_id)
|
||||
)
|
||||
item = result.scalar_one_or_none()
|
||||
if item is None:
|
||||
return None
|
||||
item.attempt_count = 0
|
||||
item.updated_at = now
|
||||
return item
|
||||
|
||||
|
||||
async def reschedule_job(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
job_id: int,
|
||||
delay_seconds: int,
|
||||
reason: str,
|
||||
error: str | None = None,
|
||||
) -> IngestionQueueItem | None:
|
||||
now = datetime.now(timezone.utc)
|
||||
result = await db_session.execute(
|
||||
select(IngestionQueueItem).where(IngestionQueueItem.id == job_id)
|
||||
)
|
||||
item = result.scalar_one_or_none()
|
||||
if item is None:
|
||||
return None
|
||||
item.next_attempt_dt = now + timedelta(seconds=max(1, int(delay_seconds)))
|
||||
item.status = QueueItemStatus.QUEUED.value
|
||||
item.reason = reason
|
||||
item.last_error = error
|
||||
item.dropped_reason = None
|
||||
item.dropped_at = None
|
||||
item.updated_at = now
|
||||
return item
|
||||
|
||||
|
||||
async def mark_retrying(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
job_id: int,
|
||||
reason: str | None = None,
|
||||
) -> IngestionQueueItem | None:
|
||||
now = datetime.now(timezone.utc)
|
||||
result = await db_session.execute(
|
||||
select(IngestionQueueItem).where(IngestionQueueItem.id == job_id)
|
||||
)
|
||||
item = result.scalar_one_or_none()
|
||||
if item is None:
|
||||
return None
|
||||
if item.status == QueueItemStatus.DROPPED.value:
|
||||
return item
|
||||
item.status = QueueItemStatus.RETRYING.value
|
||||
if reason:
|
||||
item.reason = reason
|
||||
item.updated_at = now
|
||||
return item
|
||||
|
||||
|
||||
async def mark_dropped(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
job_id: int,
|
||||
reason: str,
|
||||
error: str | None = None,
|
||||
) -> IngestionQueueItem | None:
|
||||
now = datetime.now(timezone.utc)
|
||||
result = await db_session.execute(
|
||||
select(IngestionQueueItem).where(IngestionQueueItem.id == job_id)
|
||||
)
|
||||
item = result.scalar_one_or_none()
|
||||
if item is None:
|
||||
return None
|
||||
item.status = QueueItemStatus.DROPPED.value
|
||||
item.reason = "dropped"
|
||||
item.dropped_reason = reason
|
||||
item.dropped_at = now
|
||||
if error is not None:
|
||||
item.last_error = error
|
||||
item.updated_at = now
|
||||
return item
|
||||
|
||||
|
||||
async def mark_queued_now(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
job_id: int,
|
||||
reason: str,
|
||||
reset_attempt_count: bool = False,
|
||||
) -> IngestionQueueItem | None:
|
||||
now = datetime.now(timezone.utc)
|
||||
result = await db_session.execute(
|
||||
select(IngestionQueueItem).where(IngestionQueueItem.id == job_id)
|
||||
)
|
||||
item = result.scalar_one_or_none()
|
||||
if item is None:
|
||||
return None
|
||||
item.status = QueueItemStatus.QUEUED.value
|
||||
item.reason = reason
|
||||
item.next_attempt_dt = now
|
||||
if reset_attempt_count:
|
||||
item.attempt_count = 0
|
||||
item.last_error = None
|
||||
item.dropped_reason = None
|
||||
item.dropped_at = None
|
||||
item.updated_at = now
|
||||
return item
|
||||
282
app/services/domains/ingestion/safety.py
Normal file
282
app/services/domains/ingestion/safety.py
Normal file
|
|
@ -0,0 +1,282 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any
|
||||
|
||||
from app.db.models import UserSetting
|
||||
|
||||
COOLDOWN_REASON_BLOCKED_FAILURE_THRESHOLD = "blocked_failure_threshold_exceeded"
|
||||
COOLDOWN_REASON_NETWORK_FAILURE_THRESHOLD = "network_failure_threshold_exceeded"
|
||||
|
||||
_COUNTER_CONSECUTIVE_BLOCKED_RUNS = "consecutive_blocked_runs"
|
||||
_COUNTER_CONSECUTIVE_NETWORK_RUNS = "consecutive_network_runs"
|
||||
_COUNTER_COOLDOWN_ENTRY_COUNT = "cooldown_entry_count"
|
||||
_COUNTER_BLOCKED_START_COUNT = "blocked_start_count"
|
||||
_COUNTER_LAST_BLOCKED_FAILURE_COUNT = "last_blocked_failure_count"
|
||||
_COUNTER_LAST_NETWORK_FAILURE_COUNT = "last_network_failure_count"
|
||||
_COUNTER_LAST_EVALUATED_RUN_ID = "last_evaluated_run_id"
|
||||
|
||||
|
||||
def _utcnow() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def _safe_int(value: Any, default: int = 0) -> int:
|
||||
try:
|
||||
return int(value)
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
|
||||
|
||||
def _safe_optional_int(value: Any) -> int | None:
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
return int(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _normalize_datetime(value: datetime | None) -> datetime | None:
|
||||
if value is None:
|
||||
return None
|
||||
if value.tzinfo is None:
|
||||
return value.replace(tzinfo=timezone.utc)
|
||||
return value.astimezone(timezone.utc)
|
||||
|
||||
|
||||
def _state_dict(settings: UserSetting) -> dict[str, Any]:
|
||||
state = settings.scrape_safety_state
|
||||
if isinstance(state, dict):
|
||||
return state
|
||||
return {}
|
||||
|
||||
|
||||
def _counters_from_state(settings: UserSetting) -> dict[str, Any]:
|
||||
state = _state_dict(settings)
|
||||
return {
|
||||
_COUNTER_CONSECUTIVE_BLOCKED_RUNS: max(
|
||||
0,
|
||||
_safe_int(state.get(_COUNTER_CONSECUTIVE_BLOCKED_RUNS), 0),
|
||||
),
|
||||
_COUNTER_CONSECUTIVE_NETWORK_RUNS: max(
|
||||
0,
|
||||
_safe_int(state.get(_COUNTER_CONSECUTIVE_NETWORK_RUNS), 0),
|
||||
),
|
||||
_COUNTER_COOLDOWN_ENTRY_COUNT: max(
|
||||
0,
|
||||
_safe_int(state.get(_COUNTER_COOLDOWN_ENTRY_COUNT), 0),
|
||||
),
|
||||
_COUNTER_BLOCKED_START_COUNT: max(
|
||||
0,
|
||||
_safe_int(state.get(_COUNTER_BLOCKED_START_COUNT), 0),
|
||||
),
|
||||
_COUNTER_LAST_BLOCKED_FAILURE_COUNT: max(
|
||||
0,
|
||||
_safe_int(state.get(_COUNTER_LAST_BLOCKED_FAILURE_COUNT), 0),
|
||||
),
|
||||
_COUNTER_LAST_NETWORK_FAILURE_COUNT: max(
|
||||
0,
|
||||
_safe_int(state.get(_COUNTER_LAST_NETWORK_FAILURE_COUNT), 0),
|
||||
),
|
||||
_COUNTER_LAST_EVALUATED_RUN_ID: _safe_optional_int(
|
||||
state.get(_COUNTER_LAST_EVALUATED_RUN_ID),
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def _cooldown_reason_label(reason: str | None) -> str | None:
|
||||
if reason == COOLDOWN_REASON_BLOCKED_FAILURE_THRESHOLD:
|
||||
return "Blocked responses exceeded safety threshold"
|
||||
if reason == COOLDOWN_REASON_NETWORK_FAILURE_THRESHOLD:
|
||||
return "Network failures exceeded safety threshold"
|
||||
return None
|
||||
|
||||
|
||||
def _recommended_action(reason: str | None) -> str | None:
|
||||
if reason == COOLDOWN_REASON_BLOCKED_FAILURE_THRESHOLD:
|
||||
return (
|
||||
"Google Scholar appears to be blocking requests. Wait for cooldown to expire, "
|
||||
"increase request delay, and avoid repeated manual retries."
|
||||
)
|
||||
if reason == COOLDOWN_REASON_NETWORK_FAILURE_THRESHOLD:
|
||||
return (
|
||||
"Network failures crossed the threshold. Verify connectivity and retry after cooldown."
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def is_cooldown_active(
|
||||
settings: UserSetting,
|
||||
*,
|
||||
now_utc: datetime | None = None,
|
||||
) -> bool:
|
||||
now = now_utc or _utcnow()
|
||||
cooldown_until = _normalize_datetime(settings.scrape_cooldown_until)
|
||||
if cooldown_until is None:
|
||||
return False
|
||||
return cooldown_until > now
|
||||
|
||||
|
||||
def clear_expired_cooldown(
|
||||
settings: UserSetting,
|
||||
*,
|
||||
now_utc: datetime | None = None,
|
||||
) -> bool:
|
||||
now = now_utc or _utcnow()
|
||||
cooldown_until = _normalize_datetime(settings.scrape_cooldown_until)
|
||||
if cooldown_until is None:
|
||||
return False
|
||||
if cooldown_until > now:
|
||||
return False
|
||||
settings.scrape_cooldown_until = None
|
||||
settings.scrape_cooldown_reason = None
|
||||
return True
|
||||
|
||||
|
||||
def register_cooldown_blocked_start(
|
||||
settings: UserSetting,
|
||||
*,
|
||||
now_utc: datetime | None = None,
|
||||
) -> dict[str, Any]:
|
||||
now = now_utc or _utcnow()
|
||||
counters = _counters_from_state(settings)
|
||||
counters[_COUNTER_BLOCKED_START_COUNT] = int(counters[_COUNTER_BLOCKED_START_COUNT]) + 1
|
||||
settings.scrape_safety_state = counters
|
||||
return get_safety_state_payload(settings, now_utc=now)
|
||||
|
||||
|
||||
def _update_run_counters(
|
||||
*,
|
||||
counters: dict[str, Any],
|
||||
run_id: int,
|
||||
blocked_failure_count: int,
|
||||
network_failure_count: int,
|
||||
) -> tuple[int, int]:
|
||||
bounded_blocked_failures = max(0, int(blocked_failure_count))
|
||||
bounded_network_failures = max(0, int(network_failure_count))
|
||||
counters[_COUNTER_LAST_BLOCKED_FAILURE_COUNT] = bounded_blocked_failures
|
||||
counters[_COUNTER_LAST_NETWORK_FAILURE_COUNT] = bounded_network_failures
|
||||
counters[_COUNTER_LAST_EVALUATED_RUN_ID] = int(run_id)
|
||||
counters[_COUNTER_CONSECUTIVE_BLOCKED_RUNS] = (
|
||||
int(counters[_COUNTER_CONSECUTIVE_BLOCKED_RUNS]) + 1
|
||||
if bounded_blocked_failures > 0
|
||||
else 0
|
||||
)
|
||||
counters[_COUNTER_CONSECUTIVE_NETWORK_RUNS] = (
|
||||
int(counters[_COUNTER_CONSECUTIVE_NETWORK_RUNS]) + 1
|
||||
if bounded_network_failures > 0
|
||||
else 0
|
||||
)
|
||||
return bounded_blocked_failures, bounded_network_failures
|
||||
|
||||
|
||||
def _resolve_cooldown_trigger(
|
||||
*,
|
||||
blocked_failures: int,
|
||||
network_failures: int,
|
||||
blocked_failure_threshold: int,
|
||||
network_failure_threshold: int,
|
||||
blocked_cooldown_seconds: int,
|
||||
network_cooldown_seconds: int,
|
||||
) -> tuple[str | None, int]:
|
||||
if blocked_failures >= max(1, int(blocked_failure_threshold)):
|
||||
return COOLDOWN_REASON_BLOCKED_FAILURE_THRESHOLD, max(60, int(blocked_cooldown_seconds))
|
||||
if network_failures >= max(1, int(network_failure_threshold)):
|
||||
return COOLDOWN_REASON_NETWORK_FAILURE_THRESHOLD, max(60, int(network_cooldown_seconds))
|
||||
return None, 0
|
||||
|
||||
|
||||
def _apply_cooldown_decision(
|
||||
*,
|
||||
settings: UserSetting,
|
||||
counters: dict[str, Any],
|
||||
now: datetime,
|
||||
reason: str | None,
|
||||
cooldown_seconds: int,
|
||||
) -> None:
|
||||
if reason is None:
|
||||
clear_expired_cooldown(settings, now_utc=now)
|
||||
return
|
||||
settings.scrape_cooldown_reason = reason
|
||||
settings.scrape_cooldown_until = now + timedelta(seconds=max(60, int(cooldown_seconds)))
|
||||
counters[_COUNTER_COOLDOWN_ENTRY_COUNT] = int(counters[_COUNTER_COOLDOWN_ENTRY_COUNT]) + 1
|
||||
|
||||
|
||||
def apply_run_safety_outcome(
|
||||
settings: UserSetting,
|
||||
*,
|
||||
run_id: int,
|
||||
blocked_failure_count: int,
|
||||
network_failure_count: int,
|
||||
blocked_failure_threshold: int,
|
||||
network_failure_threshold: int,
|
||||
blocked_cooldown_seconds: int,
|
||||
network_cooldown_seconds: int,
|
||||
now_utc: datetime | None = None,
|
||||
) -> tuple[dict[str, Any], str | None]:
|
||||
now = now_utc or _utcnow()
|
||||
counters = _counters_from_state(settings)
|
||||
blocked_failures, network_failures = _update_run_counters(
|
||||
counters=counters,
|
||||
run_id=run_id,
|
||||
blocked_failure_count=blocked_failure_count,
|
||||
network_failure_count=network_failure_count,
|
||||
)
|
||||
reason, cooldown_seconds = _resolve_cooldown_trigger(
|
||||
blocked_failures=blocked_failures,
|
||||
network_failures=network_failures,
|
||||
blocked_failure_threshold=blocked_failure_threshold,
|
||||
network_failure_threshold=network_failure_threshold,
|
||||
blocked_cooldown_seconds=blocked_cooldown_seconds,
|
||||
network_cooldown_seconds=network_cooldown_seconds,
|
||||
)
|
||||
_apply_cooldown_decision(
|
||||
settings=settings,
|
||||
counters=counters,
|
||||
now=now,
|
||||
reason=reason,
|
||||
cooldown_seconds=cooldown_seconds,
|
||||
)
|
||||
settings.scrape_safety_state = counters
|
||||
return get_safety_state_payload(settings, now_utc=now), reason
|
||||
|
||||
|
||||
def get_safety_state_payload(
|
||||
settings: UserSetting,
|
||||
*,
|
||||
now_utc: datetime | None = None,
|
||||
) -> dict[str, Any]:
|
||||
now = now_utc or _utcnow()
|
||||
cooldown_until = _normalize_datetime(settings.scrape_cooldown_until)
|
||||
cooldown_active = bool(cooldown_until is not None and cooldown_until > now)
|
||||
cooldown_remaining_seconds = 0
|
||||
if cooldown_active and cooldown_until is not None:
|
||||
cooldown_remaining_seconds = max(0, int((cooldown_until - now).total_seconds()))
|
||||
|
||||
reason = settings.scrape_cooldown_reason if cooldown_active else None
|
||||
|
||||
return {
|
||||
"cooldown_active": cooldown_active,
|
||||
"cooldown_reason": reason,
|
||||
"cooldown_reason_label": _cooldown_reason_label(reason),
|
||||
"cooldown_until": cooldown_until,
|
||||
"cooldown_remaining_seconds": cooldown_remaining_seconds,
|
||||
"recommended_action": _recommended_action(reason),
|
||||
"counters": _counters_from_state(settings),
|
||||
}
|
||||
|
||||
|
||||
def get_safety_event_context(
|
||||
settings: UserSetting,
|
||||
*,
|
||||
now_utc: datetime | None = None,
|
||||
) -> dict[str, Any]:
|
||||
payload = get_safety_state_payload(settings, now_utc=now_utc)
|
||||
return {
|
||||
"cooldown_active": bool(payload.get("cooldown_active")),
|
||||
"cooldown_reason": payload.get("cooldown_reason"),
|
||||
"cooldown_until": payload.get("cooldown_until"),
|
||||
"cooldown_remaining_seconds": int(payload.get("cooldown_remaining_seconds") or 0),
|
||||
"safety_counters": payload.get("counters", {}),
|
||||
}
|
||||
582
app/services/domains/ingestion/scheduler.py
Normal file
582
app/services/domains/ingestion/scheduler.py
Normal file
|
|
@ -0,0 +1,582 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta, timezone
|
||||
import logging
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.models import (
|
||||
CrawlRun,
|
||||
QueueItemStatus,
|
||||
RunTriggerType,
|
||||
ScholarProfile,
|
||||
User,
|
||||
UserSetting,
|
||||
)
|
||||
from app.db.session import get_session_factory
|
||||
from app.services.domains.ingestion import queue as queue_service
|
||||
from app.services.domains.ingestion.application import (
|
||||
RunAlreadyInProgressError,
|
||||
RunBlockedBySafetyPolicyError,
|
||||
ScholarIngestionService,
|
||||
)
|
||||
from app.services.domains.scholar.source import LiveScholarSource
|
||||
from app.settings import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _AutoRunCandidate:
|
||||
user_id: int
|
||||
run_interval_minutes: int
|
||||
request_delay_seconds: int
|
||||
cooldown_until: datetime | None
|
||||
cooldown_reason: str | None
|
||||
|
||||
|
||||
class SchedulerService:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
enabled: bool,
|
||||
tick_seconds: int,
|
||||
network_error_retries: int,
|
||||
retry_backoff_seconds: float,
|
||||
max_pages_per_scholar: int,
|
||||
page_size: int,
|
||||
continuation_queue_enabled: bool,
|
||||
continuation_base_delay_seconds: int,
|
||||
continuation_max_delay_seconds: int,
|
||||
continuation_max_attempts: int,
|
||||
queue_batch_size: int,
|
||||
) -> None:
|
||||
self._enabled = enabled
|
||||
self._tick_seconds = max(5, int(tick_seconds))
|
||||
self._network_error_retries = max(0, int(network_error_retries))
|
||||
self._retry_backoff_seconds = max(0.0, float(retry_backoff_seconds))
|
||||
self._max_pages_per_scholar = max(1, int(max_pages_per_scholar))
|
||||
self._page_size = max(1, int(page_size))
|
||||
self._continuation_queue_enabled = bool(continuation_queue_enabled)
|
||||
self._continuation_base_delay_seconds = max(1, int(continuation_base_delay_seconds))
|
||||
self._continuation_max_delay_seconds = max(
|
||||
self._continuation_base_delay_seconds,
|
||||
int(continuation_max_delay_seconds),
|
||||
)
|
||||
self._continuation_max_attempts = max(1, int(continuation_max_attempts))
|
||||
self._queue_batch_size = max(1, int(queue_batch_size))
|
||||
self._task: asyncio.Task[None] | None = None
|
||||
self._source = LiveScholarSource()
|
||||
|
||||
async def start(self) -> None:
|
||||
if not self._enabled:
|
||||
logger.info(
|
||||
"scheduler.disabled",
|
||||
extra={
|
||||
"event": "scheduler.disabled",
|
||||
},
|
||||
)
|
||||
return
|
||||
if self._task is not None:
|
||||
return
|
||||
self._task = asyncio.create_task(self._run_loop(), name="scholarr-scheduler")
|
||||
logger.info(
|
||||
"scheduler.started",
|
||||
extra={
|
||||
"event": "scheduler.started",
|
||||
"tick_seconds": self._tick_seconds,
|
||||
"network_error_retries": self._network_error_retries,
|
||||
"retry_backoff_seconds": self._retry_backoff_seconds,
|
||||
"max_pages_per_scholar": self._max_pages_per_scholar,
|
||||
"page_size": self._page_size,
|
||||
"continuation_queue_enabled": self._continuation_queue_enabled,
|
||||
"continuation_base_delay_seconds": self._continuation_base_delay_seconds,
|
||||
"continuation_max_delay_seconds": self._continuation_max_delay_seconds,
|
||||
"continuation_max_attempts": self._continuation_max_attempts,
|
||||
"queue_batch_size": self._queue_batch_size,
|
||||
},
|
||||
)
|
||||
|
||||
async def stop(self) -> None:
|
||||
if self._task is None:
|
||||
return
|
||||
self._task.cancel()
|
||||
try:
|
||||
await self._task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
finally:
|
||||
self._task = None
|
||||
logger.info("scheduler.stopped", extra={"event": "scheduler.stopped"})
|
||||
|
||||
async def _run_loop(self) -> None:
|
||||
while True:
|
||||
try:
|
||||
await self._tick_once()
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"scheduler.tick_failed",
|
||||
extra={
|
||||
"event": "scheduler.tick_failed",
|
||||
},
|
||||
)
|
||||
await asyncio.sleep(float(self._tick_seconds))
|
||||
|
||||
async def _tick_once(self) -> None:
|
||||
if self._continuation_queue_enabled:
|
||||
await self._drain_continuation_queue()
|
||||
candidates = await self._load_candidates()
|
||||
if not candidates:
|
||||
return
|
||||
now = datetime.now(timezone.utc)
|
||||
for candidate in candidates:
|
||||
if not await self._is_due(candidate, now=now):
|
||||
continue
|
||||
await self._run_candidate(candidate)
|
||||
|
||||
async def _load_candidate_rows(self) -> list[tuple]:
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as session:
|
||||
result = await session.execute(
|
||||
select(
|
||||
UserSetting.user_id,
|
||||
UserSetting.run_interval_minutes,
|
||||
UserSetting.request_delay_seconds,
|
||||
UserSetting.scrape_cooldown_until,
|
||||
UserSetting.scrape_cooldown_reason,
|
||||
)
|
||||
.join(User, User.id == UserSetting.user_id)
|
||||
.where(User.is_active.is_(True), UserSetting.auto_run_enabled.is_(True))
|
||||
.order_by(UserSetting.user_id.asc())
|
||||
)
|
||||
return result.all()
|
||||
|
||||
@staticmethod
|
||||
def _candidate_from_row(row: tuple, *, now_utc: datetime) -> _AutoRunCandidate | None:
|
||||
user_id, run_interval_minutes, request_delay_seconds, cooldown_until, cooldown_reason = row
|
||||
if cooldown_until is not None and cooldown_until.tzinfo is None:
|
||||
cooldown_until = cooldown_until.replace(tzinfo=timezone.utc)
|
||||
if cooldown_until is not None and cooldown_until > now_utc:
|
||||
logger.info(
|
||||
"scheduler.run_skipped_safety_cooldown_precheck",
|
||||
extra={
|
||||
"event": "scheduler.run_skipped_safety_cooldown_precheck",
|
||||
"user_id": int(user_id),
|
||||
"reason": cooldown_reason,
|
||||
"cooldown_until": cooldown_until,
|
||||
"cooldown_remaining_seconds": int((cooldown_until - now_utc).total_seconds()),
|
||||
"metric_name": "scheduler_run_skipped_safety_cooldown_total",
|
||||
"metric_value": 1,
|
||||
},
|
||||
)
|
||||
return None
|
||||
return _AutoRunCandidate(
|
||||
user_id=int(user_id),
|
||||
run_interval_minutes=int(run_interval_minutes),
|
||||
request_delay_seconds=int(request_delay_seconds),
|
||||
cooldown_until=cooldown_until,
|
||||
cooldown_reason=(str(cooldown_reason).strip() if cooldown_reason else None),
|
||||
)
|
||||
|
||||
async def _load_candidates(self) -> list[_AutoRunCandidate]:
|
||||
if not settings.ingestion_automation_allowed:
|
||||
return []
|
||||
rows = await self._load_candidate_rows()
|
||||
now_utc = datetime.now(timezone.utc)
|
||||
candidates: list[_AutoRunCandidate] = []
|
||||
for row in rows:
|
||||
candidate = self._candidate_from_row(row, now_utc=now_utc)
|
||||
if candidate is not None:
|
||||
candidates.append(candidate)
|
||||
return candidates
|
||||
|
||||
async def _is_due(self, candidate: _AutoRunCandidate, *, now: datetime) -> bool:
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as session:
|
||||
result = await session.execute(
|
||||
select(CrawlRun.start_dt)
|
||||
.where(
|
||||
CrawlRun.user_id == candidate.user_id,
|
||||
)
|
||||
.order_by(CrawlRun.start_dt.desc(), CrawlRun.id.desc())
|
||||
.limit(1)
|
||||
)
|
||||
last_run = result.scalar_one_or_none()
|
||||
|
||||
if last_run is None:
|
||||
return True
|
||||
|
||||
next_due_dt = last_run + timedelta(
|
||||
minutes=candidate.run_interval_minutes
|
||||
)
|
||||
return now >= next_due_dt
|
||||
|
||||
async def _run_candidate_ingestion(
|
||||
self,
|
||||
*,
|
||||
candidate: _AutoRunCandidate,
|
||||
):
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as session:
|
||||
ingestion = ScholarIngestionService(source=self._source)
|
||||
try:
|
||||
return await ingestion.run_for_user(
|
||||
session,
|
||||
user_id=candidate.user_id,
|
||||
trigger_type=RunTriggerType.SCHEDULED,
|
||||
request_delay_seconds=candidate.request_delay_seconds,
|
||||
network_error_retries=self._network_error_retries,
|
||||
retry_backoff_seconds=self._retry_backoff_seconds,
|
||||
max_pages_per_scholar=self._max_pages_per_scholar,
|
||||
page_size=self._page_size,
|
||||
auto_queue_continuations=self._continuation_queue_enabled,
|
||||
queue_delay_seconds=self._continuation_base_delay_seconds,
|
||||
alert_blocked_failure_threshold=settings.ingestion_alert_blocked_failure_threshold,
|
||||
alert_network_failure_threshold=settings.ingestion_alert_network_failure_threshold,
|
||||
alert_retry_scheduled_threshold=settings.ingestion_alert_retry_scheduled_threshold,
|
||||
)
|
||||
except RunAlreadyInProgressError:
|
||||
await session.rollback()
|
||||
logger.info("scheduler.run_skipped_locked", extra={"event": "scheduler.run_skipped_locked", "user_id": candidate.user_id})
|
||||
return None
|
||||
except RunBlockedBySafetyPolicyError as exc:
|
||||
await session.rollback()
|
||||
logger.info(
|
||||
"scheduler.run_skipped_safety_cooldown",
|
||||
extra={
|
||||
"event": "scheduler.run_skipped_safety_cooldown",
|
||||
"user_id": candidate.user_id,
|
||||
"reason": exc.safety_state.get("cooldown_reason"),
|
||||
"cooldown_until": exc.safety_state.get("cooldown_until"),
|
||||
"cooldown_remaining_seconds": exc.safety_state.get("cooldown_remaining_seconds"),
|
||||
"metric_name": "scheduler_run_skipped_safety_cooldown_total",
|
||||
"metric_value": 1,
|
||||
},
|
||||
)
|
||||
return None
|
||||
except Exception:
|
||||
await session.rollback()
|
||||
logger.exception("scheduler.run_failed", extra={"event": "scheduler.run_failed", "user_id": candidate.user_id})
|
||||
return None
|
||||
|
||||
async def _run_candidate(self, candidate: _AutoRunCandidate) -> None:
|
||||
run_summary = await self._run_candidate_ingestion(candidate=candidate)
|
||||
if run_summary is None:
|
||||
return
|
||||
logger.info(
|
||||
"scheduler.run_completed",
|
||||
extra={
|
||||
"event": "scheduler.run_completed",
|
||||
"user_id": candidate.user_id,
|
||||
"run_id": run_summary.crawl_run_id,
|
||||
"status": run_summary.status.value,
|
||||
"scholar_count": run_summary.scholar_count,
|
||||
"new_publication_count": run_summary.new_publication_count,
|
||||
},
|
||||
)
|
||||
|
||||
async def _drain_continuation_queue(self) -> None:
|
||||
now = datetime.now(timezone.utc)
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as session:
|
||||
jobs = await queue_service.list_due_jobs(
|
||||
session,
|
||||
now=now,
|
||||
limit=self._queue_batch_size,
|
||||
)
|
||||
for job in jobs:
|
||||
await self._run_queue_job(job)
|
||||
|
||||
async def _drop_queue_job_if_max_attempts(
|
||||
self,
|
||||
job: queue_service.ContinuationQueueJob,
|
||||
) -> bool:
|
||||
if job.attempt_count < self._continuation_max_attempts:
|
||||
return False
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as session:
|
||||
dropped = await queue_service.mark_dropped(
|
||||
session,
|
||||
job_id=job.id,
|
||||
reason="max_attempts_reached",
|
||||
)
|
||||
await session.commit()
|
||||
if dropped is not None:
|
||||
logger.warning(
|
||||
"scheduler.queue_item_dropped_max_attempts",
|
||||
extra={
|
||||
"event": "scheduler.queue_item_dropped_max_attempts",
|
||||
"queue_item_id": job.id,
|
||||
"user_id": job.user_id,
|
||||
"scholar_profile_id": job.scholar_profile_id,
|
||||
"attempt_count": job.attempt_count,
|
||||
"max_attempts": self._continuation_max_attempts,
|
||||
},
|
||||
)
|
||||
return True
|
||||
|
||||
async def _mark_queue_job_retrying(
|
||||
self,
|
||||
job: queue_service.ContinuationQueueJob,
|
||||
) -> bool:
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as session:
|
||||
queue_item = await queue_service.mark_retrying(session, job_id=job.id)
|
||||
await session.commit()
|
||||
if queue_item is None:
|
||||
return False
|
||||
if queue_item.status == QueueItemStatus.DROPPED.value:
|
||||
return False
|
||||
return True
|
||||
|
||||
async def _queue_job_has_available_scholar(
|
||||
self,
|
||||
job: queue_service.ContinuationQueueJob,
|
||||
) -> bool:
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as session:
|
||||
scholar_result = await session.execute(
|
||||
select(ScholarProfile.id).where(
|
||||
ScholarProfile.user_id == job.user_id,
|
||||
ScholarProfile.id == job.scholar_profile_id,
|
||||
ScholarProfile.is_enabled.is_(True),
|
||||
)
|
||||
)
|
||||
scholar_id = scholar_result.scalar_one_or_none()
|
||||
if scholar_id is not None:
|
||||
return True
|
||||
dropped = await queue_service.mark_dropped(
|
||||
session,
|
||||
job_id=job.id,
|
||||
reason="scholar_unavailable",
|
||||
)
|
||||
await session.commit()
|
||||
if dropped is not None:
|
||||
logger.info(
|
||||
"scheduler.queue_item_dropped_scholar_unavailable",
|
||||
extra={
|
||||
"event": "scheduler.queue_item_dropped_scholar_unavailable",
|
||||
"queue_item_id": job.id,
|
||||
"user_id": job.user_id,
|
||||
"scholar_profile_id": job.scholar_profile_id,
|
||||
},
|
||||
)
|
||||
return False
|
||||
|
||||
async def _reschedule_queue_job_lock_active(self, job: queue_service.ContinuationQueueJob) -> None:
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as recovery_session:
|
||||
await queue_service.reschedule_job(
|
||||
recovery_session,
|
||||
job_id=job.id,
|
||||
delay_seconds=max(self._tick_seconds, 15),
|
||||
reason="user_run_lock_active",
|
||||
error="run_already_in_progress",
|
||||
)
|
||||
await recovery_session.commit()
|
||||
logger.info(
|
||||
"scheduler.queue_item_deferred_lock",
|
||||
extra={"event": "scheduler.queue_item_deferred_lock", "queue_item_id": job.id, "user_id": job.user_id},
|
||||
)
|
||||
|
||||
async def _reschedule_queue_job_safety_cooldown(
|
||||
self,
|
||||
job: queue_service.ContinuationQueueJob,
|
||||
exc: RunBlockedBySafetyPolicyError,
|
||||
) -> None:
|
||||
cooldown_remaining_seconds = max(
|
||||
self._tick_seconds,
|
||||
int(exc.safety_state.get("cooldown_remaining_seconds") or 0),
|
||||
)
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as recovery_session:
|
||||
await queue_service.reschedule_job(
|
||||
recovery_session,
|
||||
job_id=job.id,
|
||||
delay_seconds=max(self._tick_seconds, cooldown_remaining_seconds),
|
||||
reason="scrape_safety_cooldown",
|
||||
error=str(exc.message),
|
||||
)
|
||||
await recovery_session.commit()
|
||||
logger.info(
|
||||
"scheduler.queue_item_deferred_safety_cooldown",
|
||||
extra={
|
||||
"event": "scheduler.queue_item_deferred_safety_cooldown",
|
||||
"queue_item_id": job.id,
|
||||
"user_id": job.user_id,
|
||||
"reason": exc.safety_state.get("cooldown_reason"),
|
||||
"cooldown_remaining_seconds": cooldown_remaining_seconds,
|
||||
"metric_name": "scheduler_queue_item_deferred_safety_cooldown_total",
|
||||
"metric_value": 1,
|
||||
},
|
||||
)
|
||||
|
||||
async def _reschedule_queue_job_after_exception(
|
||||
self,
|
||||
job: queue_service.ContinuationQueueJob,
|
||||
*,
|
||||
exc: Exception,
|
||||
) -> None:
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as recovery_session:
|
||||
queue_item = await queue_service.increment_attempt_count(recovery_session, job_id=job.id)
|
||||
if queue_item is None:
|
||||
await recovery_session.commit()
|
||||
return
|
||||
if int(queue_item.attempt_count) >= self._continuation_max_attempts:
|
||||
await queue_service.mark_dropped(
|
||||
recovery_session,
|
||||
job_id=job.id,
|
||||
reason="scheduler_exception_max_attempts",
|
||||
error=str(exc),
|
||||
)
|
||||
await recovery_session.commit()
|
||||
logger.warning(
|
||||
"scheduler.queue_item_dropped_after_exception",
|
||||
extra={"event": "scheduler.queue_item_dropped_after_exception", "queue_item_id": job.id, "user_id": job.user_id, "attempt_count": queue_item.attempt_count},
|
||||
)
|
||||
return
|
||||
delay_seconds = queue_service.compute_backoff_seconds(
|
||||
base_seconds=self._continuation_base_delay_seconds,
|
||||
attempt_count=int(queue_item.attempt_count),
|
||||
max_seconds=self._continuation_max_delay_seconds,
|
||||
)
|
||||
await queue_service.reschedule_job(
|
||||
recovery_session,
|
||||
job_id=job.id,
|
||||
delay_seconds=delay_seconds,
|
||||
reason="scheduler_exception",
|
||||
error=str(exc),
|
||||
)
|
||||
await recovery_session.commit()
|
||||
logger.exception("scheduler.queue_item_run_failed", extra={"event": "scheduler.queue_item_run_failed", "queue_item_id": job.id, "user_id": job.user_id})
|
||||
|
||||
async def _run_ingestion_for_queue_job(
|
||||
self,
|
||||
job: queue_service.ContinuationQueueJob,
|
||||
):
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as session:
|
||||
request_delay_seconds = await self._load_request_delay_for_user(session, user_id=job.user_id)
|
||||
ingestion = ScholarIngestionService(source=self._source)
|
||||
try:
|
||||
return await ingestion.run_for_user(
|
||||
session,
|
||||
user_id=job.user_id,
|
||||
trigger_type=RunTriggerType.SCHEDULED,
|
||||
request_delay_seconds=request_delay_seconds,
|
||||
network_error_retries=self._network_error_retries,
|
||||
retry_backoff_seconds=self._retry_backoff_seconds,
|
||||
max_pages_per_scholar=self._max_pages_per_scholar,
|
||||
page_size=self._page_size,
|
||||
scholar_profile_ids={job.scholar_profile_id},
|
||||
start_cstart_by_scholar_id={job.scholar_profile_id: job.resume_cstart},
|
||||
auto_queue_continuations=self._continuation_queue_enabled,
|
||||
queue_delay_seconds=self._continuation_base_delay_seconds,
|
||||
alert_blocked_failure_threshold=settings.ingestion_alert_blocked_failure_threshold,
|
||||
alert_network_failure_threshold=settings.ingestion_alert_network_failure_threshold,
|
||||
alert_retry_scheduled_threshold=settings.ingestion_alert_retry_scheduled_threshold,
|
||||
)
|
||||
except RunAlreadyInProgressError:
|
||||
await session.rollback()
|
||||
await self._reschedule_queue_job_lock_active(job)
|
||||
except RunBlockedBySafetyPolicyError as exc:
|
||||
await session.rollback()
|
||||
await self._reschedule_queue_job_safety_cooldown(job, exc)
|
||||
except Exception as exc:
|
||||
await session.rollback()
|
||||
await self._reschedule_queue_job_after_exception(job, exc=exc)
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _log_queue_item_resolved(
|
||||
*,
|
||||
event_name: str,
|
||||
job: queue_service.ContinuationQueueJob,
|
||||
run_summary,
|
||||
attempt_count: int | None = None,
|
||||
delay_seconds: int | None = None,
|
||||
) -> None:
|
||||
payload = {
|
||||
"event": event_name,
|
||||
"queue_item_id": job.id,
|
||||
"user_id": job.user_id,
|
||||
"run_id": run_summary.crawl_run_id,
|
||||
"status": run_summary.status.value,
|
||||
}
|
||||
if attempt_count is not None:
|
||||
payload["attempt_count"] = attempt_count
|
||||
if delay_seconds is not None:
|
||||
payload["delay_seconds"] = delay_seconds
|
||||
logger.info(event_name, extra=payload)
|
||||
|
||||
async def _finalize_queue_job_after_run(self, job: queue_service.ContinuationQueueJob, run_summary) -> None:
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as session:
|
||||
if int(run_summary.failed_count) <= 0:
|
||||
queue_item = await queue_service.reset_attempt_count(session, job_id=job.id)
|
||||
await session.commit()
|
||||
if queue_item is None:
|
||||
self._log_queue_item_resolved(event_name="scheduler.queue_item_resolved", job=job, run_summary=run_summary)
|
||||
return
|
||||
self._log_queue_item_resolved(
|
||||
event_name="scheduler.queue_item_progressed",
|
||||
job=job,
|
||||
run_summary=run_summary,
|
||||
attempt_count=int(queue_item.attempt_count),
|
||||
)
|
||||
return
|
||||
queue_item = await queue_service.increment_attempt_count(session, job_id=job.id)
|
||||
if queue_item is None:
|
||||
await session.commit()
|
||||
self._log_queue_item_resolved(event_name="scheduler.queue_item_resolved", job=job, run_summary=run_summary)
|
||||
return
|
||||
if int(queue_item.attempt_count) >= self._continuation_max_attempts:
|
||||
await queue_service.mark_dropped(session, job_id=job.id, reason="max_attempts_after_run")
|
||||
await session.commit()
|
||||
logger.warning(
|
||||
"scheduler.queue_item_dropped_max_attempts_after_run",
|
||||
extra={"event": "scheduler.queue_item_dropped_max_attempts_after_run", "queue_item_id": job.id, "user_id": job.user_id, "attempt_count": queue_item.attempt_count, "run_id": run_summary.crawl_run_id, "status": run_summary.status.value},
|
||||
)
|
||||
return
|
||||
delay_seconds = queue_service.compute_backoff_seconds(base_seconds=self._continuation_base_delay_seconds, attempt_count=int(queue_item.attempt_count), max_seconds=self._continuation_max_delay_seconds)
|
||||
await queue_service.reschedule_job(session, job_id=job.id, delay_seconds=delay_seconds, reason=queue_item.reason, error=queue_item.last_error)
|
||||
await session.commit()
|
||||
self._log_queue_item_resolved(
|
||||
event_name="scheduler.queue_item_rescheduled_failed",
|
||||
job=job,
|
||||
run_summary=run_summary,
|
||||
attempt_count=int(queue_item.attempt_count),
|
||||
delay_seconds=delay_seconds,
|
||||
)
|
||||
|
||||
async def _run_queue_job(self, job: queue_service.ContinuationQueueJob) -> None:
|
||||
if await self._drop_queue_job_if_max_attempts(job):
|
||||
return
|
||||
if not await self._mark_queue_job_retrying(job):
|
||||
return
|
||||
if not await self._queue_job_has_available_scholar(job):
|
||||
return
|
||||
run_summary = await self._run_ingestion_for_queue_job(job)
|
||||
if run_summary is None:
|
||||
return
|
||||
await self._finalize_queue_job_after_run(job, run_summary)
|
||||
|
||||
async def _load_request_delay_for_user(
|
||||
self,
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
) -> int:
|
||||
result = await db_session.execute(
|
||||
select(UserSetting.request_delay_seconds).where(UserSetting.user_id == user_id)
|
||||
)
|
||||
delay = result.scalar_one_or_none()
|
||||
if delay is None:
|
||||
return 10
|
||||
return max(1, int(delay))
|
||||
108
app/services/domains/ingestion/types.py
Normal file
108
app/services/domains/ingestion/types.py
Normal file
|
|
@ -0,0 +1,108 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from app.db.models import RunStatus
|
||||
from app.services.domains.scholar.parser import ParsedProfilePage, PublicationCandidate
|
||||
from app.services.domains.scholar.source import FetchResult
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RunExecutionSummary:
|
||||
crawl_run_id: int
|
||||
status: RunStatus
|
||||
scholar_count: int
|
||||
succeeded_count: int
|
||||
failed_count: int
|
||||
partial_count: int
|
||||
new_publication_count: int
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PagedParseResult:
|
||||
fetch_result: FetchResult
|
||||
parsed_page: ParsedProfilePage
|
||||
first_page_fetch_result: FetchResult
|
||||
first_page_parsed_page: ParsedProfilePage
|
||||
first_page_fingerprint_sha256: str | None
|
||||
publications: list[PublicationCandidate]
|
||||
attempt_log: list[dict[str, Any]]
|
||||
page_logs: list[dict[str, Any]]
|
||||
pages_fetched: int
|
||||
pages_attempted: int
|
||||
has_more_remaining: bool
|
||||
pagination_truncated_reason: str | None
|
||||
continuation_cstart: int | None
|
||||
skipped_no_change: bool
|
||||
|
||||
|
||||
@dataclass
|
||||
class RunProgress:
|
||||
succeeded_count: int = 0
|
||||
failed_count: int = 0
|
||||
partial_count: int = 0
|
||||
scholar_results: list[dict[str, Any]] = field(default_factory=list)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ScholarProcessingOutcome:
|
||||
result_entry: dict[str, Any]
|
||||
succeeded_count_delta: int
|
||||
failed_count_delta: int
|
||||
partial_count_delta: int
|
||||
discovered_publication_count: int
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RunFailureSummary:
|
||||
failed_state_counts: dict[str, int]
|
||||
failed_reason_counts: dict[str, int]
|
||||
scrape_failure_counts: dict[str, int]
|
||||
retries_scheduled_count: int
|
||||
scholars_with_retries_count: int
|
||||
retry_exhausted_count: int
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RunAlertSummary:
|
||||
blocked_failure_count: int
|
||||
network_failure_count: int
|
||||
blocked_failure_threshold: int
|
||||
network_failure_threshold: int
|
||||
retry_scheduled_threshold: int
|
||||
alert_flags: dict[str, bool]
|
||||
|
||||
|
||||
@dataclass
|
||||
class PagedLoopState:
|
||||
fetch_result: FetchResult
|
||||
parsed_page: ParsedProfilePage
|
||||
attempt_log: list[dict[str, Any]]
|
||||
page_logs: list[dict[str, Any]]
|
||||
publications: list[PublicationCandidate]
|
||||
pages_fetched: int
|
||||
pages_attempted: int
|
||||
current_cstart: int
|
||||
next_cstart: int
|
||||
has_more_remaining: bool = False
|
||||
pagination_truncated_reason: str | None = None
|
||||
continuation_cstart: int | None = None
|
||||
|
||||
|
||||
class RunAlreadyInProgressError(RuntimeError):
|
||||
"""Raised when a run lock for a user is already held by another process."""
|
||||
|
||||
|
||||
class RunBlockedBySafetyPolicyError(RuntimeError):
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
code: str,
|
||||
message: str,
|
||||
safety_state: dict[str, Any],
|
||||
) -> None:
|
||||
super().__init__(message)
|
||||
self.code = code
|
||||
self.message = message
|
||||
self.safety_state = safety_state
|
||||
1
app/services/domains/portability/__init__.py
Normal file
1
app/services/domains/portability/__init__.py
Normal file
|
|
@ -0,0 +1 @@
|
|||
from app.services.domains.portability.application import *
|
||||
67
app/services/domains/portability/application.py
Normal file
67
app/services/domains/portability/application.py
Normal file
|
|
@ -0,0 +1,67 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.models import Publication
|
||||
from app.services.domains.portability.constants import (
|
||||
EXPORT_SCHEMA_VERSION,
|
||||
MAX_IMPORT_PUBLICATIONS,
|
||||
MAX_IMPORT_SCHOLARS,
|
||||
)
|
||||
from app.services.domains.portability.exporting import export_user_data
|
||||
from app.services.domains.portability.normalize import _validate_import_sizes
|
||||
from app.services.domains.portability.publication_import import (
|
||||
_build_imported_publication_input,
|
||||
_initialize_import_counters,
|
||||
_upsert_imported_publication,
|
||||
)
|
||||
from app.services.domains.portability.scholar_import import _upsert_imported_scholars
|
||||
from app.services.domains.portability.types import ImportExportError, ImportedPublicationInput
|
||||
|
||||
|
||||
async def import_user_data(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
scholars: list[dict[str, Any]],
|
||||
publications: list[dict[str, Any]],
|
||||
) -> dict[str, int]:
|
||||
_validate_import_sizes(scholars=scholars, publications=publications)
|
||||
scholar_map, counters = await _upsert_imported_scholars(
|
||||
db_session,
|
||||
user_id=user_id,
|
||||
scholars=scholars,
|
||||
)
|
||||
cluster_cache: dict[str, Publication | None] = {}
|
||||
fingerprint_cache: dict[str, Publication | None] = {}
|
||||
_initialize_import_counters(counters)
|
||||
for item in publications:
|
||||
parsed_item = _build_imported_publication_input(
|
||||
item=item,
|
||||
scholar_map=scholar_map,
|
||||
)
|
||||
if parsed_item is None:
|
||||
counters["skipped_records"] += 1
|
||||
continue
|
||||
await _upsert_imported_publication(
|
||||
db_session,
|
||||
payload=parsed_item,
|
||||
cluster_cache=cluster_cache,
|
||||
fingerprint_cache=fingerprint_cache,
|
||||
counters=counters,
|
||||
)
|
||||
await db_session.commit()
|
||||
return counters
|
||||
|
||||
|
||||
__all__ = [
|
||||
"EXPORT_SCHEMA_VERSION",
|
||||
"MAX_IMPORT_SCHOLARS",
|
||||
"MAX_IMPORT_PUBLICATIONS",
|
||||
"ImportExportError",
|
||||
"ImportedPublicationInput",
|
||||
"export_user_data",
|
||||
"import_user_data",
|
||||
]
|
||||
9
app/services/domains/portability/constants.py
Normal file
9
app/services/domains/portability/constants.py
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
EXPORT_SCHEMA_VERSION = 1
|
||||
MAX_IMPORT_SCHOLARS = 10_000
|
||||
MAX_IMPORT_PUBLICATIONS = 100_000
|
||||
WORD_RE = re.compile(r"[a-z0-9]+")
|
||||
SHA256_RE = re.compile(r"^[0-9a-f]{64}$")
|
||||
91
app/services/domains/portability/exporting.py
Normal file
91
app/services/domains/portability/exporting.py
Normal file
|
|
@ -0,0 +1,91 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.models import Publication, ScholarProfile, ScholarPublication
|
||||
from app.services.domains.portability.constants import EXPORT_SCHEMA_VERSION
|
||||
|
||||
|
||||
def _exported_at_iso() -> str:
|
||||
return datetime.now(timezone.utc).replace(microsecond=0).isoformat()
|
||||
|
||||
|
||||
def _serialize_export_scholar(profile: ScholarProfile) -> dict[str, Any]:
|
||||
return {
|
||||
"scholar_id": profile.scholar_id,
|
||||
"display_name": profile.display_name,
|
||||
"is_enabled": bool(profile.is_enabled),
|
||||
"profile_image_override_url": profile.profile_image_override_url,
|
||||
}
|
||||
|
||||
|
||||
def _serialize_export_publication(row: tuple[Any, ...]) -> dict[str, Any]:
|
||||
(
|
||||
scholar_id,
|
||||
cluster_id,
|
||||
fingerprint_sha256,
|
||||
title_raw,
|
||||
year,
|
||||
citation_count,
|
||||
author_text,
|
||||
venue_text,
|
||||
pub_url,
|
||||
pdf_url,
|
||||
is_read,
|
||||
) = row
|
||||
return {
|
||||
"scholar_id": scholar_id,
|
||||
"cluster_id": cluster_id,
|
||||
"fingerprint_sha256": fingerprint_sha256,
|
||||
"title": title_raw,
|
||||
"year": year,
|
||||
"citation_count": int(citation_count or 0),
|
||||
"author_text": author_text,
|
||||
"venue_text": venue_text,
|
||||
"pub_url": pub_url,
|
||||
"pdf_url": pdf_url,
|
||||
"is_read": bool(is_read),
|
||||
}
|
||||
|
||||
|
||||
async def export_user_data(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
) -> dict[str, Any]:
|
||||
scholars_result = await db_session.execute(
|
||||
select(ScholarProfile)
|
||||
.where(ScholarProfile.user_id == user_id)
|
||||
.order_by(ScholarProfile.id.asc())
|
||||
)
|
||||
publication_result = await db_session.execute(
|
||||
select(
|
||||
ScholarProfile.scholar_id,
|
||||
Publication.cluster_id,
|
||||
Publication.fingerprint_sha256,
|
||||
Publication.title_raw,
|
||||
Publication.year,
|
||||
Publication.citation_count,
|
||||
Publication.author_text,
|
||||
Publication.venue_text,
|
||||
Publication.pub_url,
|
||||
Publication.pdf_url,
|
||||
ScholarPublication.is_read,
|
||||
)
|
||||
.join(ScholarPublication, ScholarPublication.scholar_profile_id == ScholarProfile.id)
|
||||
.join(Publication, Publication.id == ScholarPublication.publication_id)
|
||||
.where(ScholarProfile.user_id == user_id)
|
||||
.order_by(ScholarPublication.created_at.desc(), Publication.id.desc())
|
||||
)
|
||||
scholars = [_serialize_export_scholar(profile) for profile in scholars_result.scalars().all()]
|
||||
publications = [_serialize_export_publication(row) for row in publication_result.all()]
|
||||
return {
|
||||
"schema_version": EXPORT_SCHEMA_VERSION,
|
||||
"exported_at": _exported_at_iso(),
|
||||
"scholars": scholars,
|
||||
"publications": publications,
|
||||
}
|
||||
109
app/services/domains/portability/normalize.py
Normal file
109
app/services/domains/portability/normalize.py
Normal file
|
|
@ -0,0 +1,109 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
from typing import Any
|
||||
|
||||
from app.services.domains.ingestion.application import normalize_title
|
||||
from app.services.domains.portability.constants import (
|
||||
MAX_IMPORT_PUBLICATIONS,
|
||||
MAX_IMPORT_SCHOLARS,
|
||||
SHA256_RE,
|
||||
WORD_RE,
|
||||
)
|
||||
from app.services.domains.portability.types import ImportExportError
|
||||
|
||||
|
||||
def _normalize_optional_text(value: Any) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
normalized = str(value).strip()
|
||||
return normalized or None
|
||||
|
||||
|
||||
def _normalize_optional_year(value: Any) -> int | None:
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
year = int(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
if year < 1500 or year > 3000:
|
||||
return None
|
||||
return year
|
||||
|
||||
|
||||
def _normalize_citation_count(value: Any) -> int:
|
||||
try:
|
||||
parsed = int(value)
|
||||
except (TypeError, ValueError):
|
||||
return 0
|
||||
return max(0, parsed)
|
||||
|
||||
|
||||
def _first_author_last_name(authors_text: str | None) -> str:
|
||||
if not authors_text:
|
||||
return ""
|
||||
first_author = authors_text.split(",", maxsplit=1)[0].strip().lower()
|
||||
words = WORD_RE.findall(first_author)
|
||||
if not words:
|
||||
return ""
|
||||
return words[-1]
|
||||
|
||||
|
||||
def _first_venue_word(venue_text: str | None) -> str:
|
||||
if not venue_text:
|
||||
return ""
|
||||
words = WORD_RE.findall(venue_text.lower())
|
||||
if not words:
|
||||
return ""
|
||||
return words[0]
|
||||
|
||||
|
||||
def _build_fingerprint(
|
||||
*,
|
||||
title: str,
|
||||
year: int | None,
|
||||
author_text: str | None,
|
||||
venue_text: str | None,
|
||||
) -> str:
|
||||
canonical = "|".join(
|
||||
[
|
||||
normalize_title(title),
|
||||
str(year) if year is not None else "",
|
||||
_first_author_last_name(author_text),
|
||||
_first_venue_word(venue_text),
|
||||
]
|
||||
)
|
||||
return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _resolve_fingerprint(
|
||||
*,
|
||||
title: str,
|
||||
year: int | None,
|
||||
author_text: str | None,
|
||||
venue_text: str | None,
|
||||
provided_fingerprint: Any,
|
||||
) -> str:
|
||||
normalized = _normalize_optional_text(provided_fingerprint)
|
||||
if normalized and SHA256_RE.fullmatch(normalized.lower()):
|
||||
return normalized.lower()
|
||||
return _build_fingerprint(
|
||||
title=title,
|
||||
year=year,
|
||||
author_text=author_text,
|
||||
venue_text=venue_text,
|
||||
)
|
||||
|
||||
|
||||
def _validate_import_sizes(
|
||||
*,
|
||||
scholars: list[dict[str, Any]],
|
||||
publications: list[dict[str, Any]],
|
||||
) -> None:
|
||||
if len(scholars) > MAX_IMPORT_SCHOLARS:
|
||||
raise ImportExportError(f"Import exceeds max scholars ({MAX_IMPORT_SCHOLARS}).")
|
||||
if len(publications) > MAX_IMPORT_PUBLICATIONS:
|
||||
raise ImportExportError(
|
||||
f"Import exceeds max publications ({MAX_IMPORT_PUBLICATIONS})."
|
||||
)
|
||||
368
app/services/domains/portability/publication_import.py
Normal file
368
app/services/domains/portability/publication_import.py
Normal file
|
|
@ -0,0 +1,368 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.models import Publication, ScholarProfile, ScholarPublication
|
||||
from app.services.domains.ingestion.application import build_publication_url, normalize_title
|
||||
from app.services.domains.portability.normalize import (
|
||||
_normalize_citation_count,
|
||||
_normalize_optional_text,
|
||||
_normalize_optional_year,
|
||||
_resolve_fingerprint,
|
||||
)
|
||||
from app.services.domains.portability.types import ImportedPublicationInput
|
||||
|
||||
|
||||
async def _find_publication_by_cluster(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
cluster_id: str,
|
||||
) -> Publication | None:
|
||||
result = await db_session.execute(
|
||||
select(Publication).where(Publication.cluster_id == cluster_id)
|
||||
)
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
|
||||
async def _find_publication_by_fingerprint(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
fingerprint_sha256: str,
|
||||
) -> Publication | None:
|
||||
result = await db_session.execute(
|
||||
select(Publication).where(Publication.fingerprint_sha256 == fingerprint_sha256)
|
||||
)
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
|
||||
async def _find_linked_publication_by_title(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
scholar_profile_id: int,
|
||||
title: str,
|
||||
) -> Publication | None:
|
||||
normalized_title = normalize_title(title)
|
||||
result = await db_session.execute(
|
||||
select(Publication)
|
||||
.join(
|
||||
ScholarPublication,
|
||||
ScholarPublication.publication_id == Publication.id,
|
||||
)
|
||||
.where(
|
||||
ScholarPublication.scholar_profile_id == scholar_profile_id,
|
||||
Publication.title_normalized == normalized_title,
|
||||
)
|
||||
.order_by(Publication.id.asc())
|
||||
.limit(1)
|
||||
)
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
|
||||
def _apply_imported_publication_values(
|
||||
*,
|
||||
publication: Publication,
|
||||
title: str,
|
||||
year: int | None,
|
||||
citation_count: int,
|
||||
author_text: str | None,
|
||||
venue_text: str | None,
|
||||
pub_url: str | None,
|
||||
pdf_url: str | None,
|
||||
cluster_id: str | None,
|
||||
) -> bool:
|
||||
updated = False
|
||||
if cluster_id and publication.cluster_id != cluster_id:
|
||||
publication.cluster_id = cluster_id
|
||||
updated = True
|
||||
if publication.title_raw != title:
|
||||
publication.title_raw = title
|
||||
publication.title_normalized = normalize_title(title)
|
||||
updated = True
|
||||
if publication.year != year:
|
||||
publication.year = year
|
||||
updated = True
|
||||
if int(publication.citation_count or 0) != citation_count:
|
||||
publication.citation_count = citation_count
|
||||
updated = True
|
||||
if publication.author_text != author_text:
|
||||
publication.author_text = author_text
|
||||
updated = True
|
||||
if publication.venue_text != venue_text:
|
||||
publication.venue_text = venue_text
|
||||
updated = True
|
||||
if pub_url and publication.pub_url != pub_url:
|
||||
publication.pub_url = pub_url
|
||||
updated = True
|
||||
if pdf_url and publication.pdf_url != pdf_url:
|
||||
publication.pdf_url = pdf_url
|
||||
updated = True
|
||||
return updated
|
||||
|
||||
|
||||
def _new_publication(
|
||||
*,
|
||||
cluster_id: str | None,
|
||||
fingerprint_sha256: str,
|
||||
title: str,
|
||||
year: int | None,
|
||||
citation_count: int,
|
||||
author_text: str | None,
|
||||
venue_text: str | None,
|
||||
pub_url: str | None,
|
||||
pdf_url: str | None,
|
||||
) -> Publication:
|
||||
return Publication(
|
||||
cluster_id=cluster_id,
|
||||
fingerprint_sha256=fingerprint_sha256,
|
||||
title_raw=title,
|
||||
title_normalized=normalize_title(title),
|
||||
year=year,
|
||||
citation_count=citation_count,
|
||||
author_text=author_text,
|
||||
venue_text=venue_text,
|
||||
pub_url=pub_url,
|
||||
pdf_url=pdf_url,
|
||||
)
|
||||
|
||||
|
||||
async def _resolve_publication_for_import(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
scholar_profile_id: int,
|
||||
title: str,
|
||||
cluster_id: str | None,
|
||||
fingerprint_sha256: str,
|
||||
cluster_cache: dict[str, Publication | None],
|
||||
fingerprint_cache: dict[str, Publication | None],
|
||||
) -> Publication | None:
|
||||
if cluster_id:
|
||||
if cluster_id not in cluster_cache:
|
||||
cluster_cache[cluster_id] = await _find_publication_by_cluster(
|
||||
db_session,
|
||||
cluster_id=cluster_id,
|
||||
)
|
||||
if cluster_cache[cluster_id] is not None:
|
||||
return cluster_cache[cluster_id]
|
||||
if fingerprint_sha256 not in fingerprint_cache:
|
||||
fingerprint_cache[fingerprint_sha256] = await _find_publication_by_fingerprint(
|
||||
db_session,
|
||||
fingerprint_sha256=fingerprint_sha256,
|
||||
)
|
||||
if fingerprint_cache[fingerprint_sha256] is not None:
|
||||
return fingerprint_cache[fingerprint_sha256]
|
||||
return await _find_linked_publication_by_title(
|
||||
db_session,
|
||||
scholar_profile_id=scholar_profile_id,
|
||||
title=title,
|
||||
)
|
||||
|
||||
|
||||
async def _upsert_scholar_publication_link(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
scholar_profile_id: int,
|
||||
publication_id: int,
|
||||
is_read: bool,
|
||||
) -> tuple[bool, bool]:
|
||||
result = await db_session.execute(
|
||||
select(ScholarPublication).where(
|
||||
ScholarPublication.scholar_profile_id == scholar_profile_id,
|
||||
ScholarPublication.publication_id == publication_id,
|
||||
)
|
||||
)
|
||||
link = result.scalar_one_or_none()
|
||||
if link is None:
|
||||
db_session.add(
|
||||
ScholarPublication(
|
||||
scholar_profile_id=scholar_profile_id,
|
||||
publication_id=publication_id,
|
||||
is_read=bool(is_read),
|
||||
)
|
||||
)
|
||||
return True, False
|
||||
if bool(link.is_read) == bool(is_read):
|
||||
return False, False
|
||||
link.is_read = bool(is_read)
|
||||
return False, True
|
||||
|
||||
|
||||
def _initialize_import_counters(counters: dict[str, int]) -> None:
|
||||
counters.update(
|
||||
{
|
||||
"publications_created": 0,
|
||||
"publications_updated": 0,
|
||||
"links_created": 0,
|
||||
"links_updated": 0,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _build_imported_publication_input(
|
||||
*,
|
||||
item: dict[str, Any],
|
||||
scholar_map: dict[str, ScholarProfile],
|
||||
) -> ImportedPublicationInput | None:
|
||||
scholar_id = _normalize_optional_text(item.get("scholar_id"))
|
||||
title = _normalize_optional_text(item.get("title"))
|
||||
if not scholar_id or not title:
|
||||
return None
|
||||
|
||||
profile = scholar_map.get(scholar_id)
|
||||
if profile is None:
|
||||
return None
|
||||
|
||||
year = _normalize_optional_year(item.get("year"))
|
||||
author_text = _normalize_optional_text(item.get("author_text"))
|
||||
venue_text = _normalize_optional_text(item.get("venue_text"))
|
||||
return ImportedPublicationInput(
|
||||
profile=profile,
|
||||
title=title,
|
||||
year=year,
|
||||
citation_count=_normalize_citation_count(item.get("citation_count")),
|
||||
author_text=author_text,
|
||||
venue_text=venue_text,
|
||||
cluster_id=_normalize_optional_text(item.get("cluster_id")),
|
||||
pub_url=build_publication_url(_normalize_optional_text(item.get("pub_url"))),
|
||||
pdf_url=build_publication_url(_normalize_optional_text(item.get("pdf_url"))),
|
||||
fingerprint=_resolve_fingerprint(
|
||||
title=title,
|
||||
year=year,
|
||||
author_text=author_text,
|
||||
venue_text=venue_text,
|
||||
provided_fingerprint=item.get("fingerprint_sha256"),
|
||||
),
|
||||
is_read=bool(item.get("is_read", False)),
|
||||
)
|
||||
|
||||
|
||||
def _update_link_counters(
|
||||
*,
|
||||
counters: dict[str, int],
|
||||
link_created: bool,
|
||||
link_updated: bool,
|
||||
) -> None:
|
||||
if link_created:
|
||||
counters["links_created"] += 1
|
||||
if link_updated:
|
||||
counters["links_updated"] += 1
|
||||
|
||||
|
||||
def _cache_resolved_publication(
|
||||
*,
|
||||
publication: Publication,
|
||||
cluster_id: str | None,
|
||||
fingerprint_sha256: str,
|
||||
cluster_cache: dict[str, Publication | None],
|
||||
fingerprint_cache: dict[str, Publication | None],
|
||||
) -> None:
|
||||
if cluster_id:
|
||||
cluster_cache[cluster_id] = publication
|
||||
fingerprint_cache[fingerprint_sha256] = publication
|
||||
|
||||
|
||||
async def _create_import_publication(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
payload: ImportedPublicationInput,
|
||||
) -> Publication:
|
||||
publication = _new_publication(
|
||||
cluster_id=payload.cluster_id,
|
||||
fingerprint_sha256=payload.fingerprint,
|
||||
title=payload.title,
|
||||
year=payload.year,
|
||||
citation_count=payload.citation_count,
|
||||
author_text=payload.author_text,
|
||||
venue_text=payload.venue_text,
|
||||
pub_url=payload.pub_url,
|
||||
pdf_url=payload.pdf_url,
|
||||
)
|
||||
db_session.add(publication)
|
||||
await db_session.flush()
|
||||
return publication
|
||||
|
||||
|
||||
def _update_import_publication(
|
||||
*,
|
||||
publication: Publication,
|
||||
payload: ImportedPublicationInput,
|
||||
) -> bool:
|
||||
return _apply_imported_publication_values(
|
||||
publication=publication,
|
||||
title=payload.title,
|
||||
year=payload.year,
|
||||
citation_count=payload.citation_count,
|
||||
author_text=payload.author_text,
|
||||
venue_text=payload.venue_text,
|
||||
pub_url=payload.pub_url,
|
||||
pdf_url=payload.pdf_url,
|
||||
cluster_id=payload.cluster_id,
|
||||
)
|
||||
|
||||
|
||||
async def _upsert_publication_entity(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
payload: ImportedPublicationInput,
|
||||
cluster_cache: dict[str, Publication | None],
|
||||
fingerprint_cache: dict[str, Publication | None],
|
||||
) -> tuple[Publication, bool, bool]:
|
||||
publication = await _resolve_publication_for_import(
|
||||
db_session,
|
||||
scholar_profile_id=int(payload.profile.id),
|
||||
title=payload.title,
|
||||
cluster_id=payload.cluster_id,
|
||||
fingerprint_sha256=payload.fingerprint,
|
||||
cluster_cache=cluster_cache,
|
||||
fingerprint_cache=fingerprint_cache,
|
||||
)
|
||||
created = False
|
||||
updated = False
|
||||
if publication is None:
|
||||
publication = await _create_import_publication(db_session, payload=payload)
|
||||
created = True
|
||||
else:
|
||||
updated = _update_import_publication(publication=publication, payload=payload)
|
||||
|
||||
_cache_resolved_publication(
|
||||
publication=publication,
|
||||
cluster_id=payload.cluster_id,
|
||||
fingerprint_sha256=payload.fingerprint,
|
||||
cluster_cache=cluster_cache,
|
||||
fingerprint_cache=fingerprint_cache,
|
||||
)
|
||||
return publication, created, updated
|
||||
|
||||
|
||||
async def _upsert_imported_publication(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
payload: ImportedPublicationInput,
|
||||
cluster_cache: dict[str, Publication | None],
|
||||
fingerprint_cache: dict[str, Publication | None],
|
||||
counters: dict[str, int],
|
||||
) -> None:
|
||||
publication, created, updated = await _upsert_publication_entity(
|
||||
db_session,
|
||||
payload=payload,
|
||||
cluster_cache=cluster_cache,
|
||||
fingerprint_cache=fingerprint_cache,
|
||||
)
|
||||
if created:
|
||||
counters["publications_created"] += 1
|
||||
if updated:
|
||||
counters["publications_updated"] += 1
|
||||
|
||||
link_created, link_updated = await _upsert_scholar_publication_link(
|
||||
db_session,
|
||||
scholar_profile_id=int(payload.profile.id),
|
||||
publication_id=int(publication.id),
|
||||
is_read=payload.is_read,
|
||||
)
|
||||
_update_link_counters(
|
||||
counters=counters,
|
||||
link_created=link_created,
|
||||
link_updated=link_updated,
|
||||
)
|
||||
107
app/services/domains/portability/scholar_import.py
Normal file
107
app/services/domains/portability/scholar_import.py
Normal file
|
|
@ -0,0 +1,107 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.models import ScholarProfile
|
||||
from app.services.domains.portability.normalize import _normalize_optional_text
|
||||
from app.services.domains.scholars import application as scholar_service
|
||||
|
||||
|
||||
async def _load_user_scholar_map(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
) -> dict[str, ScholarProfile]:
|
||||
result = await db_session.execute(
|
||||
select(ScholarProfile).where(ScholarProfile.user_id == user_id)
|
||||
)
|
||||
profiles = list(result.scalars().all())
|
||||
return {profile.scholar_id: profile for profile in profiles}
|
||||
|
||||
|
||||
def _apply_imported_scholar_values(
|
||||
*,
|
||||
profile: ScholarProfile,
|
||||
display_name: str | None,
|
||||
profile_image_override_url: str | None,
|
||||
is_enabled: bool,
|
||||
) -> bool:
|
||||
updated = False
|
||||
if display_name and profile.display_name != display_name:
|
||||
profile.display_name = display_name
|
||||
updated = True
|
||||
if profile.profile_image_override_url != profile_image_override_url:
|
||||
profile.profile_image_override_url = profile_image_override_url
|
||||
updated = True
|
||||
if bool(profile.is_enabled) != bool(is_enabled):
|
||||
profile.is_enabled = bool(is_enabled)
|
||||
updated = True
|
||||
return updated
|
||||
|
||||
|
||||
def _new_scholar_profile(
|
||||
*,
|
||||
user_id: int,
|
||||
scholar_id: str,
|
||||
display_name: str | None,
|
||||
profile_image_override_url: str | None,
|
||||
is_enabled: bool,
|
||||
) -> ScholarProfile:
|
||||
return ScholarProfile(
|
||||
user_id=user_id,
|
||||
scholar_id=scholar_id,
|
||||
display_name=display_name,
|
||||
profile_image_override_url=profile_image_override_url,
|
||||
is_enabled=bool(is_enabled),
|
||||
)
|
||||
|
||||
|
||||
async def _upsert_imported_scholars(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
scholars: list[dict[str, Any]],
|
||||
) -> tuple[dict[str, ScholarProfile], dict[str, int]]:
|
||||
scholar_map = await _load_user_scholar_map(db_session, user_id=user_id)
|
||||
counters = {"scholars_created": 0, "scholars_updated": 0, "skipped_records": 0}
|
||||
for item in scholars:
|
||||
try:
|
||||
scholar_id = scholar_service.validate_scholar_id(str(item["scholar_id"]))
|
||||
display_name = scholar_service.normalize_display_name(
|
||||
str(item.get("display_name") or "")
|
||||
)
|
||||
override_url = scholar_service.normalize_profile_image_url(
|
||||
_normalize_optional_text(item.get("profile_image_override_url"))
|
||||
)
|
||||
except (KeyError, scholar_service.ScholarServiceError):
|
||||
counters["skipped_records"] += 1
|
||||
continue
|
||||
|
||||
is_enabled = bool(item.get("is_enabled", True))
|
||||
existing = scholar_map.get(scholar_id)
|
||||
if existing is None:
|
||||
profile = _new_scholar_profile(
|
||||
user_id=user_id,
|
||||
scholar_id=scholar_id,
|
||||
display_name=display_name,
|
||||
profile_image_override_url=override_url,
|
||||
is_enabled=is_enabled,
|
||||
)
|
||||
db_session.add(profile)
|
||||
scholar_map[scholar_id] = profile
|
||||
counters["scholars_created"] += 1
|
||||
continue
|
||||
|
||||
if _apply_imported_scholar_values(
|
||||
profile=existing,
|
||||
display_name=display_name,
|
||||
profile_image_override_url=override_url,
|
||||
is_enabled=is_enabled,
|
||||
):
|
||||
counters["scholars_updated"] += 1
|
||||
|
||||
await db_session.flush()
|
||||
return scholar_map, counters
|
||||
24
app/services/domains/portability/types.py
Normal file
24
app/services/domains/portability/types.py
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
from app.db.models import ScholarProfile
|
||||
|
||||
|
||||
class ImportExportError(ValueError):
|
||||
"""Raised when import/export payload constraints are violated."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ImportedPublicationInput:
|
||||
profile: ScholarProfile
|
||||
title: str
|
||||
year: int | None
|
||||
citation_count: int
|
||||
author_text: str | None
|
||||
venue_text: str | None
|
||||
cluster_id: str | None
|
||||
pub_url: str | None
|
||||
pdf_url: str | None
|
||||
fingerprint: str
|
||||
is_read: bool
|
||||
1
app/services/domains/publications/__init__.py
Normal file
1
app/services/domains/publications/__init__.py
Normal file
|
|
@ -0,0 +1 @@
|
|||
from app.services.domains.publications.application import *
|
||||
50
app/services/domains/publications/application.py
Normal file
50
app/services/domains/publications/application.py
Normal file
|
|
@ -0,0 +1,50 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from app.services.domains.publications.counts import (
|
||||
count_for_user,
|
||||
count_latest_for_user,
|
||||
count_unread_for_user,
|
||||
)
|
||||
from app.services.domains.publications.listing import (
|
||||
list_for_user,
|
||||
list_new_for_latest_run_for_user,
|
||||
list_unread_for_user,
|
||||
)
|
||||
from app.services.domains.publications.modes import (
|
||||
MODE_ALL,
|
||||
MODE_LATEST,
|
||||
MODE_NEW,
|
||||
MODE_UNREAD,
|
||||
resolve_mode,
|
||||
resolve_publication_view_mode,
|
||||
)
|
||||
from app.services.domains.publications.queries import (
|
||||
get_latest_completed_run_id_for_user,
|
||||
publications_query,
|
||||
)
|
||||
from app.services.domains.publications.read_state import (
|
||||
mark_all_unread_as_read_for_user,
|
||||
mark_selected_as_read_for_user,
|
||||
)
|
||||
from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem
|
||||
|
||||
__all__ = [
|
||||
"MODE_ALL",
|
||||
"MODE_UNREAD",
|
||||
"MODE_LATEST",
|
||||
"MODE_NEW",
|
||||
"PublicationListItem",
|
||||
"UnreadPublicationItem",
|
||||
"resolve_publication_view_mode",
|
||||
"resolve_mode",
|
||||
"get_latest_completed_run_id_for_user",
|
||||
"publications_query",
|
||||
"list_for_user",
|
||||
"list_unread_for_user",
|
||||
"list_new_for_latest_run_for_user",
|
||||
"count_for_user",
|
||||
"count_unread_for_user",
|
||||
"count_latest_for_user",
|
||||
"mark_all_unread_as_read_for_user",
|
||||
"mark_selected_as_read_for_user",
|
||||
]
|
||||
68
app/services/domains/publications/counts.py
Normal file
68
app/services/domains/publications/counts.py
Normal file
|
|
@ -0,0 +1,68 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.models import ScholarProfile, ScholarPublication
|
||||
from app.services.domains.publications.modes import (
|
||||
MODE_ALL,
|
||||
MODE_LATEST,
|
||||
MODE_UNREAD,
|
||||
resolve_publication_view_mode,
|
||||
)
|
||||
from app.services.domains.publications.queries import get_latest_completed_run_id_for_user
|
||||
|
||||
|
||||
async def count_for_user(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
mode: str = MODE_ALL,
|
||||
scholar_profile_id: int | None = None,
|
||||
) -> int:
|
||||
resolved_mode = resolve_publication_view_mode(mode)
|
||||
latest_run_id = await get_latest_completed_run_id_for_user(db_session, user_id=user_id)
|
||||
stmt = (
|
||||
select(func.count())
|
||||
.select_from(ScholarPublication)
|
||||
.join(ScholarProfile, ScholarProfile.id == ScholarPublication.scholar_profile_id)
|
||||
.where(ScholarProfile.user_id == user_id)
|
||||
)
|
||||
if scholar_profile_id is not None:
|
||||
stmt = stmt.where(ScholarProfile.id == scholar_profile_id)
|
||||
if resolved_mode == MODE_UNREAD:
|
||||
stmt = stmt.where(ScholarPublication.is_read.is_(False))
|
||||
if resolved_mode == MODE_LATEST:
|
||||
if latest_run_id is None:
|
||||
return 0
|
||||
stmt = stmt.where(ScholarPublication.first_seen_run_id == latest_run_id)
|
||||
result = await db_session.execute(stmt)
|
||||
return int(result.scalar_one() or 0)
|
||||
|
||||
|
||||
async def count_unread_for_user(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
scholar_profile_id: int | None = None,
|
||||
) -> int:
|
||||
return await count_for_user(
|
||||
db_session,
|
||||
user_id=user_id,
|
||||
mode=MODE_UNREAD,
|
||||
scholar_profile_id=scholar_profile_id,
|
||||
)
|
||||
|
||||
|
||||
async def count_latest_for_user(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
scholar_profile_id: int | None = None,
|
||||
) -> int:
|
||||
return await count_for_user(
|
||||
db_session,
|
||||
user_id=user_id,
|
||||
mode=MODE_LATEST,
|
||||
scholar_profile_id=scholar_profile_id,
|
||||
)
|
||||
89
app/services/domains/publications/listing.py
Normal file
89
app/services/domains/publications/listing.py
Normal file
|
|
@ -0,0 +1,89 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.services.domains.publications.modes import (
|
||||
MODE_ALL,
|
||||
MODE_LATEST,
|
||||
MODE_UNREAD,
|
||||
resolve_publication_view_mode,
|
||||
)
|
||||
from app.services.domains.publications.queries import (
|
||||
get_latest_completed_run_id_for_user,
|
||||
publication_list_item_from_row,
|
||||
publications_query,
|
||||
unread_item_from_row,
|
||||
)
|
||||
from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem
|
||||
|
||||
|
||||
async def list_for_user(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
mode: str = MODE_ALL,
|
||||
scholar_profile_id: int | None = None,
|
||||
limit: int = 300,
|
||||
) -> list[PublicationListItem]:
|
||||
resolved_mode = resolve_publication_view_mode(mode)
|
||||
latest_run_id = await get_latest_completed_run_id_for_user(db_session, user_id=user_id)
|
||||
result = await db_session.execute(
|
||||
publications_query(
|
||||
user_id=user_id,
|
||||
mode=resolved_mode,
|
||||
latest_run_id=latest_run_id,
|
||||
scholar_profile_id=scholar_profile_id,
|
||||
limit=limit,
|
||||
)
|
||||
)
|
||||
return [
|
||||
publication_list_item_from_row(row, latest_run_id=latest_run_id)
|
||||
for row in result.all()
|
||||
]
|
||||
|
||||
|
||||
async def list_unread_for_user(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
limit: int = 100,
|
||||
) -> list[UnreadPublicationItem]:
|
||||
result = await db_session.execute(
|
||||
publications_query(
|
||||
user_id=user_id,
|
||||
mode=MODE_UNREAD,
|
||||
latest_run_id=None,
|
||||
scholar_profile_id=None,
|
||||
limit=limit,
|
||||
)
|
||||
)
|
||||
return [unread_item_from_row(row) for row in result.all()]
|
||||
|
||||
|
||||
async def list_new_for_latest_run_for_user(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
limit: int = 100,
|
||||
) -> list[UnreadPublicationItem]:
|
||||
rows = await list_for_user(
|
||||
db_session,
|
||||
user_id=user_id,
|
||||
mode=MODE_LATEST,
|
||||
scholar_profile_id=None,
|
||||
limit=limit,
|
||||
)
|
||||
return [
|
||||
UnreadPublicationItem(
|
||||
publication_id=row.publication_id,
|
||||
scholar_profile_id=row.scholar_profile_id,
|
||||
scholar_label=row.scholar_label,
|
||||
title=row.title,
|
||||
year=row.year,
|
||||
citation_count=row.citation_count,
|
||||
venue_text=row.venue_text,
|
||||
pub_url=row.pub_url,
|
||||
pdf_url=row.pdf_url,
|
||||
)
|
||||
for row in rows
|
||||
]
|
||||
18
app/services/domains/publications/modes.py
Normal file
18
app/services/domains/publications/modes.py
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
from __future__ import annotations
|
||||
|
||||
MODE_ALL = "all"
|
||||
MODE_UNREAD = "unread"
|
||||
MODE_LATEST = "latest"
|
||||
MODE_NEW = "new" # compatibility alias for MODE_LATEST
|
||||
|
||||
|
||||
def resolve_publication_view_mode(value: str | None) -> str:
|
||||
if value == MODE_UNREAD:
|
||||
return MODE_UNREAD
|
||||
if value in {MODE_LATEST, MODE_NEW}:
|
||||
return MODE_LATEST
|
||||
return MODE_ALL
|
||||
|
||||
|
||||
def resolve_mode(value: str | None) -> str:
|
||||
return resolve_publication_view_mode(value)
|
||||
139
app/services/domains/publications/queries.py
Normal file
139
app/services/domains/publications/queries.py
Normal file
|
|
@ -0,0 +1,139 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from sqlalchemy import Select, func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.models import CrawlRun, Publication, RunStatus, ScholarProfile, ScholarPublication
|
||||
from app.services.domains.publications.modes import MODE_LATEST, MODE_UNREAD
|
||||
from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem
|
||||
|
||||
|
||||
def _normalized_citation_count(value: object) -> int:
|
||||
try:
|
||||
return int(value or 0)
|
||||
except (TypeError, ValueError):
|
||||
return 0
|
||||
|
||||
|
||||
async def get_latest_completed_run_id_for_user(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
) -> int | None:
|
||||
result = await db_session.execute(
|
||||
select(func.max(CrawlRun.id)).where(
|
||||
CrawlRun.user_id == user_id,
|
||||
CrawlRun.status != RunStatus.RUNNING,
|
||||
)
|
||||
)
|
||||
latest_run_id = result.scalar_one_or_none()
|
||||
return int(latest_run_id) if latest_run_id is not None else None
|
||||
|
||||
|
||||
def publications_query(
|
||||
*,
|
||||
user_id: int,
|
||||
mode: str,
|
||||
latest_run_id: int | None,
|
||||
scholar_profile_id: int | None,
|
||||
limit: int,
|
||||
) -> Select[tuple]:
|
||||
scholar_label = ScholarProfile.display_name
|
||||
stmt = (
|
||||
select(
|
||||
Publication.id,
|
||||
ScholarProfile.id,
|
||||
scholar_label,
|
||||
ScholarProfile.scholar_id,
|
||||
Publication.title_raw,
|
||||
Publication.year,
|
||||
Publication.citation_count,
|
||||
Publication.venue_text,
|
||||
Publication.pub_url,
|
||||
Publication.pdf_url,
|
||||
ScholarPublication.is_read,
|
||||
ScholarPublication.first_seen_run_id,
|
||||
ScholarPublication.created_at,
|
||||
)
|
||||
.join(ScholarPublication, ScholarPublication.publication_id == Publication.id)
|
||||
.join(ScholarProfile, ScholarProfile.id == ScholarPublication.scholar_profile_id)
|
||||
.where(ScholarProfile.user_id == user_id)
|
||||
.order_by(ScholarPublication.created_at.desc(), Publication.id.desc())
|
||||
.limit(limit)
|
||||
)
|
||||
if scholar_profile_id is not None:
|
||||
stmt = stmt.where(ScholarProfile.id == scholar_profile_id)
|
||||
if mode == MODE_UNREAD:
|
||||
stmt = stmt.where(ScholarPublication.is_read.is_(False))
|
||||
if mode == MODE_LATEST:
|
||||
if latest_run_id is None:
|
||||
return stmt.where(False)
|
||||
stmt = stmt.where(ScholarPublication.first_seen_run_id == latest_run_id)
|
||||
return stmt
|
||||
|
||||
|
||||
def publication_list_item_from_row(
|
||||
row: tuple,
|
||||
*,
|
||||
latest_run_id: int | None,
|
||||
) -> PublicationListItem:
|
||||
(
|
||||
publication_id,
|
||||
scholar_profile_id,
|
||||
display_name,
|
||||
scholar_id,
|
||||
title_raw,
|
||||
year,
|
||||
citation_count,
|
||||
venue_text,
|
||||
pub_url,
|
||||
pdf_url,
|
||||
is_read,
|
||||
first_seen_run_id,
|
||||
created_at,
|
||||
) = row
|
||||
return PublicationListItem(
|
||||
publication_id=int(publication_id),
|
||||
scholar_profile_id=int(scholar_profile_id),
|
||||
scholar_label=(display_name or scholar_id),
|
||||
title=title_raw,
|
||||
year=year,
|
||||
citation_count=_normalized_citation_count(citation_count),
|
||||
venue_text=venue_text,
|
||||
pub_url=pub_url,
|
||||
pdf_url=pdf_url,
|
||||
is_read=bool(is_read),
|
||||
first_seen_at=created_at,
|
||||
is_new_in_latest_run=(
|
||||
latest_run_id is not None and int(first_seen_run_id or 0) == latest_run_id
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def unread_item_from_row(row: tuple) -> UnreadPublicationItem:
|
||||
(
|
||||
publication_id,
|
||||
scholar_profile_id,
|
||||
display_name,
|
||||
scholar_id,
|
||||
title_raw,
|
||||
year,
|
||||
citation_count,
|
||||
venue_text,
|
||||
pub_url,
|
||||
pdf_url,
|
||||
_is_read,
|
||||
_first_seen_run_id,
|
||||
_created_at,
|
||||
) = row
|
||||
return UnreadPublicationItem(
|
||||
publication_id=int(publication_id),
|
||||
scholar_profile_id=int(scholar_profile_id),
|
||||
scholar_label=(display_name or scholar_id),
|
||||
title=title_raw,
|
||||
year=year,
|
||||
citation_count=_normalized_citation_count(citation_count),
|
||||
venue_text=venue_text,
|
||||
pub_url=pub_url,
|
||||
pdf_url=pdf_url,
|
||||
)
|
||||
71
app/services/domains/publications/read_state.py
Normal file
71
app/services/domains/publications/read_state.py
Normal file
|
|
@ -0,0 +1,71 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from sqlalchemy import select, tuple_, update
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.models import ScholarProfile, ScholarPublication
|
||||
|
||||
|
||||
def _normalized_selection_pairs(selections: list[tuple[int, int]]) -> set[tuple[int, int]]:
|
||||
pairs: set[tuple[int, int]] = set()
|
||||
for scholar_profile_id, publication_id in selections:
|
||||
normalized = (int(scholar_profile_id), int(publication_id))
|
||||
if normalized[0] <= 0 or normalized[1] <= 0:
|
||||
continue
|
||||
pairs.add(normalized)
|
||||
return pairs
|
||||
|
||||
|
||||
async def mark_all_unread_as_read_for_user(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
) -> int:
|
||||
scholar_ids = (
|
||||
select(ScholarProfile.id)
|
||||
.where(ScholarProfile.user_id == user_id)
|
||||
.scalar_subquery()
|
||||
)
|
||||
stmt = (
|
||||
update(ScholarPublication)
|
||||
.where(
|
||||
ScholarPublication.scholar_profile_id.in_(scholar_ids),
|
||||
ScholarPublication.is_read.is_(False),
|
||||
)
|
||||
.values(is_read=True)
|
||||
)
|
||||
result = await db_session.execute(stmt)
|
||||
await db_session.commit()
|
||||
return int(result.rowcount or 0)
|
||||
|
||||
|
||||
async def mark_selected_as_read_for_user(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
selections: list[tuple[int, int]],
|
||||
) -> int:
|
||||
normalized_pairs = _normalized_selection_pairs(selections)
|
||||
if not normalized_pairs:
|
||||
return 0
|
||||
|
||||
scholar_ids = (
|
||||
select(ScholarProfile.id)
|
||||
.where(ScholarProfile.user_id == user_id)
|
||||
.scalar_subquery()
|
||||
)
|
||||
stmt = (
|
||||
update(ScholarPublication)
|
||||
.where(
|
||||
ScholarPublication.scholar_profile_id.in_(scholar_ids),
|
||||
tuple_(
|
||||
ScholarPublication.scholar_profile_id,
|
||||
ScholarPublication.publication_id,
|
||||
).in_(list(normalized_pairs)),
|
||||
ScholarPublication.is_read.is_(False),
|
||||
)
|
||||
.values(is_read=True)
|
||||
)
|
||||
result = await db_session.execute(stmt)
|
||||
await db_session.commit()
|
||||
return int(result.rowcount or 0)
|
||||
33
app/services/domains/publications/types.py
Normal file
33
app/services/domains/publications/types.py
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PublicationListItem:
|
||||
publication_id: int
|
||||
scholar_profile_id: int
|
||||
scholar_label: str
|
||||
title: str
|
||||
year: int | None
|
||||
citation_count: int
|
||||
venue_text: str | None
|
||||
pub_url: str | None
|
||||
pdf_url: str | None
|
||||
is_read: bool
|
||||
first_seen_at: datetime
|
||||
is_new_in_latest_run: bool
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class UnreadPublicationItem:
|
||||
publication_id: int
|
||||
scholar_profile_id: int
|
||||
scholar_label: str
|
||||
title: str
|
||||
year: int | None
|
||||
citation_count: int
|
||||
venue_text: str | None
|
||||
pub_url: str | None
|
||||
pdf_url: str | None
|
||||
1
app/services/domains/runs/__init__.py
Normal file
1
app/services/domains/runs/__init__.py
Normal file
|
|
@ -0,0 +1 @@
|
|||
from app.services.domains.runs.application import *
|
||||
45
app/services/domains/runs/application.py
Normal file
45
app/services/domains/runs/application.py
Normal file
|
|
@ -0,0 +1,45 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from app.services.domains.runs.queue_service import (
|
||||
clear_queue_item_for_user,
|
||||
drop_queue_item_for_user,
|
||||
get_queue_item_for_user,
|
||||
list_queue_items_for_user,
|
||||
queue_status_counts_for_user,
|
||||
retry_queue_item_for_user,
|
||||
)
|
||||
from app.services.domains.runs.runs_service import (
|
||||
get_manual_run_by_idempotency_key,
|
||||
get_run_for_user,
|
||||
list_recent_runs_for_user,
|
||||
list_runs_for_user,
|
||||
)
|
||||
from app.services.domains.runs.summary import extract_run_summary
|
||||
from app.services.domains.runs.types import (
|
||||
QUEUE_STATUS_DROPPED,
|
||||
QUEUE_STATUS_QUEUED,
|
||||
QUEUE_STATUS_RETRYING,
|
||||
QueueClearResult,
|
||||
QueueListItem,
|
||||
QueueTransitionError,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"QUEUE_STATUS_QUEUED",
|
||||
"QUEUE_STATUS_RETRYING",
|
||||
"QUEUE_STATUS_DROPPED",
|
||||
"QueueListItem",
|
||||
"QueueClearResult",
|
||||
"QueueTransitionError",
|
||||
"extract_run_summary",
|
||||
"list_recent_runs_for_user",
|
||||
"list_runs_for_user",
|
||||
"get_run_for_user",
|
||||
"get_manual_run_by_idempotency_key",
|
||||
"list_queue_items_for_user",
|
||||
"get_queue_item_for_user",
|
||||
"retry_queue_item_for_user",
|
||||
"drop_queue_item_for_user",
|
||||
"clear_queue_item_for_user",
|
||||
"queue_status_counts_for_user",
|
||||
]
|
||||
70
app/services/domains/runs/queue_queries.py
Normal file
70
app/services/domains/runs/queue_queries.py
Normal file
|
|
@ -0,0 +1,70 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from sqlalchemy import and_, select
|
||||
|
||||
from app.db.models import IngestionQueueItem, ScholarProfile
|
||||
from app.services.domains.runs.types import QueueListItem
|
||||
|
||||
|
||||
def queue_item_columns() -> tuple:
|
||||
return (
|
||||
IngestionQueueItem.id,
|
||||
IngestionQueueItem.scholar_profile_id,
|
||||
ScholarProfile.display_name,
|
||||
ScholarProfile.scholar_id,
|
||||
IngestionQueueItem.status,
|
||||
IngestionQueueItem.reason,
|
||||
IngestionQueueItem.dropped_reason,
|
||||
IngestionQueueItem.attempt_count,
|
||||
IngestionQueueItem.resume_cstart,
|
||||
IngestionQueueItem.next_attempt_dt,
|
||||
IngestionQueueItem.updated_at,
|
||||
IngestionQueueItem.last_error,
|
||||
IngestionQueueItem.last_run_id,
|
||||
)
|
||||
|
||||
|
||||
def queue_item_select(*, user_id: int):
|
||||
return (
|
||||
select(*queue_item_columns())
|
||||
.join(
|
||||
ScholarProfile,
|
||||
and_(
|
||||
ScholarProfile.id == IngestionQueueItem.scholar_profile_id,
|
||||
ScholarProfile.user_id == IngestionQueueItem.user_id,
|
||||
),
|
||||
)
|
||||
.where(IngestionQueueItem.user_id == user_id)
|
||||
)
|
||||
|
||||
|
||||
def queue_list_item_from_row(row: tuple) -> QueueListItem:
|
||||
(
|
||||
item_id,
|
||||
scholar_profile_id,
|
||||
display_name,
|
||||
scholar_id,
|
||||
status,
|
||||
reason,
|
||||
dropped_reason,
|
||||
attempt_count,
|
||||
resume_cstart,
|
||||
next_attempt_dt,
|
||||
updated_at,
|
||||
last_error,
|
||||
last_run_id,
|
||||
) = row
|
||||
return QueueListItem(
|
||||
id=int(item_id),
|
||||
scholar_profile_id=int(scholar_profile_id),
|
||||
scholar_label=(display_name or scholar_id),
|
||||
status=str(status),
|
||||
reason=str(reason),
|
||||
dropped_reason=dropped_reason,
|
||||
attempt_count=int(attempt_count or 0),
|
||||
resume_cstart=int(resume_cstart or 0),
|
||||
next_attempt_dt=next_attempt_dt,
|
||||
updated_at=updated_at,
|
||||
last_error=last_error,
|
||||
last_run_id=int(last_run_id) if last_run_id is not None else None,
|
||||
)
|
||||
182
app/services/domains/runs/queue_service.py
Normal file
182
app/services/domains/runs/queue_service.py
Normal file
|
|
@ -0,0 +1,182 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from sqlalchemy import case, func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.models import IngestionQueueItem
|
||||
from app.services.domains.ingestion import queue as queue_mutations
|
||||
from app.services.domains.runs.queue_queries import queue_item_select, queue_list_item_from_row
|
||||
from app.services.domains.runs.types import (
|
||||
QUEUE_STATUS_DROPPED,
|
||||
QUEUE_STATUS_QUEUED,
|
||||
QUEUE_STATUS_RETRYING,
|
||||
QueueClearResult,
|
||||
QueueListItem,
|
||||
QueueTransitionError,
|
||||
)
|
||||
|
||||
|
||||
async def list_queue_items_for_user(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
limit: int = 200,
|
||||
) -> list[QueueListItem]:
|
||||
result = await db_session.execute(
|
||||
queue_item_select(user_id=user_id)
|
||||
.order_by(
|
||||
case((IngestionQueueItem.status == QUEUE_STATUS_DROPPED, 1), else_=0).asc(),
|
||||
IngestionQueueItem.next_attempt_dt.asc(),
|
||||
IngestionQueueItem.id.asc(),
|
||||
)
|
||||
.limit(limit)
|
||||
)
|
||||
return [queue_list_item_from_row(row) for row in result.all()]
|
||||
|
||||
|
||||
async def get_queue_item_for_user(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
queue_item_id: int,
|
||||
) -> QueueListItem | None:
|
||||
result = await db_session.execute(
|
||||
queue_item_select(user_id=user_id)
|
||||
.where(IngestionQueueItem.id == queue_item_id)
|
||||
.limit(1)
|
||||
)
|
||||
row = result.one_or_none()
|
||||
if row is None:
|
||||
return None
|
||||
return queue_list_item_from_row(row)
|
||||
|
||||
|
||||
async def retry_queue_item_for_user(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
queue_item_id: int,
|
||||
) -> QueueListItem | None:
|
||||
result = await db_session.execute(
|
||||
select(IngestionQueueItem).where(
|
||||
IngestionQueueItem.id == queue_item_id,
|
||||
IngestionQueueItem.user_id == user_id,
|
||||
)
|
||||
)
|
||||
item = result.scalar_one_or_none()
|
||||
if item is None:
|
||||
return None
|
||||
if item.status == QUEUE_STATUS_QUEUED:
|
||||
raise QueueTransitionError(
|
||||
code="queue_item_already_queued",
|
||||
message="Queue item is already queued.",
|
||||
current_status=item.status,
|
||||
)
|
||||
if item.status == QUEUE_STATUS_RETRYING:
|
||||
raise QueueTransitionError(
|
||||
code="queue_item_retrying",
|
||||
message="Queue item is currently retrying.",
|
||||
current_status=item.status,
|
||||
)
|
||||
|
||||
await queue_mutations.mark_queued_now(
|
||||
db_session,
|
||||
job_id=item.id,
|
||||
reason="manual_retry",
|
||||
reset_attempt_count=(item.status == QUEUE_STATUS_DROPPED),
|
||||
)
|
||||
await db_session.commit()
|
||||
return await get_queue_item_for_user(
|
||||
db_session,
|
||||
user_id=user_id,
|
||||
queue_item_id=queue_item_id,
|
||||
)
|
||||
|
||||
|
||||
async def drop_queue_item_for_user(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
queue_item_id: int,
|
||||
) -> QueueListItem | None:
|
||||
result = await db_session.execute(
|
||||
select(IngestionQueueItem).where(
|
||||
IngestionQueueItem.id == queue_item_id,
|
||||
IngestionQueueItem.user_id == user_id,
|
||||
)
|
||||
)
|
||||
item = result.scalar_one_or_none()
|
||||
if item is None:
|
||||
return None
|
||||
if item.status == QUEUE_STATUS_DROPPED:
|
||||
raise QueueTransitionError(
|
||||
code="queue_item_already_dropped",
|
||||
message="Queue item is already dropped.",
|
||||
current_status=item.status,
|
||||
)
|
||||
|
||||
await queue_mutations.mark_dropped(
|
||||
db_session,
|
||||
job_id=int(item.id),
|
||||
reason="manual_drop",
|
||||
)
|
||||
await db_session.commit()
|
||||
return await get_queue_item_for_user(
|
||||
db_session,
|
||||
user_id=user_id,
|
||||
queue_item_id=queue_item_id,
|
||||
)
|
||||
|
||||
|
||||
async def clear_queue_item_for_user(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
queue_item_id: int,
|
||||
) -> QueueClearResult | None:
|
||||
result = await db_session.execute(
|
||||
select(IngestionQueueItem).where(
|
||||
IngestionQueueItem.id == queue_item_id,
|
||||
IngestionQueueItem.user_id == user_id,
|
||||
)
|
||||
)
|
||||
item = result.scalar_one_or_none()
|
||||
if item is None:
|
||||
return None
|
||||
if item.status != QUEUE_STATUS_DROPPED:
|
||||
raise QueueTransitionError(
|
||||
code="queue_item_not_dropped",
|
||||
message="Queue item can only be cleared after it is dropped.",
|
||||
current_status=item.status,
|
||||
)
|
||||
|
||||
item_id = int(item.id)
|
||||
previous_status = str(item.status)
|
||||
deleted = await queue_mutations.delete_job_by_id(db_session, job_id=item_id)
|
||||
await db_session.commit()
|
||||
if not deleted:
|
||||
return None
|
||||
return QueueClearResult(queue_item_id=item_id, previous_status=previous_status)
|
||||
|
||||
|
||||
async def queue_status_counts_for_user(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
) -> dict[str, int]:
|
||||
result = await db_session.execute(
|
||||
select(
|
||||
IngestionQueueItem.status,
|
||||
func.count(IngestionQueueItem.id),
|
||||
)
|
||||
.where(IngestionQueueItem.user_id == user_id)
|
||||
.group_by(IngestionQueueItem.status)
|
||||
)
|
||||
counts: dict[str, int] = {
|
||||
QUEUE_STATUS_QUEUED: 0,
|
||||
QUEUE_STATUS_RETRYING: 0,
|
||||
QUEUE_STATUS_DROPPED: 0,
|
||||
}
|
||||
for status, count in result.all():
|
||||
counts[str(status)] = int(count or 0)
|
||||
return counts
|
||||
79
app/services/domains/runs/runs_service.py
Normal file
79
app/services/domains/runs/runs_service.py
Normal file
|
|
@ -0,0 +1,79 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from sqlalchemy import or_, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.models import CrawlRun, RunStatus, RunTriggerType
|
||||
|
||||
|
||||
async def list_recent_runs_for_user(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
limit: int = 20,
|
||||
) -> list[CrawlRun]:
|
||||
result = await db_session.execute(
|
||||
select(CrawlRun)
|
||||
.where(CrawlRun.user_id == user_id)
|
||||
.order_by(CrawlRun.start_dt.desc(), CrawlRun.id.desc())
|
||||
.limit(limit)
|
||||
)
|
||||
return list(result.scalars().all())
|
||||
|
||||
|
||||
async def list_runs_for_user(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
limit: int = 100,
|
||||
failed_only: bool = False,
|
||||
) -> list[CrawlRun]:
|
||||
stmt = (
|
||||
select(CrawlRun)
|
||||
.where(CrawlRun.user_id == user_id)
|
||||
.order_by(CrawlRun.start_dt.desc(), CrawlRun.id.desc())
|
||||
.limit(limit)
|
||||
)
|
||||
if failed_only:
|
||||
stmt = stmt.where(
|
||||
CrawlRun.status.in_([RunStatus.FAILED, RunStatus.PARTIAL_FAILURE])
|
||||
)
|
||||
result = await db_session.execute(stmt)
|
||||
return list(result.scalars().all())
|
||||
|
||||
|
||||
async def get_run_for_user(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
run_id: int,
|
||||
) -> CrawlRun | None:
|
||||
result = await db_session.execute(
|
||||
select(CrawlRun).where(
|
||||
CrawlRun.user_id == user_id,
|
||||
CrawlRun.id == run_id,
|
||||
)
|
||||
)
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
|
||||
async def get_manual_run_by_idempotency_key(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
idempotency_key: str,
|
||||
) -> CrawlRun | None:
|
||||
result = await db_session.execute(
|
||||
select(CrawlRun)
|
||||
.where(
|
||||
CrawlRun.user_id == user_id,
|
||||
CrawlRun.trigger_type == RunTriggerType.MANUAL,
|
||||
or_(
|
||||
CrawlRun.idempotency_key == idempotency_key,
|
||||
CrawlRun.error_log["meta"]["idempotency_key"].astext == idempotency_key,
|
||||
),
|
||||
)
|
||||
.order_by(CrawlRun.start_dt.desc(), CrawlRun.id.desc())
|
||||
.limit(1)
|
||||
)
|
||||
return result.scalar_one_or_none()
|
||||
76
app/services/domains/runs/summary.py
Normal file
76
app/services/domains/runs/summary.py
Normal file
|
|
@ -0,0 +1,76 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
|
||||
def _safe_int(value: object, default: int = 0) -> int:
|
||||
try:
|
||||
return int(value)
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
|
||||
|
||||
def _summary_dict(error_log: object) -> dict[str, Any]:
|
||||
if not isinstance(error_log, dict):
|
||||
return {}
|
||||
summary = error_log.get("summary")
|
||||
if not isinstance(summary, dict):
|
||||
return {}
|
||||
return summary
|
||||
|
||||
|
||||
def _summary_int_dict(summary: dict[str, Any], key: str) -> dict[str, int]:
|
||||
value = summary.get(key)
|
||||
if not isinstance(value, dict):
|
||||
return {}
|
||||
return {
|
||||
str(item_key): _safe_int(item_value, 0)
|
||||
for item_key, item_value in value.items()
|
||||
if isinstance(item_key, str)
|
||||
}
|
||||
|
||||
|
||||
def _summary_bool_dict(summary: dict[str, Any], key: str) -> dict[str, bool]:
|
||||
value = summary.get(key)
|
||||
if not isinstance(value, dict):
|
||||
return {}
|
||||
return {
|
||||
str(item_key): bool(item_value)
|
||||
for item_key, item_value in value.items()
|
||||
if isinstance(item_key, str)
|
||||
}
|
||||
|
||||
|
||||
def _retry_counts(summary: dict[str, Any]) -> dict[str, int]:
|
||||
retry_counts = summary.get("retry_counts")
|
||||
if not isinstance(retry_counts, dict):
|
||||
retry_counts = {}
|
||||
return {
|
||||
"retries_scheduled_count": _safe_int(
|
||||
retry_counts.get("retries_scheduled_count", 0),
|
||||
0,
|
||||
),
|
||||
"scholars_with_retries_count": _safe_int(
|
||||
retry_counts.get("scholars_with_retries_count", 0),
|
||||
0,
|
||||
),
|
||||
"retry_exhausted_count": _safe_int(
|
||||
retry_counts.get("retry_exhausted_count", 0),
|
||||
0,
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def extract_run_summary(error_log: object) -> dict[str, Any]:
|
||||
summary = _summary_dict(error_log)
|
||||
return {
|
||||
"succeeded_count": _safe_int(summary.get("succeeded_count", 0)),
|
||||
"failed_count": _safe_int(summary.get("failed_count", 0)),
|
||||
"partial_count": _safe_int(summary.get("partial_count", 0)),
|
||||
"failed_state_counts": _summary_int_dict(summary, "failed_state_counts"),
|
||||
"failed_reason_counts": _summary_int_dict(summary, "failed_reason_counts"),
|
||||
"scrape_failure_counts": _summary_int_dict(summary, "scrape_failure_counts"),
|
||||
"retry_counts": _retry_counts(summary),
|
||||
"alert_thresholds": _summary_int_dict(summary, "alert_thresholds"),
|
||||
"alert_flags": _summary_bool_dict(summary, "alert_flags"),
|
||||
}
|
||||
38
app/services/domains/runs/types.py
Normal file
38
app/services/domains/runs/types.py
Normal file
|
|
@ -0,0 +1,38 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
|
||||
QUEUE_STATUS_QUEUED = "queued"
|
||||
QUEUE_STATUS_RETRYING = "retrying"
|
||||
QUEUE_STATUS_DROPPED = "dropped"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class QueueListItem:
|
||||
id: int
|
||||
scholar_profile_id: int
|
||||
scholar_label: str
|
||||
status: str
|
||||
reason: str
|
||||
dropped_reason: str | None
|
||||
attempt_count: int
|
||||
resume_cstart: int
|
||||
next_attempt_dt: datetime | None
|
||||
updated_at: datetime
|
||||
last_error: str | None
|
||||
last_run_id: int | None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class QueueClearResult:
|
||||
queue_item_id: int
|
||||
previous_status: str
|
||||
|
||||
|
||||
class QueueTransitionError(RuntimeError):
|
||||
def __init__(self, *, code: str, message: str, current_status: str) -> None:
|
||||
super().__init__(message)
|
||||
self.code = code
|
||||
self.message = message
|
||||
self.current_status = current_status
|
||||
1
app/services/domains/scholar/__init__.py
Normal file
1
app/services/domains/scholar/__init__.py
Normal file
|
|
@ -0,0 +1 @@
|
|||
from __future__ import annotations
|
||||
202
app/services/domains/scholar/author_rows.py
Normal file
202
app/services/domains/scholar/author_rows.py
Normal file
|
|
@ -0,0 +1,202 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from html.parser import HTMLParser
|
||||
from typing import Any
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
|
||||
from app.services.domains.scholar.parser_constants import AUTHOR_SEARCH_MARKER_KEYS
|
||||
from app.services.domains.scholar.parser_types import ScholarSearchCandidate
|
||||
from app.services.domains.scholar.parser_utils import (
|
||||
attr_class,
|
||||
attr_href,
|
||||
attr_src,
|
||||
build_absolute_scholar_url,
|
||||
normalize_space,
|
||||
)
|
||||
|
||||
|
||||
def parse_scholar_id_from_href(href: str | None) -> str | None:
|
||||
if not href:
|
||||
return None
|
||||
parsed = urlparse(href)
|
||||
query = parse_qs(parsed.query)
|
||||
user_values = query.get("user")
|
||||
if not user_values:
|
||||
return None
|
||||
candidate = user_values[0].strip()
|
||||
return candidate or None
|
||||
|
||||
|
||||
def _extract_verified_email_domain(value: str | None) -> str | None:
|
||||
if not value:
|
||||
return None
|
||||
match = re.search(r"verified email at\s+(.+)$", value.strip(), re.I)
|
||||
if not match:
|
||||
return None
|
||||
domain = normalize_space(match.group(1))
|
||||
return domain or None
|
||||
|
||||
|
||||
class ScholarAuthorSearchParser(HTMLParser):
|
||||
def __init__(self) -> None:
|
||||
super().__init__(convert_charrefs=True)
|
||||
self.candidates: list[ScholarSearchCandidate] = []
|
||||
self._candidate: dict[str, Any] | None = None
|
||||
|
||||
def _begin_candidate(self) -> None:
|
||||
self._candidate = {
|
||||
"depth": 1,
|
||||
"name_href": None,
|
||||
"name_parts": [],
|
||||
"aff_depth": 0,
|
||||
"aff_parts": [],
|
||||
"name_depth": 0,
|
||||
"eml_depth": 0,
|
||||
"eml_parts": [],
|
||||
"cby_depth": 0,
|
||||
"cby_parts": [],
|
||||
"interest_depth": 0,
|
||||
"interest_parts": [],
|
||||
"interests": [],
|
||||
"image_src": None,
|
||||
}
|
||||
|
||||
def _increment_capture_depths(self) -> None:
|
||||
if self._candidate is None:
|
||||
return
|
||||
for key in ("name_depth", "aff_depth", "eml_depth", "cby_depth", "interest_depth"):
|
||||
if self._candidate[key] > 0:
|
||||
self._candidate[key] += 1
|
||||
|
||||
def _finalize_candidate(self) -> None:
|
||||
if self._candidate is None:
|
||||
return
|
||||
|
||||
name = normalize_space("".join(self._candidate["name_parts"]))
|
||||
scholar_id = parse_scholar_id_from_href(self._candidate["name_href"])
|
||||
if not name or not scholar_id:
|
||||
return
|
||||
|
||||
affiliation = normalize_space("".join(self._candidate["aff_parts"])) or None
|
||||
email_domain = _extract_verified_email_domain(
|
||||
normalize_space("".join(self._candidate["eml_parts"])) or None
|
||||
)
|
||||
cited_by_text = normalize_space("".join(self._candidate["cby_parts"]))
|
||||
cited_by_match = re.search(r"\d+", cited_by_text)
|
||||
cited_by_count = int(cited_by_match.group(0)) if cited_by_match else None
|
||||
|
||||
seen_interests: set[str] = set()
|
||||
interests: list[str] = []
|
||||
for interest in self._candidate["interests"]:
|
||||
normalized = normalize_space(interest)
|
||||
if not normalized or normalized in seen_interests:
|
||||
continue
|
||||
seen_interests.add(normalized)
|
||||
interests.append(normalized)
|
||||
|
||||
profile_url = build_absolute_scholar_url(self._candidate["name_href"])
|
||||
if not profile_url:
|
||||
profile_url = (
|
||||
"https://scholar.google.com/citations"
|
||||
f"?hl=en&user={scholar_id}"
|
||||
)
|
||||
|
||||
self.candidates.append(
|
||||
ScholarSearchCandidate(
|
||||
scholar_id=scholar_id,
|
||||
display_name=name,
|
||||
affiliation=affiliation,
|
||||
email_domain=email_domain,
|
||||
cited_by_count=cited_by_count,
|
||||
interests=interests,
|
||||
profile_url=profile_url,
|
||||
profile_image_url=build_absolute_scholar_url(self._candidate["image_src"]),
|
||||
)
|
||||
)
|
||||
|
||||
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
||||
classes = attr_class(attrs)
|
||||
|
||||
if self._candidate is None:
|
||||
if tag == "div" and "gsc_1usr" in classes:
|
||||
self._begin_candidate()
|
||||
return
|
||||
|
||||
self._candidate["depth"] += 1
|
||||
self._increment_capture_depths()
|
||||
|
||||
if tag == "a" and "gs_ai_name" in classes:
|
||||
self._candidate["name_depth"] = 1
|
||||
self._candidate["name_href"] = attr_href(attrs)
|
||||
return
|
||||
|
||||
if tag == "div" and "gs_ai_aff" in classes:
|
||||
self._candidate["aff_depth"] = 1
|
||||
return
|
||||
|
||||
if tag == "div" and "gs_ai_eml" in classes:
|
||||
self._candidate["eml_depth"] = 1
|
||||
return
|
||||
|
||||
if tag == "div" and "gs_ai_cby" in classes:
|
||||
self._candidate["cby_depth"] = 1
|
||||
return
|
||||
|
||||
if tag == "a" and "gs_ai_one_int" in classes:
|
||||
self._candidate["interest_depth"] = 1
|
||||
self._candidate["interest_parts"] = []
|
||||
return
|
||||
|
||||
if tag == "img" and self._candidate["image_src"] is None:
|
||||
self._candidate["image_src"] = attr_src(attrs)
|
||||
|
||||
def handle_data(self, data: str) -> None:
|
||||
if self._candidate is None:
|
||||
return
|
||||
if self._candidate["name_depth"] > 0:
|
||||
self._candidate["name_parts"].append(data)
|
||||
if self._candidate["aff_depth"] > 0:
|
||||
self._candidate["aff_parts"].append(data)
|
||||
if self._candidate["eml_depth"] > 0:
|
||||
self._candidate["eml_parts"].append(data)
|
||||
if self._candidate["cby_depth"] > 0:
|
||||
self._candidate["cby_parts"].append(data)
|
||||
if self._candidate["interest_depth"] > 0:
|
||||
self._candidate["interest_parts"].append(data)
|
||||
|
||||
def _decrement_capture_depth(self, key: str) -> bool:
|
||||
if self._candidate is None:
|
||||
return False
|
||||
if self._candidate[key] <= 0:
|
||||
return False
|
||||
self._candidate[key] -= 1
|
||||
return self._candidate[key] == 0
|
||||
|
||||
def handle_endtag(self, _tag: str) -> None:
|
||||
if self._candidate is None:
|
||||
return
|
||||
|
||||
interest_closed = self._decrement_capture_depth("interest_depth")
|
||||
self._decrement_capture_depth("name_depth")
|
||||
self._decrement_capture_depth("aff_depth")
|
||||
self._decrement_capture_depth("eml_depth")
|
||||
self._decrement_capture_depth("cby_depth")
|
||||
|
||||
if interest_closed:
|
||||
interest_text = normalize_space("".join(self._candidate["interest_parts"]))
|
||||
if interest_text:
|
||||
self._candidate["interests"].append(interest_text)
|
||||
self._candidate["interest_parts"] = []
|
||||
|
||||
self._candidate["depth"] -= 1
|
||||
if self._candidate["depth"] > 0:
|
||||
return
|
||||
|
||||
self._finalize_candidate()
|
||||
self._candidate = None
|
||||
|
||||
|
||||
def count_author_search_markers(html: str) -> dict[str, int]:
|
||||
lowered = html.lower()
|
||||
return {key: lowered.count(key.lower()) for key in AUTHOR_SEARCH_MARKER_KEYS}
|
||||
110
app/services/domains/scholar/parser.py
Normal file
110
app/services/domains/scholar/parser.py
Normal file
|
|
@ -0,0 +1,110 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from app.services.domains.scholar.author_rows import (
|
||||
ScholarAuthorSearchParser,
|
||||
count_author_search_markers,
|
||||
parse_scholar_id_from_href,
|
||||
)
|
||||
from app.services.domains.scholar.parser_constants import SCRIPT_STYLE_RE
|
||||
from app.services.domains.scholar.parser_types import (
|
||||
ParseState,
|
||||
ParsedAuthorSearchPage,
|
||||
ParsedProfilePage,
|
||||
PublicationCandidate,
|
||||
ScholarSearchCandidate,
|
||||
)
|
||||
from app.services.domains.scholar.parser_utils import (
|
||||
attr_class,
|
||||
attr_href,
|
||||
attr_src,
|
||||
build_absolute_scholar_url,
|
||||
normalize_space,
|
||||
strip_tags,
|
||||
)
|
||||
from app.services.domains.scholar.profile_rows import (
|
||||
ScholarRowParser,
|
||||
count_markers,
|
||||
extract_articles_range,
|
||||
extract_profile_image_url,
|
||||
extract_profile_name,
|
||||
extract_rows,
|
||||
has_operation_error_banner,
|
||||
has_show_more_button,
|
||||
parse_citation_count,
|
||||
parse_cluster_id_from_href,
|
||||
parse_publications,
|
||||
parse_year,
|
||||
)
|
||||
from app.services.domains.scholar.source import FetchResult
|
||||
from app.services.domains.scholar.state_detection import (
|
||||
classify_block_or_captcha_reason,
|
||||
classify_network_error_reason,
|
||||
detect_author_search_state,
|
||||
detect_state,
|
||||
)
|
||||
|
||||
|
||||
def parse_profile_page(fetch_result: FetchResult) -> ParsedProfilePage:
|
||||
publications, warnings = parse_publications(fetch_result.body)
|
||||
marker_counts = count_markers(fetch_result.body)
|
||||
visible_text = strip_tags(SCRIPT_STYLE_RE.sub(" ", fetch_result.body)).lower()
|
||||
|
||||
show_more = has_show_more_button(fetch_result.body)
|
||||
operation_error_banner = has_operation_error_banner(fetch_result.body)
|
||||
articles_range = extract_articles_range(fetch_result.body)
|
||||
|
||||
if show_more:
|
||||
warnings.append("possible_partial_page_show_more_present")
|
||||
if operation_error_banner:
|
||||
warnings.append("operation_error_banner_present")
|
||||
|
||||
warnings = sorted(set(warnings))
|
||||
|
||||
state, state_reason = detect_state(
|
||||
fetch_result,
|
||||
publications,
|
||||
marker_counts,
|
||||
warnings=warnings,
|
||||
has_show_more_button_flag=show_more,
|
||||
articles_range=articles_range,
|
||||
visible_text=visible_text,
|
||||
)
|
||||
|
||||
return ParsedProfilePage(
|
||||
state=state,
|
||||
state_reason=state_reason,
|
||||
profile_name=extract_profile_name(fetch_result.body),
|
||||
profile_image_url=extract_profile_image_url(fetch_result.body),
|
||||
publications=publications,
|
||||
marker_counts=marker_counts,
|
||||
warnings=warnings,
|
||||
has_show_more_button=show_more,
|
||||
has_operation_error_banner=operation_error_banner,
|
||||
articles_range=articles_range,
|
||||
)
|
||||
|
||||
|
||||
def parse_author_search_page(fetch_result: FetchResult) -> ParsedAuthorSearchPage:
|
||||
parser = ScholarAuthorSearchParser()
|
||||
parser.feed(fetch_result.body)
|
||||
|
||||
marker_counts = count_author_search_markers(fetch_result.body)
|
||||
visible_text = strip_tags(SCRIPT_STYLE_RE.sub(" ", fetch_result.body)).lower()
|
||||
warnings: list[str] = []
|
||||
if not parser.candidates:
|
||||
warnings.append("no_author_candidates_detected")
|
||||
|
||||
state, state_reason = detect_author_search_state(
|
||||
fetch_result,
|
||||
parser.candidates,
|
||||
marker_counts,
|
||||
visible_text=visible_text,
|
||||
)
|
||||
|
||||
return ParsedAuthorSearchPage(
|
||||
state=state,
|
||||
state_reason=state_reason,
|
||||
candidates=parser.candidates,
|
||||
marker_counts=marker_counts,
|
||||
warnings=warnings,
|
||||
)
|
||||
87
app/services/domains/scholar/parser_constants.py
Normal file
87
app/services/domains/scholar/parser_constants.py
Normal file
|
|
@ -0,0 +1,87 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
BLOCKED_KEYWORDS = [
|
||||
"unusual traffic",
|
||||
"sorry/index",
|
||||
"not a robot",
|
||||
"our systems have detected",
|
||||
"automated queries",
|
||||
"recaptcha",
|
||||
"captcha",
|
||||
]
|
||||
|
||||
NO_RESULTS_KEYWORDS = [
|
||||
"didn't match any articles",
|
||||
"did not match any articles",
|
||||
"no articles",
|
||||
"no documents",
|
||||
]
|
||||
|
||||
NO_AUTHOR_RESULTS_KEYWORDS = [
|
||||
"didn't match any user profiles",
|
||||
"did not match any user profiles",
|
||||
"didn't match any scholars",
|
||||
"did not match any scholars",
|
||||
"no user profiles",
|
||||
]
|
||||
|
||||
MARKER_KEYS = [
|
||||
"gsc_a_tr",
|
||||
"gsc_a_at",
|
||||
"gsc_a_ac",
|
||||
"gsc_a_h",
|
||||
"gsc_a_y",
|
||||
"gs_gray",
|
||||
"gsc_prf_in",
|
||||
"gsc_rsb_st",
|
||||
]
|
||||
|
||||
AUTHOR_SEARCH_MARKER_KEYS = [
|
||||
"gsc_1usr",
|
||||
"gs_ai_name",
|
||||
"gs_ai_aff",
|
||||
"gs_ai_eml",
|
||||
"gs_ai_cby",
|
||||
"gs_ai_one_int",
|
||||
]
|
||||
|
||||
NETWORK_DNS_ERROR_KEYWORDS = [
|
||||
"temporary failure in name resolution",
|
||||
"name or service not known",
|
||||
"nodename nor servname provided",
|
||||
"getaddrinfo failed",
|
||||
]
|
||||
|
||||
NETWORK_TIMEOUT_KEYWORDS = [
|
||||
"timed out",
|
||||
"timeout",
|
||||
]
|
||||
|
||||
NETWORK_TLS_ERROR_KEYWORDS = [
|
||||
"ssl",
|
||||
"tls",
|
||||
"certificate verify failed",
|
||||
]
|
||||
|
||||
TAG_RE = re.compile(r"<[^>]+>", re.S)
|
||||
SCRIPT_STYLE_RE = re.compile(r"<(script|style)\b[^>]*>.*?</\1>", re.I | re.S)
|
||||
SHOW_MORE_BUTTON_RE = re.compile(
|
||||
r"<button\b[^>]*\bid\s*=\s*['\"]gsc_bpf_more['\"][^>]*>",
|
||||
re.I | re.S,
|
||||
)
|
||||
|
||||
PROFILE_ROW_PARSER_DIRECT_MARKERS = (
|
||||
"gs_ggs",
|
||||
"gs_ggsd",
|
||||
"gs_ggsa",
|
||||
"gs_or_ggsm",
|
||||
)
|
||||
|
||||
PROFILE_ROW_DIRECT_LABEL_TOKENS = (
|
||||
"pdf",
|
||||
"[pdf]",
|
||||
"full text",
|
||||
"download",
|
||||
)
|
||||
59
app/services/domains/scholar/parser_types.py
Normal file
59
app/services/domains/scholar/parser_types.py
Normal file
|
|
@ -0,0 +1,59 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from enum import StrEnum
|
||||
|
||||
|
||||
class ParseState(StrEnum):
|
||||
OK = "ok"
|
||||
NO_RESULTS = "no_results"
|
||||
BLOCKED_OR_CAPTCHA = "blocked_or_captcha"
|
||||
LAYOUT_CHANGED = "layout_changed"
|
||||
NETWORK_ERROR = "network_error"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PublicationCandidate:
|
||||
title: str
|
||||
title_url: str | None
|
||||
cluster_id: str | None
|
||||
year: int | None
|
||||
citation_count: int | None
|
||||
authors_text: str | None
|
||||
venue_text: str | None
|
||||
pdf_url: str | None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ScholarSearchCandidate:
|
||||
scholar_id: str
|
||||
display_name: str
|
||||
affiliation: str | None
|
||||
email_domain: str | None
|
||||
cited_by_count: int | None
|
||||
interests: list[str]
|
||||
profile_url: str
|
||||
profile_image_url: str | None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ParsedProfilePage:
|
||||
state: ParseState
|
||||
state_reason: str
|
||||
profile_name: str | None
|
||||
profile_image_url: str | None
|
||||
publications: list[PublicationCandidate]
|
||||
marker_counts: dict[str, int]
|
||||
warnings: list[str]
|
||||
has_show_more_button: bool
|
||||
has_operation_error_banner: bool
|
||||
articles_range: str | None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ParsedAuthorSearchPage:
|
||||
state: ParseState
|
||||
state_reason: str
|
||||
candidates: list[ScholarSearchCandidate]
|
||||
marker_counts: dict[str, int]
|
||||
warnings: list[str]
|
||||
42
app/services/domains/scholar/parser_utils.py
Normal file
42
app/services/domains/scholar/parser_utils.py
Normal file
|
|
@ -0,0 +1,42 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from html import unescape
|
||||
|
||||
from app.services.domains.scholar.parser_constants import TAG_RE
|
||||
|
||||
|
||||
def normalize_space(value: str) -> str:
|
||||
return " ".join(unescape(value).split())
|
||||
|
||||
|
||||
def strip_tags(value: str) -> str:
|
||||
return normalize_space(TAG_RE.sub(" ", value))
|
||||
|
||||
|
||||
def attr_class(attrs: list[tuple[str, str | None]]) -> str:
|
||||
for name, raw_value in attrs:
|
||||
if name.lower() == "class":
|
||||
return raw_value or ""
|
||||
return ""
|
||||
|
||||
|
||||
def attr_href(attrs: list[tuple[str, str | None]]) -> str | None:
|
||||
for name, raw_value in attrs:
|
||||
if name.lower() == "href":
|
||||
return raw_value
|
||||
return None
|
||||
|
||||
|
||||
def attr_src(attrs: list[tuple[str, str | None]]) -> str | None:
|
||||
for name, raw_value in attrs:
|
||||
if name.lower() == "src":
|
||||
return raw_value
|
||||
return None
|
||||
|
||||
|
||||
def build_absolute_scholar_url(path_or_url: str | None) -> str | None:
|
||||
if not path_or_url:
|
||||
return None
|
||||
from urllib.parse import urljoin
|
||||
|
||||
return urljoin("https://scholar.google.com", path_or_url)
|
||||
324
app/services/domains/scholar/profile_rows.py
Normal file
324
app/services/domains/scholar/profile_rows.py
Normal file
|
|
@ -0,0 +1,324 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from html.parser import HTMLParser
|
||||
from typing import Any
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
|
||||
from app.services.domains.scholar.parser_constants import (
|
||||
MARKER_KEYS,
|
||||
PROFILE_ROW_DIRECT_LABEL_TOKENS,
|
||||
PROFILE_ROW_PARSER_DIRECT_MARKERS,
|
||||
SHOW_MORE_BUTTON_RE,
|
||||
)
|
||||
from app.services.domains.scholar.parser_types import PublicationCandidate
|
||||
from app.services.domains.scholar.parser_utils import (
|
||||
attr_class,
|
||||
attr_href,
|
||||
attr_src,
|
||||
build_absolute_scholar_url,
|
||||
normalize_space,
|
||||
strip_tags,
|
||||
)
|
||||
|
||||
|
||||
class ScholarRowParser(HTMLParser):
|
||||
def __init__(self) -> None:
|
||||
super().__init__(convert_charrefs=True)
|
||||
self.title_href: str | None = None
|
||||
self.direct_download_href: str | None = None
|
||||
self.title_parts: list[str] = []
|
||||
self.citation_parts: list[str] = []
|
||||
self.year_parts: list[str] = []
|
||||
self.gray_texts: list[str] = []
|
||||
|
||||
self._title_depth = 0
|
||||
self._citation_depth = 0
|
||||
self._year_depth = 0
|
||||
self._gray_stack: list[dict[str, Any]] = []
|
||||
self._direct_marker_depth = 0
|
||||
self._aux_link_stack: list[dict[str, Any]] = []
|
||||
|
||||
@staticmethod
|
||||
def _contains_direct_marker(classes: str) -> bool:
|
||||
lowered = classes.lower()
|
||||
return any(marker in lowered for marker in PROFILE_ROW_PARSER_DIRECT_MARKERS)
|
||||
|
||||
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
||||
if self._title_depth > 0:
|
||||
self._title_depth += 1
|
||||
if self._citation_depth > 0:
|
||||
self._citation_depth += 1
|
||||
if self._year_depth > 0:
|
||||
self._year_depth += 1
|
||||
if self._gray_stack:
|
||||
self._gray_stack[-1]["depth"] += 1
|
||||
if self._direct_marker_depth > 0:
|
||||
self._direct_marker_depth += 1
|
||||
if self._aux_link_stack:
|
||||
self._aux_link_stack[-1]["depth"] += 1
|
||||
|
||||
classes = attr_class(attrs)
|
||||
if tag in {"div", "span"} and self._contains_direct_marker(classes):
|
||||
self._direct_marker_depth = 1
|
||||
|
||||
if tag == "a" and "gsc_a_at" in classes:
|
||||
self._title_depth = 1
|
||||
self.title_href = attr_href(attrs)
|
||||
return
|
||||
|
||||
if tag == "a" and "gsc_a_ac" in classes:
|
||||
self._citation_depth = 1
|
||||
return
|
||||
|
||||
if tag in {"span", "a"} and ("gsc_a_h" in classes or "gsc_a_y" in classes):
|
||||
self._year_depth = 1
|
||||
return
|
||||
|
||||
if tag == "div" and "gs_gray" in classes:
|
||||
self._gray_stack.append({"depth": 1, "parts": []})
|
||||
return
|
||||
|
||||
if tag == "a":
|
||||
href = attr_href(attrs)
|
||||
self._aux_link_stack.append(
|
||||
{
|
||||
"depth": 1,
|
||||
"classes": classes,
|
||||
"href": href,
|
||||
"parts": [],
|
||||
}
|
||||
)
|
||||
|
||||
def handle_data(self, data: str) -> None:
|
||||
if self._title_depth > 0:
|
||||
self.title_parts.append(data)
|
||||
if self._citation_depth > 0:
|
||||
self.citation_parts.append(data)
|
||||
if self._year_depth > 0:
|
||||
self.year_parts.append(data)
|
||||
if self._gray_stack:
|
||||
self._gray_stack[-1]["parts"].append(data)
|
||||
if self._aux_link_stack:
|
||||
self._aux_link_stack[-1]["parts"].append(data)
|
||||
|
||||
def _capture_direct_download_href(self, link_info: dict[str, Any]) -> None:
|
||||
if self.direct_download_href:
|
||||
return
|
||||
href = link_info.get("href")
|
||||
if not href:
|
||||
return
|
||||
classes = str(link_info.get("classes") or "")
|
||||
label = normalize_space("".join(link_info.get("parts") or [])).lower()
|
||||
if "gs_ggsd" in classes or "gs_ggs" in classes or "gs_ggsa" in classes:
|
||||
self.direct_download_href = href
|
||||
return
|
||||
label_match = any(token in label for token in PROFILE_ROW_DIRECT_LABEL_TOKENS)
|
||||
if self._direct_marker_depth > 0 and label_match:
|
||||
self.direct_download_href = href
|
||||
|
||||
def handle_endtag(self, tag: str) -> None:
|
||||
if self._title_depth > 0:
|
||||
self._title_depth -= 1
|
||||
if self._citation_depth > 0:
|
||||
self._citation_depth -= 1
|
||||
if self._year_depth > 0:
|
||||
self._year_depth -= 1
|
||||
if self._gray_stack:
|
||||
self._gray_stack[-1]["depth"] -= 1
|
||||
if self._gray_stack[-1]["depth"] == 0:
|
||||
text_value = normalize_space("".join(self._gray_stack[-1]["parts"]))
|
||||
if text_value:
|
||||
self.gray_texts.append(text_value)
|
||||
self._gray_stack.pop()
|
||||
if self._aux_link_stack:
|
||||
self._aux_link_stack[-1]["depth"] -= 1
|
||||
if self._aux_link_stack[-1]["depth"] == 0:
|
||||
link_info = self._aux_link_stack.pop()
|
||||
self._capture_direct_download_href(link_info)
|
||||
if self._direct_marker_depth > 0:
|
||||
self._direct_marker_depth -= 1
|
||||
|
||||
|
||||
def extract_rows(html: str) -> list[str]:
|
||||
pattern = re.compile(
|
||||
r"<tr\b(?=[^>]*\bclass\s*=\s*['\"][^'\"]*\bgsc_a_tr\b[^'\"]*['\"])[^>]*>(.*?)</tr>",
|
||||
re.I | re.S,
|
||||
)
|
||||
return [match.group(1) for match in pattern.finditer(html)]
|
||||
|
||||
|
||||
def parse_cluster_id_from_href(href: str | None) -> str | None:
|
||||
if not href:
|
||||
return None
|
||||
parsed = urlparse(href)
|
||||
query = parse_qs(parsed.query)
|
||||
|
||||
citation_for_view = query.get("citation_for_view")
|
||||
if citation_for_view:
|
||||
token = citation_for_view[0].strip()
|
||||
if token:
|
||||
if ":" in token:
|
||||
return token.rsplit(":", 1)[-1] or None
|
||||
return token
|
||||
|
||||
cluster = query.get("cluster")
|
||||
if cluster:
|
||||
token = cluster[0].strip()
|
||||
if token:
|
||||
return token
|
||||
return None
|
||||
|
||||
|
||||
def parse_year(parts: list[str]) -> int | None:
|
||||
text = normalize_space(" ".join(parts))
|
||||
match = re.search(r"\b(19|20)\d{2}\b", text)
|
||||
if not match:
|
||||
return None
|
||||
try:
|
||||
return int(match.group(0))
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def parse_citation_count(parts: list[str]) -> int | None:
|
||||
text = normalize_space(" ".join(parts))
|
||||
if not text:
|
||||
return 0
|
||||
match = re.search(r"\d+", text)
|
||||
if not match:
|
||||
return None
|
||||
return int(match.group(0))
|
||||
|
||||
|
||||
def _parse_publication_row(row_html: str) -> tuple[PublicationCandidate | None, list[str]]:
|
||||
parser = ScholarRowParser()
|
||||
parser.feed(row_html)
|
||||
warnings: list[str] = []
|
||||
title = normalize_space("".join(parser.title_parts))
|
||||
if not title:
|
||||
warnings.append("row_missing_title")
|
||||
return None, warnings
|
||||
if not parser.title_href:
|
||||
warnings.append("row_missing_title_href")
|
||||
|
||||
citation_text = normalize_space(" ".join(parser.citation_parts))
|
||||
citation_count = parse_citation_count(parser.citation_parts)
|
||||
if citation_text and citation_count is None:
|
||||
warnings.append("layout_row_citation_unparseable")
|
||||
|
||||
year_text = normalize_space(" ".join(parser.year_parts))
|
||||
year = parse_year(parser.year_parts)
|
||||
if year_text and year is None:
|
||||
warnings.append("layout_row_year_unparseable")
|
||||
|
||||
authors_text = parser.gray_texts[0] if len(parser.gray_texts) > 0 else None
|
||||
venue_text = parser.gray_texts[1] if len(parser.gray_texts) > 1 else None
|
||||
return (
|
||||
PublicationCandidate(
|
||||
title=title,
|
||||
title_url=parser.title_href,
|
||||
cluster_id=parse_cluster_id_from_href(parser.title_href),
|
||||
year=year,
|
||||
citation_count=citation_count,
|
||||
authors_text=authors_text,
|
||||
venue_text=venue_text,
|
||||
pdf_url=build_absolute_scholar_url(parser.direct_download_href),
|
||||
),
|
||||
warnings,
|
||||
)
|
||||
|
||||
|
||||
def parse_publications(html: str) -> tuple[list[PublicationCandidate], list[str]]:
|
||||
rows = extract_rows(html)
|
||||
warnings: list[str] = []
|
||||
publications: list[PublicationCandidate] = []
|
||||
|
||||
for row_html in rows:
|
||||
publication, row_warnings = _parse_publication_row(row_html)
|
||||
warnings.extend(row_warnings)
|
||||
if publication is None:
|
||||
continue
|
||||
publications.append(publication)
|
||||
|
||||
if not rows:
|
||||
warnings.append("no_rows_detected")
|
||||
if rows and not publications:
|
||||
warnings.append("layout_all_rows_unparseable")
|
||||
|
||||
return publications, sorted(set(warnings))
|
||||
|
||||
|
||||
def extract_profile_name(html: str) -> str | None:
|
||||
pattern = re.compile(
|
||||
r"<[^>]*\bid\s*=\s*['\"]gsc_prf_in['\"][^>]*>(.*?)</[^>]+>",
|
||||
re.I | re.S,
|
||||
)
|
||||
match = pattern.search(html)
|
||||
if not match:
|
||||
return None
|
||||
value = strip_tags(match.group(1))
|
||||
return value or None
|
||||
|
||||
|
||||
def extract_profile_image_url(html: str) -> str | None:
|
||||
og_image_pattern = re.compile(
|
||||
r"<meta[^>]+property=['\"]og:image['\"][^>]+content=['\"]([^'\"]+)['\"][^>]*>",
|
||||
re.I | re.S,
|
||||
)
|
||||
og_match = og_image_pattern.search(html)
|
||||
if og_match:
|
||||
value = normalize_space(og_match.group(1))
|
||||
absolute = build_absolute_scholar_url(value)
|
||||
if absolute:
|
||||
return absolute
|
||||
|
||||
image_pattern = re.compile(
|
||||
r"<img[^>]*\bid=['\"]gsc_prf_pup-img['\"][^>]*\bsrc=['\"]([^'\"]+)['\"][^>]*>",
|
||||
re.I | re.S,
|
||||
)
|
||||
image_match = image_pattern.search(html)
|
||||
if not image_match:
|
||||
return None
|
||||
|
||||
value = normalize_space(image_match.group(1))
|
||||
return build_absolute_scholar_url(value)
|
||||
|
||||
|
||||
def extract_articles_range(html: str) -> str | None:
|
||||
pattern = re.compile(
|
||||
r"<[^>]*\bid\s*=\s*['\"]gsc_a_nn['\"][^>]*>(.*?)</[^>]+>",
|
||||
re.I | re.S,
|
||||
)
|
||||
match = pattern.search(html)
|
||||
if not match:
|
||||
return None
|
||||
value = strip_tags(match.group(1))
|
||||
return value or None
|
||||
|
||||
|
||||
def has_show_more_button(html: str) -> bool:
|
||||
match = SHOW_MORE_BUTTON_RE.search(html)
|
||||
if match is None:
|
||||
return False
|
||||
|
||||
button_tag = match.group(0).lower()
|
||||
if "disabled" in button_tag:
|
||||
return False
|
||||
if 'aria-disabled="true"' in button_tag or "aria-disabled='true'" in button_tag:
|
||||
return False
|
||||
if "gs_dis" in button_tag:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def has_operation_error_banner(html: str) -> bool:
|
||||
lowered = html.lower()
|
||||
if "id=\"gsc_a_err\"" not in lowered and "id='gsc_a_err'" not in lowered:
|
||||
return False
|
||||
return "can't perform the operation now" in lowered or "cannot perform the operation now" in lowered
|
||||
|
||||
|
||||
def count_markers(html: str) -> dict[str, int]:
|
||||
lowered = html.lower()
|
||||
return {key: lowered.count(key.lower()) for key in MARKER_KEYS}
|
||||
225
app/services/domains/scholar/source.py
Normal file
225
app/services/domains/scholar/source.py
Normal file
|
|
@ -0,0 +1,225 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import random
|
||||
from dataclasses import dataclass
|
||||
from typing import Protocol
|
||||
from urllib.error import HTTPError, URLError
|
||||
from urllib.parse import urlencode
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
SCHOLAR_PROFILE_URL = "https://scholar.google.com/citations"
|
||||
DEFAULT_PAGE_SIZE = 100
|
||||
|
||||
DEFAULT_USER_AGENTS = [
|
||||
(
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
|
||||
),
|
||||
(
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 14_7) AppleWebKit/605.1.15 "
|
||||
"(KHTML, like Gecko) Version/18.1 Safari/605.1.15"
|
||||
),
|
||||
(
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:131.0) "
|
||||
"Gecko/20100101 Firefox/131.0"
|
||||
),
|
||||
]
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FetchResult:
|
||||
requested_url: str
|
||||
status_code: int | None
|
||||
final_url: str | None
|
||||
body: str
|
||||
error: str | None
|
||||
|
||||
|
||||
class ScholarSource(Protocol):
|
||||
async def fetch_profile_html(self, scholar_id: str) -> FetchResult:
|
||||
...
|
||||
|
||||
async def fetch_profile_page_html(
|
||||
self,
|
||||
scholar_id: str,
|
||||
*,
|
||||
cstart: int,
|
||||
pagesize: int,
|
||||
) -> FetchResult:
|
||||
...
|
||||
|
||||
async def fetch_author_search_html(
|
||||
self,
|
||||
query: str,
|
||||
*,
|
||||
start: int,
|
||||
) -> FetchResult:
|
||||
...
|
||||
|
||||
|
||||
class LiveScholarSource:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
timeout_seconds: float = 25.0,
|
||||
user_agents: list[str] | None = None,
|
||||
) -> None:
|
||||
self._timeout_seconds = timeout_seconds
|
||||
self._user_agents = user_agents or DEFAULT_USER_AGENTS
|
||||
|
||||
async def fetch_profile_html(self, scholar_id: str) -> FetchResult:
|
||||
return await self.fetch_profile_page_html(
|
||||
scholar_id,
|
||||
cstart=0,
|
||||
pagesize=DEFAULT_PAGE_SIZE,
|
||||
)
|
||||
|
||||
async def fetch_profile_page_html(
|
||||
self,
|
||||
scholar_id: str,
|
||||
*,
|
||||
cstart: int,
|
||||
pagesize: int = DEFAULT_PAGE_SIZE,
|
||||
) -> FetchResult:
|
||||
requested_url = _build_profile_url(
|
||||
scholar_id=scholar_id,
|
||||
cstart=cstart,
|
||||
pagesize=pagesize,
|
||||
)
|
||||
logger.debug(
|
||||
"scholar_source.fetch_started",
|
||||
extra={
|
||||
"event": "scholar_source.fetch_started",
|
||||
"scholar_id": scholar_id,
|
||||
"requested_url": requested_url,
|
||||
"cstart": cstart,
|
||||
"pagesize": pagesize,
|
||||
},
|
||||
)
|
||||
return await asyncio.to_thread(self._fetch_sync, requested_url)
|
||||
|
||||
async def fetch_author_search_html(
|
||||
self,
|
||||
query: str,
|
||||
*,
|
||||
start: int = 0,
|
||||
) -> FetchResult:
|
||||
requested_url = _build_author_search_url(
|
||||
query=query,
|
||||
start=start,
|
||||
)
|
||||
logger.debug(
|
||||
"scholar_source.search_fetch_started",
|
||||
extra={
|
||||
"event": "scholar_source.search_fetch_started",
|
||||
"query": query,
|
||||
"requested_url": requested_url,
|
||||
"start": start,
|
||||
},
|
||||
)
|
||||
return await asyncio.to_thread(self._fetch_sync, requested_url)
|
||||
|
||||
def _build_request(self, requested_url: str) -> Request:
|
||||
return Request(
|
||||
requested_url,
|
||||
headers={
|
||||
"User-Agent": random.choice(self._user_agents),
|
||||
"Accept": "text/html,application/xhtml+xml",
|
||||
"Accept-Language": "en-US,en;q=0.9",
|
||||
"Connection": "close",
|
||||
},
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _http_error_body(exc: HTTPError) -> str:
|
||||
try:
|
||||
return exc.read().decode("utf-8", errors="replace")
|
||||
except Exception:
|
||||
return ""
|
||||
|
||||
@staticmethod
|
||||
def _network_error_result(requested_url: str, exc: URLError) -> FetchResult:
|
||||
logger.warning(
|
||||
"scholar_source.fetch_network_error",
|
||||
extra={"event": "scholar_source.fetch_network_error", "requested_url": requested_url},
|
||||
)
|
||||
return FetchResult(
|
||||
requested_url=requested_url,
|
||||
status_code=None,
|
||||
final_url=None,
|
||||
body="",
|
||||
error=str(exc),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _http_error_result(requested_url: str, exc: HTTPError) -> FetchResult:
|
||||
logger.warning(
|
||||
"scholar_source.fetch_http_error",
|
||||
extra={
|
||||
"event": "scholar_source.fetch_http_error",
|
||||
"requested_url": requested_url,
|
||||
"status_code": exc.code,
|
||||
},
|
||||
)
|
||||
return FetchResult(
|
||||
requested_url=requested_url,
|
||||
status_code=exc.code,
|
||||
final_url=exc.geturl(),
|
||||
body=LiveScholarSource._http_error_body(exc),
|
||||
error=str(exc),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _success_result(requested_url: str, response) -> FetchResult:
|
||||
body = response.read().decode("utf-8", errors="replace")
|
||||
status_code = getattr(response, "status", 200)
|
||||
logger.debug(
|
||||
"scholar_source.fetch_succeeded",
|
||||
extra={
|
||||
"event": "scholar_source.fetch_succeeded",
|
||||
"requested_url": requested_url,
|
||||
"status_code": status_code,
|
||||
},
|
||||
)
|
||||
return FetchResult(
|
||||
requested_url=requested_url,
|
||||
status_code=status_code,
|
||||
final_url=response.geturl(),
|
||||
body=body,
|
||||
error=None,
|
||||
)
|
||||
|
||||
def _fetch_sync(self, requested_url: str) -> FetchResult:
|
||||
request = self._build_request(requested_url)
|
||||
|
||||
try:
|
||||
with urlopen(request, timeout=self._timeout_seconds) as response:
|
||||
return self._success_result(requested_url, response)
|
||||
except HTTPError as exc:
|
||||
return self._http_error_result(requested_url, exc)
|
||||
except URLError as exc:
|
||||
return self._network_error_result(requested_url, exc)
|
||||
|
||||
|
||||
def _build_profile_url(*, scholar_id: str, cstart: int, pagesize: int) -> str:
|
||||
query: dict[str, int | str] = {"hl": "en", "user": scholar_id}
|
||||
if cstart > 0:
|
||||
query["cstart"] = int(cstart)
|
||||
if pagesize > 0:
|
||||
query["pagesize"] = int(pagesize)
|
||||
return f"{SCHOLAR_PROFILE_URL}?{urlencode(query)}"
|
||||
|
||||
|
||||
def _build_author_search_url(*, query: str, start: int) -> str:
|
||||
params: dict[str, int | str] = {
|
||||
"hl": "en",
|
||||
"view_op": "search_authors",
|
||||
"mauthors": query,
|
||||
}
|
||||
if start > 0:
|
||||
params["astart"] = int(start)
|
||||
return f"{SCHOLAR_PROFILE_URL}?{urlencode(params)}"
|
||||
164
app/services/domains/scholar/state_detection.py
Normal file
164
app/services/domains/scholar/state_detection.py
Normal file
|
|
@ -0,0 +1,164 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from app.services.domains.scholar.parser_constants import (
|
||||
BLOCKED_KEYWORDS,
|
||||
NETWORK_DNS_ERROR_KEYWORDS,
|
||||
NETWORK_TIMEOUT_KEYWORDS,
|
||||
NETWORK_TLS_ERROR_KEYWORDS,
|
||||
NO_AUTHOR_RESULTS_KEYWORDS,
|
||||
NO_RESULTS_KEYWORDS,
|
||||
)
|
||||
from app.services.domains.scholar.parser_types import ParseState, PublicationCandidate, ScholarSearchCandidate
|
||||
from app.services.domains.scholar.source import FetchResult
|
||||
|
||||
|
||||
def classify_network_error_reason(fetch_error: str | None) -> str:
|
||||
lowered = (fetch_error or "").lower()
|
||||
if lowered:
|
||||
if any(keyword in lowered for keyword in NETWORK_DNS_ERROR_KEYWORDS):
|
||||
return "network_dns_resolution_failed"
|
||||
if any(keyword in lowered for keyword in NETWORK_TIMEOUT_KEYWORDS):
|
||||
return "network_timeout"
|
||||
if any(keyword in lowered for keyword in NETWORK_TLS_ERROR_KEYWORDS):
|
||||
return "network_tls_error"
|
||||
if "connection reset" in lowered:
|
||||
return "network_connection_reset"
|
||||
if "connection refused" in lowered:
|
||||
return "network_connection_refused"
|
||||
if "network is unreachable" in lowered:
|
||||
return "network_unreachable"
|
||||
return "network_error_missing_status_code"
|
||||
|
||||
|
||||
def classify_block_or_captcha_reason(
|
||||
*,
|
||||
status_code: int,
|
||||
final_url: str,
|
||||
body_lowered: str,
|
||||
) -> str | None:
|
||||
if "accounts.google.com" in final_url and ("signin" in final_url or "servicelogin" in final_url):
|
||||
return "blocked_accounts_redirect"
|
||||
if status_code == 429:
|
||||
return "blocked_http_429_rate_limited"
|
||||
if status_code == 403:
|
||||
if "recaptcha" in body_lowered or "captcha" in body_lowered or "sorry/index" in final_url:
|
||||
return "blocked_http_403_captcha_challenge"
|
||||
return "blocked_http_403_forbidden"
|
||||
if "sorry/index" in final_url or "sorry/index" in body_lowered:
|
||||
return "blocked_google_sorry_challenge"
|
||||
if "our systems have detected" in body_lowered or "unusual traffic" in body_lowered:
|
||||
return "blocked_unusual_traffic_detected"
|
||||
if "automated queries" in body_lowered:
|
||||
return "blocked_automated_queries_detected"
|
||||
if "not a robot" in body_lowered:
|
||||
return "blocked_not_a_robot_challenge"
|
||||
if "recaptcha" in body_lowered:
|
||||
return "blocked_recaptcha_challenge"
|
||||
if "captcha" in body_lowered:
|
||||
return "blocked_captcha_challenge"
|
||||
if any(keyword in body_lowered for keyword in BLOCKED_KEYWORDS):
|
||||
return "blocked_keyword_detected"
|
||||
return None
|
||||
|
||||
|
||||
def _warnings_contain(warnings: list[str], code: str) -> bool:
|
||||
return any(item == code for item in warnings)
|
||||
|
||||
|
||||
def _has_layout_row_failure(marker_counts: dict[str, int], warnings: list[str]) -> bool:
|
||||
if _warnings_contain(warnings, "layout_all_rows_unparseable"):
|
||||
return True
|
||||
if marker_counts.get("gsc_a_tr", 0) <= 0:
|
||||
return False
|
||||
if _warnings_contain(warnings, "row_missing_title"):
|
||||
return True
|
||||
return marker_counts.get("gsc_a_at", 0) <= 0
|
||||
|
||||
|
||||
def _first_layout_warning(warnings: list[str]) -> str | None:
|
||||
for warning in warnings:
|
||||
if warning.startswith("layout_"):
|
||||
return warning
|
||||
return None
|
||||
|
||||
|
||||
def detect_state(
|
||||
fetch_result: FetchResult,
|
||||
publications: list[PublicationCandidate],
|
||||
marker_counts: dict[str, int],
|
||||
*,
|
||||
warnings: list[str],
|
||||
has_show_more_button_flag: bool,
|
||||
articles_range: str | None,
|
||||
visible_text: str,
|
||||
) -> tuple[ParseState, str]:
|
||||
if fetch_result.status_code is None:
|
||||
return ParseState.NETWORK_ERROR, classify_network_error_reason(fetch_result.error)
|
||||
|
||||
lowered = fetch_result.body.lower()
|
||||
final = (fetch_result.final_url or "").lower()
|
||||
status_code = int(fetch_result.status_code)
|
||||
|
||||
block_reason = classify_block_or_captcha_reason(
|
||||
status_code=status_code,
|
||||
final_url=final,
|
||||
body_lowered=lowered,
|
||||
)
|
||||
if block_reason is not None:
|
||||
return ParseState.BLOCKED_OR_CAPTCHA, block_reason
|
||||
|
||||
if not publications and any(keyword in visible_text for keyword in NO_RESULTS_KEYWORDS):
|
||||
return ParseState.NO_RESULTS, "no_results_keyword_detected"
|
||||
|
||||
layout_warning = _first_layout_warning(warnings)
|
||||
if layout_warning is not None:
|
||||
return ParseState.LAYOUT_CHANGED, layout_warning
|
||||
|
||||
if _has_layout_row_failure(marker_counts, warnings):
|
||||
return ParseState.LAYOUT_CHANGED, "layout_publication_rows_unparseable"
|
||||
|
||||
if has_show_more_button_flag and not articles_range:
|
||||
return ParseState.LAYOUT_CHANGED, "layout_show_more_without_articles_range"
|
||||
|
||||
if not publications:
|
||||
has_profile_markers = marker_counts.get("gsc_prf_in", 0) > 0
|
||||
has_table_markers = marker_counts.get("gsc_a_tr", 0) > 0 or marker_counts.get("gsc_a_at", 0) > 0
|
||||
if not has_profile_markers and not has_table_markers:
|
||||
return ParseState.LAYOUT_CHANGED, "layout_markers_missing"
|
||||
return ParseState.OK, "no_rows_with_known_markers"
|
||||
|
||||
return ParseState.OK, "publications_extracted"
|
||||
|
||||
|
||||
def detect_author_search_state(
|
||||
fetch_result: FetchResult,
|
||||
candidates: list[ScholarSearchCandidate],
|
||||
marker_counts: dict[str, int],
|
||||
*,
|
||||
visible_text: str,
|
||||
) -> tuple[ParseState, str]:
|
||||
if fetch_result.status_code is None:
|
||||
return ParseState.NETWORK_ERROR, classify_network_error_reason(fetch_result.error)
|
||||
|
||||
lowered = fetch_result.body.lower()
|
||||
final = (fetch_result.final_url or "").lower()
|
||||
status_code = int(fetch_result.status_code)
|
||||
|
||||
block_reason = classify_block_or_captcha_reason(
|
||||
status_code=status_code,
|
||||
final_url=final,
|
||||
body_lowered=lowered,
|
||||
)
|
||||
if block_reason is not None:
|
||||
return ParseState.BLOCKED_OR_CAPTCHA, block_reason
|
||||
|
||||
if not candidates and any(keyword in visible_text for keyword in NO_AUTHOR_RESULTS_KEYWORDS):
|
||||
return ParseState.NO_RESULTS, "no_results_keyword_detected"
|
||||
|
||||
if not candidates:
|
||||
has_search_markers = marker_counts.get("gsc_1usr", 0) > 0 or marker_counts.get("gs_ai_name", 0) > 0
|
||||
if not has_search_markers:
|
||||
return ParseState.NO_RESULTS, "no_search_candidates_detected"
|
||||
return ParseState.LAYOUT_CHANGED, "layout_author_candidates_unparseable"
|
||||
|
||||
return ParseState.OK, "author_candidates_extracted"
|
||||
1
app/services/domains/scholars/__init__.py
Normal file
1
app/services/domains/scholars/__init__.py
Normal file
|
|
@ -0,0 +1 @@
|
|||
from __future__ import annotations
|
||||
966
app/services/domains/scholars/application.py
Normal file
966
app/services/domains/scholars/application.py
Normal file
|
|
@ -0,0 +1,966 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from dataclasses import replace
|
||||
from datetime import datetime
|
||||
from datetime import timedelta
|
||||
from datetime import timezone
|
||||
import logging
|
||||
import os
|
||||
import random
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import delete, func, select, text
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.models import AuthorSearchCacheEntry, AuthorSearchRuntimeState, ScholarProfile
|
||||
from app.services.domains.scholar.parser import (
|
||||
ParseState,
|
||||
ParsedAuthorSearchPage,
|
||||
parse_author_search_page,
|
||||
parse_profile_page,
|
||||
)
|
||||
from app.services.domains.scholar.source import ScholarSource
|
||||
from app.services.domains.scholars.constants import (
|
||||
ALLOWED_IMAGE_UPLOAD_CONTENT_TYPES,
|
||||
AUTHOR_SEARCH_LOCK_KEY,
|
||||
AUTHOR_SEARCH_LOCK_NAMESPACE,
|
||||
AUTHOR_SEARCH_RUNTIME_STATE_KEY,
|
||||
DEFAULT_AUTHOR_SEARCH_BLOCKED_CACHE_TTL_SECONDS,
|
||||
DEFAULT_AUTHOR_SEARCH_CACHE_MAX_ENTRIES,
|
||||
DEFAULT_AUTHOR_SEARCH_COOLDOWN_BLOCK_THRESHOLD,
|
||||
DEFAULT_AUTHOR_SEARCH_COOLDOWN_REJECTION_ALERT_THRESHOLD,
|
||||
DEFAULT_AUTHOR_SEARCH_COOLDOWN_SECONDS,
|
||||
DEFAULT_AUTHOR_SEARCH_INTERVAL_JITTER_SECONDS,
|
||||
MAX_AUTHOR_SEARCH_LIMIT,
|
||||
DEFAULT_AUTHOR_SEARCH_MIN_INTERVAL_SECONDS,
|
||||
DEFAULT_AUTHOR_SEARCH_RETRY_ALERT_THRESHOLD,
|
||||
SEARCH_CACHED_BLOCK_REASON,
|
||||
SEARCH_COOLDOWN_REASON,
|
||||
SEARCH_DISABLED_REASON,
|
||||
)
|
||||
from app.services.domains.scholars.exceptions import ScholarServiceError
|
||||
from app.services.domains.scholars.search_hints import (
|
||||
_merge_warnings,
|
||||
_policy_blocked_author_search_result,
|
||||
_trim_author_search_result,
|
||||
resolve_profile_image,
|
||||
scrape_state_hint,
|
||||
)
|
||||
from app.services.domains.scholars.uploads import (
|
||||
_ensure_upload_root,
|
||||
_resolve_upload_path,
|
||||
_safe_remove_upload,
|
||||
resolve_upload_file_path,
|
||||
)
|
||||
from app.services.domains.scholars.validators import (
|
||||
normalize_display_name,
|
||||
normalize_profile_image_url,
|
||||
validate_scholar_id,
|
||||
)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def _acquire_author_search_lock(db_session: AsyncSession) -> None:
|
||||
await db_session.execute(
|
||||
text("SELECT pg_advisory_xact_lock(:namespace, :lock_key)"),
|
||||
{
|
||||
"namespace": AUTHOR_SEARCH_LOCK_NAMESPACE,
|
||||
"lock_key": AUTHOR_SEARCH_LOCK_KEY,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
async def _load_runtime_state_for_update(
|
||||
db_session: AsyncSession,
|
||||
) -> AuthorSearchRuntimeState:
|
||||
result = await db_session.execute(
|
||||
select(AuthorSearchRuntimeState)
|
||||
.where(AuthorSearchRuntimeState.state_key == AUTHOR_SEARCH_RUNTIME_STATE_KEY)
|
||||
.with_for_update()
|
||||
)
|
||||
state = result.scalar_one_or_none()
|
||||
if state is not None:
|
||||
return state
|
||||
|
||||
state = AuthorSearchRuntimeState(state_key=AUTHOR_SEARCH_RUNTIME_STATE_KEY)
|
||||
db_session.add(state)
|
||||
await db_session.flush()
|
||||
return state
|
||||
|
||||
|
||||
def _serialize_parsed_author_search_page(parsed: ParsedAuthorSearchPage) -> dict:
|
||||
return {
|
||||
"state": parsed.state.value,
|
||||
"state_reason": parsed.state_reason,
|
||||
"marker_counts": {str(key): int(value) for key, value in parsed.marker_counts.items()},
|
||||
"warnings": [str(value) for value in parsed.warnings],
|
||||
"candidates": [
|
||||
{
|
||||
"scholar_id": candidate.scholar_id,
|
||||
"display_name": candidate.display_name,
|
||||
"affiliation": candidate.affiliation,
|
||||
"email_domain": candidate.email_domain,
|
||||
"cited_by_count": candidate.cited_by_count,
|
||||
"interests": [str(interest) for interest in candidate.interests],
|
||||
"profile_url": candidate.profile_url,
|
||||
"profile_image_url": candidate.profile_image_url,
|
||||
}
|
||||
for candidate in parsed.candidates
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _payload_state(payload: dict[str, object]) -> ParseState | None:
|
||||
state_raw = str(payload.get("state", "")).strip()
|
||||
try:
|
||||
return ParseState(state_raw)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _payload_marker_counts(payload: dict[str, object]) -> dict[str, int]:
|
||||
marker_counts_payload = payload.get("marker_counts")
|
||||
if not isinstance(marker_counts_payload, dict):
|
||||
return {}
|
||||
parsed: dict[str, int] = {}
|
||||
for key, value in marker_counts_payload.items():
|
||||
try:
|
||||
parsed[str(key)] = int(value)
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
return parsed
|
||||
|
||||
|
||||
def _payload_warnings(payload: dict[str, object]) -> list[str]:
|
||||
warnings_payload = payload.get("warnings")
|
||||
if not isinstance(warnings_payload, list):
|
||||
return []
|
||||
return [str(value) for value in warnings_payload if isinstance(value, str)]
|
||||
|
||||
|
||||
def _parse_optional_string(value: object) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
normalized = str(value).strip()
|
||||
return normalized or None
|
||||
|
||||
|
||||
def _parse_optional_int(value: object) -> int | None:
|
||||
if isinstance(value, int):
|
||||
return value
|
||||
if isinstance(value, str) and value.strip():
|
||||
try:
|
||||
return int(value)
|
||||
except ValueError:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def _normalize_interests(value: object) -> list[str]:
|
||||
if not isinstance(value, list):
|
||||
return []
|
||||
return [str(item) for item in value if isinstance(item, str)]
|
||||
|
||||
|
||||
def _deserialize_candidate_payload(value: object) -> dict[str, object] | None:
|
||||
if not isinstance(value, dict):
|
||||
return None
|
||||
scholar_id = str(value.get("scholar_id", "")).strip()
|
||||
display_name = str(value.get("display_name", "")).strip()
|
||||
profile_url = str(value.get("profile_url", "")).strip()
|
||||
if not scholar_id or not display_name or not profile_url:
|
||||
return None
|
||||
return {
|
||||
"scholar_id": scholar_id,
|
||||
"display_name": display_name,
|
||||
"affiliation": _parse_optional_string(value.get("affiliation")),
|
||||
"email_domain": _parse_optional_string(value.get("email_domain")),
|
||||
"cited_by_count": _parse_optional_int(value.get("cited_by_count")),
|
||||
"interests": _normalize_interests(value.get("interests")),
|
||||
"profile_url": profile_url,
|
||||
"profile_image_url": _parse_optional_string(value.get("profile_image_url")),
|
||||
}
|
||||
|
||||
|
||||
def _deserialize_candidates(payload: dict[str, object]) -> list[dict[str, object]]:
|
||||
candidates_payload = payload.get("candidates")
|
||||
if not isinstance(candidates_payload, list):
|
||||
return []
|
||||
normalized: list[dict[str, object]] = []
|
||||
for value in candidates_payload:
|
||||
candidate = _deserialize_candidate_payload(value)
|
||||
if candidate is not None:
|
||||
normalized.append(candidate)
|
||||
return normalized
|
||||
|
||||
|
||||
def _deserialize_parsed_author_search_page(payload: object) -> ParsedAuthorSearchPage | None:
|
||||
if not isinstance(payload, dict):
|
||||
return None
|
||||
|
||||
state = _payload_state(payload)
|
||||
if state is None:
|
||||
return None
|
||||
|
||||
marker_counts = _payload_marker_counts(payload)
|
||||
warnings = _payload_warnings(payload)
|
||||
from app.services.domains.scholar.parser import ScholarSearchCandidate
|
||||
|
||||
normalized_candidates = _deserialize_candidates(payload)
|
||||
|
||||
return ParsedAuthorSearchPage(
|
||||
state=state,
|
||||
state_reason=str(payload.get("state_reason", "")).strip() or "unknown",
|
||||
candidates=[
|
||||
ScholarSearchCandidate(
|
||||
scholar_id=item["scholar_id"],
|
||||
display_name=item["display_name"],
|
||||
affiliation=item["affiliation"],
|
||||
email_domain=item["email_domain"],
|
||||
cited_by_count=item["cited_by_count"],
|
||||
interests=item["interests"],
|
||||
profile_url=item["profile_url"],
|
||||
profile_image_url=item["profile_image_url"],
|
||||
)
|
||||
for item in normalized_candidates
|
||||
],
|
||||
marker_counts=marker_counts,
|
||||
warnings=warnings,
|
||||
)
|
||||
|
||||
|
||||
async def _cache_get_author_search_result(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
query_key: str,
|
||||
now_utc: datetime,
|
||||
) -> ParsedAuthorSearchPage | None:
|
||||
result = await db_session.execute(
|
||||
select(AuthorSearchCacheEntry).where(AuthorSearchCacheEntry.query_key == query_key)
|
||||
)
|
||||
entry = result.scalar_one_or_none()
|
||||
if entry is None:
|
||||
return None
|
||||
expires_at = entry.expires_at
|
||||
if expires_at.tzinfo is None:
|
||||
expires_at = expires_at.replace(tzinfo=timezone.utc)
|
||||
if expires_at <= now_utc:
|
||||
await db_session.delete(entry)
|
||||
return None
|
||||
parsed = _deserialize_parsed_author_search_page(entry.payload)
|
||||
if parsed is None:
|
||||
await db_session.delete(entry)
|
||||
return None
|
||||
return parsed
|
||||
|
||||
|
||||
async def _cache_set_author_search_result(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
query_key: str,
|
||||
parsed: ParsedAuthorSearchPage,
|
||||
ttl_seconds: float,
|
||||
max_entries: int,
|
||||
now_utc: datetime,
|
||||
) -> None:
|
||||
ttl = max(float(ttl_seconds), 0.0)
|
||||
existing_result = await db_session.execute(
|
||||
select(AuthorSearchCacheEntry).where(AuthorSearchCacheEntry.query_key == query_key)
|
||||
)
|
||||
existing = existing_result.scalar_one_or_none()
|
||||
|
||||
if ttl <= 0.0:
|
||||
if existing is not None:
|
||||
await db_session.delete(existing)
|
||||
return
|
||||
|
||||
expires_at = now_utc + timedelta(seconds=ttl)
|
||||
payload = _serialize_parsed_author_search_page(parsed)
|
||||
if existing is None:
|
||||
db_session.add(
|
||||
AuthorSearchCacheEntry(
|
||||
query_key=query_key,
|
||||
payload=payload,
|
||||
expires_at=expires_at,
|
||||
cached_at=now_utc,
|
||||
updated_at=now_utc,
|
||||
)
|
||||
)
|
||||
else:
|
||||
existing.payload = payload
|
||||
existing.expires_at = expires_at
|
||||
existing.cached_at = now_utc
|
||||
existing.updated_at = now_utc
|
||||
|
||||
await _prune_author_search_cache(db_session, now_utc=now_utc, max_entries=max_entries)
|
||||
|
||||
|
||||
async def _prune_author_search_cache(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
now_utc: datetime,
|
||||
max_entries: int,
|
||||
) -> None:
|
||||
await db_session.execute(
|
||||
delete(AuthorSearchCacheEntry).where(AuthorSearchCacheEntry.expires_at <= now_utc)
|
||||
)
|
||||
bounded_max_entries = max(1, int(max_entries))
|
||||
count_result = await db_session.execute(
|
||||
select(func.count()).select_from(AuthorSearchCacheEntry)
|
||||
)
|
||||
entry_count = int(count_result.scalar_one() or 0)
|
||||
overflow = max(0, entry_count - bounded_max_entries)
|
||||
if overflow <= 0:
|
||||
return
|
||||
stale_keys_result = await db_session.execute(
|
||||
select(AuthorSearchCacheEntry.query_key)
|
||||
.order_by(AuthorSearchCacheEntry.cached_at.asc())
|
||||
.limit(overflow)
|
||||
)
|
||||
stale_keys = [str(row[0]) for row in stale_keys_result.all()]
|
||||
if stale_keys:
|
||||
await db_session.execute(
|
||||
delete(AuthorSearchCacheEntry).where(AuthorSearchCacheEntry.query_key.in_(stale_keys))
|
||||
)
|
||||
|
||||
|
||||
def _is_author_search_block_state(parsed: ParsedAuthorSearchPage) -> bool:
|
||||
return parsed.state == ParseState.BLOCKED_OR_CAPTCHA
|
||||
|
||||
|
||||
def _author_search_cooldown_remaining_seconds(
|
||||
runtime_state: AuthorSearchRuntimeState,
|
||||
now_utc: datetime,
|
||||
) -> int:
|
||||
cooldown_until = runtime_state.cooldown_until
|
||||
if cooldown_until is None:
|
||||
return 0
|
||||
if cooldown_until.tzinfo is None:
|
||||
cooldown_until = cooldown_until.replace(tzinfo=timezone.utc)
|
||||
remaining_seconds = int((cooldown_until - now_utc).total_seconds())
|
||||
return max(0, remaining_seconds)
|
||||
|
||||
|
||||
def _reset_author_search_runtime_state_for_tests() -> None:
|
||||
# Runtime state now lives in the database; tests should reset via DB fixtures.
|
||||
return None
|
||||
|
||||
|
||||
async def list_scholars_for_user(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
) -> list[ScholarProfile]:
|
||||
result = await db_session.execute(
|
||||
select(ScholarProfile)
|
||||
.where(ScholarProfile.user_id == user_id)
|
||||
.order_by(ScholarProfile.created_at.desc(), ScholarProfile.id.desc())
|
||||
)
|
||||
return list(result.scalars().all())
|
||||
|
||||
|
||||
async def create_scholar_for_user(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
scholar_id: str,
|
||||
display_name: str,
|
||||
profile_image_url: str | None = None,
|
||||
) -> ScholarProfile:
|
||||
profile = ScholarProfile(
|
||||
user_id=user_id,
|
||||
scholar_id=validate_scholar_id(scholar_id),
|
||||
display_name=normalize_display_name(display_name),
|
||||
profile_image_url=normalize_profile_image_url(profile_image_url),
|
||||
)
|
||||
db_session.add(profile)
|
||||
try:
|
||||
await db_session.commit()
|
||||
except IntegrityError as exc:
|
||||
await db_session.rollback()
|
||||
raise ScholarServiceError("That scholar is already tracked for this account.") from exc
|
||||
await db_session.refresh(profile)
|
||||
return profile
|
||||
|
||||
|
||||
async def get_user_scholar_by_id(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
scholar_profile_id: int,
|
||||
) -> ScholarProfile | None:
|
||||
result = await db_session.execute(
|
||||
select(ScholarProfile).where(
|
||||
ScholarProfile.id == scholar_profile_id,
|
||||
ScholarProfile.user_id == user_id,
|
||||
)
|
||||
)
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
|
||||
async def toggle_scholar_enabled(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
profile: ScholarProfile,
|
||||
) -> ScholarProfile:
|
||||
profile.is_enabled = not profile.is_enabled
|
||||
await db_session.commit()
|
||||
await db_session.refresh(profile)
|
||||
return profile
|
||||
|
||||
|
||||
async def delete_scholar(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
profile: ScholarProfile,
|
||||
upload_dir: str | None = None,
|
||||
) -> None:
|
||||
if upload_dir:
|
||||
upload_root = _ensure_upload_root(upload_dir, create=True)
|
||||
_safe_remove_upload(upload_root, profile.profile_image_upload_path)
|
||||
|
||||
await db_session.delete(profile)
|
||||
await db_session.commit()
|
||||
|
||||
|
||||
def _normalize_author_search_inputs(query: str, limit: int) -> tuple[str, int, str]:
|
||||
normalized_query = query.strip()
|
||||
if len(normalized_query) < 2:
|
||||
raise ScholarServiceError("Search query must be at least 2 characters.")
|
||||
bounded_limit = max(1, min(int(limit), MAX_AUTHOR_SEARCH_LIMIT))
|
||||
return normalized_query, bounded_limit, normalized_query.casefold()
|
||||
|
||||
|
||||
def _disabled_search_result(*, normalized_query: str, bounded_limit: int) -> ParsedAuthorSearchPage:
|
||||
logger.warning(
|
||||
"scholar_search.disabled_by_configuration",
|
||||
extra={"event": "scholar_search.disabled_by_configuration", "query": normalized_query},
|
||||
)
|
||||
return _policy_blocked_author_search_result(
|
||||
reason=SEARCH_DISABLED_REASON,
|
||||
warning_codes=["author_search_disabled_by_configuration"],
|
||||
limit=bounded_limit,
|
||||
)
|
||||
|
||||
|
||||
def _normalize_runtime_cooldown_state(
|
||||
runtime_state: AuthorSearchRuntimeState,
|
||||
*,
|
||||
now_utc: datetime,
|
||||
) -> bool:
|
||||
if runtime_state.cooldown_until is None:
|
||||
return False
|
||||
cooldown_until = runtime_state.cooldown_until
|
||||
updated = False
|
||||
if cooldown_until.tzinfo is None:
|
||||
cooldown_until = cooldown_until.replace(tzinfo=timezone.utc)
|
||||
runtime_state.cooldown_until = cooldown_until
|
||||
updated = True
|
||||
if now_utc < cooldown_until:
|
||||
return updated
|
||||
logger.info(
|
||||
"scholar_search.cooldown_expired",
|
||||
extra={"event": "scholar_search.cooldown_expired", "cooldown_until_utc": cooldown_until.isoformat()},
|
||||
)
|
||||
runtime_state.cooldown_until = None
|
||||
runtime_state.cooldown_rejection_count = 0
|
||||
runtime_state.cooldown_alert_emitted = False
|
||||
return True
|
||||
|
||||
|
||||
def _cooldown_warning_codes(
|
||||
*,
|
||||
runtime_state: AuthorSearchRuntimeState,
|
||||
cooldown_remaining_seconds: int,
|
||||
) -> list[str]:
|
||||
warning_codes = [
|
||||
"author_search_cooldown_active",
|
||||
f"author_search_cooldown_remaining_{cooldown_remaining_seconds}s",
|
||||
]
|
||||
if bool(runtime_state.cooldown_alert_emitted):
|
||||
warning_codes.append("author_search_cooldown_alert_threshold_exceeded")
|
||||
return warning_codes
|
||||
|
||||
|
||||
def _emit_cooldown_threshold_alert(
|
||||
*,
|
||||
runtime_state: AuthorSearchRuntimeState,
|
||||
normalized_query: str,
|
||||
cooldown_rejection_alert_threshold: int,
|
||||
) -> bool:
|
||||
runtime_state.cooldown_rejection_count = int(runtime_state.cooldown_rejection_count) + 1
|
||||
threshold = max(1, int(cooldown_rejection_alert_threshold))
|
||||
if int(runtime_state.cooldown_rejection_count) < threshold:
|
||||
return True
|
||||
if bool(runtime_state.cooldown_alert_emitted):
|
||||
return True
|
||||
logger.error(
|
||||
"scholar_search.cooldown_rejection_threshold_exceeded",
|
||||
extra={
|
||||
"event": "scholar_search.cooldown_rejection_threshold_exceeded",
|
||||
"query": normalized_query,
|
||||
"cooldown_rejection_count": int(runtime_state.cooldown_rejection_count),
|
||||
"threshold": threshold,
|
||||
"cooldown_until_utc": runtime_state.cooldown_until.isoformat() if runtime_state.cooldown_until else None,
|
||||
},
|
||||
)
|
||||
runtime_state.cooldown_alert_emitted = True
|
||||
return True
|
||||
|
||||
|
||||
def _cooldown_block_result(
|
||||
*,
|
||||
runtime_state: AuthorSearchRuntimeState,
|
||||
normalized_query: str,
|
||||
bounded_limit: int,
|
||||
cooldown_rejection_alert_threshold: int,
|
||||
cooldown_remaining_seconds: int,
|
||||
) -> ParsedAuthorSearchPage:
|
||||
_emit_cooldown_threshold_alert(
|
||||
runtime_state=runtime_state,
|
||||
normalized_query=normalized_query,
|
||||
cooldown_rejection_alert_threshold=cooldown_rejection_alert_threshold,
|
||||
)
|
||||
logger.warning(
|
||||
"scholar_search.cooldown_active",
|
||||
extra={
|
||||
"event": "scholar_search.cooldown_active",
|
||||
"query": normalized_query,
|
||||
"cooldown_remaining_seconds": cooldown_remaining_seconds,
|
||||
"cooldown_until_utc": runtime_state.cooldown_until.isoformat() if runtime_state.cooldown_until else None,
|
||||
},
|
||||
)
|
||||
return _policy_blocked_author_search_result(
|
||||
reason=SEARCH_COOLDOWN_REASON,
|
||||
warning_codes=_cooldown_warning_codes(
|
||||
runtime_state=runtime_state,
|
||||
cooldown_remaining_seconds=cooldown_remaining_seconds,
|
||||
),
|
||||
limit=bounded_limit,
|
||||
)
|
||||
|
||||
|
||||
async def _cache_hit_result(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
query_key: str,
|
||||
now_utc: datetime,
|
||||
normalized_query: str,
|
||||
bounded_limit: int,
|
||||
) -> ParsedAuthorSearchPage | None:
|
||||
cached = await _cache_get_author_search_result(
|
||||
db_session,
|
||||
query_key=query_key,
|
||||
now_utc=now_utc,
|
||||
)
|
||||
if cached is None:
|
||||
return None
|
||||
logger.info(
|
||||
"scholar_search.cache_hit",
|
||||
extra={
|
||||
"event": "scholar_search.cache_hit",
|
||||
"query": normalized_query,
|
||||
"state": cached.state.value,
|
||||
"state_reason": cached.state_reason,
|
||||
},
|
||||
)
|
||||
state_reason_override = SEARCH_CACHED_BLOCK_REASON if _is_author_search_block_state(cached) else None
|
||||
return _trim_author_search_result(
|
||||
cached,
|
||||
limit=bounded_limit,
|
||||
extra_warnings=["author_search_served_from_cache"],
|
||||
state_reason_override=state_reason_override,
|
||||
)
|
||||
|
||||
|
||||
def _throttle_sleep_seconds(
|
||||
*,
|
||||
runtime_state: AuthorSearchRuntimeState,
|
||||
now_utc: datetime,
|
||||
min_interval_seconds: float,
|
||||
interval_jitter_seconds: float,
|
||||
) -> tuple[float, bool]:
|
||||
updated = False
|
||||
if runtime_state.last_live_request_at is None:
|
||||
enforced_wait_seconds = 0.0
|
||||
else:
|
||||
last_live_request_at = runtime_state.last_live_request_at
|
||||
if last_live_request_at.tzinfo is None:
|
||||
last_live_request_at = last_live_request_at.replace(tzinfo=timezone.utc)
|
||||
runtime_state.last_live_request_at = last_live_request_at
|
||||
updated = True
|
||||
enforced_wait_seconds = (
|
||||
last_live_request_at + timedelta(seconds=max(float(min_interval_seconds), 0.0)) - now_utc
|
||||
).total_seconds()
|
||||
jitter_seconds = random.uniform(0.0, max(float(interval_jitter_seconds), 0.0))
|
||||
return max(0.0, float(enforced_wait_seconds)) + jitter_seconds, updated
|
||||
|
||||
|
||||
async def _wait_for_author_search_throttle(
|
||||
*,
|
||||
runtime_state: AuthorSearchRuntimeState,
|
||||
normalized_query: str,
|
||||
now_utc: datetime,
|
||||
min_interval_seconds: float,
|
||||
interval_jitter_seconds: float,
|
||||
) -> bool:
|
||||
sleep_seconds, updated = _throttle_sleep_seconds(
|
||||
runtime_state=runtime_state,
|
||||
now_utc=now_utc,
|
||||
min_interval_seconds=min_interval_seconds,
|
||||
interval_jitter_seconds=interval_jitter_seconds,
|
||||
)
|
||||
if sleep_seconds <= 0.0:
|
||||
return updated
|
||||
logger.info(
|
||||
"scholar_search.throttle_wait",
|
||||
extra={"event": "scholar_search.throttle_wait", "query": normalized_query, "sleep_seconds": round(sleep_seconds, 3)},
|
||||
)
|
||||
await asyncio.sleep(sleep_seconds)
|
||||
return True
|
||||
|
||||
|
||||
async def _fetch_author_search_with_retries(
|
||||
*,
|
||||
source: ScholarSource,
|
||||
normalized_query: str,
|
||||
network_error_retries: int,
|
||||
retry_backoff_seconds: float,
|
||||
) -> tuple[ParsedAuthorSearchPage, int, list[str]]:
|
||||
max_attempts = max(1, int(network_error_retries) + 1)
|
||||
parsed: ParsedAuthorSearchPage | None = None
|
||||
retry_warnings: list[str] = []
|
||||
retry_scheduled_count = 0
|
||||
for attempt_index in range(max_attempts):
|
||||
fetch_result = await source.fetch_author_search_html(normalized_query, start=0)
|
||||
parsed = parse_author_search_page(fetch_result)
|
||||
if parsed.state != ParseState.NETWORK_ERROR or attempt_index >= max_attempts - 1:
|
||||
break
|
||||
retry_warnings.append("network_retry_scheduled_for_author_search")
|
||||
retry_scheduled_count += 1
|
||||
retry_sleep_seconds = max(float(retry_backoff_seconds), 0.0) * (2**attempt_index)
|
||||
if retry_sleep_seconds > 0:
|
||||
await asyncio.sleep(retry_sleep_seconds)
|
||||
if parsed is None:
|
||||
raise ScholarServiceError("Unable to complete scholar author search.")
|
||||
return parsed, retry_scheduled_count, retry_warnings
|
||||
|
||||
|
||||
def _with_retry_warnings(
|
||||
parsed: ParsedAuthorSearchPage,
|
||||
*,
|
||||
retry_warnings: list[str],
|
||||
retry_scheduled_count: int,
|
||||
retry_alert_threshold: int,
|
||||
normalized_query: str,
|
||||
) -> ParsedAuthorSearchPage:
|
||||
merged = replace(parsed, warnings=_merge_warnings(parsed.warnings, retry_warnings))
|
||||
threshold = max(1, int(retry_alert_threshold))
|
||||
if retry_scheduled_count < threshold:
|
||||
return merged
|
||||
logger.warning(
|
||||
"scholar_search.retry_threshold_exceeded",
|
||||
extra={
|
||||
"event": "scholar_search.retry_threshold_exceeded",
|
||||
"query": normalized_query,
|
||||
"retry_scheduled_count": retry_scheduled_count,
|
||||
"threshold": threshold,
|
||||
"final_state": merged.state.value,
|
||||
"final_state_reason": merged.state_reason,
|
||||
},
|
||||
)
|
||||
return replace(
|
||||
merged,
|
||||
warnings=_merge_warnings(
|
||||
merged.warnings,
|
||||
[f"author_search_retry_threshold_exceeded_{retry_scheduled_count}"],
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _apply_block_circuit_breaker(
|
||||
*,
|
||||
runtime_state: AuthorSearchRuntimeState,
|
||||
merged_parsed: ParsedAuthorSearchPage,
|
||||
cooldown_block_threshold: int,
|
||||
cooldown_seconds: int,
|
||||
normalized_query: str,
|
||||
) -> ParsedAuthorSearchPage:
|
||||
if not _is_author_search_block_state(merged_parsed):
|
||||
runtime_state.consecutive_blocked_count = 0
|
||||
return merged_parsed
|
||||
runtime_state.consecutive_blocked_count = int(runtime_state.consecutive_blocked_count) + 1
|
||||
logger.warning(
|
||||
"scholar_search.block_detected",
|
||||
extra={
|
||||
"event": "scholar_search.block_detected",
|
||||
"query": normalized_query,
|
||||
"state_reason": merged_parsed.state_reason,
|
||||
"consecutive_blocked_count": int(runtime_state.consecutive_blocked_count),
|
||||
},
|
||||
)
|
||||
if int(runtime_state.consecutive_blocked_count) < max(1, int(cooldown_block_threshold)):
|
||||
return merged_parsed
|
||||
runtime_state.cooldown_until = datetime.now(timezone.utc) + timedelta(seconds=max(60, int(cooldown_seconds)))
|
||||
runtime_state.consecutive_blocked_count = 0
|
||||
runtime_state.cooldown_rejection_count = 0
|
||||
runtime_state.cooldown_alert_emitted = False
|
||||
logger.error(
|
||||
"scholar_search.cooldown_activated",
|
||||
extra={
|
||||
"event": "scholar_search.cooldown_activated",
|
||||
"query": normalized_query,
|
||||
"cooldown_until_utc": runtime_state.cooldown_until.isoformat() if runtime_state.cooldown_until else None,
|
||||
},
|
||||
)
|
||||
return replace(
|
||||
merged_parsed,
|
||||
warnings=_merge_warnings(merged_parsed.warnings, ["author_search_circuit_breaker_armed"]),
|
||||
)
|
||||
|
||||
|
||||
def _resolve_author_search_cache_ttl_seconds(
|
||||
*,
|
||||
merged_parsed: ParsedAuthorSearchPage,
|
||||
blocked_cache_ttl_seconds: int,
|
||||
cache_ttl_seconds: int,
|
||||
) -> int:
|
||||
if _is_author_search_block_state(merged_parsed):
|
||||
return min(max(1, int(blocked_cache_ttl_seconds)), max(1, int(cache_ttl_seconds)))
|
||||
return max(1, int(cache_ttl_seconds))
|
||||
|
||||
|
||||
async def _load_locked_runtime_state(
|
||||
db_session: AsyncSession,
|
||||
) -> AuthorSearchRuntimeState:
|
||||
await _acquire_author_search_lock(db_session)
|
||||
return await _load_runtime_state_for_update(db_session)
|
||||
|
||||
|
||||
async def _cooldown_or_cache_result(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
runtime_state: AuthorSearchRuntimeState,
|
||||
query_key: str,
|
||||
normalized_query: str,
|
||||
bounded_limit: int,
|
||||
cooldown_rejection_alert_threshold: int,
|
||||
) -> tuple[ParsedAuthorSearchPage | None, bool]:
|
||||
runtime_state_updated = _normalize_runtime_cooldown_state(
|
||||
runtime_state,
|
||||
now_utc=datetime.now(timezone.utc),
|
||||
)
|
||||
cooldown_remaining_seconds = _author_search_cooldown_remaining_seconds(
|
||||
runtime_state,
|
||||
datetime.now(timezone.utc),
|
||||
)
|
||||
if cooldown_remaining_seconds > 0:
|
||||
return (
|
||||
_cooldown_block_result(
|
||||
runtime_state=runtime_state,
|
||||
normalized_query=normalized_query,
|
||||
bounded_limit=bounded_limit,
|
||||
cooldown_rejection_alert_threshold=cooldown_rejection_alert_threshold,
|
||||
cooldown_remaining_seconds=cooldown_remaining_seconds,
|
||||
),
|
||||
True,
|
||||
)
|
||||
cached_result = await _cache_hit_result(
|
||||
db_session,
|
||||
query_key=query_key,
|
||||
now_utc=datetime.now(timezone.utc),
|
||||
normalized_query=normalized_query,
|
||||
bounded_limit=bounded_limit,
|
||||
)
|
||||
return cached_result, runtime_state_updated
|
||||
|
||||
|
||||
async def _perform_live_author_search(db_session: AsyncSession, *, source: ScholarSource, runtime_state: AuthorSearchRuntimeState, normalized_query: str, query_key: str, network_error_retries: int, retry_backoff_seconds: float, min_interval_seconds: float, interval_jitter_seconds: float, retry_alert_threshold: int, cooldown_block_threshold: int, cooldown_seconds: int, blocked_cache_ttl_seconds: int, cache_ttl_seconds: int, cache_max_entries: int) -> tuple[ParsedAuthorSearchPage, bool]:
|
||||
runtime_state_updated = await _wait_for_author_search_throttle(
|
||||
runtime_state=runtime_state,
|
||||
normalized_query=normalized_query,
|
||||
now_utc=datetime.now(timezone.utc),
|
||||
min_interval_seconds=min_interval_seconds,
|
||||
interval_jitter_seconds=interval_jitter_seconds,
|
||||
)
|
||||
parsed, retry_count, retry_warnings = await _fetch_author_search_with_retries(
|
||||
source=source,
|
||||
normalized_query=normalized_query,
|
||||
network_error_retries=network_error_retries,
|
||||
retry_backoff_seconds=retry_backoff_seconds,
|
||||
)
|
||||
runtime_state.last_live_request_at = datetime.now(timezone.utc)
|
||||
merged = _with_retry_warnings(
|
||||
parsed,
|
||||
retry_warnings=retry_warnings,
|
||||
retry_scheduled_count=retry_count,
|
||||
retry_alert_threshold=retry_alert_threshold,
|
||||
normalized_query=normalized_query,
|
||||
)
|
||||
merged = _apply_block_circuit_breaker(
|
||||
runtime_state=runtime_state,
|
||||
merged_parsed=merged,
|
||||
cooldown_block_threshold=cooldown_block_threshold,
|
||||
cooldown_seconds=cooldown_seconds,
|
||||
normalized_query=normalized_query,
|
||||
)
|
||||
ttl_seconds = _resolve_author_search_cache_ttl_seconds(
|
||||
merged_parsed=merged,
|
||||
blocked_cache_ttl_seconds=blocked_cache_ttl_seconds,
|
||||
cache_ttl_seconds=cache_ttl_seconds,
|
||||
)
|
||||
await _cache_set_author_search_result(
|
||||
db_session,
|
||||
query_key=query_key,
|
||||
parsed=merged,
|
||||
ttl_seconds=float(ttl_seconds),
|
||||
max_entries=cache_max_entries,
|
||||
now_utc=datetime.now(timezone.utc),
|
||||
)
|
||||
return merged, True
|
||||
|
||||
|
||||
async def search_author_candidates(*, source: ScholarSource, db_session: AsyncSession, query: str, limit: int, network_error_retries: int = 1, retry_backoff_seconds: float = 1.0, search_enabled: bool = True, cache_ttl_seconds: int = 21_600, blocked_cache_ttl_seconds: int = DEFAULT_AUTHOR_SEARCH_BLOCKED_CACHE_TTL_SECONDS, cache_max_entries: int = DEFAULT_AUTHOR_SEARCH_CACHE_MAX_ENTRIES, min_interval_seconds: float = DEFAULT_AUTHOR_SEARCH_MIN_INTERVAL_SECONDS, interval_jitter_seconds: float = DEFAULT_AUTHOR_SEARCH_INTERVAL_JITTER_SECONDS, cooldown_block_threshold: int = DEFAULT_AUTHOR_SEARCH_COOLDOWN_BLOCK_THRESHOLD, cooldown_seconds: int = DEFAULT_AUTHOR_SEARCH_COOLDOWN_SECONDS, retry_alert_threshold: int = DEFAULT_AUTHOR_SEARCH_RETRY_ALERT_THRESHOLD, cooldown_rejection_alert_threshold: int = DEFAULT_AUTHOR_SEARCH_COOLDOWN_REJECTION_ALERT_THRESHOLD) -> ParsedAuthorSearchPage:
|
||||
normalized_query, bounded_limit, query_key = _normalize_author_search_inputs(query, limit)
|
||||
if not search_enabled:
|
||||
return _disabled_search_result(
|
||||
normalized_query=normalized_query,
|
||||
bounded_limit=bounded_limit,
|
||||
)
|
||||
|
||||
runtime_state = await _load_locked_runtime_state(db_session)
|
||||
early_result, runtime_state_updated = await _cooldown_or_cache_result(
|
||||
db_session,
|
||||
runtime_state=runtime_state,
|
||||
query_key=query_key,
|
||||
normalized_query=normalized_query,
|
||||
bounded_limit=bounded_limit,
|
||||
cooldown_rejection_alert_threshold=cooldown_rejection_alert_threshold,
|
||||
)
|
||||
if early_result is not None:
|
||||
await db_session.commit()
|
||||
return early_result
|
||||
|
||||
merged_parsed, live_runtime_updated = await _perform_live_author_search(
|
||||
db_session,
|
||||
source=source,
|
||||
runtime_state=runtime_state,
|
||||
normalized_query=normalized_query,
|
||||
query_key=query_key,
|
||||
network_error_retries=network_error_retries,
|
||||
retry_backoff_seconds=retry_backoff_seconds,
|
||||
min_interval_seconds=min_interval_seconds,
|
||||
interval_jitter_seconds=interval_jitter_seconds,
|
||||
retry_alert_threshold=retry_alert_threshold,
|
||||
cooldown_block_threshold=cooldown_block_threshold,
|
||||
cooldown_seconds=cooldown_seconds,
|
||||
blocked_cache_ttl_seconds=blocked_cache_ttl_seconds,
|
||||
cache_ttl_seconds=cache_ttl_seconds,
|
||||
cache_max_entries=cache_max_entries,
|
||||
)
|
||||
runtime_state_updated = runtime_state_updated or live_runtime_updated
|
||||
if runtime_state_updated:
|
||||
await db_session.commit()
|
||||
return _trim_author_search_result(merged_parsed, limit=bounded_limit)
|
||||
|
||||
|
||||
async def hydrate_profile_metadata(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
profile: ScholarProfile,
|
||||
source: ScholarSource,
|
||||
) -> ScholarProfile:
|
||||
fetch_result = await source.fetch_profile_html(profile.scholar_id)
|
||||
parsed_page = parse_profile_page(fetch_result)
|
||||
|
||||
if parsed_page.profile_name and not (profile.display_name or "").strip():
|
||||
profile.display_name = parsed_page.profile_name
|
||||
if parsed_page.profile_image_url and not profile.profile_image_url:
|
||||
profile.profile_image_url = parsed_page.profile_image_url
|
||||
|
||||
await db_session.commit()
|
||||
await db_session.refresh(profile)
|
||||
return profile
|
||||
|
||||
|
||||
async def set_profile_image_override_url(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
profile: ScholarProfile,
|
||||
image_url: str | None,
|
||||
upload_dir: str,
|
||||
) -> ScholarProfile:
|
||||
upload_root = _ensure_upload_root(upload_dir, create=True)
|
||||
_safe_remove_upload(upload_root, profile.profile_image_upload_path)
|
||||
|
||||
profile.profile_image_upload_path = None
|
||||
profile.profile_image_override_url = normalize_profile_image_url(image_url)
|
||||
|
||||
await db_session.commit()
|
||||
await db_session.refresh(profile)
|
||||
return profile
|
||||
|
||||
|
||||
async def clear_profile_image_customization(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
profile: ScholarProfile,
|
||||
upload_dir: str,
|
||||
) -> ScholarProfile:
|
||||
upload_root = _ensure_upload_root(upload_dir, create=True)
|
||||
_safe_remove_upload(upload_root, profile.profile_image_upload_path)
|
||||
|
||||
profile.profile_image_upload_path = None
|
||||
profile.profile_image_override_url = None
|
||||
|
||||
await db_session.commit()
|
||||
await db_session.refresh(profile)
|
||||
return profile
|
||||
|
||||
|
||||
async def set_profile_image_upload(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
profile: ScholarProfile,
|
||||
content_type: str | None,
|
||||
image_bytes: bytes,
|
||||
upload_dir: str,
|
||||
max_upload_bytes: int,
|
||||
) -> ScholarProfile:
|
||||
normalized_content_type = (content_type or "").strip().lower()
|
||||
extension = ALLOWED_IMAGE_UPLOAD_CONTENT_TYPES.get(normalized_content_type)
|
||||
if extension is None:
|
||||
raise ScholarServiceError(
|
||||
"Unsupported image type. Use JPEG, PNG, WEBP, or GIF."
|
||||
)
|
||||
|
||||
if not image_bytes:
|
||||
raise ScholarServiceError("Uploaded image file is empty.")
|
||||
|
||||
if len(image_bytes) > max_upload_bytes:
|
||||
raise ScholarServiceError(
|
||||
f"Uploaded image exceeds {max_upload_bytes} bytes."
|
||||
)
|
||||
|
||||
upload_root = _ensure_upload_root(upload_dir, create=True)
|
||||
user_dir = upload_root / str(profile.user_id)
|
||||
user_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
filename = f"{profile.id}_{uuid4().hex}{extension}"
|
||||
relative_path = os.path.join(str(profile.user_id), filename)
|
||||
absolute_path = _resolve_upload_path(upload_root, relative_path)
|
||||
absolute_path.write_bytes(image_bytes)
|
||||
|
||||
old_path = profile.profile_image_upload_path
|
||||
profile.profile_image_upload_path = relative_path
|
||||
profile.profile_image_override_url = None
|
||||
|
||||
await db_session.commit()
|
||||
await db_session.refresh(profile)
|
||||
|
||||
if old_path and old_path != relative_path:
|
||||
_safe_remove_upload(upload_root, old_path)
|
||||
|
||||
return profile
|
||||
70
app/services/domains/scholars/constants.py
Normal file
70
app/services/domains/scholars/constants.py
Normal file
|
|
@ -0,0 +1,70 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
SCHOLAR_ID_PATTERN = re.compile(r"^[a-zA-Z0-9_-]{12}$")
|
||||
MAX_IMAGE_URL_LENGTH = 2048
|
||||
MAX_AUTHOR_SEARCH_LIMIT = 25
|
||||
|
||||
DEFAULT_AUTHOR_SEARCH_CACHE_MAX_ENTRIES = 512
|
||||
DEFAULT_AUTHOR_SEARCH_BLOCKED_CACHE_TTL_SECONDS = 300
|
||||
DEFAULT_AUTHOR_SEARCH_COOLDOWN_BLOCK_THRESHOLD = 1
|
||||
DEFAULT_AUTHOR_SEARCH_COOLDOWN_SECONDS = 1800
|
||||
DEFAULT_AUTHOR_SEARCH_MIN_INTERVAL_SECONDS = 3.0
|
||||
DEFAULT_AUTHOR_SEARCH_INTERVAL_JITTER_SECONDS = 1.0
|
||||
DEFAULT_AUTHOR_SEARCH_RETRY_ALERT_THRESHOLD = 2
|
||||
DEFAULT_AUTHOR_SEARCH_COOLDOWN_REJECTION_ALERT_THRESHOLD = 3
|
||||
|
||||
AUTHOR_SEARCH_RUNTIME_STATE_KEY = "global"
|
||||
AUTHOR_SEARCH_LOCK_NAMESPACE = 3901
|
||||
AUTHOR_SEARCH_LOCK_KEY = 1
|
||||
|
||||
ALLOWED_IMAGE_UPLOAD_CONTENT_TYPES = {
|
||||
"image/jpeg": ".jpg",
|
||||
"image/png": ".png",
|
||||
"image/webp": ".webp",
|
||||
"image/gif": ".gif",
|
||||
}
|
||||
|
||||
SEARCH_DISABLED_REASON = "search_disabled_by_configuration"
|
||||
SEARCH_COOLDOWN_REASON = "search_temporarily_disabled_due_to_repeated_blocks"
|
||||
SEARCH_CACHED_BLOCK_REASON = "search_temporarily_disabled_from_cached_blocked_response"
|
||||
|
||||
STATE_REASON_HINTS: dict[str, str] = {
|
||||
SEARCH_DISABLED_REASON: (
|
||||
"Scholar name search is currently disabled by service policy. "
|
||||
"Add scholars by profile URL or Scholar ID."
|
||||
),
|
||||
SEARCH_COOLDOWN_REASON: (
|
||||
"Scholar name search is temporarily paused after repeated block responses. "
|
||||
"Use Scholar URL/ID adds until cooldown expires."
|
||||
),
|
||||
SEARCH_CACHED_BLOCK_REASON: (
|
||||
"A recent blocked response was cached to reduce traffic. "
|
||||
"Retry later or add by Scholar URL/ID."
|
||||
),
|
||||
"network_dns_resolution_failed": (
|
||||
"DNS resolution failed while reaching scholar.google.com. "
|
||||
"Verify container DNS/network and retry."
|
||||
),
|
||||
"network_timeout": (
|
||||
"Request timed out before Google Scholar responded. "
|
||||
"Increase delay/backoff and retry."
|
||||
),
|
||||
"network_tls_error": (
|
||||
"TLS handshake/certificate validation failed. "
|
||||
"Verify outbound TLS/network configuration."
|
||||
),
|
||||
"blocked_http_429_rate_limited": (
|
||||
"Google Scholar rate-limited the request. "
|
||||
"Slow request cadence and retry later."
|
||||
),
|
||||
"blocked_unusual_traffic_detected": (
|
||||
"Google Scholar flagged traffic as unusual. "
|
||||
"Increase delay/jitter and reduce concurrent scraping."
|
||||
),
|
||||
"blocked_accounts_redirect": (
|
||||
"Request was redirected to Google Account sign-in. "
|
||||
"Treat as access block and retry later."
|
||||
),
|
||||
}
|
||||
5
app/services/domains/scholars/exceptions.py
Normal file
5
app/services/domains/scholars/exceptions.py
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
from __future__ import annotations
|
||||
|
||||
|
||||
class ScholarServiceError(ValueError):
|
||||
"""Raised for expected scholar-management validation failures."""
|
||||
67
app/services/domains/scholars/search_hints.py
Normal file
67
app/services/domains/scholars/search_hints.py
Normal file
|
|
@ -0,0 +1,67 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from app.db.models import ScholarProfile
|
||||
from app.services.domains.scholar.parser import ParseState, ParsedAuthorSearchPage
|
||||
from app.services.domains.scholars.constants import (
|
||||
MAX_AUTHOR_SEARCH_LIMIT,
|
||||
STATE_REASON_HINTS,
|
||||
)
|
||||
|
||||
|
||||
def resolve_profile_image(
|
||||
profile: ScholarProfile,
|
||||
*,
|
||||
uploaded_image_url: str | None,
|
||||
) -> tuple[str | None, str]:
|
||||
if profile.profile_image_upload_path and uploaded_image_url:
|
||||
return uploaded_image_url, "upload"
|
||||
if profile.profile_image_override_url:
|
||||
return profile.profile_image_override_url, "override"
|
||||
if profile.profile_image_url:
|
||||
return profile.profile_image_url, "scraped"
|
||||
return None, "none"
|
||||
|
||||
|
||||
def scrape_state_hint(*, state: ParseState, state_reason: str) -> str | None:
|
||||
if state not in {ParseState.NETWORK_ERROR, ParseState.BLOCKED_OR_CAPTCHA}:
|
||||
return None
|
||||
return STATE_REASON_HINTS.get(state_reason)
|
||||
|
||||
|
||||
def _merge_warnings(base: list[str], extra: list[str]) -> list[str]:
|
||||
if not extra:
|
||||
return sorted(set(base))
|
||||
return sorted(set(base + extra))
|
||||
|
||||
|
||||
def _trim_author_search_result(
|
||||
parsed: ParsedAuthorSearchPage,
|
||||
*,
|
||||
limit: int,
|
||||
extra_warnings: list[str] | None = None,
|
||||
state_reason_override: str | None = None,
|
||||
) -> ParsedAuthorSearchPage:
|
||||
bounded_limit = max(1, min(int(limit), MAX_AUTHOR_SEARCH_LIMIT))
|
||||
return ParsedAuthorSearchPage(
|
||||
state=parsed.state,
|
||||
state_reason=state_reason_override or parsed.state_reason,
|
||||
candidates=parsed.candidates[:bounded_limit],
|
||||
marker_counts=parsed.marker_counts,
|
||||
warnings=_merge_warnings(parsed.warnings, extra_warnings or []),
|
||||
)
|
||||
|
||||
|
||||
def _policy_blocked_author_search_result(
|
||||
*,
|
||||
reason: str,
|
||||
warning_codes: list[str],
|
||||
limit: int,
|
||||
) -> ParsedAuthorSearchPage:
|
||||
_ = limit
|
||||
return ParsedAuthorSearchPage(
|
||||
state=ParseState.BLOCKED_OR_CAPTCHA,
|
||||
state_reason=reason,
|
||||
candidates=[],
|
||||
marker_counts={},
|
||||
warnings=_merge_warnings([], warning_codes),
|
||||
)
|
||||
39
app/services/domains/scholars/uploads.py
Normal file
39
app/services/domains/scholars/uploads.py
Normal file
|
|
@ -0,0 +1,39 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from app.services.domains.scholars.exceptions import ScholarServiceError
|
||||
|
||||
|
||||
def _ensure_upload_root(upload_dir: str, *, create: bool) -> Path:
|
||||
root = Path(upload_dir).expanduser().resolve()
|
||||
if create:
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
return root
|
||||
|
||||
|
||||
def _resolve_upload_path(upload_root: Path, relative_path: str) -> Path:
|
||||
candidate = (upload_root / relative_path).resolve()
|
||||
if upload_root != candidate and upload_root not in candidate.parents:
|
||||
raise ScholarServiceError("Invalid scholar image path.")
|
||||
return candidate
|
||||
|
||||
|
||||
def _safe_remove_upload(upload_root: Path, relative_path: str | None) -> None:
|
||||
if not relative_path:
|
||||
return
|
||||
try:
|
||||
file_path = _resolve_upload_path(upload_root, relative_path)
|
||||
except ScholarServiceError:
|
||||
return
|
||||
|
||||
try:
|
||||
if file_path.exists() and file_path.is_file():
|
||||
file_path.unlink()
|
||||
except OSError:
|
||||
return
|
||||
|
||||
|
||||
def resolve_upload_file_path(*, upload_dir: str, relative_path: str) -> Path:
|
||||
root = _ensure_upload_root(upload_dir, create=False)
|
||||
return _resolve_upload_path(root, relative_path)
|
||||
38
app/services/domains/scholars/validators.py
Normal file
38
app/services/domains/scholars/validators.py
Normal file
|
|
@ -0,0 +1,38 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from app.services.domains.scholars.constants import MAX_IMAGE_URL_LENGTH, SCHOLAR_ID_PATTERN
|
||||
from app.services.domains.scholars.exceptions import ScholarServiceError
|
||||
|
||||
|
||||
def validate_scholar_id(value: str) -> str:
|
||||
scholar_id = value.strip()
|
||||
if not SCHOLAR_ID_PATTERN.fullmatch(scholar_id):
|
||||
raise ScholarServiceError("Scholar ID must match [a-zA-Z0-9_-]{12}.")
|
||||
return scholar_id
|
||||
|
||||
|
||||
def normalize_display_name(value: str) -> str | None:
|
||||
normalized = value.strip()
|
||||
return normalized if normalized else None
|
||||
|
||||
|
||||
def normalize_profile_image_url(value: str | None) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
|
||||
candidate = value.strip()
|
||||
if not candidate:
|
||||
return None
|
||||
|
||||
if len(candidate) > MAX_IMAGE_URL_LENGTH:
|
||||
raise ScholarServiceError(
|
||||
f"Image URL must be {MAX_IMAGE_URL_LENGTH} characters or fewer."
|
||||
)
|
||||
|
||||
parsed = urlparse(candidate)
|
||||
if parsed.scheme.lower() not in {"http", "https"} or not parsed.netloc:
|
||||
raise ScholarServiceError("Image URL must be an absolute http(s) URL.")
|
||||
|
||||
return candidate
|
||||
1
app/services/domains/settings/__init__.py
Normal file
1
app/services/domains/settings/__init__.py
Normal file
|
|
@ -0,0 +1 @@
|
|||
from __future__ import annotations
|
||||
146
app/services/domains/settings/application.py
Normal file
146
app/services/domains/settings/application.py
Normal file
|
|
@ -0,0 +1,146 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.models import UserSetting
|
||||
|
||||
|
||||
class UserSettingsServiceError(ValueError):
|
||||
"""Raised for expected settings-validation failures."""
|
||||
|
||||
|
||||
NAV_PAGE_DASHBOARD = "dashboard"
|
||||
NAV_PAGE_SCHOLARS = "scholars"
|
||||
NAV_PAGE_PUBLICATIONS = "publications"
|
||||
NAV_PAGE_SETTINGS = "settings"
|
||||
NAV_PAGE_STYLE_GUIDE = "style-guide"
|
||||
NAV_PAGE_RUNS = "runs"
|
||||
NAV_PAGE_USERS = "users"
|
||||
|
||||
ALLOWED_NAV_PAGES = (
|
||||
NAV_PAGE_DASHBOARD,
|
||||
NAV_PAGE_SCHOLARS,
|
||||
NAV_PAGE_PUBLICATIONS,
|
||||
NAV_PAGE_SETTINGS,
|
||||
NAV_PAGE_STYLE_GUIDE,
|
||||
NAV_PAGE_RUNS,
|
||||
NAV_PAGE_USERS,
|
||||
)
|
||||
REQUIRED_NAV_PAGES = (
|
||||
NAV_PAGE_DASHBOARD,
|
||||
NAV_PAGE_SCHOLARS,
|
||||
NAV_PAGE_SETTINGS,
|
||||
)
|
||||
DEFAULT_NAV_VISIBLE_PAGES = list(ALLOWED_NAV_PAGES)
|
||||
HARD_MIN_RUN_INTERVAL_MINUTES = 15
|
||||
HARD_MIN_REQUEST_DELAY_SECONDS = 2
|
||||
|
||||
|
||||
def resolve_run_interval_minimum(configured_minimum: int | None) -> int:
|
||||
try:
|
||||
parsed = int(configured_minimum) if configured_minimum is not None else HARD_MIN_RUN_INTERVAL_MINUTES
|
||||
except (TypeError, ValueError):
|
||||
parsed = HARD_MIN_RUN_INTERVAL_MINUTES
|
||||
return max(HARD_MIN_RUN_INTERVAL_MINUTES, parsed)
|
||||
|
||||
|
||||
def resolve_request_delay_minimum(configured_minimum: int | None) -> int:
|
||||
try:
|
||||
parsed = int(configured_minimum) if configured_minimum is not None else HARD_MIN_REQUEST_DELAY_SECONDS
|
||||
except (TypeError, ValueError):
|
||||
parsed = HARD_MIN_REQUEST_DELAY_SECONDS
|
||||
return max(HARD_MIN_REQUEST_DELAY_SECONDS, parsed)
|
||||
|
||||
|
||||
def parse_run_interval_minutes(value: str, *, minimum: int = HARD_MIN_RUN_INTERVAL_MINUTES) -> int:
|
||||
try:
|
||||
parsed = int(value)
|
||||
except ValueError as exc:
|
||||
raise UserSettingsServiceError("Check interval must be a whole number.") from exc
|
||||
effective_minimum = resolve_run_interval_minimum(minimum)
|
||||
if parsed < effective_minimum:
|
||||
raise UserSettingsServiceError(
|
||||
f"Check interval must be at least {effective_minimum} minutes."
|
||||
)
|
||||
return parsed
|
||||
|
||||
|
||||
def parse_request_delay_seconds(value: str, *, minimum: int = HARD_MIN_REQUEST_DELAY_SECONDS) -> int:
|
||||
try:
|
||||
parsed = int(value)
|
||||
except ValueError as exc:
|
||||
raise UserSettingsServiceError("Request delay must be a whole number.") from exc
|
||||
effective_minimum = resolve_request_delay_minimum(minimum)
|
||||
if parsed < effective_minimum:
|
||||
raise UserSettingsServiceError(
|
||||
f"Request delay must be at least {effective_minimum} seconds."
|
||||
)
|
||||
return parsed
|
||||
|
||||
|
||||
def parse_nav_visible_pages(value: object) -> list[str]:
|
||||
if not isinstance(value, list):
|
||||
raise UserSettingsServiceError("Navigation visibility must be a list of page ids.")
|
||||
|
||||
deduped: list[str] = []
|
||||
seen: set[str] = set()
|
||||
|
||||
for raw_page in value:
|
||||
if not isinstance(raw_page, str):
|
||||
raise UserSettingsServiceError("Navigation visibility entries must be strings.")
|
||||
|
||||
page_id = raw_page.strip()
|
||||
if page_id not in ALLOWED_NAV_PAGES:
|
||||
raise UserSettingsServiceError(f"Unsupported navigation page id: {page_id}")
|
||||
|
||||
if page_id in seen:
|
||||
continue
|
||||
|
||||
seen.add(page_id)
|
||||
deduped.append(page_id)
|
||||
|
||||
missing_required = [page for page in REQUIRED_NAV_PAGES if page not in seen]
|
||||
if missing_required:
|
||||
raise UserSettingsServiceError(
|
||||
"Dashboard, Scholars, and Settings must remain visible."
|
||||
)
|
||||
|
||||
return deduped
|
||||
|
||||
|
||||
async def get_or_create_settings(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
) -> UserSetting:
|
||||
result = await db_session.execute(
|
||||
select(UserSetting).where(UserSetting.user_id == user_id)
|
||||
)
|
||||
settings = result.scalar_one_or_none()
|
||||
if settings is not None:
|
||||
return settings
|
||||
|
||||
settings = UserSetting(user_id=user_id)
|
||||
db_session.add(settings)
|
||||
await db_session.commit()
|
||||
await db_session.refresh(settings)
|
||||
return settings
|
||||
|
||||
|
||||
async def update_settings(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
settings: UserSetting,
|
||||
auto_run_enabled: bool,
|
||||
run_interval_minutes: int,
|
||||
request_delay_seconds: int,
|
||||
nav_visible_pages: list[str],
|
||||
) -> UserSetting:
|
||||
settings.auto_run_enabled = auto_run_enabled
|
||||
settings.run_interval_minutes = run_interval_minutes
|
||||
settings.request_delay_seconds = request_delay_seconds
|
||||
settings.nav_visible_pages = nav_visible_pages
|
||||
await db_session.commit()
|
||||
await db_session.refresh(settings)
|
||||
return settings
|
||||
1
app/services/domains/users/__init__.py
Normal file
1
app/services/domains/users/__init__.py
Normal file
|
|
@ -0,0 +1 @@
|
|||
from __future__ import annotations
|
||||
98
app/services/domains/users/application.py
Normal file
98
app/services/domains/users/application.py
Normal file
|
|
@ -0,0 +1,98 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.models import User
|
||||
|
||||
EMAIL_PATTERN = re.compile(r"^[^@\s]+@[^@\s]+\.[^@\s]+$")
|
||||
|
||||
|
||||
class UserServiceError(ValueError):
|
||||
"""Raised for expected user-management validation failures."""
|
||||
|
||||
|
||||
def normalize_email(value: str) -> str:
|
||||
return value.strip().lower()
|
||||
|
||||
|
||||
def validate_email(value: str) -> str:
|
||||
email = normalize_email(value)
|
||||
if not EMAIL_PATTERN.fullmatch(email):
|
||||
raise UserServiceError("Enter a valid email address.")
|
||||
return email
|
||||
|
||||
|
||||
def validate_password(value: str) -> str:
|
||||
password = value.strip()
|
||||
if len(password) < 8:
|
||||
raise UserServiceError("Password must be at least 8 characters.")
|
||||
return password
|
||||
|
||||
|
||||
async def get_user_by_id(db_session: AsyncSession, user_id: int) -> User | None:
|
||||
result = await db_session.execute(select(User).where(User.id == user_id))
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
|
||||
async def get_user_by_email(db_session: AsyncSession, email: str) -> User | None:
|
||||
result = await db_session.execute(
|
||||
select(User).where(User.email == normalize_email(email))
|
||||
)
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
|
||||
async def list_users(db_session: AsyncSession) -> list[User]:
|
||||
result = await db_session.execute(select(User).order_by(User.email.asc()))
|
||||
return list(result.scalars().all())
|
||||
|
||||
|
||||
async def create_user(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
email: str,
|
||||
password_hash: str,
|
||||
is_admin: bool,
|
||||
) -> User:
|
||||
user = User(
|
||||
email=validate_email(email),
|
||||
password_hash=password_hash,
|
||||
is_admin=is_admin,
|
||||
is_active=True,
|
||||
)
|
||||
db_session.add(user)
|
||||
try:
|
||||
await db_session.commit()
|
||||
except IntegrityError as exc:
|
||||
await db_session.rollback()
|
||||
raise UserServiceError("A user with that email already exists.") from exc
|
||||
await db_session.refresh(user)
|
||||
return user
|
||||
|
||||
|
||||
async def set_user_active(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user: User,
|
||||
is_active: bool,
|
||||
) -> User:
|
||||
user.is_active = is_active
|
||||
await db_session.commit()
|
||||
await db_session.refresh(user)
|
||||
return user
|
||||
|
||||
|
||||
async def set_user_password_hash(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user: User,
|
||||
password_hash: str,
|
||||
) -> User:
|
||||
user.password_hash = password_hash
|
||||
await db_session.commit()
|
||||
await db_session.refresh(user)
|
||||
return user
|
||||
|
||||
|
|
@ -1,667 +1,3 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
import hashlib
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.models import Publication, ScholarProfile, ScholarPublication
|
||||
from app.services import scholars as scholar_service
|
||||
from app.services.ingestion import build_publication_url, normalize_title
|
||||
|
||||
EXPORT_SCHEMA_VERSION = 1
|
||||
MAX_IMPORT_SCHOLARS = 10_000
|
||||
MAX_IMPORT_PUBLICATIONS = 100_000
|
||||
WORD_RE = re.compile(r"[a-z0-9]+")
|
||||
SHA256_RE = re.compile(r"^[0-9a-f]{64}$")
|
||||
|
||||
|
||||
class ImportExportError(ValueError):
|
||||
"""Raised when import/export payload constraints are violated."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ImportedPublicationInput:
|
||||
profile: ScholarProfile
|
||||
title: str
|
||||
year: int | None
|
||||
citation_count: int
|
||||
author_text: str | None
|
||||
venue_text: str | None
|
||||
cluster_id: str | None
|
||||
pub_url: str | None
|
||||
pdf_url: str | None
|
||||
fingerprint: str
|
||||
is_read: bool
|
||||
|
||||
|
||||
def _normalize_optional_text(value: Any) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
normalized = str(value).strip()
|
||||
return normalized or None
|
||||
|
||||
|
||||
def _normalize_optional_year(value: Any) -> int | None:
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
year = int(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
if year < 1500 or year > 3000:
|
||||
return None
|
||||
return year
|
||||
|
||||
|
||||
def _normalize_citation_count(value: Any) -> int:
|
||||
try:
|
||||
parsed = int(value)
|
||||
except (TypeError, ValueError):
|
||||
return 0
|
||||
return max(0, parsed)
|
||||
|
||||
|
||||
def _first_author_last_name(authors_text: str | None) -> str:
|
||||
if not authors_text:
|
||||
return ""
|
||||
first_author = authors_text.split(",", maxsplit=1)[0].strip().lower()
|
||||
words = WORD_RE.findall(first_author)
|
||||
if not words:
|
||||
return ""
|
||||
return words[-1]
|
||||
|
||||
|
||||
def _first_venue_word(venue_text: str | None) -> str:
|
||||
if not venue_text:
|
||||
return ""
|
||||
words = WORD_RE.findall(venue_text.lower())
|
||||
if not words:
|
||||
return ""
|
||||
return words[0]
|
||||
|
||||
|
||||
def _build_fingerprint(
|
||||
*,
|
||||
title: str,
|
||||
year: int | None,
|
||||
author_text: str | None,
|
||||
venue_text: str | None,
|
||||
) -> str:
|
||||
canonical = "|".join(
|
||||
[
|
||||
normalize_title(title),
|
||||
str(year) if year is not None else "",
|
||||
_first_author_last_name(author_text),
|
||||
_first_venue_word(venue_text),
|
||||
]
|
||||
)
|
||||
return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _resolve_fingerprint(
|
||||
*,
|
||||
title: str,
|
||||
year: int | None,
|
||||
author_text: str | None,
|
||||
venue_text: str | None,
|
||||
provided_fingerprint: Any,
|
||||
) -> str:
|
||||
normalized = _normalize_optional_text(provided_fingerprint)
|
||||
if normalized and SHA256_RE.fullmatch(normalized.lower()):
|
||||
return normalized.lower()
|
||||
return _build_fingerprint(
|
||||
title=title,
|
||||
year=year,
|
||||
author_text=author_text,
|
||||
venue_text=venue_text,
|
||||
)
|
||||
|
||||
|
||||
def _validate_import_sizes(
|
||||
*,
|
||||
scholars: list[dict[str, Any]],
|
||||
publications: list[dict[str, Any]],
|
||||
) -> None:
|
||||
if len(scholars) > MAX_IMPORT_SCHOLARS:
|
||||
raise ImportExportError(f"Import exceeds max scholars ({MAX_IMPORT_SCHOLARS}).")
|
||||
if len(publications) > MAX_IMPORT_PUBLICATIONS:
|
||||
raise ImportExportError(
|
||||
f"Import exceeds max publications ({MAX_IMPORT_PUBLICATIONS})."
|
||||
)
|
||||
|
||||
|
||||
async def _load_user_scholar_map(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
) -> dict[str, ScholarProfile]:
|
||||
result = await db_session.execute(
|
||||
select(ScholarProfile).where(ScholarProfile.user_id == user_id)
|
||||
)
|
||||
profiles = list(result.scalars().all())
|
||||
return {profile.scholar_id: profile for profile in profiles}
|
||||
|
||||
|
||||
def _apply_imported_scholar_values(
|
||||
*,
|
||||
profile: ScholarProfile,
|
||||
display_name: str | None,
|
||||
profile_image_override_url: str | None,
|
||||
is_enabled: bool,
|
||||
) -> bool:
|
||||
updated = False
|
||||
if display_name and profile.display_name != display_name:
|
||||
profile.display_name = display_name
|
||||
updated = True
|
||||
if profile.profile_image_override_url != profile_image_override_url:
|
||||
profile.profile_image_override_url = profile_image_override_url
|
||||
updated = True
|
||||
if bool(profile.is_enabled) != bool(is_enabled):
|
||||
profile.is_enabled = bool(is_enabled)
|
||||
updated = True
|
||||
return updated
|
||||
|
||||
|
||||
def _new_scholar_profile(
|
||||
*,
|
||||
user_id: int,
|
||||
scholar_id: str,
|
||||
display_name: str | None,
|
||||
profile_image_override_url: str | None,
|
||||
is_enabled: bool,
|
||||
) -> ScholarProfile:
|
||||
return ScholarProfile(
|
||||
user_id=user_id,
|
||||
scholar_id=scholar_id,
|
||||
display_name=display_name,
|
||||
profile_image_override_url=profile_image_override_url,
|
||||
is_enabled=bool(is_enabled),
|
||||
)
|
||||
|
||||
|
||||
async def _upsert_imported_scholars(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
scholars: list[dict[str, Any]],
|
||||
) -> tuple[dict[str, ScholarProfile], dict[str, int]]:
|
||||
scholar_map = await _load_user_scholar_map(db_session, user_id=user_id)
|
||||
counters = {"scholars_created": 0, "scholars_updated": 0, "skipped_records": 0}
|
||||
for item in scholars:
|
||||
try:
|
||||
scholar_id = scholar_service.validate_scholar_id(str(item["scholar_id"]))
|
||||
display_name = scholar_service.normalize_display_name(str(item.get("display_name") or ""))
|
||||
override_url = scholar_service.normalize_profile_image_url(
|
||||
_normalize_optional_text(item.get("profile_image_override_url"))
|
||||
)
|
||||
except (KeyError, scholar_service.ScholarServiceError):
|
||||
counters["skipped_records"] += 1
|
||||
continue
|
||||
is_enabled = bool(item.get("is_enabled", True))
|
||||
existing = scholar_map.get(scholar_id)
|
||||
if existing is None:
|
||||
profile = _new_scholar_profile(
|
||||
user_id=user_id,
|
||||
scholar_id=scholar_id,
|
||||
display_name=display_name,
|
||||
profile_image_override_url=override_url,
|
||||
is_enabled=is_enabled,
|
||||
)
|
||||
db_session.add(profile)
|
||||
scholar_map[scholar_id] = profile
|
||||
counters["scholars_created"] += 1
|
||||
continue
|
||||
if _apply_imported_scholar_values(
|
||||
profile=existing,
|
||||
display_name=display_name,
|
||||
profile_image_override_url=override_url,
|
||||
is_enabled=is_enabled,
|
||||
):
|
||||
counters["scholars_updated"] += 1
|
||||
await db_session.flush()
|
||||
return scholar_map, counters
|
||||
|
||||
|
||||
async def _find_publication_by_cluster(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
cluster_id: str,
|
||||
) -> Publication | None:
|
||||
result = await db_session.execute(
|
||||
select(Publication).where(Publication.cluster_id == cluster_id)
|
||||
)
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
|
||||
async def _find_publication_by_fingerprint(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
fingerprint_sha256: str,
|
||||
) -> Publication | None:
|
||||
result = await db_session.execute(
|
||||
select(Publication).where(Publication.fingerprint_sha256 == fingerprint_sha256)
|
||||
)
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
|
||||
def _apply_imported_publication_values(
|
||||
*,
|
||||
publication: Publication,
|
||||
title: str,
|
||||
year: int | None,
|
||||
citation_count: int,
|
||||
author_text: str | None,
|
||||
venue_text: str | None,
|
||||
pub_url: str | None,
|
||||
pdf_url: str | None,
|
||||
cluster_id: str | None,
|
||||
) -> bool:
|
||||
updated = False
|
||||
if cluster_id and publication.cluster_id != cluster_id:
|
||||
publication.cluster_id = cluster_id
|
||||
updated = True
|
||||
if publication.title_raw != title:
|
||||
publication.title_raw = title
|
||||
publication.title_normalized = normalize_title(title)
|
||||
updated = True
|
||||
if publication.year != year:
|
||||
publication.year = year
|
||||
updated = True
|
||||
if int(publication.citation_count or 0) != citation_count:
|
||||
publication.citation_count = citation_count
|
||||
updated = True
|
||||
if publication.author_text != author_text:
|
||||
publication.author_text = author_text
|
||||
updated = True
|
||||
if publication.venue_text != venue_text:
|
||||
publication.venue_text = venue_text
|
||||
updated = True
|
||||
if pub_url and publication.pub_url != pub_url:
|
||||
publication.pub_url = pub_url
|
||||
updated = True
|
||||
if pdf_url and publication.pdf_url != pdf_url:
|
||||
publication.pdf_url = pdf_url
|
||||
updated = True
|
||||
return updated
|
||||
|
||||
|
||||
def _new_publication(
|
||||
*,
|
||||
cluster_id: str | None,
|
||||
fingerprint_sha256: str,
|
||||
title: str,
|
||||
year: int | None,
|
||||
citation_count: int,
|
||||
author_text: str | None,
|
||||
venue_text: str | None,
|
||||
pub_url: str | None,
|
||||
pdf_url: str | None,
|
||||
) -> Publication:
|
||||
return Publication(
|
||||
cluster_id=cluster_id,
|
||||
fingerprint_sha256=fingerprint_sha256,
|
||||
title_raw=title,
|
||||
title_normalized=normalize_title(title),
|
||||
year=year,
|
||||
citation_count=citation_count,
|
||||
author_text=author_text,
|
||||
venue_text=venue_text,
|
||||
pub_url=pub_url,
|
||||
pdf_url=pdf_url,
|
||||
)
|
||||
|
||||
|
||||
async def _resolve_publication_for_import(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
cluster_id: str | None,
|
||||
fingerprint_sha256: str,
|
||||
cluster_cache: dict[str, Publication | None],
|
||||
fingerprint_cache: dict[str, Publication | None],
|
||||
) -> Publication | None:
|
||||
if cluster_id:
|
||||
if cluster_id not in cluster_cache:
|
||||
cluster_cache[cluster_id] = await _find_publication_by_cluster(
|
||||
db_session,
|
||||
cluster_id=cluster_id,
|
||||
)
|
||||
if cluster_cache[cluster_id] is not None:
|
||||
return cluster_cache[cluster_id]
|
||||
if fingerprint_sha256 not in fingerprint_cache:
|
||||
fingerprint_cache[fingerprint_sha256] = await _find_publication_by_fingerprint(
|
||||
db_session,
|
||||
fingerprint_sha256=fingerprint_sha256,
|
||||
)
|
||||
return fingerprint_cache[fingerprint_sha256]
|
||||
|
||||
|
||||
async def _upsert_scholar_publication_link(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
scholar_profile_id: int,
|
||||
publication_id: int,
|
||||
is_read: bool,
|
||||
) -> tuple[bool, bool]:
|
||||
result = await db_session.execute(
|
||||
select(ScholarPublication).where(
|
||||
ScholarPublication.scholar_profile_id == scholar_profile_id,
|
||||
ScholarPublication.publication_id == publication_id,
|
||||
)
|
||||
)
|
||||
link = result.scalar_one_or_none()
|
||||
if link is None:
|
||||
db_session.add(
|
||||
ScholarPublication(
|
||||
scholar_profile_id=scholar_profile_id,
|
||||
publication_id=publication_id,
|
||||
is_read=bool(is_read),
|
||||
)
|
||||
)
|
||||
return True, False
|
||||
if bool(link.is_read) == bool(is_read):
|
||||
return False, False
|
||||
link.is_read = bool(is_read)
|
||||
return False, True
|
||||
|
||||
|
||||
def _initialize_import_counters(counters: dict[str, int]) -> None:
|
||||
counters.update(
|
||||
{
|
||||
"publications_created": 0,
|
||||
"publications_updated": 0,
|
||||
"links_created": 0,
|
||||
"links_updated": 0,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _build_imported_publication_input(
|
||||
*,
|
||||
item: dict[str, Any],
|
||||
scholar_map: dict[str, ScholarProfile],
|
||||
) -> ImportedPublicationInput | None:
|
||||
scholar_id = _normalize_optional_text(item.get("scholar_id"))
|
||||
title = _normalize_optional_text(item.get("title"))
|
||||
if not scholar_id or not title:
|
||||
return None
|
||||
profile = scholar_map.get(scholar_id)
|
||||
if profile is None:
|
||||
return None
|
||||
year = _normalize_optional_year(item.get("year"))
|
||||
author_text = _normalize_optional_text(item.get("author_text"))
|
||||
venue_text = _normalize_optional_text(item.get("venue_text"))
|
||||
return ImportedPublicationInput(
|
||||
profile=profile,
|
||||
title=title,
|
||||
year=year,
|
||||
citation_count=_normalize_citation_count(item.get("citation_count")),
|
||||
author_text=author_text,
|
||||
venue_text=venue_text,
|
||||
cluster_id=_normalize_optional_text(item.get("cluster_id")),
|
||||
pub_url=build_publication_url(_normalize_optional_text(item.get("pub_url"))),
|
||||
pdf_url=build_publication_url(_normalize_optional_text(item.get("pdf_url"))),
|
||||
fingerprint=_resolve_fingerprint(
|
||||
title=title,
|
||||
year=year,
|
||||
author_text=author_text,
|
||||
venue_text=venue_text,
|
||||
provided_fingerprint=item.get("fingerprint_sha256"),
|
||||
),
|
||||
is_read=bool(item.get("is_read", False)),
|
||||
)
|
||||
|
||||
|
||||
def _update_link_counters(
|
||||
*,
|
||||
counters: dict[str, int],
|
||||
link_created: bool,
|
||||
link_updated: bool,
|
||||
) -> None:
|
||||
if link_created:
|
||||
counters["links_created"] += 1
|
||||
if link_updated:
|
||||
counters["links_updated"] += 1
|
||||
|
||||
|
||||
def _cache_resolved_publication(
|
||||
*,
|
||||
publication: Publication,
|
||||
cluster_id: str | None,
|
||||
fingerprint_sha256: str,
|
||||
cluster_cache: dict[str, Publication | None],
|
||||
fingerprint_cache: dict[str, Publication | None],
|
||||
) -> None:
|
||||
if cluster_id:
|
||||
cluster_cache[cluster_id] = publication
|
||||
fingerprint_cache[fingerprint_sha256] = publication
|
||||
|
||||
|
||||
async def _create_import_publication(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
payload: ImportedPublicationInput,
|
||||
) -> Publication:
|
||||
publication = _new_publication(
|
||||
cluster_id=payload.cluster_id,
|
||||
fingerprint_sha256=payload.fingerprint,
|
||||
title=payload.title,
|
||||
year=payload.year,
|
||||
citation_count=payload.citation_count,
|
||||
author_text=payload.author_text,
|
||||
venue_text=payload.venue_text,
|
||||
pub_url=payload.pub_url,
|
||||
pdf_url=payload.pdf_url,
|
||||
)
|
||||
db_session.add(publication)
|
||||
await db_session.flush()
|
||||
return publication
|
||||
|
||||
|
||||
def _update_import_publication(
|
||||
*,
|
||||
publication: Publication,
|
||||
payload: ImportedPublicationInput,
|
||||
) -> bool:
|
||||
return _apply_imported_publication_values(
|
||||
publication=publication,
|
||||
title=payload.title,
|
||||
year=payload.year,
|
||||
citation_count=payload.citation_count,
|
||||
author_text=payload.author_text,
|
||||
venue_text=payload.venue_text,
|
||||
pub_url=payload.pub_url,
|
||||
pdf_url=payload.pdf_url,
|
||||
cluster_id=payload.cluster_id,
|
||||
)
|
||||
|
||||
|
||||
async def _upsert_publication_entity(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
payload: ImportedPublicationInput,
|
||||
cluster_cache: dict[str, Publication | None],
|
||||
fingerprint_cache: dict[str, Publication | None],
|
||||
) -> tuple[Publication, bool, bool]:
|
||||
publication = await _resolve_publication_for_import(
|
||||
db_session,
|
||||
cluster_id=payload.cluster_id,
|
||||
fingerprint_sha256=payload.fingerprint,
|
||||
cluster_cache=cluster_cache,
|
||||
fingerprint_cache=fingerprint_cache,
|
||||
)
|
||||
created = False
|
||||
updated = False
|
||||
if publication is None:
|
||||
publication = await _create_import_publication(
|
||||
db_session,
|
||||
payload=payload,
|
||||
)
|
||||
created = True
|
||||
else:
|
||||
updated = _update_import_publication(
|
||||
publication=publication,
|
||||
payload=payload,
|
||||
)
|
||||
_cache_resolved_publication(
|
||||
publication=publication,
|
||||
cluster_id=payload.cluster_id,
|
||||
fingerprint_sha256=payload.fingerprint,
|
||||
cluster_cache=cluster_cache,
|
||||
fingerprint_cache=fingerprint_cache,
|
||||
)
|
||||
return publication, created, updated
|
||||
|
||||
|
||||
async def _upsert_imported_publication(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
payload: ImportedPublicationInput,
|
||||
cluster_cache: dict[str, Publication | None],
|
||||
fingerprint_cache: dict[str, Publication | None],
|
||||
counters: dict[str, int],
|
||||
) -> None:
|
||||
publication, created, updated = await _upsert_publication_entity(
|
||||
db_session,
|
||||
payload=payload,
|
||||
cluster_cache=cluster_cache,
|
||||
fingerprint_cache=fingerprint_cache,
|
||||
)
|
||||
if created:
|
||||
counters["publications_created"] += 1
|
||||
if updated:
|
||||
counters["publications_updated"] += 1
|
||||
link_created, link_updated = await _upsert_scholar_publication_link(
|
||||
db_session,
|
||||
scholar_profile_id=int(payload.profile.id),
|
||||
publication_id=int(publication.id),
|
||||
is_read=payload.is_read,
|
||||
)
|
||||
_update_link_counters(
|
||||
counters=counters,
|
||||
link_created=link_created,
|
||||
link_updated=link_updated,
|
||||
)
|
||||
|
||||
|
||||
def _exported_at_iso() -> str:
|
||||
return datetime.now(timezone.utc).replace(microsecond=0).isoformat()
|
||||
|
||||
|
||||
def _serialize_export_scholar(profile: ScholarProfile) -> dict[str, Any]:
|
||||
return {
|
||||
"scholar_id": profile.scholar_id,
|
||||
"display_name": profile.display_name,
|
||||
"is_enabled": bool(profile.is_enabled),
|
||||
"profile_image_override_url": profile.profile_image_override_url,
|
||||
}
|
||||
|
||||
|
||||
def _serialize_export_publication(row: tuple[Any, ...]) -> dict[str, Any]:
|
||||
(
|
||||
scholar_id,
|
||||
cluster_id,
|
||||
fingerprint_sha256,
|
||||
title_raw,
|
||||
year,
|
||||
citation_count,
|
||||
author_text,
|
||||
venue_text,
|
||||
pub_url,
|
||||
pdf_url,
|
||||
is_read,
|
||||
) = row
|
||||
return {
|
||||
"scholar_id": scholar_id,
|
||||
"cluster_id": cluster_id,
|
||||
"fingerprint_sha256": fingerprint_sha256,
|
||||
"title": title_raw,
|
||||
"year": year,
|
||||
"citation_count": int(citation_count or 0),
|
||||
"author_text": author_text,
|
||||
"venue_text": venue_text,
|
||||
"pub_url": pub_url,
|
||||
"pdf_url": pdf_url,
|
||||
"is_read": bool(is_read),
|
||||
}
|
||||
|
||||
|
||||
async def export_user_data(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
) -> dict[str, Any]:
|
||||
scholars_result = await db_session.execute(
|
||||
select(ScholarProfile)
|
||||
.where(ScholarProfile.user_id == user_id)
|
||||
.order_by(ScholarProfile.id.asc())
|
||||
)
|
||||
publication_result = await db_session.execute(
|
||||
select(
|
||||
ScholarProfile.scholar_id,
|
||||
Publication.cluster_id,
|
||||
Publication.fingerprint_sha256,
|
||||
Publication.title_raw,
|
||||
Publication.year,
|
||||
Publication.citation_count,
|
||||
Publication.author_text,
|
||||
Publication.venue_text,
|
||||
Publication.pub_url,
|
||||
Publication.pdf_url,
|
||||
ScholarPublication.is_read,
|
||||
)
|
||||
.join(ScholarPublication, ScholarPublication.scholar_profile_id == ScholarProfile.id)
|
||||
.join(Publication, Publication.id == ScholarPublication.publication_id)
|
||||
.where(ScholarProfile.user_id == user_id)
|
||||
.order_by(ScholarPublication.created_at.desc(), Publication.id.desc())
|
||||
)
|
||||
scholars = [_serialize_export_scholar(profile) for profile in scholars_result.scalars().all()]
|
||||
publications = [
|
||||
_serialize_export_publication(row)
|
||||
for row in publication_result.all()
|
||||
]
|
||||
return {
|
||||
"schema_version": EXPORT_SCHEMA_VERSION,
|
||||
"exported_at": _exported_at_iso(),
|
||||
"scholars": scholars,
|
||||
"publications": publications,
|
||||
}
|
||||
|
||||
|
||||
async def import_user_data(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
scholars: list[dict[str, Any]],
|
||||
publications: list[dict[str, Any]],
|
||||
) -> dict[str, int]:
|
||||
_validate_import_sizes(scholars=scholars, publications=publications)
|
||||
scholar_map, counters = await _upsert_imported_scholars(
|
||||
db_session,
|
||||
user_id=user_id,
|
||||
scholars=scholars,
|
||||
)
|
||||
cluster_cache: dict[str, Publication | None] = {}
|
||||
fingerprint_cache: dict[str, Publication | None] = {}
|
||||
_initialize_import_counters(counters)
|
||||
for item in publications:
|
||||
parsed_item = _build_imported_publication_input(
|
||||
item=item,
|
||||
scholar_map=scholar_map,
|
||||
)
|
||||
if parsed_item is None:
|
||||
counters["skipped_records"] += 1
|
||||
continue
|
||||
await _upsert_imported_publication(
|
||||
db_session,
|
||||
payload=parsed_item,
|
||||
cluster_cache=cluster_cache,
|
||||
fingerprint_cache=fingerprint_cache,
|
||||
counters=counters,
|
||||
)
|
||||
await db_session.commit()
|
||||
return counters
|
||||
from app.services.domains.portability.application import *
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -1,385 +1,3 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import Select, func, select, tuple_, update
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.models import (
|
||||
CrawlRun,
|
||||
Publication,
|
||||
RunStatus,
|
||||
ScholarProfile,
|
||||
ScholarPublication,
|
||||
)
|
||||
|
||||
MODE_ALL = "all"
|
||||
MODE_UNREAD = "unread"
|
||||
MODE_LATEST = "latest"
|
||||
MODE_NEW = "new" # compatibility alias for MODE_LATEST
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PublicationListItem:
|
||||
publication_id: int
|
||||
scholar_profile_id: int
|
||||
scholar_label: str
|
||||
title: str
|
||||
year: int | None
|
||||
citation_count: int
|
||||
venue_text: str | None
|
||||
pub_url: str | None
|
||||
pdf_url: str | None
|
||||
is_read: bool
|
||||
first_seen_at: datetime
|
||||
is_new_in_latest_run: bool
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class UnreadPublicationItem:
|
||||
publication_id: int
|
||||
scholar_profile_id: int
|
||||
scholar_label: str
|
||||
title: str
|
||||
year: int | None
|
||||
citation_count: int
|
||||
venue_text: str | None
|
||||
pub_url: str | None
|
||||
pdf_url: str | None
|
||||
|
||||
|
||||
def resolve_publication_view_mode(value: str | None) -> str:
|
||||
if value == MODE_UNREAD:
|
||||
return MODE_UNREAD
|
||||
if value in {MODE_LATEST, MODE_NEW}:
|
||||
return MODE_LATEST
|
||||
return MODE_ALL
|
||||
|
||||
|
||||
def resolve_mode(value: str | None) -> str:
|
||||
return resolve_publication_view_mode(value)
|
||||
|
||||
|
||||
async def get_latest_completed_run_id_for_user(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
) -> int | None:
|
||||
result = await db_session.execute(
|
||||
select(func.max(CrawlRun.id)).where(
|
||||
CrawlRun.user_id == user_id,
|
||||
CrawlRun.status != RunStatus.RUNNING,
|
||||
)
|
||||
)
|
||||
latest_run_id = result.scalar_one_or_none()
|
||||
return int(latest_run_id) if latest_run_id is not None else None
|
||||
|
||||
|
||||
def publications_query(
|
||||
*,
|
||||
user_id: int,
|
||||
mode: str,
|
||||
latest_run_id: int | None,
|
||||
scholar_profile_id: int | None,
|
||||
limit: int,
|
||||
) -> Select[tuple]:
|
||||
scholar_label = ScholarProfile.display_name
|
||||
|
||||
stmt = (
|
||||
select(
|
||||
Publication.id,
|
||||
ScholarProfile.id,
|
||||
scholar_label,
|
||||
ScholarProfile.scholar_id,
|
||||
Publication.title_raw,
|
||||
Publication.year,
|
||||
Publication.citation_count,
|
||||
Publication.venue_text,
|
||||
Publication.pub_url,
|
||||
Publication.pdf_url,
|
||||
ScholarPublication.is_read,
|
||||
ScholarPublication.first_seen_run_id,
|
||||
ScholarPublication.created_at,
|
||||
)
|
||||
.join(ScholarPublication, ScholarPublication.publication_id == Publication.id)
|
||||
.join(ScholarProfile, ScholarProfile.id == ScholarPublication.scholar_profile_id)
|
||||
.where(ScholarProfile.user_id == user_id)
|
||||
.order_by(ScholarPublication.created_at.desc(), Publication.id.desc())
|
||||
.limit(limit)
|
||||
)
|
||||
if scholar_profile_id is not None:
|
||||
stmt = stmt.where(ScholarProfile.id == scholar_profile_id)
|
||||
if mode == MODE_UNREAD:
|
||||
stmt = stmt.where(ScholarPublication.is_read.is_(False))
|
||||
if mode == MODE_LATEST:
|
||||
# "Latest" means discovered in the latest completed run.
|
||||
if latest_run_id is None:
|
||||
stmt = stmt.where(False)
|
||||
else:
|
||||
stmt = stmt.where(ScholarPublication.first_seen_run_id == latest_run_id)
|
||||
return stmt
|
||||
|
||||
|
||||
def _publication_list_item_from_row(
|
||||
row: tuple,
|
||||
*,
|
||||
latest_run_id: int | None,
|
||||
) -> PublicationListItem:
|
||||
(
|
||||
publication_id,
|
||||
scholar_profile_id,
|
||||
display_name,
|
||||
scholar_id,
|
||||
title_raw,
|
||||
year,
|
||||
citation_count,
|
||||
venue_text,
|
||||
pub_url,
|
||||
pdf_url,
|
||||
is_read,
|
||||
first_seen_run_id,
|
||||
created_at,
|
||||
) = row
|
||||
return PublicationListItem(
|
||||
publication_id=int(publication_id),
|
||||
scholar_profile_id=int(scholar_profile_id),
|
||||
scholar_label=(display_name or scholar_id),
|
||||
title=title_raw,
|
||||
year=year,
|
||||
citation_count=int(citation_count or 0),
|
||||
venue_text=venue_text,
|
||||
pub_url=pub_url,
|
||||
pdf_url=pdf_url,
|
||||
is_read=bool(is_read),
|
||||
first_seen_at=created_at,
|
||||
is_new_in_latest_run=(
|
||||
latest_run_id is not None and int(first_seen_run_id or 0) == latest_run_id
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
async def list_for_user(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
mode: str = MODE_ALL,
|
||||
scholar_profile_id: int | None = None,
|
||||
limit: int = 300,
|
||||
) -> list[PublicationListItem]:
|
||||
resolved_mode = resolve_publication_view_mode(mode)
|
||||
latest_run_id = await get_latest_completed_run_id_for_user(
|
||||
db_session,
|
||||
user_id=user_id,
|
||||
)
|
||||
result = await db_session.execute(
|
||||
publications_query(
|
||||
user_id=user_id,
|
||||
mode=resolved_mode,
|
||||
latest_run_id=latest_run_id,
|
||||
scholar_profile_id=scholar_profile_id,
|
||||
limit=limit,
|
||||
)
|
||||
)
|
||||
return [
|
||||
_publication_list_item_from_row(row, latest_run_id=latest_run_id)
|
||||
for row in result.all()
|
||||
]
|
||||
|
||||
|
||||
async def list_unread_for_user(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
limit: int = 100,
|
||||
) -> list[UnreadPublicationItem]:
|
||||
result = await db_session.execute(
|
||||
publications_query(
|
||||
user_id=user_id,
|
||||
mode=MODE_UNREAD,
|
||||
latest_run_id=None,
|
||||
scholar_profile_id=None,
|
||||
limit=limit,
|
||||
)
|
||||
)
|
||||
rows = result.all()
|
||||
items: list[UnreadPublicationItem] = []
|
||||
for row in rows:
|
||||
(
|
||||
publication_id,
|
||||
scholar_profile_id,
|
||||
display_name,
|
||||
scholar_id,
|
||||
title_raw,
|
||||
year,
|
||||
citation_count,
|
||||
venue_text,
|
||||
pub_url,
|
||||
pdf_url,
|
||||
_is_read,
|
||||
_first_seen_run_id,
|
||||
_created_at,
|
||||
) = row
|
||||
items.append(
|
||||
UnreadPublicationItem(
|
||||
publication_id=int(publication_id),
|
||||
scholar_profile_id=int(scholar_profile_id),
|
||||
scholar_label=(display_name or scholar_id),
|
||||
title=title_raw,
|
||||
year=year,
|
||||
citation_count=int(citation_count or 0),
|
||||
venue_text=venue_text,
|
||||
pub_url=pub_url,
|
||||
pdf_url=pdf_url,
|
||||
)
|
||||
)
|
||||
return items
|
||||
|
||||
|
||||
async def list_new_for_latest_run_for_user(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
limit: int = 100,
|
||||
) -> list[UnreadPublicationItem]:
|
||||
rows = await list_for_user(
|
||||
db_session,
|
||||
user_id=user_id,
|
||||
mode=MODE_LATEST,
|
||||
scholar_profile_id=None,
|
||||
limit=limit,
|
||||
)
|
||||
return [
|
||||
UnreadPublicationItem(
|
||||
publication_id=row.publication_id,
|
||||
scholar_profile_id=row.scholar_profile_id,
|
||||
scholar_label=row.scholar_label,
|
||||
title=row.title,
|
||||
year=row.year,
|
||||
citation_count=row.citation_count,
|
||||
venue_text=row.venue_text,
|
||||
pub_url=row.pub_url,
|
||||
pdf_url=row.pdf_url,
|
||||
)
|
||||
for row in rows
|
||||
]
|
||||
|
||||
|
||||
async def count_for_user(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
mode: str = MODE_ALL,
|
||||
scholar_profile_id: int | None = None,
|
||||
) -> int:
|
||||
resolved_mode = resolve_publication_view_mode(mode)
|
||||
latest_run_id = await get_latest_completed_run_id_for_user(
|
||||
db_session,
|
||||
user_id=user_id,
|
||||
)
|
||||
stmt = (
|
||||
select(func.count())
|
||||
.select_from(ScholarPublication)
|
||||
.join(ScholarProfile, ScholarProfile.id == ScholarPublication.scholar_profile_id)
|
||||
.where(ScholarProfile.user_id == user_id)
|
||||
)
|
||||
if scholar_profile_id is not None:
|
||||
stmt = stmt.where(ScholarProfile.id == scholar_profile_id)
|
||||
if resolved_mode == MODE_UNREAD:
|
||||
stmt = stmt.where(ScholarPublication.is_read.is_(False))
|
||||
if resolved_mode == MODE_LATEST:
|
||||
if latest_run_id is None:
|
||||
return 0
|
||||
stmt = stmt.where(ScholarPublication.first_seen_run_id == latest_run_id)
|
||||
result = await db_session.execute(stmt)
|
||||
return int(result.scalar_one() or 0)
|
||||
|
||||
|
||||
async def count_unread_for_user(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
scholar_profile_id: int | None = None,
|
||||
) -> int:
|
||||
return await count_for_user(
|
||||
db_session,
|
||||
user_id=user_id,
|
||||
mode=MODE_UNREAD,
|
||||
scholar_profile_id=scholar_profile_id,
|
||||
)
|
||||
|
||||
|
||||
async def count_latest_for_user(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
scholar_profile_id: int | None = None,
|
||||
) -> int:
|
||||
return await count_for_user(
|
||||
db_session,
|
||||
user_id=user_id,
|
||||
mode=MODE_LATEST,
|
||||
scholar_profile_id=scholar_profile_id,
|
||||
)
|
||||
|
||||
|
||||
async def mark_all_unread_as_read_for_user(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
) -> int:
|
||||
scholar_ids = (
|
||||
select(ScholarProfile.id)
|
||||
.where(ScholarProfile.user_id == user_id)
|
||||
.scalar_subquery()
|
||||
)
|
||||
|
||||
stmt = (
|
||||
update(ScholarPublication)
|
||||
.where(
|
||||
ScholarPublication.scholar_profile_id.in_(scholar_ids),
|
||||
ScholarPublication.is_read.is_(False),
|
||||
)
|
||||
.values(is_read=True)
|
||||
)
|
||||
result = await db_session.execute(stmt)
|
||||
await db_session.commit()
|
||||
|
||||
rowcount = result.rowcount
|
||||
return int(rowcount or 0)
|
||||
|
||||
|
||||
async def mark_selected_as_read_for_user(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
selections: list[tuple[int, int]],
|
||||
) -> int:
|
||||
normalized_pairs = {
|
||||
(int(scholar_profile_id), int(publication_id))
|
||||
for scholar_profile_id, publication_id in selections
|
||||
if int(scholar_profile_id) > 0 and int(publication_id) > 0
|
||||
}
|
||||
if not normalized_pairs:
|
||||
return 0
|
||||
|
||||
scholar_ids = (
|
||||
select(ScholarProfile.id)
|
||||
.where(ScholarProfile.user_id == user_id)
|
||||
.scalar_subquery()
|
||||
)
|
||||
stmt = (
|
||||
update(ScholarPublication)
|
||||
.where(
|
||||
ScholarPublication.scholar_profile_id.in_(scholar_ids),
|
||||
tuple_(
|
||||
ScholarPublication.scholar_profile_id,
|
||||
ScholarPublication.publication_id,
|
||||
).in_(list(normalized_pairs)),
|
||||
ScholarPublication.is_read.is_(False),
|
||||
)
|
||||
.values(is_read=True)
|
||||
)
|
||||
result = await db_session.execute(stmt)
|
||||
await db_session.commit()
|
||||
return int(result.rowcount or 0)
|
||||
from app.services.domains.publications.application import *
|
||||
|
|
|
|||
|
|
@ -1,282 +1,3 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any
|
||||
|
||||
from app.db.models import UserSetting
|
||||
|
||||
COOLDOWN_REASON_BLOCKED_FAILURE_THRESHOLD = "blocked_failure_threshold_exceeded"
|
||||
COOLDOWN_REASON_NETWORK_FAILURE_THRESHOLD = "network_failure_threshold_exceeded"
|
||||
|
||||
_COUNTER_CONSECUTIVE_BLOCKED_RUNS = "consecutive_blocked_runs"
|
||||
_COUNTER_CONSECUTIVE_NETWORK_RUNS = "consecutive_network_runs"
|
||||
_COUNTER_COOLDOWN_ENTRY_COUNT = "cooldown_entry_count"
|
||||
_COUNTER_BLOCKED_START_COUNT = "blocked_start_count"
|
||||
_COUNTER_LAST_BLOCKED_FAILURE_COUNT = "last_blocked_failure_count"
|
||||
_COUNTER_LAST_NETWORK_FAILURE_COUNT = "last_network_failure_count"
|
||||
_COUNTER_LAST_EVALUATED_RUN_ID = "last_evaluated_run_id"
|
||||
|
||||
|
||||
def _utcnow() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def _safe_int(value: Any, default: int = 0) -> int:
|
||||
try:
|
||||
return int(value)
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
|
||||
|
||||
def _safe_optional_int(value: Any) -> int | None:
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
return int(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _normalize_datetime(value: datetime | None) -> datetime | None:
|
||||
if value is None:
|
||||
return None
|
||||
if value.tzinfo is None:
|
||||
return value.replace(tzinfo=timezone.utc)
|
||||
return value.astimezone(timezone.utc)
|
||||
|
||||
|
||||
def _state_dict(settings: UserSetting) -> dict[str, Any]:
|
||||
state = settings.scrape_safety_state
|
||||
if isinstance(state, dict):
|
||||
return state
|
||||
return {}
|
||||
|
||||
|
||||
def _counters_from_state(settings: UserSetting) -> dict[str, Any]:
|
||||
state = _state_dict(settings)
|
||||
return {
|
||||
_COUNTER_CONSECUTIVE_BLOCKED_RUNS: max(
|
||||
0,
|
||||
_safe_int(state.get(_COUNTER_CONSECUTIVE_BLOCKED_RUNS), 0),
|
||||
),
|
||||
_COUNTER_CONSECUTIVE_NETWORK_RUNS: max(
|
||||
0,
|
||||
_safe_int(state.get(_COUNTER_CONSECUTIVE_NETWORK_RUNS), 0),
|
||||
),
|
||||
_COUNTER_COOLDOWN_ENTRY_COUNT: max(
|
||||
0,
|
||||
_safe_int(state.get(_COUNTER_COOLDOWN_ENTRY_COUNT), 0),
|
||||
),
|
||||
_COUNTER_BLOCKED_START_COUNT: max(
|
||||
0,
|
||||
_safe_int(state.get(_COUNTER_BLOCKED_START_COUNT), 0),
|
||||
),
|
||||
_COUNTER_LAST_BLOCKED_FAILURE_COUNT: max(
|
||||
0,
|
||||
_safe_int(state.get(_COUNTER_LAST_BLOCKED_FAILURE_COUNT), 0),
|
||||
),
|
||||
_COUNTER_LAST_NETWORK_FAILURE_COUNT: max(
|
||||
0,
|
||||
_safe_int(state.get(_COUNTER_LAST_NETWORK_FAILURE_COUNT), 0),
|
||||
),
|
||||
_COUNTER_LAST_EVALUATED_RUN_ID: _safe_optional_int(
|
||||
state.get(_COUNTER_LAST_EVALUATED_RUN_ID),
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def _cooldown_reason_label(reason: str | None) -> str | None:
|
||||
if reason == COOLDOWN_REASON_BLOCKED_FAILURE_THRESHOLD:
|
||||
return "Blocked responses exceeded safety threshold"
|
||||
if reason == COOLDOWN_REASON_NETWORK_FAILURE_THRESHOLD:
|
||||
return "Network failures exceeded safety threshold"
|
||||
return None
|
||||
|
||||
|
||||
def _recommended_action(reason: str | None) -> str | None:
|
||||
if reason == COOLDOWN_REASON_BLOCKED_FAILURE_THRESHOLD:
|
||||
return (
|
||||
"Google Scholar appears to be blocking requests. Wait for cooldown to expire, "
|
||||
"increase request delay, and avoid repeated manual retries."
|
||||
)
|
||||
if reason == COOLDOWN_REASON_NETWORK_FAILURE_THRESHOLD:
|
||||
return (
|
||||
"Network failures crossed the threshold. Verify connectivity and retry after cooldown."
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def is_cooldown_active(
|
||||
settings: UserSetting,
|
||||
*,
|
||||
now_utc: datetime | None = None,
|
||||
) -> bool:
|
||||
now = now_utc or _utcnow()
|
||||
cooldown_until = _normalize_datetime(settings.scrape_cooldown_until)
|
||||
if cooldown_until is None:
|
||||
return False
|
||||
return cooldown_until > now
|
||||
|
||||
|
||||
def clear_expired_cooldown(
|
||||
settings: UserSetting,
|
||||
*,
|
||||
now_utc: datetime | None = None,
|
||||
) -> bool:
|
||||
now = now_utc or _utcnow()
|
||||
cooldown_until = _normalize_datetime(settings.scrape_cooldown_until)
|
||||
if cooldown_until is None:
|
||||
return False
|
||||
if cooldown_until > now:
|
||||
return False
|
||||
settings.scrape_cooldown_until = None
|
||||
settings.scrape_cooldown_reason = None
|
||||
return True
|
||||
|
||||
|
||||
def register_cooldown_blocked_start(
|
||||
settings: UserSetting,
|
||||
*,
|
||||
now_utc: datetime | None = None,
|
||||
) -> dict[str, Any]:
|
||||
now = now_utc or _utcnow()
|
||||
counters = _counters_from_state(settings)
|
||||
counters[_COUNTER_BLOCKED_START_COUNT] = int(counters[_COUNTER_BLOCKED_START_COUNT]) + 1
|
||||
settings.scrape_safety_state = counters
|
||||
return get_safety_state_payload(settings, now_utc=now)
|
||||
|
||||
|
||||
def _update_run_counters(
|
||||
*,
|
||||
counters: dict[str, Any],
|
||||
run_id: int,
|
||||
blocked_failure_count: int,
|
||||
network_failure_count: int,
|
||||
) -> tuple[int, int]:
|
||||
bounded_blocked_failures = max(0, int(blocked_failure_count))
|
||||
bounded_network_failures = max(0, int(network_failure_count))
|
||||
counters[_COUNTER_LAST_BLOCKED_FAILURE_COUNT] = bounded_blocked_failures
|
||||
counters[_COUNTER_LAST_NETWORK_FAILURE_COUNT] = bounded_network_failures
|
||||
counters[_COUNTER_LAST_EVALUATED_RUN_ID] = int(run_id)
|
||||
counters[_COUNTER_CONSECUTIVE_BLOCKED_RUNS] = (
|
||||
int(counters[_COUNTER_CONSECUTIVE_BLOCKED_RUNS]) + 1
|
||||
if bounded_blocked_failures > 0
|
||||
else 0
|
||||
)
|
||||
counters[_COUNTER_CONSECUTIVE_NETWORK_RUNS] = (
|
||||
int(counters[_COUNTER_CONSECUTIVE_NETWORK_RUNS]) + 1
|
||||
if bounded_network_failures > 0
|
||||
else 0
|
||||
)
|
||||
return bounded_blocked_failures, bounded_network_failures
|
||||
|
||||
|
||||
def _resolve_cooldown_trigger(
|
||||
*,
|
||||
blocked_failures: int,
|
||||
network_failures: int,
|
||||
blocked_failure_threshold: int,
|
||||
network_failure_threshold: int,
|
||||
blocked_cooldown_seconds: int,
|
||||
network_cooldown_seconds: int,
|
||||
) -> tuple[str | None, int]:
|
||||
if blocked_failures >= max(1, int(blocked_failure_threshold)):
|
||||
return COOLDOWN_REASON_BLOCKED_FAILURE_THRESHOLD, max(60, int(blocked_cooldown_seconds))
|
||||
if network_failures >= max(1, int(network_failure_threshold)):
|
||||
return COOLDOWN_REASON_NETWORK_FAILURE_THRESHOLD, max(60, int(network_cooldown_seconds))
|
||||
return None, 0
|
||||
|
||||
|
||||
def _apply_cooldown_decision(
|
||||
*,
|
||||
settings: UserSetting,
|
||||
counters: dict[str, Any],
|
||||
now: datetime,
|
||||
reason: str | None,
|
||||
cooldown_seconds: int,
|
||||
) -> None:
|
||||
if reason is None:
|
||||
clear_expired_cooldown(settings, now_utc=now)
|
||||
return
|
||||
settings.scrape_cooldown_reason = reason
|
||||
settings.scrape_cooldown_until = now + timedelta(seconds=max(60, int(cooldown_seconds)))
|
||||
counters[_COUNTER_COOLDOWN_ENTRY_COUNT] = int(counters[_COUNTER_COOLDOWN_ENTRY_COUNT]) + 1
|
||||
|
||||
|
||||
def apply_run_safety_outcome(
|
||||
settings: UserSetting,
|
||||
*,
|
||||
run_id: int,
|
||||
blocked_failure_count: int,
|
||||
network_failure_count: int,
|
||||
blocked_failure_threshold: int,
|
||||
network_failure_threshold: int,
|
||||
blocked_cooldown_seconds: int,
|
||||
network_cooldown_seconds: int,
|
||||
now_utc: datetime | None = None,
|
||||
) -> tuple[dict[str, Any], str | None]:
|
||||
now = now_utc or _utcnow()
|
||||
counters = _counters_from_state(settings)
|
||||
blocked_failures, network_failures = _update_run_counters(
|
||||
counters=counters,
|
||||
run_id=run_id,
|
||||
blocked_failure_count=blocked_failure_count,
|
||||
network_failure_count=network_failure_count,
|
||||
)
|
||||
reason, cooldown_seconds = _resolve_cooldown_trigger(
|
||||
blocked_failures=blocked_failures,
|
||||
network_failures=network_failures,
|
||||
blocked_failure_threshold=blocked_failure_threshold,
|
||||
network_failure_threshold=network_failure_threshold,
|
||||
blocked_cooldown_seconds=blocked_cooldown_seconds,
|
||||
network_cooldown_seconds=network_cooldown_seconds,
|
||||
)
|
||||
_apply_cooldown_decision(
|
||||
settings=settings,
|
||||
counters=counters,
|
||||
now=now,
|
||||
reason=reason,
|
||||
cooldown_seconds=cooldown_seconds,
|
||||
)
|
||||
settings.scrape_safety_state = counters
|
||||
return get_safety_state_payload(settings, now_utc=now), reason
|
||||
|
||||
|
||||
def get_safety_state_payload(
|
||||
settings: UserSetting,
|
||||
*,
|
||||
now_utc: datetime | None = None,
|
||||
) -> dict[str, Any]:
|
||||
now = now_utc or _utcnow()
|
||||
cooldown_until = _normalize_datetime(settings.scrape_cooldown_until)
|
||||
cooldown_active = bool(cooldown_until is not None and cooldown_until > now)
|
||||
cooldown_remaining_seconds = 0
|
||||
if cooldown_active and cooldown_until is not None:
|
||||
cooldown_remaining_seconds = max(0, int((cooldown_until - now).total_seconds()))
|
||||
|
||||
reason = settings.scrape_cooldown_reason if cooldown_active else None
|
||||
|
||||
return {
|
||||
"cooldown_active": cooldown_active,
|
||||
"cooldown_reason": reason,
|
||||
"cooldown_reason_label": _cooldown_reason_label(reason),
|
||||
"cooldown_until": cooldown_until,
|
||||
"cooldown_remaining_seconds": cooldown_remaining_seconds,
|
||||
"recommended_action": _recommended_action(reason),
|
||||
"counters": _counters_from_state(settings),
|
||||
}
|
||||
|
||||
|
||||
def get_safety_event_context(
|
||||
settings: UserSetting,
|
||||
*,
|
||||
now_utc: datetime | None = None,
|
||||
) -> dict[str, Any]:
|
||||
payload = get_safety_state_payload(settings, now_utc=now_utc)
|
||||
return {
|
||||
"cooldown_active": bool(payload.get("cooldown_active")),
|
||||
"cooldown_reason": payload.get("cooldown_reason"),
|
||||
"cooldown_until": payload.get("cooldown_until"),
|
||||
"cooldown_remaining_seconds": int(payload.get("cooldown_remaining_seconds") or 0),
|
||||
"safety_counters": payload.get("counters", {}),
|
||||
}
|
||||
from app.services.domains.ingestion.safety import *
|
||||
|
|
|
|||
|
|
@ -1,435 +1,3 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import and_, case, func, or_, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.models import (
|
||||
CrawlRun,
|
||||
IngestionQueueItem,
|
||||
RunStatus,
|
||||
RunTriggerType,
|
||||
ScholarProfile,
|
||||
)
|
||||
from app.services import continuation_queue as queue_service
|
||||
|
||||
QUEUE_STATUS_QUEUED = "queued"
|
||||
QUEUE_STATUS_RETRYING = "retrying"
|
||||
QUEUE_STATUS_DROPPED = "dropped"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class QueueListItem:
|
||||
id: int
|
||||
scholar_profile_id: int
|
||||
scholar_label: str
|
||||
status: str
|
||||
reason: str
|
||||
dropped_reason: str | None
|
||||
attempt_count: int
|
||||
resume_cstart: int
|
||||
next_attempt_dt: datetime | None
|
||||
updated_at: datetime
|
||||
last_error: str | None
|
||||
last_run_id: int | None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class QueueClearResult:
|
||||
queue_item_id: int
|
||||
previous_status: str
|
||||
|
||||
|
||||
class QueueTransitionError(RuntimeError):
|
||||
def __init__(self, *, code: str, message: str, current_status: str) -> None:
|
||||
super().__init__(message)
|
||||
self.code = code
|
||||
self.message = message
|
||||
self.current_status = current_status
|
||||
|
||||
|
||||
def _safe_int(value: object, default: int = 0) -> int:
|
||||
try:
|
||||
return int(value)
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
|
||||
|
||||
def _summary_dict(error_log: object) -> dict[str, Any]:
|
||||
if not isinstance(error_log, dict):
|
||||
return {}
|
||||
summary = error_log.get("summary")
|
||||
if not isinstance(summary, dict):
|
||||
return {}
|
||||
return summary
|
||||
|
||||
|
||||
def _summary_int_dict(summary: dict[str, Any], key: str) -> dict[str, int]:
|
||||
value = summary.get(key)
|
||||
if not isinstance(value, dict):
|
||||
return {}
|
||||
return {
|
||||
str(item_key): _safe_int(item_value, 0)
|
||||
for item_key, item_value in value.items()
|
||||
if isinstance(item_key, str)
|
||||
}
|
||||
|
||||
|
||||
def _summary_bool_dict(summary: dict[str, Any], key: str) -> dict[str, bool]:
|
||||
value = summary.get(key)
|
||||
if not isinstance(value, dict):
|
||||
return {}
|
||||
return {
|
||||
str(item_key): bool(item_value)
|
||||
for item_key, item_value in value.items()
|
||||
if isinstance(item_key, str)
|
||||
}
|
||||
|
||||
|
||||
def extract_run_summary(error_log: object) -> dict[str, Any]:
|
||||
summary = _summary_dict(error_log)
|
||||
return {
|
||||
"succeeded_count": _safe_int(summary.get("succeeded_count", 0)),
|
||||
"failed_count": _safe_int(summary.get("failed_count", 0)),
|
||||
"partial_count": _safe_int(summary.get("partial_count", 0)),
|
||||
"failed_state_counts": _summary_int_dict(summary, "failed_state_counts"),
|
||||
"failed_reason_counts": _summary_int_dict(summary, "failed_reason_counts"),
|
||||
"scrape_failure_counts": _summary_int_dict(summary, "scrape_failure_counts"),
|
||||
"retry_counts": {
|
||||
"retries_scheduled_count": _safe_int(
|
||||
(
|
||||
summary.get("retry_counts", {}).get("retries_scheduled_count")
|
||||
if isinstance(summary.get("retry_counts"), dict)
|
||||
else 0
|
||||
),
|
||||
0,
|
||||
),
|
||||
"scholars_with_retries_count": _safe_int(
|
||||
(
|
||||
summary.get("retry_counts", {}).get("scholars_with_retries_count")
|
||||
if isinstance(summary.get("retry_counts"), dict)
|
||||
else 0
|
||||
),
|
||||
0,
|
||||
),
|
||||
"retry_exhausted_count": _safe_int(
|
||||
(
|
||||
summary.get("retry_counts", {}).get("retry_exhausted_count")
|
||||
if isinstance(summary.get("retry_counts"), dict)
|
||||
else 0
|
||||
),
|
||||
0,
|
||||
),
|
||||
},
|
||||
"alert_thresholds": _summary_int_dict(summary, "alert_thresholds"),
|
||||
"alert_flags": _summary_bool_dict(summary, "alert_flags"),
|
||||
}
|
||||
|
||||
|
||||
def _queue_item_columns() -> tuple:
|
||||
return (
|
||||
IngestionQueueItem.id,
|
||||
IngestionQueueItem.scholar_profile_id,
|
||||
ScholarProfile.display_name,
|
||||
ScholarProfile.scholar_id,
|
||||
IngestionQueueItem.status,
|
||||
IngestionQueueItem.reason,
|
||||
IngestionQueueItem.dropped_reason,
|
||||
IngestionQueueItem.attempt_count,
|
||||
IngestionQueueItem.resume_cstart,
|
||||
IngestionQueueItem.next_attempt_dt,
|
||||
IngestionQueueItem.updated_at,
|
||||
IngestionQueueItem.last_error,
|
||||
IngestionQueueItem.last_run_id,
|
||||
)
|
||||
|
||||
|
||||
def _queue_item_select(*, user_id: int):
|
||||
return (
|
||||
select(*_queue_item_columns())
|
||||
.join(
|
||||
ScholarProfile,
|
||||
and_(
|
||||
ScholarProfile.id == IngestionQueueItem.scholar_profile_id,
|
||||
ScholarProfile.user_id == IngestionQueueItem.user_id,
|
||||
),
|
||||
)
|
||||
.where(IngestionQueueItem.user_id == user_id)
|
||||
)
|
||||
|
||||
|
||||
def _queue_list_item_from_row(row: tuple) -> QueueListItem:
|
||||
(
|
||||
item_id,
|
||||
scholar_profile_id,
|
||||
display_name,
|
||||
scholar_id,
|
||||
status,
|
||||
reason,
|
||||
dropped_reason,
|
||||
attempt_count,
|
||||
resume_cstart,
|
||||
next_attempt_dt,
|
||||
updated_at,
|
||||
last_error,
|
||||
last_run_id,
|
||||
) = row
|
||||
return QueueListItem(
|
||||
id=int(item_id),
|
||||
scholar_profile_id=int(scholar_profile_id),
|
||||
scholar_label=(display_name or scholar_id),
|
||||
status=str(status),
|
||||
reason=str(reason),
|
||||
dropped_reason=dropped_reason,
|
||||
attempt_count=int(attempt_count or 0),
|
||||
resume_cstart=int(resume_cstart or 0),
|
||||
next_attempt_dt=next_attempt_dt,
|
||||
updated_at=updated_at,
|
||||
last_error=last_error,
|
||||
last_run_id=int(last_run_id) if last_run_id is not None else None,
|
||||
)
|
||||
|
||||
|
||||
async def list_recent_runs_for_user(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
limit: int = 20,
|
||||
) -> list[CrawlRun]:
|
||||
result = await db_session.execute(
|
||||
select(CrawlRun)
|
||||
.where(CrawlRun.user_id == user_id)
|
||||
.order_by(CrawlRun.start_dt.desc(), CrawlRun.id.desc())
|
||||
.limit(limit)
|
||||
)
|
||||
return list(result.scalars().all())
|
||||
|
||||
|
||||
async def list_runs_for_user(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
limit: int = 100,
|
||||
failed_only: bool = False,
|
||||
) -> list[CrawlRun]:
|
||||
stmt = (
|
||||
select(CrawlRun)
|
||||
.where(CrawlRun.user_id == user_id)
|
||||
.order_by(CrawlRun.start_dt.desc(), CrawlRun.id.desc())
|
||||
.limit(limit)
|
||||
)
|
||||
if failed_only:
|
||||
stmt = stmt.where(
|
||||
CrawlRun.status.in_([RunStatus.FAILED, RunStatus.PARTIAL_FAILURE])
|
||||
)
|
||||
result = await db_session.execute(stmt)
|
||||
return list(result.scalars().all())
|
||||
|
||||
|
||||
async def get_run_for_user(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
run_id: int,
|
||||
) -> CrawlRun | None:
|
||||
result = await db_session.execute(
|
||||
select(CrawlRun).where(
|
||||
CrawlRun.user_id == user_id,
|
||||
CrawlRun.id == run_id,
|
||||
)
|
||||
)
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
|
||||
async def get_manual_run_by_idempotency_key(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
idempotency_key: str,
|
||||
) -> CrawlRun | None:
|
||||
result = await db_session.execute(
|
||||
select(CrawlRun)
|
||||
.where(
|
||||
CrawlRun.user_id == user_id,
|
||||
CrawlRun.trigger_type == RunTriggerType.MANUAL,
|
||||
or_(
|
||||
CrawlRun.idempotency_key == idempotency_key,
|
||||
CrawlRun.error_log["meta"]["idempotency_key"].astext == idempotency_key,
|
||||
),
|
||||
)
|
||||
.order_by(CrawlRun.start_dt.desc(), CrawlRun.id.desc())
|
||||
.limit(1)
|
||||
)
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
|
||||
async def list_queue_items_for_user(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
limit: int = 200,
|
||||
) -> list[QueueListItem]:
|
||||
result = await db_session.execute(
|
||||
_queue_item_select(user_id=user_id)
|
||||
.order_by(
|
||||
case((IngestionQueueItem.status == "dropped", 1), else_=0).asc(),
|
||||
IngestionQueueItem.next_attempt_dt.asc(),
|
||||
IngestionQueueItem.id.asc(),
|
||||
)
|
||||
.limit(limit)
|
||||
)
|
||||
return [_queue_list_item_from_row(row) for row in result.all()]
|
||||
|
||||
|
||||
async def get_queue_item_for_user(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
queue_item_id: int,
|
||||
) -> QueueListItem | None:
|
||||
result = await db_session.execute(
|
||||
_queue_item_select(user_id=user_id)
|
||||
.where(IngestionQueueItem.id == queue_item_id)
|
||||
.limit(1)
|
||||
)
|
||||
row = result.one_or_none()
|
||||
if row is None:
|
||||
return None
|
||||
return _queue_list_item_from_row(row)
|
||||
|
||||
|
||||
async def retry_queue_item_for_user(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
queue_item_id: int,
|
||||
) -> QueueListItem | None:
|
||||
result = await db_session.execute(
|
||||
select(IngestionQueueItem).where(
|
||||
IngestionQueueItem.id == queue_item_id,
|
||||
IngestionQueueItem.user_id == user_id,
|
||||
)
|
||||
)
|
||||
item = result.scalar_one_or_none()
|
||||
if item is None:
|
||||
return None
|
||||
if item.status == QUEUE_STATUS_QUEUED:
|
||||
raise QueueTransitionError(
|
||||
code="queue_item_already_queued",
|
||||
message="Queue item is already queued.",
|
||||
current_status=item.status,
|
||||
)
|
||||
if item.status == QUEUE_STATUS_RETRYING:
|
||||
raise QueueTransitionError(
|
||||
code="queue_item_retrying",
|
||||
message="Queue item is currently retrying.",
|
||||
current_status=item.status,
|
||||
)
|
||||
|
||||
await queue_service.mark_queued_now(
|
||||
db_session,
|
||||
job_id=item.id,
|
||||
reason="manual_retry",
|
||||
reset_attempt_count=(item.status == QUEUE_STATUS_DROPPED),
|
||||
)
|
||||
await db_session.commit()
|
||||
|
||||
return await get_queue_item_for_user(
|
||||
db_session,
|
||||
user_id=user_id,
|
||||
queue_item_id=queue_item_id,
|
||||
)
|
||||
|
||||
|
||||
async def drop_queue_item_for_user(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
queue_item_id: int,
|
||||
) -> QueueListItem | None:
|
||||
result = await db_session.execute(
|
||||
select(IngestionQueueItem).where(
|
||||
IngestionQueueItem.id == queue_item_id,
|
||||
IngestionQueueItem.user_id == user_id,
|
||||
)
|
||||
)
|
||||
item = result.scalar_one_or_none()
|
||||
if item is None:
|
||||
return None
|
||||
if item.status == QUEUE_STATUS_DROPPED:
|
||||
raise QueueTransitionError(
|
||||
code="queue_item_already_dropped",
|
||||
message="Queue item is already dropped.",
|
||||
current_status=item.status,
|
||||
)
|
||||
|
||||
await queue_service.mark_dropped(
|
||||
db_session,
|
||||
job_id=int(item.id),
|
||||
reason="manual_drop",
|
||||
)
|
||||
await db_session.commit()
|
||||
return await get_queue_item_for_user(
|
||||
db_session,
|
||||
user_id=user_id,
|
||||
queue_item_id=queue_item_id,
|
||||
)
|
||||
|
||||
|
||||
async def clear_queue_item_for_user(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
queue_item_id: int,
|
||||
) -> QueueClearResult | None:
|
||||
result = await db_session.execute(
|
||||
select(IngestionQueueItem).where(
|
||||
IngestionQueueItem.id == queue_item_id,
|
||||
IngestionQueueItem.user_id == user_id,
|
||||
)
|
||||
)
|
||||
item = result.scalar_one_or_none()
|
||||
if item is None:
|
||||
return None
|
||||
if item.status != QUEUE_STATUS_DROPPED:
|
||||
raise QueueTransitionError(
|
||||
code="queue_item_not_dropped",
|
||||
message="Queue item can only be cleared after it is dropped.",
|
||||
current_status=item.status,
|
||||
)
|
||||
|
||||
item_id = int(item.id)
|
||||
previous_status = str(item.status)
|
||||
deleted = await queue_service.delete_job_by_id(
|
||||
db_session,
|
||||
job_id=item_id,
|
||||
)
|
||||
await db_session.commit()
|
||||
if not deleted:
|
||||
return None
|
||||
return QueueClearResult(
|
||||
queue_item_id=item_id,
|
||||
previous_status=previous_status,
|
||||
)
|
||||
|
||||
|
||||
async def queue_status_counts_for_user(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
) -> dict[str, int]:
|
||||
result = await db_session.execute(
|
||||
select(
|
||||
IngestionQueueItem.status,
|
||||
func.count(IngestionQueueItem.id),
|
||||
)
|
||||
.where(IngestionQueueItem.user_id == user_id)
|
||||
.group_by(IngestionQueueItem.status)
|
||||
)
|
||||
counts: dict[str, int] = {"queued": 0, "retrying": 0, "dropped": 0}
|
||||
for status, count in result.all():
|
||||
counts[str(status)] = int(count or 0)
|
||||
return counts
|
||||
from app.services.domains.runs.application import *
|
||||
|
|
|
|||
|
|
@ -1,582 +1,3 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta, timezone
|
||||
import logging
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.models import (
|
||||
CrawlRun,
|
||||
QueueItemStatus,
|
||||
RunTriggerType,
|
||||
ScholarProfile,
|
||||
User,
|
||||
UserSetting,
|
||||
)
|
||||
from app.db.session import get_session_factory
|
||||
from app.services import continuation_queue as queue_service
|
||||
from app.services.ingestion import (
|
||||
RunAlreadyInProgressError,
|
||||
RunBlockedBySafetyPolicyError,
|
||||
ScholarIngestionService,
|
||||
)
|
||||
from app.services.scholar_source import LiveScholarSource
|
||||
from app.settings import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _AutoRunCandidate:
|
||||
user_id: int
|
||||
run_interval_minutes: int
|
||||
request_delay_seconds: int
|
||||
cooldown_until: datetime | None
|
||||
cooldown_reason: str | None
|
||||
|
||||
|
||||
class SchedulerService:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
enabled: bool,
|
||||
tick_seconds: int,
|
||||
network_error_retries: int,
|
||||
retry_backoff_seconds: float,
|
||||
max_pages_per_scholar: int,
|
||||
page_size: int,
|
||||
continuation_queue_enabled: bool,
|
||||
continuation_base_delay_seconds: int,
|
||||
continuation_max_delay_seconds: int,
|
||||
continuation_max_attempts: int,
|
||||
queue_batch_size: int,
|
||||
) -> None:
|
||||
self._enabled = enabled
|
||||
self._tick_seconds = max(5, int(tick_seconds))
|
||||
self._network_error_retries = max(0, int(network_error_retries))
|
||||
self._retry_backoff_seconds = max(0.0, float(retry_backoff_seconds))
|
||||
self._max_pages_per_scholar = max(1, int(max_pages_per_scholar))
|
||||
self._page_size = max(1, int(page_size))
|
||||
self._continuation_queue_enabled = bool(continuation_queue_enabled)
|
||||
self._continuation_base_delay_seconds = max(1, int(continuation_base_delay_seconds))
|
||||
self._continuation_max_delay_seconds = max(
|
||||
self._continuation_base_delay_seconds,
|
||||
int(continuation_max_delay_seconds),
|
||||
)
|
||||
self._continuation_max_attempts = max(1, int(continuation_max_attempts))
|
||||
self._queue_batch_size = max(1, int(queue_batch_size))
|
||||
self._task: asyncio.Task[None] | None = None
|
||||
self._source = LiveScholarSource()
|
||||
|
||||
async def start(self) -> None:
|
||||
if not self._enabled:
|
||||
logger.info(
|
||||
"scheduler.disabled",
|
||||
extra={
|
||||
"event": "scheduler.disabled",
|
||||
},
|
||||
)
|
||||
return
|
||||
if self._task is not None:
|
||||
return
|
||||
self._task = asyncio.create_task(self._run_loop(), name="scholarr-scheduler")
|
||||
logger.info(
|
||||
"scheduler.started",
|
||||
extra={
|
||||
"event": "scheduler.started",
|
||||
"tick_seconds": self._tick_seconds,
|
||||
"network_error_retries": self._network_error_retries,
|
||||
"retry_backoff_seconds": self._retry_backoff_seconds,
|
||||
"max_pages_per_scholar": self._max_pages_per_scholar,
|
||||
"page_size": self._page_size,
|
||||
"continuation_queue_enabled": self._continuation_queue_enabled,
|
||||
"continuation_base_delay_seconds": self._continuation_base_delay_seconds,
|
||||
"continuation_max_delay_seconds": self._continuation_max_delay_seconds,
|
||||
"continuation_max_attempts": self._continuation_max_attempts,
|
||||
"queue_batch_size": self._queue_batch_size,
|
||||
},
|
||||
)
|
||||
|
||||
async def stop(self) -> None:
|
||||
if self._task is None:
|
||||
return
|
||||
self._task.cancel()
|
||||
try:
|
||||
await self._task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
finally:
|
||||
self._task = None
|
||||
logger.info("scheduler.stopped", extra={"event": "scheduler.stopped"})
|
||||
|
||||
async def _run_loop(self) -> None:
|
||||
while True:
|
||||
try:
|
||||
await self._tick_once()
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"scheduler.tick_failed",
|
||||
extra={
|
||||
"event": "scheduler.tick_failed",
|
||||
},
|
||||
)
|
||||
await asyncio.sleep(float(self._tick_seconds))
|
||||
|
||||
async def _tick_once(self) -> None:
|
||||
if self._continuation_queue_enabled:
|
||||
await self._drain_continuation_queue()
|
||||
candidates = await self._load_candidates()
|
||||
if not candidates:
|
||||
return
|
||||
now = datetime.now(timezone.utc)
|
||||
for candidate in candidates:
|
||||
if not await self._is_due(candidate, now=now):
|
||||
continue
|
||||
await self._run_candidate(candidate)
|
||||
|
||||
async def _load_candidate_rows(self) -> list[tuple]:
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as session:
|
||||
result = await session.execute(
|
||||
select(
|
||||
UserSetting.user_id,
|
||||
UserSetting.run_interval_minutes,
|
||||
UserSetting.request_delay_seconds,
|
||||
UserSetting.scrape_cooldown_until,
|
||||
UserSetting.scrape_cooldown_reason,
|
||||
)
|
||||
.join(User, User.id == UserSetting.user_id)
|
||||
.where(User.is_active.is_(True), UserSetting.auto_run_enabled.is_(True))
|
||||
.order_by(UserSetting.user_id.asc())
|
||||
)
|
||||
return result.all()
|
||||
|
||||
@staticmethod
|
||||
def _candidate_from_row(row: tuple, *, now_utc: datetime) -> _AutoRunCandidate | None:
|
||||
user_id, run_interval_minutes, request_delay_seconds, cooldown_until, cooldown_reason = row
|
||||
if cooldown_until is not None and cooldown_until.tzinfo is None:
|
||||
cooldown_until = cooldown_until.replace(tzinfo=timezone.utc)
|
||||
if cooldown_until is not None and cooldown_until > now_utc:
|
||||
logger.info(
|
||||
"scheduler.run_skipped_safety_cooldown_precheck",
|
||||
extra={
|
||||
"event": "scheduler.run_skipped_safety_cooldown_precheck",
|
||||
"user_id": int(user_id),
|
||||
"reason": cooldown_reason,
|
||||
"cooldown_until": cooldown_until,
|
||||
"cooldown_remaining_seconds": int((cooldown_until - now_utc).total_seconds()),
|
||||
"metric_name": "scheduler_run_skipped_safety_cooldown_total",
|
||||
"metric_value": 1,
|
||||
},
|
||||
)
|
||||
return None
|
||||
return _AutoRunCandidate(
|
||||
user_id=int(user_id),
|
||||
run_interval_minutes=int(run_interval_minutes),
|
||||
request_delay_seconds=int(request_delay_seconds),
|
||||
cooldown_until=cooldown_until,
|
||||
cooldown_reason=(str(cooldown_reason).strip() if cooldown_reason else None),
|
||||
)
|
||||
|
||||
async def _load_candidates(self) -> list[_AutoRunCandidate]:
|
||||
if not settings.ingestion_automation_allowed:
|
||||
return []
|
||||
rows = await self._load_candidate_rows()
|
||||
now_utc = datetime.now(timezone.utc)
|
||||
candidates: list[_AutoRunCandidate] = []
|
||||
for row in rows:
|
||||
candidate = self._candidate_from_row(row, now_utc=now_utc)
|
||||
if candidate is not None:
|
||||
candidates.append(candidate)
|
||||
return candidates
|
||||
|
||||
async def _is_due(self, candidate: _AutoRunCandidate, *, now: datetime) -> bool:
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as session:
|
||||
result = await session.execute(
|
||||
select(CrawlRun.start_dt)
|
||||
.where(
|
||||
CrawlRun.user_id == candidate.user_id,
|
||||
)
|
||||
.order_by(CrawlRun.start_dt.desc(), CrawlRun.id.desc())
|
||||
.limit(1)
|
||||
)
|
||||
last_run = result.scalar_one_or_none()
|
||||
|
||||
if last_run is None:
|
||||
return True
|
||||
|
||||
next_due_dt = last_run + timedelta(
|
||||
minutes=candidate.run_interval_minutes
|
||||
)
|
||||
return now >= next_due_dt
|
||||
|
||||
async def _run_candidate_ingestion(
|
||||
self,
|
||||
*,
|
||||
candidate: _AutoRunCandidate,
|
||||
):
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as session:
|
||||
ingestion = ScholarIngestionService(source=self._source)
|
||||
try:
|
||||
return await ingestion.run_for_user(
|
||||
session,
|
||||
user_id=candidate.user_id,
|
||||
trigger_type=RunTriggerType.SCHEDULED,
|
||||
request_delay_seconds=candidate.request_delay_seconds,
|
||||
network_error_retries=self._network_error_retries,
|
||||
retry_backoff_seconds=self._retry_backoff_seconds,
|
||||
max_pages_per_scholar=self._max_pages_per_scholar,
|
||||
page_size=self._page_size,
|
||||
auto_queue_continuations=self._continuation_queue_enabled,
|
||||
queue_delay_seconds=self._continuation_base_delay_seconds,
|
||||
alert_blocked_failure_threshold=settings.ingestion_alert_blocked_failure_threshold,
|
||||
alert_network_failure_threshold=settings.ingestion_alert_network_failure_threshold,
|
||||
alert_retry_scheduled_threshold=settings.ingestion_alert_retry_scheduled_threshold,
|
||||
)
|
||||
except RunAlreadyInProgressError:
|
||||
await session.rollback()
|
||||
logger.info("scheduler.run_skipped_locked", extra={"event": "scheduler.run_skipped_locked", "user_id": candidate.user_id})
|
||||
return None
|
||||
except RunBlockedBySafetyPolicyError as exc:
|
||||
await session.rollback()
|
||||
logger.info(
|
||||
"scheduler.run_skipped_safety_cooldown",
|
||||
extra={
|
||||
"event": "scheduler.run_skipped_safety_cooldown",
|
||||
"user_id": candidate.user_id,
|
||||
"reason": exc.safety_state.get("cooldown_reason"),
|
||||
"cooldown_until": exc.safety_state.get("cooldown_until"),
|
||||
"cooldown_remaining_seconds": exc.safety_state.get("cooldown_remaining_seconds"),
|
||||
"metric_name": "scheduler_run_skipped_safety_cooldown_total",
|
||||
"metric_value": 1,
|
||||
},
|
||||
)
|
||||
return None
|
||||
except Exception:
|
||||
await session.rollback()
|
||||
logger.exception("scheduler.run_failed", extra={"event": "scheduler.run_failed", "user_id": candidate.user_id})
|
||||
return None
|
||||
|
||||
async def _run_candidate(self, candidate: _AutoRunCandidate) -> None:
|
||||
run_summary = await self._run_candidate_ingestion(candidate=candidate)
|
||||
if run_summary is None:
|
||||
return
|
||||
logger.info(
|
||||
"scheduler.run_completed",
|
||||
extra={
|
||||
"event": "scheduler.run_completed",
|
||||
"user_id": candidate.user_id,
|
||||
"run_id": run_summary.crawl_run_id,
|
||||
"status": run_summary.status.value,
|
||||
"scholar_count": run_summary.scholar_count,
|
||||
"new_publication_count": run_summary.new_publication_count,
|
||||
},
|
||||
)
|
||||
|
||||
async def _drain_continuation_queue(self) -> None:
|
||||
now = datetime.now(timezone.utc)
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as session:
|
||||
jobs = await queue_service.list_due_jobs(
|
||||
session,
|
||||
now=now,
|
||||
limit=self._queue_batch_size,
|
||||
)
|
||||
for job in jobs:
|
||||
await self._run_queue_job(job)
|
||||
|
||||
async def _drop_queue_job_if_max_attempts(
|
||||
self,
|
||||
job: queue_service.ContinuationQueueJob,
|
||||
) -> bool:
|
||||
if job.attempt_count < self._continuation_max_attempts:
|
||||
return False
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as session:
|
||||
dropped = await queue_service.mark_dropped(
|
||||
session,
|
||||
job_id=job.id,
|
||||
reason="max_attempts_reached",
|
||||
)
|
||||
await session.commit()
|
||||
if dropped is not None:
|
||||
logger.warning(
|
||||
"scheduler.queue_item_dropped_max_attempts",
|
||||
extra={
|
||||
"event": "scheduler.queue_item_dropped_max_attempts",
|
||||
"queue_item_id": job.id,
|
||||
"user_id": job.user_id,
|
||||
"scholar_profile_id": job.scholar_profile_id,
|
||||
"attempt_count": job.attempt_count,
|
||||
"max_attempts": self._continuation_max_attempts,
|
||||
},
|
||||
)
|
||||
return True
|
||||
|
||||
async def _mark_queue_job_retrying(
|
||||
self,
|
||||
job: queue_service.ContinuationQueueJob,
|
||||
) -> bool:
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as session:
|
||||
queue_item = await queue_service.mark_retrying(session, job_id=job.id)
|
||||
await session.commit()
|
||||
if queue_item is None:
|
||||
return False
|
||||
if queue_item.status == QueueItemStatus.DROPPED.value:
|
||||
return False
|
||||
return True
|
||||
|
||||
async def _queue_job_has_available_scholar(
|
||||
self,
|
||||
job: queue_service.ContinuationQueueJob,
|
||||
) -> bool:
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as session:
|
||||
scholar_result = await session.execute(
|
||||
select(ScholarProfile.id).where(
|
||||
ScholarProfile.user_id == job.user_id,
|
||||
ScholarProfile.id == job.scholar_profile_id,
|
||||
ScholarProfile.is_enabled.is_(True),
|
||||
)
|
||||
)
|
||||
scholar_id = scholar_result.scalar_one_or_none()
|
||||
if scholar_id is not None:
|
||||
return True
|
||||
dropped = await queue_service.mark_dropped(
|
||||
session,
|
||||
job_id=job.id,
|
||||
reason="scholar_unavailable",
|
||||
)
|
||||
await session.commit()
|
||||
if dropped is not None:
|
||||
logger.info(
|
||||
"scheduler.queue_item_dropped_scholar_unavailable",
|
||||
extra={
|
||||
"event": "scheduler.queue_item_dropped_scholar_unavailable",
|
||||
"queue_item_id": job.id,
|
||||
"user_id": job.user_id,
|
||||
"scholar_profile_id": job.scholar_profile_id,
|
||||
},
|
||||
)
|
||||
return False
|
||||
|
||||
async def _reschedule_queue_job_lock_active(self, job: queue_service.ContinuationQueueJob) -> None:
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as recovery_session:
|
||||
await queue_service.reschedule_job(
|
||||
recovery_session,
|
||||
job_id=job.id,
|
||||
delay_seconds=max(self._tick_seconds, 15),
|
||||
reason="user_run_lock_active",
|
||||
error="run_already_in_progress",
|
||||
)
|
||||
await recovery_session.commit()
|
||||
logger.info(
|
||||
"scheduler.queue_item_deferred_lock",
|
||||
extra={"event": "scheduler.queue_item_deferred_lock", "queue_item_id": job.id, "user_id": job.user_id},
|
||||
)
|
||||
|
||||
async def _reschedule_queue_job_safety_cooldown(
|
||||
self,
|
||||
job: queue_service.ContinuationQueueJob,
|
||||
exc: RunBlockedBySafetyPolicyError,
|
||||
) -> None:
|
||||
cooldown_remaining_seconds = max(
|
||||
self._tick_seconds,
|
||||
int(exc.safety_state.get("cooldown_remaining_seconds") or 0),
|
||||
)
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as recovery_session:
|
||||
await queue_service.reschedule_job(
|
||||
recovery_session,
|
||||
job_id=job.id,
|
||||
delay_seconds=max(self._tick_seconds, cooldown_remaining_seconds),
|
||||
reason="scrape_safety_cooldown",
|
||||
error=str(exc.message),
|
||||
)
|
||||
await recovery_session.commit()
|
||||
logger.info(
|
||||
"scheduler.queue_item_deferred_safety_cooldown",
|
||||
extra={
|
||||
"event": "scheduler.queue_item_deferred_safety_cooldown",
|
||||
"queue_item_id": job.id,
|
||||
"user_id": job.user_id,
|
||||
"reason": exc.safety_state.get("cooldown_reason"),
|
||||
"cooldown_remaining_seconds": cooldown_remaining_seconds,
|
||||
"metric_name": "scheduler_queue_item_deferred_safety_cooldown_total",
|
||||
"metric_value": 1,
|
||||
},
|
||||
)
|
||||
|
||||
async def _reschedule_queue_job_after_exception(
|
||||
self,
|
||||
job: queue_service.ContinuationQueueJob,
|
||||
*,
|
||||
exc: Exception,
|
||||
) -> None:
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as recovery_session:
|
||||
queue_item = await queue_service.increment_attempt_count(recovery_session, job_id=job.id)
|
||||
if queue_item is None:
|
||||
await recovery_session.commit()
|
||||
return
|
||||
if int(queue_item.attempt_count) >= self._continuation_max_attempts:
|
||||
await queue_service.mark_dropped(
|
||||
recovery_session,
|
||||
job_id=job.id,
|
||||
reason="scheduler_exception_max_attempts",
|
||||
error=str(exc),
|
||||
)
|
||||
await recovery_session.commit()
|
||||
logger.warning(
|
||||
"scheduler.queue_item_dropped_after_exception",
|
||||
extra={"event": "scheduler.queue_item_dropped_after_exception", "queue_item_id": job.id, "user_id": job.user_id, "attempt_count": queue_item.attempt_count},
|
||||
)
|
||||
return
|
||||
delay_seconds = queue_service.compute_backoff_seconds(
|
||||
base_seconds=self._continuation_base_delay_seconds,
|
||||
attempt_count=int(queue_item.attempt_count),
|
||||
max_seconds=self._continuation_max_delay_seconds,
|
||||
)
|
||||
await queue_service.reschedule_job(
|
||||
recovery_session,
|
||||
job_id=job.id,
|
||||
delay_seconds=delay_seconds,
|
||||
reason="scheduler_exception",
|
||||
error=str(exc),
|
||||
)
|
||||
await recovery_session.commit()
|
||||
logger.exception("scheduler.queue_item_run_failed", extra={"event": "scheduler.queue_item_run_failed", "queue_item_id": job.id, "user_id": job.user_id})
|
||||
|
||||
async def _run_ingestion_for_queue_job(
|
||||
self,
|
||||
job: queue_service.ContinuationQueueJob,
|
||||
):
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as session:
|
||||
request_delay_seconds = await self._load_request_delay_for_user(session, user_id=job.user_id)
|
||||
ingestion = ScholarIngestionService(source=self._source)
|
||||
try:
|
||||
return await ingestion.run_for_user(
|
||||
session,
|
||||
user_id=job.user_id,
|
||||
trigger_type=RunTriggerType.SCHEDULED,
|
||||
request_delay_seconds=request_delay_seconds,
|
||||
network_error_retries=self._network_error_retries,
|
||||
retry_backoff_seconds=self._retry_backoff_seconds,
|
||||
max_pages_per_scholar=self._max_pages_per_scholar,
|
||||
page_size=self._page_size,
|
||||
scholar_profile_ids={job.scholar_profile_id},
|
||||
start_cstart_by_scholar_id={job.scholar_profile_id: job.resume_cstart},
|
||||
auto_queue_continuations=self._continuation_queue_enabled,
|
||||
queue_delay_seconds=self._continuation_base_delay_seconds,
|
||||
alert_blocked_failure_threshold=settings.ingestion_alert_blocked_failure_threshold,
|
||||
alert_network_failure_threshold=settings.ingestion_alert_network_failure_threshold,
|
||||
alert_retry_scheduled_threshold=settings.ingestion_alert_retry_scheduled_threshold,
|
||||
)
|
||||
except RunAlreadyInProgressError:
|
||||
await session.rollback()
|
||||
await self._reschedule_queue_job_lock_active(job)
|
||||
except RunBlockedBySafetyPolicyError as exc:
|
||||
await session.rollback()
|
||||
await self._reschedule_queue_job_safety_cooldown(job, exc)
|
||||
except Exception as exc:
|
||||
await session.rollback()
|
||||
await self._reschedule_queue_job_after_exception(job, exc=exc)
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _log_queue_item_resolved(
|
||||
*,
|
||||
event_name: str,
|
||||
job: queue_service.ContinuationQueueJob,
|
||||
run_summary,
|
||||
attempt_count: int | None = None,
|
||||
delay_seconds: int | None = None,
|
||||
) -> None:
|
||||
payload = {
|
||||
"event": event_name,
|
||||
"queue_item_id": job.id,
|
||||
"user_id": job.user_id,
|
||||
"run_id": run_summary.crawl_run_id,
|
||||
"status": run_summary.status.value,
|
||||
}
|
||||
if attempt_count is not None:
|
||||
payload["attempt_count"] = attempt_count
|
||||
if delay_seconds is not None:
|
||||
payload["delay_seconds"] = delay_seconds
|
||||
logger.info(event_name, extra=payload)
|
||||
|
||||
async def _finalize_queue_job_after_run(self, job: queue_service.ContinuationQueueJob, run_summary) -> None:
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as session:
|
||||
if int(run_summary.failed_count) <= 0:
|
||||
queue_item = await queue_service.reset_attempt_count(session, job_id=job.id)
|
||||
await session.commit()
|
||||
if queue_item is None:
|
||||
self._log_queue_item_resolved(event_name="scheduler.queue_item_resolved", job=job, run_summary=run_summary)
|
||||
return
|
||||
self._log_queue_item_resolved(
|
||||
event_name="scheduler.queue_item_progressed",
|
||||
job=job,
|
||||
run_summary=run_summary,
|
||||
attempt_count=int(queue_item.attempt_count),
|
||||
)
|
||||
return
|
||||
queue_item = await queue_service.increment_attempt_count(session, job_id=job.id)
|
||||
if queue_item is None:
|
||||
await session.commit()
|
||||
self._log_queue_item_resolved(event_name="scheduler.queue_item_resolved", job=job, run_summary=run_summary)
|
||||
return
|
||||
if int(queue_item.attempt_count) >= self._continuation_max_attempts:
|
||||
await queue_service.mark_dropped(session, job_id=job.id, reason="max_attempts_after_run")
|
||||
await session.commit()
|
||||
logger.warning(
|
||||
"scheduler.queue_item_dropped_max_attempts_after_run",
|
||||
extra={"event": "scheduler.queue_item_dropped_max_attempts_after_run", "queue_item_id": job.id, "user_id": job.user_id, "attempt_count": queue_item.attempt_count, "run_id": run_summary.crawl_run_id, "status": run_summary.status.value},
|
||||
)
|
||||
return
|
||||
delay_seconds = queue_service.compute_backoff_seconds(base_seconds=self._continuation_base_delay_seconds, attempt_count=int(queue_item.attempt_count), max_seconds=self._continuation_max_delay_seconds)
|
||||
await queue_service.reschedule_job(session, job_id=job.id, delay_seconds=delay_seconds, reason=queue_item.reason, error=queue_item.last_error)
|
||||
await session.commit()
|
||||
self._log_queue_item_resolved(
|
||||
event_name="scheduler.queue_item_rescheduled_failed",
|
||||
job=job,
|
||||
run_summary=run_summary,
|
||||
attempt_count=int(queue_item.attempt_count),
|
||||
delay_seconds=delay_seconds,
|
||||
)
|
||||
|
||||
async def _run_queue_job(self, job: queue_service.ContinuationQueueJob) -> None:
|
||||
if await self._drop_queue_job_if_max_attempts(job):
|
||||
return
|
||||
if not await self._mark_queue_job_retrying(job):
|
||||
return
|
||||
if not await self._queue_job_has_available_scholar(job):
|
||||
return
|
||||
run_summary = await self._run_ingestion_for_queue_job(job)
|
||||
if run_summary is None:
|
||||
return
|
||||
await self._finalize_queue_job_after_run(job, run_summary)
|
||||
|
||||
async def _load_request_delay_for_user(
|
||||
self,
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
) -> int:
|
||||
result = await db_session.execute(
|
||||
select(UserSetting.request_delay_seconds).where(UserSetting.user_id == user_id)
|
||||
)
|
||||
delay = result.scalar_one_or_none()
|
||||
if delay is None:
|
||||
return 10
|
||||
return max(1, int(delay))
|
||||
from app.services.domains.ingestion.scheduler import *
|
||||
|
|
|
|||
|
|
@ -1,886 +1,3 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from enum import StrEnum
|
||||
from html import unescape
|
||||
from html.parser import HTMLParser
|
||||
from typing import Any
|
||||
from urllib.parse import parse_qs, urljoin, urlparse
|
||||
|
||||
from app.services.scholar_source import FetchResult
|
||||
|
||||
BLOCKED_KEYWORDS = [
|
||||
"unusual traffic",
|
||||
"sorry/index",
|
||||
"not a robot",
|
||||
"our systems have detected",
|
||||
"automated queries",
|
||||
"recaptcha",
|
||||
"captcha",
|
||||
]
|
||||
|
||||
NO_RESULTS_KEYWORDS = [
|
||||
"didn't match any articles",
|
||||
"did not match any articles",
|
||||
"no articles",
|
||||
"no documents",
|
||||
]
|
||||
|
||||
NO_AUTHOR_RESULTS_KEYWORDS = [
|
||||
"didn't match any user profiles",
|
||||
"did not match any user profiles",
|
||||
"didn't match any scholars",
|
||||
"did not match any scholars",
|
||||
"no user profiles",
|
||||
]
|
||||
|
||||
MARKER_KEYS = [
|
||||
"gsc_a_tr",
|
||||
"gsc_a_at",
|
||||
"gsc_a_ac",
|
||||
"gsc_a_h",
|
||||
"gsc_a_y",
|
||||
"gs_gray",
|
||||
"gsc_prf_in",
|
||||
"gsc_rsb_st",
|
||||
]
|
||||
|
||||
AUTHOR_SEARCH_MARKER_KEYS = [
|
||||
"gsc_1usr",
|
||||
"gs_ai_name",
|
||||
"gs_ai_aff",
|
||||
"gs_ai_eml",
|
||||
"gs_ai_cby",
|
||||
"gs_ai_one_int",
|
||||
]
|
||||
|
||||
NETWORK_DNS_ERROR_KEYWORDS = [
|
||||
"temporary failure in name resolution",
|
||||
"name or service not known",
|
||||
"nodename nor servname provided",
|
||||
"getaddrinfo failed",
|
||||
]
|
||||
|
||||
NETWORK_TIMEOUT_KEYWORDS = [
|
||||
"timed out",
|
||||
"timeout",
|
||||
]
|
||||
|
||||
NETWORK_TLS_ERROR_KEYWORDS = [
|
||||
"ssl",
|
||||
"tls",
|
||||
"certificate verify failed",
|
||||
]
|
||||
|
||||
TAG_RE = re.compile(r"<[^>]+>", re.S)
|
||||
SCRIPT_STYLE_RE = re.compile(r"<(script|style)\b[^>]*>.*?</\1>", re.I | re.S)
|
||||
SHOW_MORE_BUTTON_RE = re.compile(
|
||||
r"<button\b[^>]*\bid\s*=\s*['\"]gsc_bpf_more['\"][^>]*>",
|
||||
re.I | re.S,
|
||||
)
|
||||
PROFILE_ROW_PARSER_DIRECT_MARKERS = (
|
||||
"gs_ggs",
|
||||
"gs_ggsd",
|
||||
"gs_ggsa",
|
||||
"gs_or_ggsm",
|
||||
)
|
||||
PROFILE_ROW_DIRECT_LABEL_TOKENS = (
|
||||
"pdf",
|
||||
"[pdf]",
|
||||
"full text",
|
||||
"download",
|
||||
)
|
||||
|
||||
|
||||
class ParseState(StrEnum):
|
||||
OK = "ok"
|
||||
NO_RESULTS = "no_results"
|
||||
BLOCKED_OR_CAPTCHA = "blocked_or_captcha"
|
||||
LAYOUT_CHANGED = "layout_changed"
|
||||
NETWORK_ERROR = "network_error"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PublicationCandidate:
|
||||
title: str
|
||||
title_url: str | None
|
||||
cluster_id: str | None
|
||||
year: int | None
|
||||
citation_count: int | None
|
||||
authors_text: str | None
|
||||
venue_text: str | None
|
||||
pdf_url: str | None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ScholarSearchCandidate:
|
||||
scholar_id: str
|
||||
display_name: str
|
||||
affiliation: str | None
|
||||
email_domain: str | None
|
||||
cited_by_count: int | None
|
||||
interests: list[str]
|
||||
profile_url: str
|
||||
profile_image_url: str | None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ParsedProfilePage:
|
||||
state: ParseState
|
||||
state_reason: str
|
||||
profile_name: str | None
|
||||
profile_image_url: str | None
|
||||
publications: list[PublicationCandidate]
|
||||
marker_counts: dict[str, int]
|
||||
warnings: list[str]
|
||||
has_show_more_button: bool
|
||||
has_operation_error_banner: bool
|
||||
articles_range: str | None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ParsedAuthorSearchPage:
|
||||
state: ParseState
|
||||
state_reason: str
|
||||
candidates: list[ScholarSearchCandidate]
|
||||
marker_counts: dict[str, int]
|
||||
warnings: list[str]
|
||||
|
||||
|
||||
def normalize_space(value: str) -> str:
|
||||
return " ".join(unescape(value).split())
|
||||
|
||||
|
||||
def strip_tags(value: str) -> str:
|
||||
return normalize_space(TAG_RE.sub(" ", value))
|
||||
|
||||
|
||||
def attr_class(attrs: list[tuple[str, str | None]]) -> str:
|
||||
for name, raw_value in attrs:
|
||||
if name.lower() == "class":
|
||||
return raw_value or ""
|
||||
return ""
|
||||
|
||||
|
||||
def attr_href(attrs: list[tuple[str, str | None]]) -> str | None:
|
||||
for name, raw_value in attrs:
|
||||
if name.lower() == "href":
|
||||
return raw_value
|
||||
return None
|
||||
|
||||
|
||||
def attr_src(attrs: list[tuple[str, str | None]]) -> str | None:
|
||||
for name, raw_value in attrs:
|
||||
if name.lower() == "src":
|
||||
return raw_value
|
||||
return None
|
||||
|
||||
|
||||
def build_absolute_scholar_url(path_or_url: str | None) -> str | None:
|
||||
if not path_or_url:
|
||||
return None
|
||||
return urljoin("https://scholar.google.com", path_or_url)
|
||||
|
||||
|
||||
class ScholarRowParser(HTMLParser):
|
||||
def __init__(self) -> None:
|
||||
super().__init__(convert_charrefs=True)
|
||||
self.title_href: str | None = None
|
||||
self.direct_download_href: str | None = None
|
||||
self.title_parts: list[str] = []
|
||||
self.citation_parts: list[str] = []
|
||||
self.year_parts: list[str] = []
|
||||
self.gray_texts: list[str] = []
|
||||
|
||||
self._title_depth = 0
|
||||
self._citation_depth = 0
|
||||
self._year_depth = 0
|
||||
self._gray_stack: list[dict[str, Any]] = []
|
||||
self._direct_marker_depth = 0
|
||||
self._aux_link_stack: list[dict[str, Any]] = []
|
||||
|
||||
@staticmethod
|
||||
def _contains_direct_marker(classes: str) -> bool:
|
||||
lowered = classes.lower()
|
||||
return any(marker in lowered for marker in PROFILE_ROW_PARSER_DIRECT_MARKERS)
|
||||
|
||||
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
||||
if self._title_depth > 0:
|
||||
self._title_depth += 1
|
||||
if self._citation_depth > 0:
|
||||
self._citation_depth += 1
|
||||
if self._year_depth > 0:
|
||||
self._year_depth += 1
|
||||
if self._gray_stack:
|
||||
self._gray_stack[-1]["depth"] += 1
|
||||
if self._direct_marker_depth > 0:
|
||||
self._direct_marker_depth += 1
|
||||
if self._aux_link_stack:
|
||||
self._aux_link_stack[-1]["depth"] += 1
|
||||
|
||||
classes = attr_class(attrs)
|
||||
if tag in {"div", "span"} and self._contains_direct_marker(classes):
|
||||
self._direct_marker_depth = 1
|
||||
|
||||
if tag == "a" and "gsc_a_at" in classes:
|
||||
self._title_depth = 1
|
||||
self.title_href = attr_href(attrs)
|
||||
return
|
||||
|
||||
if tag == "a" and "gsc_a_ac" in classes:
|
||||
self._citation_depth = 1
|
||||
return
|
||||
|
||||
if tag in {"span", "a"} and ("gsc_a_h" in classes or "gsc_a_y" in classes):
|
||||
self._year_depth = 1
|
||||
return
|
||||
|
||||
if tag == "div" and "gs_gray" in classes:
|
||||
self._gray_stack.append({"depth": 1, "parts": []})
|
||||
return
|
||||
|
||||
if tag == "a":
|
||||
self._aux_link_stack.append(
|
||||
{
|
||||
"depth": 1,
|
||||
"href": attr_href(attrs),
|
||||
"classes": classes,
|
||||
"parts": [],
|
||||
}
|
||||
)
|
||||
|
||||
def handle_data(self, data: str) -> None:
|
||||
if self._title_depth > 0:
|
||||
self.title_parts.append(data)
|
||||
if self._citation_depth > 0:
|
||||
self.citation_parts.append(data)
|
||||
if self._year_depth > 0:
|
||||
self.year_parts.append(data)
|
||||
if self._gray_stack:
|
||||
self._gray_stack[-1]["parts"].append(data)
|
||||
if self._aux_link_stack:
|
||||
self._aux_link_stack[-1]["parts"].append(data)
|
||||
|
||||
def _capture_direct_download_href(self, link: dict[str, Any]) -> None:
|
||||
if self.direct_download_href:
|
||||
return
|
||||
href = link.get("href")
|
||||
if not isinstance(href, str) or not href.strip():
|
||||
return
|
||||
label = normalize_space("".join(link.get("parts", []))).lower()
|
||||
classes = str(link.get("classes", "")).lower()
|
||||
label_match = any(token in label for token in PROFILE_ROW_DIRECT_LABEL_TOKENS)
|
||||
marker_match = self._contains_direct_marker(classes) or self._direct_marker_depth > 0
|
||||
if label_match or marker_match:
|
||||
self.direct_download_href = href.strip()
|
||||
|
||||
def handle_endtag(self, _tag: str) -> None:
|
||||
if self._title_depth > 0:
|
||||
self._title_depth -= 1
|
||||
if self._citation_depth > 0:
|
||||
self._citation_depth -= 1
|
||||
if self._year_depth > 0:
|
||||
self._year_depth -= 1
|
||||
if self._gray_stack:
|
||||
self._gray_stack[-1]["depth"] -= 1
|
||||
if self._gray_stack[-1]["depth"] == 0:
|
||||
text = normalize_space("".join(self._gray_stack[-1]["parts"]))
|
||||
if text:
|
||||
self.gray_texts.append(text)
|
||||
self._gray_stack.pop()
|
||||
if self._aux_link_stack:
|
||||
self._aux_link_stack[-1]["depth"] -= 1
|
||||
if self._aux_link_stack[-1]["depth"] == 0:
|
||||
self._capture_direct_download_href(self._aux_link_stack[-1])
|
||||
self._aux_link_stack.pop()
|
||||
if self._direct_marker_depth > 0:
|
||||
self._direct_marker_depth -= 1
|
||||
|
||||
|
||||
def extract_rows(html: str) -> list[str]:
|
||||
pattern = re.compile(
|
||||
r"<tr\b(?=[^>]*\bclass\s*=\s*['\"][^'\"]*\bgsc_a_tr\b[^'\"]*['\"])[^>]*>(.*?)</tr>",
|
||||
re.I | re.S,
|
||||
)
|
||||
return [match.group(1) for match in pattern.finditer(html)]
|
||||
|
||||
|
||||
def parse_cluster_id_from_href(href: str | None) -> str | None:
|
||||
if not href:
|
||||
return None
|
||||
parsed = urlparse(href)
|
||||
query = parse_qs(parsed.query)
|
||||
|
||||
citation_for_view = query.get("citation_for_view")
|
||||
if citation_for_view:
|
||||
token = citation_for_view[0].strip()
|
||||
if token:
|
||||
if ":" in token:
|
||||
return token.rsplit(":", 1)[-1] or None
|
||||
return token
|
||||
|
||||
cluster = query.get("cluster")
|
||||
if cluster:
|
||||
token = cluster[0].strip()
|
||||
if token:
|
||||
return token
|
||||
return None
|
||||
|
||||
|
||||
def parse_scholar_id_from_href(href: str | None) -> str | None:
|
||||
if not href:
|
||||
return None
|
||||
parsed = urlparse(href)
|
||||
query = parse_qs(parsed.query)
|
||||
user_values = query.get("user")
|
||||
if not user_values:
|
||||
return None
|
||||
candidate = user_values[0].strip()
|
||||
return candidate or None
|
||||
|
||||
|
||||
def parse_year(parts: list[str]) -> int | None:
|
||||
text = normalize_space(" ".join(parts))
|
||||
match = re.search(r"\b(19|20)\d{2}\b", text)
|
||||
if not match:
|
||||
return None
|
||||
try:
|
||||
return int(match.group(0))
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def parse_citation_count(parts: list[str]) -> int | None:
|
||||
text = normalize_space(" ".join(parts))
|
||||
if not text:
|
||||
return 0
|
||||
match = re.search(r"\d+", text)
|
||||
if not match:
|
||||
return None
|
||||
return int(match.group(0))
|
||||
|
||||
|
||||
def _parse_publication_row(row_html: str) -> tuple[PublicationCandidate | None, list[str]]:
|
||||
parser = ScholarRowParser()
|
||||
parser.feed(row_html)
|
||||
warnings: list[str] = []
|
||||
title = normalize_space("".join(parser.title_parts))
|
||||
if not title:
|
||||
warnings.append("row_missing_title")
|
||||
return None, warnings
|
||||
if not parser.title_href:
|
||||
warnings.append("row_missing_title_href")
|
||||
|
||||
citation_text = normalize_space(" ".join(parser.citation_parts))
|
||||
citation_count = parse_citation_count(parser.citation_parts)
|
||||
if citation_text and citation_count is None:
|
||||
warnings.append("layout_row_citation_unparseable")
|
||||
|
||||
year_text = normalize_space(" ".join(parser.year_parts))
|
||||
year = parse_year(parser.year_parts)
|
||||
if year_text and year is None:
|
||||
warnings.append("layout_row_year_unparseable")
|
||||
|
||||
authors_text = parser.gray_texts[0] if len(parser.gray_texts) > 0 else None
|
||||
venue_text = parser.gray_texts[1] if len(parser.gray_texts) > 1 else None
|
||||
return (
|
||||
PublicationCandidate(
|
||||
title=title,
|
||||
title_url=parser.title_href,
|
||||
cluster_id=parse_cluster_id_from_href(parser.title_href),
|
||||
year=year,
|
||||
citation_count=citation_count,
|
||||
authors_text=authors_text,
|
||||
venue_text=venue_text,
|
||||
pdf_url=build_absolute_scholar_url(parser.direct_download_href),
|
||||
),
|
||||
warnings,
|
||||
)
|
||||
|
||||
|
||||
def parse_publications(html: str) -> tuple[list[PublicationCandidate], list[str]]:
|
||||
rows = extract_rows(html)
|
||||
warnings: list[str] = []
|
||||
publications: list[PublicationCandidate] = []
|
||||
|
||||
for row_html in rows:
|
||||
publication, row_warnings = _parse_publication_row(row_html)
|
||||
warnings.extend(row_warnings)
|
||||
if publication is None:
|
||||
continue
|
||||
publications.append(publication)
|
||||
|
||||
if not rows:
|
||||
warnings.append("no_rows_detected")
|
||||
if rows and not publications:
|
||||
warnings.append("layout_all_rows_unparseable")
|
||||
|
||||
return publications, sorted(set(warnings))
|
||||
|
||||
|
||||
def extract_profile_name(html: str) -> str | None:
|
||||
pattern = re.compile(
|
||||
r"<[^>]*\bid\s*=\s*['\"]gsc_prf_in['\"][^>]*>(.*?)</[^>]+>",
|
||||
re.I | re.S,
|
||||
)
|
||||
match = pattern.search(html)
|
||||
if not match:
|
||||
return None
|
||||
value = strip_tags(match.group(1))
|
||||
return value or None
|
||||
|
||||
|
||||
def extract_profile_image_url(html: str) -> str | None:
|
||||
og_image_pattern = re.compile(
|
||||
r"<meta[^>]+property=['\"]og:image['\"][^>]+content=['\"]([^'\"]+)['\"][^>]*>",
|
||||
re.I | re.S,
|
||||
)
|
||||
og_match = og_image_pattern.search(html)
|
||||
if og_match:
|
||||
value = normalize_space(og_match.group(1))
|
||||
absolute = build_absolute_scholar_url(value)
|
||||
if absolute:
|
||||
return absolute
|
||||
|
||||
image_pattern = re.compile(
|
||||
r"<img[^>]*\bid=['\"]gsc_prf_pup-img['\"][^>]*\bsrc=['\"]([^'\"]+)['\"][^>]*>",
|
||||
re.I | re.S,
|
||||
)
|
||||
image_match = image_pattern.search(html)
|
||||
if not image_match:
|
||||
return None
|
||||
|
||||
value = normalize_space(image_match.group(1))
|
||||
return build_absolute_scholar_url(value)
|
||||
|
||||
|
||||
def extract_articles_range(html: str) -> str | None:
|
||||
pattern = re.compile(
|
||||
r"<[^>]*\bid\s*=\s*['\"]gsc_a_nn['\"][^>]*>(.*?)</[^>]+>",
|
||||
re.I | re.S,
|
||||
)
|
||||
match = pattern.search(html)
|
||||
if not match:
|
||||
return None
|
||||
value = strip_tags(match.group(1))
|
||||
return value or None
|
||||
|
||||
|
||||
def has_show_more_button(html: str) -> bool:
|
||||
match = SHOW_MORE_BUTTON_RE.search(html)
|
||||
if match is None:
|
||||
return False
|
||||
|
||||
button_tag = match.group(0).lower()
|
||||
if "disabled" in button_tag:
|
||||
return False
|
||||
if 'aria-disabled="true"' in button_tag or "aria-disabled='true'" in button_tag:
|
||||
return False
|
||||
if "gs_dis" in button_tag:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def has_operation_error_banner(html: str) -> bool:
|
||||
lowered = html.lower()
|
||||
if "id=\"gsc_a_err\"" not in lowered and "id='gsc_a_err'" not in lowered:
|
||||
return False
|
||||
return "can't perform the operation now" in lowered or "cannot perform the operation now" in lowered
|
||||
|
||||
|
||||
def count_markers(html: str) -> dict[str, int]:
|
||||
lowered = html.lower()
|
||||
return {key: lowered.count(key.lower()) for key in MARKER_KEYS}
|
||||
|
||||
|
||||
def count_author_search_markers(html: str) -> dict[str, int]:
|
||||
lowered = html.lower()
|
||||
return {key: lowered.count(key.lower()) for key in AUTHOR_SEARCH_MARKER_KEYS}
|
||||
|
||||
|
||||
def _extract_verified_email_domain(value: str | None) -> str | None:
|
||||
if not value:
|
||||
return None
|
||||
match = re.search(r"verified email at\s+(.+)$", value.strip(), re.I)
|
||||
if not match:
|
||||
return None
|
||||
domain = normalize_space(match.group(1))
|
||||
return domain or None
|
||||
|
||||
|
||||
class ScholarAuthorSearchParser(HTMLParser):
|
||||
def __init__(self) -> None:
|
||||
super().__init__(convert_charrefs=True)
|
||||
self.candidates: list[ScholarSearchCandidate] = []
|
||||
self._candidate: dict[str, Any] | None = None
|
||||
|
||||
def _begin_candidate(self) -> None:
|
||||
self._candidate = {
|
||||
"depth": 1,
|
||||
"name_href": None,
|
||||
"name_parts": [],
|
||||
"aff_depth": 0,
|
||||
"aff_parts": [],
|
||||
"name_depth": 0,
|
||||
"eml_depth": 0,
|
||||
"eml_parts": [],
|
||||
"cby_depth": 0,
|
||||
"cby_parts": [],
|
||||
"interest_depth": 0,
|
||||
"interest_parts": [],
|
||||
"interests": [],
|
||||
"image_src": None,
|
||||
}
|
||||
|
||||
def _increment_capture_depths(self) -> None:
|
||||
if self._candidate is None:
|
||||
return
|
||||
for key in ("name_depth", "aff_depth", "eml_depth", "cby_depth", "interest_depth"):
|
||||
if self._candidate[key] > 0:
|
||||
self._candidate[key] += 1
|
||||
|
||||
def _finalize_candidate(self) -> None:
|
||||
if self._candidate is None:
|
||||
return
|
||||
|
||||
name = normalize_space("".join(self._candidate["name_parts"]))
|
||||
scholar_id = parse_scholar_id_from_href(self._candidate["name_href"])
|
||||
if not name or not scholar_id:
|
||||
return
|
||||
|
||||
affiliation = normalize_space("".join(self._candidate["aff_parts"])) or None
|
||||
email_domain = _extract_verified_email_domain(
|
||||
normalize_space("".join(self._candidate["eml_parts"])) or None
|
||||
)
|
||||
cited_by_text = normalize_space("".join(self._candidate["cby_parts"]))
|
||||
cited_by_match = re.search(r"\d+", cited_by_text)
|
||||
cited_by_count = int(cited_by_match.group(0)) if cited_by_match else None
|
||||
|
||||
seen_interests: set[str] = set()
|
||||
interests: list[str] = []
|
||||
for interest in self._candidate["interests"]:
|
||||
normalized = normalize_space(interest)
|
||||
if not normalized or normalized in seen_interests:
|
||||
continue
|
||||
seen_interests.add(normalized)
|
||||
interests.append(normalized)
|
||||
|
||||
profile_url = build_absolute_scholar_url(self._candidate["name_href"])
|
||||
if not profile_url:
|
||||
profile_url = (
|
||||
"https://scholar.google.com/citations"
|
||||
f"?hl=en&user={scholar_id}"
|
||||
)
|
||||
|
||||
self.candidates.append(
|
||||
ScholarSearchCandidate(
|
||||
scholar_id=scholar_id,
|
||||
display_name=name,
|
||||
affiliation=affiliation,
|
||||
email_domain=email_domain,
|
||||
cited_by_count=cited_by_count,
|
||||
interests=interests,
|
||||
profile_url=profile_url,
|
||||
profile_image_url=build_absolute_scholar_url(self._candidate["image_src"]),
|
||||
)
|
||||
)
|
||||
|
||||
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
||||
classes = attr_class(attrs)
|
||||
|
||||
if self._candidate is None:
|
||||
if tag == "div" and "gsc_1usr" in classes:
|
||||
self._begin_candidate()
|
||||
return
|
||||
|
||||
self._candidate["depth"] += 1
|
||||
self._increment_capture_depths()
|
||||
|
||||
if tag == "a" and "gs_ai_name" in classes:
|
||||
self._candidate["name_depth"] = 1
|
||||
self._candidate["name_href"] = attr_href(attrs)
|
||||
return
|
||||
|
||||
if tag == "div" and "gs_ai_aff" in classes:
|
||||
self._candidate["aff_depth"] = 1
|
||||
return
|
||||
|
||||
if tag == "div" and "gs_ai_eml" in classes:
|
||||
self._candidate["eml_depth"] = 1
|
||||
return
|
||||
|
||||
if tag == "div" and "gs_ai_cby" in classes:
|
||||
self._candidate["cby_depth"] = 1
|
||||
return
|
||||
|
||||
if tag == "a" and "gs_ai_one_int" in classes:
|
||||
self._candidate["interest_depth"] = 1
|
||||
self._candidate["interest_parts"] = []
|
||||
return
|
||||
|
||||
if tag == "img" and self._candidate["image_src"] is None:
|
||||
self._candidate["image_src"] = attr_src(attrs)
|
||||
|
||||
def handle_data(self, data: str) -> None:
|
||||
if self._candidate is None:
|
||||
return
|
||||
if self._candidate["name_depth"] > 0:
|
||||
self._candidate["name_parts"].append(data)
|
||||
if self._candidate["aff_depth"] > 0:
|
||||
self._candidate["aff_parts"].append(data)
|
||||
if self._candidate["eml_depth"] > 0:
|
||||
self._candidate["eml_parts"].append(data)
|
||||
if self._candidate["cby_depth"] > 0:
|
||||
self._candidate["cby_parts"].append(data)
|
||||
if self._candidate["interest_depth"] > 0:
|
||||
self._candidate["interest_parts"].append(data)
|
||||
|
||||
def _decrement_capture_depth(self, key: str) -> bool:
|
||||
if self._candidate is None:
|
||||
return False
|
||||
if self._candidate[key] <= 0:
|
||||
return False
|
||||
self._candidate[key] -= 1
|
||||
return self._candidate[key] == 0
|
||||
|
||||
def handle_endtag(self, _tag: str) -> None:
|
||||
if self._candidate is None:
|
||||
return
|
||||
|
||||
interest_closed = self._decrement_capture_depth("interest_depth")
|
||||
self._decrement_capture_depth("name_depth")
|
||||
self._decrement_capture_depth("aff_depth")
|
||||
self._decrement_capture_depth("eml_depth")
|
||||
self._decrement_capture_depth("cby_depth")
|
||||
|
||||
if interest_closed:
|
||||
interest_text = normalize_space("".join(self._candidate["interest_parts"]))
|
||||
if interest_text:
|
||||
self._candidate["interests"].append(interest_text)
|
||||
self._candidate["interest_parts"] = []
|
||||
|
||||
self._candidate["depth"] -= 1
|
||||
if self._candidate["depth"] > 0:
|
||||
return
|
||||
|
||||
self._finalize_candidate()
|
||||
self._candidate = None
|
||||
|
||||
|
||||
def classify_network_error_reason(fetch_error: str | None) -> str:
|
||||
lowered = (fetch_error or "").lower()
|
||||
if lowered:
|
||||
if any(keyword in lowered for keyword in NETWORK_DNS_ERROR_KEYWORDS):
|
||||
return "network_dns_resolution_failed"
|
||||
if any(keyword in lowered for keyword in NETWORK_TIMEOUT_KEYWORDS):
|
||||
return "network_timeout"
|
||||
if any(keyword in lowered for keyword in NETWORK_TLS_ERROR_KEYWORDS):
|
||||
return "network_tls_error"
|
||||
if "connection reset" in lowered:
|
||||
return "network_connection_reset"
|
||||
if "connection refused" in lowered:
|
||||
return "network_connection_refused"
|
||||
if "network is unreachable" in lowered:
|
||||
return "network_unreachable"
|
||||
return "network_error_missing_status_code"
|
||||
|
||||
|
||||
def classify_block_or_captcha_reason(
|
||||
*,
|
||||
status_code: int,
|
||||
final_url: str,
|
||||
body_lowered: str,
|
||||
) -> str | None:
|
||||
if "accounts.google.com" in final_url and ("signin" in final_url or "servicelogin" in final_url):
|
||||
return "blocked_accounts_redirect"
|
||||
if status_code == 429:
|
||||
return "blocked_http_429_rate_limited"
|
||||
if status_code == 403:
|
||||
if "recaptcha" in body_lowered or "captcha" in body_lowered or "sorry/index" in final_url:
|
||||
return "blocked_http_403_captcha_challenge"
|
||||
return "blocked_http_403_forbidden"
|
||||
if "sorry/index" in final_url or "sorry/index" in body_lowered:
|
||||
return "blocked_google_sorry_challenge"
|
||||
if "our systems have detected" in body_lowered or "unusual traffic" in body_lowered:
|
||||
return "blocked_unusual_traffic_detected"
|
||||
if "automated queries" in body_lowered:
|
||||
return "blocked_automated_queries_detected"
|
||||
if "not a robot" in body_lowered:
|
||||
return "blocked_not_a_robot_challenge"
|
||||
if "recaptcha" in body_lowered:
|
||||
return "blocked_recaptcha_challenge"
|
||||
if "captcha" in body_lowered:
|
||||
return "blocked_captcha_challenge"
|
||||
if any(keyword in body_lowered for keyword in BLOCKED_KEYWORDS):
|
||||
return "blocked_keyword_detected"
|
||||
return None
|
||||
|
||||
|
||||
def _warnings_contain(warnings: list[str], code: str) -> bool:
|
||||
return any(item == code for item in warnings)
|
||||
|
||||
|
||||
def _has_layout_row_failure(marker_counts: dict[str, int], warnings: list[str]) -> bool:
|
||||
if _warnings_contain(warnings, "layout_all_rows_unparseable"):
|
||||
return True
|
||||
if marker_counts.get("gsc_a_tr", 0) <= 0:
|
||||
return False
|
||||
if _warnings_contain(warnings, "row_missing_title"):
|
||||
return True
|
||||
return marker_counts.get("gsc_a_at", 0) <= 0
|
||||
|
||||
|
||||
def _first_layout_warning(warnings: list[str]) -> str | None:
|
||||
for warning in warnings:
|
||||
if warning.startswith("layout_"):
|
||||
return warning
|
||||
return None
|
||||
|
||||
|
||||
def detect_state(
|
||||
fetch_result: FetchResult,
|
||||
publications: list[PublicationCandidate],
|
||||
marker_counts: dict[str, int],
|
||||
*,
|
||||
warnings: list[str],
|
||||
has_show_more_button_flag: bool,
|
||||
articles_range: str | None,
|
||||
visible_text: str,
|
||||
) -> tuple[ParseState, str]:
|
||||
if fetch_result.status_code is None:
|
||||
return ParseState.NETWORK_ERROR, classify_network_error_reason(fetch_result.error)
|
||||
|
||||
lowered = fetch_result.body.lower()
|
||||
final = (fetch_result.final_url or "").lower()
|
||||
status_code = int(fetch_result.status_code)
|
||||
|
||||
block_reason = classify_block_or_captcha_reason(
|
||||
status_code=status_code,
|
||||
final_url=final,
|
||||
body_lowered=lowered,
|
||||
)
|
||||
if block_reason is not None:
|
||||
return ParseState.BLOCKED_OR_CAPTCHA, block_reason
|
||||
|
||||
if not publications and any(keyword in visible_text for keyword in NO_RESULTS_KEYWORDS):
|
||||
return ParseState.NO_RESULTS, "no_results_keyword_detected"
|
||||
|
||||
layout_warning = _first_layout_warning(warnings)
|
||||
if layout_warning is not None:
|
||||
return ParseState.LAYOUT_CHANGED, layout_warning
|
||||
|
||||
if _has_layout_row_failure(marker_counts, warnings):
|
||||
return ParseState.LAYOUT_CHANGED, "layout_publication_rows_unparseable"
|
||||
|
||||
if has_show_more_button_flag and not articles_range:
|
||||
return ParseState.LAYOUT_CHANGED, "layout_show_more_without_articles_range"
|
||||
|
||||
if not publications:
|
||||
has_profile_markers = marker_counts.get("gsc_prf_in", 0) > 0
|
||||
has_table_markers = marker_counts.get("gsc_a_tr", 0) > 0 or marker_counts.get("gsc_a_at", 0) > 0
|
||||
if not has_profile_markers and not has_table_markers:
|
||||
return ParseState.LAYOUT_CHANGED, "layout_markers_missing"
|
||||
return ParseState.OK, "no_rows_with_known_markers"
|
||||
|
||||
return ParseState.OK, "publications_extracted"
|
||||
|
||||
|
||||
def detect_author_search_state(
|
||||
fetch_result: FetchResult,
|
||||
candidates: list[ScholarSearchCandidate],
|
||||
marker_counts: dict[str, int],
|
||||
*,
|
||||
visible_text: str,
|
||||
) -> tuple[ParseState, str]:
|
||||
if fetch_result.status_code is None:
|
||||
return ParseState.NETWORK_ERROR, classify_network_error_reason(fetch_result.error)
|
||||
|
||||
lowered = fetch_result.body.lower()
|
||||
final = (fetch_result.final_url or "").lower()
|
||||
status_code = int(fetch_result.status_code)
|
||||
|
||||
block_reason = classify_block_or_captcha_reason(
|
||||
status_code=status_code,
|
||||
final_url=final,
|
||||
body_lowered=lowered,
|
||||
)
|
||||
if block_reason is not None:
|
||||
return ParseState.BLOCKED_OR_CAPTCHA, block_reason
|
||||
|
||||
if not candidates and any(keyword in visible_text for keyword in NO_AUTHOR_RESULTS_KEYWORDS):
|
||||
return ParseState.NO_RESULTS, "no_results_keyword_detected"
|
||||
|
||||
if not candidates:
|
||||
has_search_markers = marker_counts.get("gsc_1usr", 0) > 0 or marker_counts.get("gs_ai_name", 0) > 0
|
||||
if not has_search_markers:
|
||||
return ParseState.NO_RESULTS, "no_search_candidates_detected"
|
||||
return ParseState.LAYOUT_CHANGED, "layout_author_candidates_unparseable"
|
||||
|
||||
return ParseState.OK, "author_candidates_extracted"
|
||||
|
||||
|
||||
def parse_profile_page(fetch_result: FetchResult) -> ParsedProfilePage:
|
||||
publications, warnings = parse_publications(fetch_result.body)
|
||||
marker_counts = count_markers(fetch_result.body)
|
||||
visible_text = strip_tags(SCRIPT_STYLE_RE.sub(" ", fetch_result.body)).lower()
|
||||
|
||||
show_more = has_show_more_button(fetch_result.body)
|
||||
operation_error_banner = has_operation_error_banner(fetch_result.body)
|
||||
articles_range = extract_articles_range(fetch_result.body)
|
||||
|
||||
if show_more:
|
||||
warnings.append("possible_partial_page_show_more_present")
|
||||
if operation_error_banner:
|
||||
warnings.append("operation_error_banner_present")
|
||||
|
||||
warnings = sorted(set(warnings))
|
||||
|
||||
state, state_reason = detect_state(
|
||||
fetch_result,
|
||||
publications,
|
||||
marker_counts,
|
||||
warnings=warnings,
|
||||
has_show_more_button_flag=show_more,
|
||||
articles_range=articles_range,
|
||||
visible_text=visible_text,
|
||||
)
|
||||
|
||||
return ParsedProfilePage(
|
||||
state=state,
|
||||
state_reason=state_reason,
|
||||
profile_name=extract_profile_name(fetch_result.body),
|
||||
profile_image_url=extract_profile_image_url(fetch_result.body),
|
||||
publications=publications,
|
||||
marker_counts=marker_counts,
|
||||
warnings=warnings,
|
||||
has_show_more_button=show_more,
|
||||
has_operation_error_banner=operation_error_banner,
|
||||
articles_range=articles_range,
|
||||
)
|
||||
|
||||
|
||||
def parse_author_search_page(fetch_result: FetchResult) -> ParsedAuthorSearchPage:
|
||||
parser = ScholarAuthorSearchParser()
|
||||
parser.feed(fetch_result.body)
|
||||
|
||||
marker_counts = count_author_search_markers(fetch_result.body)
|
||||
visible_text = strip_tags(SCRIPT_STYLE_RE.sub(" ", fetch_result.body)).lower()
|
||||
warnings: list[str] = []
|
||||
if not parser.candidates:
|
||||
warnings.append("no_author_candidates_detected")
|
||||
|
||||
state, state_reason = detect_author_search_state(
|
||||
fetch_result,
|
||||
parser.candidates,
|
||||
marker_counts,
|
||||
visible_text=visible_text,
|
||||
)
|
||||
|
||||
return ParsedAuthorSearchPage(
|
||||
state=state,
|
||||
state_reason=state_reason,
|
||||
candidates=parser.candidates,
|
||||
marker_counts=marker_counts,
|
||||
warnings=warnings,
|
||||
)
|
||||
from app.services.domains.scholar.parser import *
|
||||
|
|
|
|||
|
|
@ -1,225 +1,4 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import random
|
||||
from dataclasses import dataclass
|
||||
from typing import Protocol
|
||||
from urllib.error import HTTPError, URLError
|
||||
from urllib.parse import urlencode
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
SCHOLAR_PROFILE_URL = "https://scholar.google.com/citations"
|
||||
DEFAULT_PAGE_SIZE = 100
|
||||
|
||||
DEFAULT_USER_AGENTS = [
|
||||
(
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
|
||||
),
|
||||
(
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 14_7) AppleWebKit/605.1.15 "
|
||||
"(KHTML, like Gecko) Version/18.1 Safari/605.1.15"
|
||||
),
|
||||
(
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:131.0) "
|
||||
"Gecko/20100101 Firefox/131.0"
|
||||
),
|
||||
]
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FetchResult:
|
||||
requested_url: str
|
||||
status_code: int | None
|
||||
final_url: str | None
|
||||
body: str
|
||||
error: str | None
|
||||
|
||||
|
||||
class ScholarSource(Protocol):
|
||||
async def fetch_profile_html(self, scholar_id: str) -> FetchResult:
|
||||
...
|
||||
|
||||
async def fetch_profile_page_html(
|
||||
self,
|
||||
scholar_id: str,
|
||||
*,
|
||||
cstart: int,
|
||||
pagesize: int,
|
||||
) -> FetchResult:
|
||||
...
|
||||
|
||||
async def fetch_author_search_html(
|
||||
self,
|
||||
query: str,
|
||||
*,
|
||||
start: int,
|
||||
) -> FetchResult:
|
||||
...
|
||||
|
||||
|
||||
class LiveScholarSource:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
timeout_seconds: float = 25.0,
|
||||
user_agents: list[str] | None = None,
|
||||
) -> None:
|
||||
self._timeout_seconds = timeout_seconds
|
||||
self._user_agents = user_agents or DEFAULT_USER_AGENTS
|
||||
|
||||
async def fetch_profile_html(self, scholar_id: str) -> FetchResult:
|
||||
return await self.fetch_profile_page_html(
|
||||
scholar_id,
|
||||
cstart=0,
|
||||
pagesize=DEFAULT_PAGE_SIZE,
|
||||
)
|
||||
|
||||
async def fetch_profile_page_html(
|
||||
self,
|
||||
scholar_id: str,
|
||||
*,
|
||||
cstart: int,
|
||||
pagesize: int = DEFAULT_PAGE_SIZE,
|
||||
) -> FetchResult:
|
||||
requested_url = _build_profile_url(
|
||||
scholar_id=scholar_id,
|
||||
cstart=cstart,
|
||||
pagesize=pagesize,
|
||||
)
|
||||
logger.debug(
|
||||
"scholar_source.fetch_started",
|
||||
extra={
|
||||
"event": "scholar_source.fetch_started",
|
||||
"scholar_id": scholar_id,
|
||||
"requested_url": requested_url,
|
||||
"cstart": cstart,
|
||||
"pagesize": pagesize,
|
||||
},
|
||||
)
|
||||
return await asyncio.to_thread(self._fetch_sync, requested_url)
|
||||
|
||||
async def fetch_author_search_html(
|
||||
self,
|
||||
query: str,
|
||||
*,
|
||||
start: int = 0,
|
||||
) -> FetchResult:
|
||||
requested_url = _build_author_search_url(
|
||||
query=query,
|
||||
start=start,
|
||||
)
|
||||
logger.debug(
|
||||
"scholar_source.search_fetch_started",
|
||||
extra={
|
||||
"event": "scholar_source.search_fetch_started",
|
||||
"query": query,
|
||||
"requested_url": requested_url,
|
||||
"start": start,
|
||||
},
|
||||
)
|
||||
return await asyncio.to_thread(self._fetch_sync, requested_url)
|
||||
|
||||
def _build_request(self, requested_url: str) -> Request:
|
||||
return Request(
|
||||
requested_url,
|
||||
headers={
|
||||
"User-Agent": random.choice(self._user_agents),
|
||||
"Accept": "text/html,application/xhtml+xml",
|
||||
"Accept-Language": "en-US,en;q=0.9",
|
||||
"Connection": "close",
|
||||
},
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _http_error_body(exc: HTTPError) -> str:
|
||||
try:
|
||||
return exc.read().decode("utf-8", errors="replace")
|
||||
except Exception:
|
||||
return ""
|
||||
|
||||
@staticmethod
|
||||
def _network_error_result(requested_url: str, exc: URLError) -> FetchResult:
|
||||
logger.warning(
|
||||
"scholar_source.fetch_network_error",
|
||||
extra={"event": "scholar_source.fetch_network_error", "requested_url": requested_url},
|
||||
)
|
||||
return FetchResult(
|
||||
requested_url=requested_url,
|
||||
status_code=None,
|
||||
final_url=None,
|
||||
body="",
|
||||
error=str(exc),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _http_error_result(requested_url: str, exc: HTTPError) -> FetchResult:
|
||||
logger.warning(
|
||||
"scholar_source.fetch_http_error",
|
||||
extra={
|
||||
"event": "scholar_source.fetch_http_error",
|
||||
"requested_url": requested_url,
|
||||
"status_code": exc.code,
|
||||
},
|
||||
)
|
||||
return FetchResult(
|
||||
requested_url=requested_url,
|
||||
status_code=exc.code,
|
||||
final_url=exc.geturl(),
|
||||
body=LiveScholarSource._http_error_body(exc),
|
||||
error=str(exc),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _success_result(requested_url: str, response) -> FetchResult:
|
||||
body = response.read().decode("utf-8", errors="replace")
|
||||
status_code = getattr(response, "status", 200)
|
||||
logger.debug(
|
||||
"scholar_source.fetch_succeeded",
|
||||
extra={
|
||||
"event": "scholar_source.fetch_succeeded",
|
||||
"requested_url": requested_url,
|
||||
"status_code": status_code,
|
||||
},
|
||||
)
|
||||
return FetchResult(
|
||||
requested_url=requested_url,
|
||||
status_code=status_code,
|
||||
final_url=response.geturl(),
|
||||
body=body,
|
||||
error=None,
|
||||
)
|
||||
|
||||
def _fetch_sync(self, requested_url: str) -> FetchResult:
|
||||
request = self._build_request(requested_url)
|
||||
|
||||
try:
|
||||
with urlopen(request, timeout=self._timeout_seconds) as response:
|
||||
return self._success_result(requested_url, response)
|
||||
except HTTPError as exc:
|
||||
return self._http_error_result(requested_url, exc)
|
||||
except URLError as exc:
|
||||
return self._network_error_result(requested_url, exc)
|
||||
|
||||
|
||||
def _build_profile_url(*, scholar_id: str, cstart: int, pagesize: int) -> str:
|
||||
query: dict[str, int | str] = {"hl": "en", "user": scholar_id}
|
||||
if cstart > 0:
|
||||
query["cstart"] = int(cstart)
|
||||
if pagesize > 0:
|
||||
query["pagesize"] = int(pagesize)
|
||||
return f"{SCHOLAR_PROFILE_URL}?{urlencode(query)}"
|
||||
|
||||
|
||||
def _build_author_search_url(*, query: str, start: int) -> str:
|
||||
params: dict[str, int | str] = {
|
||||
"hl": "en",
|
||||
"view_op": "search_authors",
|
||||
"mauthors": query,
|
||||
}
|
||||
if start > 0:
|
||||
params["astart"] = int(start)
|
||||
return f"{SCHOLAR_PROFILE_URL}?{urlencode(params)}"
|
||||
from app.services.domains.scholar.source import *
|
||||
from app.services.domains.scholar.source import _build_profile_url
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -1,146 +1,3 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.models import UserSetting
|
||||
|
||||
|
||||
class UserSettingsServiceError(ValueError):
|
||||
"""Raised for expected settings-validation failures."""
|
||||
|
||||
|
||||
NAV_PAGE_DASHBOARD = "dashboard"
|
||||
NAV_PAGE_SCHOLARS = "scholars"
|
||||
NAV_PAGE_PUBLICATIONS = "publications"
|
||||
NAV_PAGE_SETTINGS = "settings"
|
||||
NAV_PAGE_STYLE_GUIDE = "style-guide"
|
||||
NAV_PAGE_RUNS = "runs"
|
||||
NAV_PAGE_USERS = "users"
|
||||
|
||||
ALLOWED_NAV_PAGES = (
|
||||
NAV_PAGE_DASHBOARD,
|
||||
NAV_PAGE_SCHOLARS,
|
||||
NAV_PAGE_PUBLICATIONS,
|
||||
NAV_PAGE_SETTINGS,
|
||||
NAV_PAGE_STYLE_GUIDE,
|
||||
NAV_PAGE_RUNS,
|
||||
NAV_PAGE_USERS,
|
||||
)
|
||||
REQUIRED_NAV_PAGES = (
|
||||
NAV_PAGE_DASHBOARD,
|
||||
NAV_PAGE_SCHOLARS,
|
||||
NAV_PAGE_SETTINGS,
|
||||
)
|
||||
DEFAULT_NAV_VISIBLE_PAGES = list(ALLOWED_NAV_PAGES)
|
||||
HARD_MIN_RUN_INTERVAL_MINUTES = 15
|
||||
HARD_MIN_REQUEST_DELAY_SECONDS = 2
|
||||
|
||||
|
||||
def resolve_run_interval_minimum(configured_minimum: int | None) -> int:
|
||||
try:
|
||||
parsed = int(configured_minimum) if configured_minimum is not None else HARD_MIN_RUN_INTERVAL_MINUTES
|
||||
except (TypeError, ValueError):
|
||||
parsed = HARD_MIN_RUN_INTERVAL_MINUTES
|
||||
return max(HARD_MIN_RUN_INTERVAL_MINUTES, parsed)
|
||||
|
||||
|
||||
def resolve_request_delay_minimum(configured_minimum: int | None) -> int:
|
||||
try:
|
||||
parsed = int(configured_minimum) if configured_minimum is not None else HARD_MIN_REQUEST_DELAY_SECONDS
|
||||
except (TypeError, ValueError):
|
||||
parsed = HARD_MIN_REQUEST_DELAY_SECONDS
|
||||
return max(HARD_MIN_REQUEST_DELAY_SECONDS, parsed)
|
||||
|
||||
|
||||
def parse_run_interval_minutes(value: str, *, minimum: int = HARD_MIN_RUN_INTERVAL_MINUTES) -> int:
|
||||
try:
|
||||
parsed = int(value)
|
||||
except ValueError as exc:
|
||||
raise UserSettingsServiceError("Check interval must be a whole number.") from exc
|
||||
effective_minimum = resolve_run_interval_minimum(minimum)
|
||||
if parsed < effective_minimum:
|
||||
raise UserSettingsServiceError(
|
||||
f"Check interval must be at least {effective_minimum} minutes."
|
||||
)
|
||||
return parsed
|
||||
|
||||
|
||||
def parse_request_delay_seconds(value: str, *, minimum: int = HARD_MIN_REQUEST_DELAY_SECONDS) -> int:
|
||||
try:
|
||||
parsed = int(value)
|
||||
except ValueError as exc:
|
||||
raise UserSettingsServiceError("Request delay must be a whole number.") from exc
|
||||
effective_minimum = resolve_request_delay_minimum(minimum)
|
||||
if parsed < effective_minimum:
|
||||
raise UserSettingsServiceError(
|
||||
f"Request delay must be at least {effective_minimum} seconds."
|
||||
)
|
||||
return parsed
|
||||
|
||||
|
||||
def parse_nav_visible_pages(value: object) -> list[str]:
|
||||
if not isinstance(value, list):
|
||||
raise UserSettingsServiceError("Navigation visibility must be a list of page ids.")
|
||||
|
||||
deduped: list[str] = []
|
||||
seen: set[str] = set()
|
||||
|
||||
for raw_page in value:
|
||||
if not isinstance(raw_page, str):
|
||||
raise UserSettingsServiceError("Navigation visibility entries must be strings.")
|
||||
|
||||
page_id = raw_page.strip()
|
||||
if page_id not in ALLOWED_NAV_PAGES:
|
||||
raise UserSettingsServiceError(f"Unsupported navigation page id: {page_id}")
|
||||
|
||||
if page_id in seen:
|
||||
continue
|
||||
|
||||
seen.add(page_id)
|
||||
deduped.append(page_id)
|
||||
|
||||
missing_required = [page for page in REQUIRED_NAV_PAGES if page not in seen]
|
||||
if missing_required:
|
||||
raise UserSettingsServiceError(
|
||||
"Dashboard, Scholars, and Settings must remain visible."
|
||||
)
|
||||
|
||||
return deduped
|
||||
|
||||
|
||||
async def get_or_create_settings(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
) -> UserSetting:
|
||||
result = await db_session.execute(
|
||||
select(UserSetting).where(UserSetting.user_id == user_id)
|
||||
)
|
||||
settings = result.scalar_one_or_none()
|
||||
if settings is not None:
|
||||
return settings
|
||||
|
||||
settings = UserSetting(user_id=user_id)
|
||||
db_session.add(settings)
|
||||
await db_session.commit()
|
||||
await db_session.refresh(settings)
|
||||
return settings
|
||||
|
||||
|
||||
async def update_settings(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
settings: UserSetting,
|
||||
auto_run_enabled: bool,
|
||||
run_interval_minutes: int,
|
||||
request_delay_seconds: int,
|
||||
nav_visible_pages: list[str],
|
||||
) -> UserSetting:
|
||||
settings.auto_run_enabled = auto_run_enabled
|
||||
settings.run_interval_minutes = run_interval_minutes
|
||||
settings.request_delay_seconds = request_delay_seconds
|
||||
settings.nav_visible_pages = nav_visible_pages
|
||||
await db_session.commit()
|
||||
await db_session.refresh(settings)
|
||||
return settings
|
||||
from app.services.domains.settings.application import *
|
||||
|
|
|
|||
|
|
@ -1,98 +1,3 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.models import User
|
||||
|
||||
EMAIL_PATTERN = re.compile(r"^[^@\s]+@[^@\s]+\.[^@\s]+$")
|
||||
|
||||
|
||||
class UserServiceError(ValueError):
|
||||
"""Raised for expected user-management validation failures."""
|
||||
|
||||
|
||||
def normalize_email(value: str) -> str:
|
||||
return value.strip().lower()
|
||||
|
||||
|
||||
def validate_email(value: str) -> str:
|
||||
email = normalize_email(value)
|
||||
if not EMAIL_PATTERN.fullmatch(email):
|
||||
raise UserServiceError("Enter a valid email address.")
|
||||
return email
|
||||
|
||||
|
||||
def validate_password(value: str) -> str:
|
||||
password = value.strip()
|
||||
if len(password) < 8:
|
||||
raise UserServiceError("Password must be at least 8 characters.")
|
||||
return password
|
||||
|
||||
|
||||
async def get_user_by_id(db_session: AsyncSession, user_id: int) -> User | None:
|
||||
result = await db_session.execute(select(User).where(User.id == user_id))
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
|
||||
async def get_user_by_email(db_session: AsyncSession, email: str) -> User | None:
|
||||
result = await db_session.execute(
|
||||
select(User).where(User.email == normalize_email(email))
|
||||
)
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
|
||||
async def list_users(db_session: AsyncSession) -> list[User]:
|
||||
result = await db_session.execute(select(User).order_by(User.email.asc()))
|
||||
return list(result.scalars().all())
|
||||
|
||||
|
||||
async def create_user(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
email: str,
|
||||
password_hash: str,
|
||||
is_admin: bool,
|
||||
) -> User:
|
||||
user = User(
|
||||
email=validate_email(email),
|
||||
password_hash=password_hash,
|
||||
is_admin=is_admin,
|
||||
is_active=True,
|
||||
)
|
||||
db_session.add(user)
|
||||
try:
|
||||
await db_session.commit()
|
||||
except IntegrityError as exc:
|
||||
await db_session.rollback()
|
||||
raise UserServiceError("A user with that email already exists.") from exc
|
||||
await db_session.refresh(user)
|
||||
return user
|
||||
|
||||
|
||||
async def set_user_active(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user: User,
|
||||
is_active: bool,
|
||||
) -> User:
|
||||
user.is_active = is_active
|
||||
await db_session.commit()
|
||||
await db_session.refresh(user)
|
||||
return user
|
||||
|
||||
|
||||
async def set_user_password_hash(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user: User,
|
||||
password_hash: str,
|
||||
) -> User:
|
||||
user.password_hash = password_hash
|
||||
await db_session.commit()
|
||||
await db_session.refresh(user)
|
||||
return user
|
||||
|
||||
from app.services.domains.users.application import *
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue