First product
This commit is contained in:
parent
778da9e2fc
commit
4433d7d2c4
157 changed files with 23975 additions and 0 deletions
2
app/services/__init__.py
Normal file
2
app/services/__init__.py
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
"""Service layer for scholarr application workflows."""
|
||||
|
||||
278
app/services/continuation_queue.py
Normal file
278
app/services/continuation_queue.py
Normal file
|
|
@ -0,0 +1,278 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.models import IngestionQueueItem, QueueItemStatus
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ContinuationQueueJob:
|
||||
id: int
|
||||
user_id: int
|
||||
scholar_profile_id: int
|
||||
resume_cstart: int
|
||||
reason: str
|
||||
status: str
|
||||
attempt_count: int
|
||||
next_attempt_dt: datetime
|
||||
|
||||
|
||||
ACTIVE_QUEUE_STATUSES: tuple[str, ...] = (
|
||||
QueueItemStatus.QUEUED.value,
|
||||
QueueItemStatus.RETRYING.value,
|
||||
)
|
||||
|
||||
|
||||
def normalize_cstart(value: int | None) -> int:
|
||||
if value is None:
|
||||
return 0
|
||||
return max(0, int(value))
|
||||
|
||||
|
||||
def compute_backoff_seconds(*, base_seconds: int, attempt_count: int, max_seconds: int) -> int:
|
||||
base = max(1, int(base_seconds))
|
||||
attempts = max(1, int(attempt_count))
|
||||
maximum = max(base, int(max_seconds))
|
||||
seconds = base * (2 ** max(0, attempts - 1))
|
||||
return min(seconds, maximum)
|
||||
|
||||
|
||||
async def upsert_job(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
scholar_profile_id: int,
|
||||
resume_cstart: int,
|
||||
reason: str,
|
||||
run_id: int | None,
|
||||
delay_seconds: int,
|
||||
) -> IngestionQueueItem:
|
||||
now = datetime.now(timezone.utc)
|
||||
next_attempt_dt = now + timedelta(seconds=max(0, int(delay_seconds)))
|
||||
result = await db_session.execute(
|
||||
select(IngestionQueueItem).where(
|
||||
IngestionQueueItem.user_id == user_id,
|
||||
IngestionQueueItem.scholar_profile_id == scholar_profile_id,
|
||||
)
|
||||
)
|
||||
item = result.scalar_one_or_none()
|
||||
normalized_cstart = normalize_cstart(resume_cstart)
|
||||
if item is None:
|
||||
item = IngestionQueueItem(
|
||||
user_id=user_id,
|
||||
scholar_profile_id=scholar_profile_id,
|
||||
resume_cstart=normalized_cstart,
|
||||
reason=reason,
|
||||
status=QueueItemStatus.QUEUED.value,
|
||||
attempt_count=0,
|
||||
next_attempt_dt=next_attempt_dt,
|
||||
last_run_id=run_id,
|
||||
last_error=None,
|
||||
dropped_reason=None,
|
||||
dropped_at=None,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
db_session.add(item)
|
||||
return item
|
||||
|
||||
item.resume_cstart = normalized_cstart
|
||||
item.reason = reason
|
||||
if item.status == QueueItemStatus.DROPPED.value:
|
||||
item.attempt_count = 0
|
||||
item.status = QueueItemStatus.QUEUED.value
|
||||
item.next_attempt_dt = next_attempt_dt
|
||||
item.last_run_id = run_id
|
||||
item.last_error = None
|
||||
item.dropped_reason = None
|
||||
item.dropped_at = None
|
||||
item.updated_at = now
|
||||
return item
|
||||
|
||||
|
||||
async def clear_job_for_scholar(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
scholar_profile_id: int,
|
||||
) -> bool:
|
||||
result = await db_session.execute(
|
||||
select(IngestionQueueItem).where(
|
||||
IngestionQueueItem.user_id == user_id,
|
||||
IngestionQueueItem.scholar_profile_id == scholar_profile_id,
|
||||
)
|
||||
)
|
||||
item = result.scalar_one_or_none()
|
||||
if item is None:
|
||||
return False
|
||||
await db_session.delete(item)
|
||||
return True
|
||||
|
||||
|
||||
async def delete_job_by_id(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
job_id: int,
|
||||
) -> bool:
|
||||
result = await db_session.execute(
|
||||
select(IngestionQueueItem).where(IngestionQueueItem.id == job_id)
|
||||
)
|
||||
item = result.scalar_one_or_none()
|
||||
if item is None:
|
||||
return False
|
||||
await db_session.delete(item)
|
||||
return True
|
||||
|
||||
|
||||
async def list_due_jobs(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
now: datetime,
|
||||
limit: int,
|
||||
) -> list[ContinuationQueueJob]:
|
||||
result = await db_session.execute(
|
||||
select(IngestionQueueItem)
|
||||
.where(
|
||||
IngestionQueueItem.next_attempt_dt <= now,
|
||||
IngestionQueueItem.status.in_(ACTIVE_QUEUE_STATUSES),
|
||||
)
|
||||
.order_by(
|
||||
IngestionQueueItem.next_attempt_dt.asc(),
|
||||
IngestionQueueItem.id.asc(),
|
||||
)
|
||||
.limit(limit)
|
||||
)
|
||||
rows = list(result.scalars().all())
|
||||
jobs: list[ContinuationQueueJob] = []
|
||||
for row in rows:
|
||||
jobs.append(
|
||||
ContinuationQueueJob(
|
||||
id=int(row.id),
|
||||
user_id=int(row.user_id),
|
||||
scholar_profile_id=int(row.scholar_profile_id),
|
||||
resume_cstart=normalize_cstart(row.resume_cstart),
|
||||
reason=row.reason,
|
||||
status=row.status,
|
||||
attempt_count=int(row.attempt_count),
|
||||
next_attempt_dt=row.next_attempt_dt,
|
||||
)
|
||||
)
|
||||
return jobs
|
||||
|
||||
|
||||
async def increment_attempt_count(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
job_id: int,
|
||||
) -> IngestionQueueItem | None:
|
||||
now = datetime.now(timezone.utc)
|
||||
result = await db_session.execute(
|
||||
select(IngestionQueueItem).where(IngestionQueueItem.id == job_id)
|
||||
)
|
||||
item = result.scalar_one_or_none()
|
||||
if item is None:
|
||||
return None
|
||||
item.attempt_count = int(item.attempt_count or 0) + 1
|
||||
item.updated_at = now
|
||||
return item
|
||||
|
||||
|
||||
async def reschedule_job(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
job_id: int,
|
||||
delay_seconds: int,
|
||||
reason: str,
|
||||
error: str | None = None,
|
||||
) -> IngestionQueueItem | None:
|
||||
now = datetime.now(timezone.utc)
|
||||
result = await db_session.execute(
|
||||
select(IngestionQueueItem).where(IngestionQueueItem.id == job_id)
|
||||
)
|
||||
item = result.scalar_one_or_none()
|
||||
if item is None:
|
||||
return None
|
||||
item.next_attempt_dt = now + timedelta(seconds=max(1, int(delay_seconds)))
|
||||
item.status = QueueItemStatus.QUEUED.value
|
||||
item.reason = reason
|
||||
item.last_error = error
|
||||
item.dropped_reason = None
|
||||
item.dropped_at = None
|
||||
item.updated_at = now
|
||||
return item
|
||||
|
||||
|
||||
async def mark_retrying(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
job_id: int,
|
||||
reason: str | None = None,
|
||||
) -> IngestionQueueItem | None:
|
||||
now = datetime.now(timezone.utc)
|
||||
result = await db_session.execute(
|
||||
select(IngestionQueueItem).where(IngestionQueueItem.id == job_id)
|
||||
)
|
||||
item = result.scalar_one_or_none()
|
||||
if item is None:
|
||||
return None
|
||||
if item.status == QueueItemStatus.DROPPED.value:
|
||||
return item
|
||||
item.status = QueueItemStatus.RETRYING.value
|
||||
if reason:
|
||||
item.reason = reason
|
||||
item.updated_at = now
|
||||
return item
|
||||
|
||||
|
||||
async def mark_dropped(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
job_id: int,
|
||||
reason: str,
|
||||
error: str | None = None,
|
||||
) -> IngestionQueueItem | None:
|
||||
now = datetime.now(timezone.utc)
|
||||
result = await db_session.execute(
|
||||
select(IngestionQueueItem).where(IngestionQueueItem.id == job_id)
|
||||
)
|
||||
item = result.scalar_one_or_none()
|
||||
if item is None:
|
||||
return None
|
||||
item.status = QueueItemStatus.DROPPED.value
|
||||
item.reason = "dropped"
|
||||
item.dropped_reason = reason
|
||||
item.dropped_at = now
|
||||
if error is not None:
|
||||
item.last_error = error
|
||||
item.updated_at = now
|
||||
return item
|
||||
|
||||
|
||||
async def mark_queued_now(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
job_id: int,
|
||||
reason: str,
|
||||
reset_attempt_count: bool = False,
|
||||
) -> IngestionQueueItem | None:
|
||||
now = datetime.now(timezone.utc)
|
||||
result = await db_session.execute(
|
||||
select(IngestionQueueItem).where(IngestionQueueItem.id == job_id)
|
||||
)
|
||||
item = result.scalar_one_or_none()
|
||||
if item is None:
|
||||
return None
|
||||
item.status = QueueItemStatus.QUEUED.value
|
||||
item.reason = reason
|
||||
item.next_attempt_dt = now
|
||||
if reset_attempt_count:
|
||||
item.attempt_count = 0
|
||||
item.last_error = None
|
||||
item.dropped_reason = None
|
||||
item.dropped_at = None
|
||||
item.updated_at = now
|
||||
return item
|
||||
967
app/services/ingestion.py
Normal file
967
app/services/ingestion.py
Normal file
|
|
@ -0,0 +1,967 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
import hashlib
|
||||
import logging
|
||||
import re
|
||||
from typing import Any
|
||||
from urllib.parse import urljoin
|
||||
|
||||
from sqlalchemy import select, text
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.models import (
|
||||
CrawlRun,
|
||||
Publication,
|
||||
RunStatus,
|
||||
RunTriggerType,
|
||||
ScholarProfile,
|
||||
ScholarPublication,
|
||||
)
|
||||
from app.services import continuation_queue as queue_service
|
||||
from app.services.scholar_parser import (
|
||||
ParseState,
|
||||
ParsedProfilePage,
|
||||
PublicationCandidate,
|
||||
parse_profile_page,
|
||||
)
|
||||
from app.services.scholar_source import FetchResult, ScholarSource
|
||||
|
||||
TITLE_ALNUM_RE = re.compile(r"[^a-z0-9]+")
|
||||
WORD_RE = re.compile(r"[a-z0-9]+")
|
||||
HTML_TAG_RE = re.compile(r"<[^>]+>", re.S)
|
||||
SPACE_RE = re.compile(r"\s+")
|
||||
FAILED_STATES = {
|
||||
ParseState.BLOCKED_OR_CAPTCHA.value,
|
||||
ParseState.LAYOUT_CHANGED.value,
|
||||
ParseState.NETWORK_ERROR.value,
|
||||
"ingestion_error",
|
||||
}
|
||||
RUN_LOCK_NAMESPACE = 8217
|
||||
RESUMABLE_PARTIAL_REASONS = {
|
||||
"max_pages_reached",
|
||||
"pagination_cursor_stalled",
|
||||
}
|
||||
RESUMABLE_PARTIAL_REASON_PREFIXES = ("page_state_network_error",)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RunExecutionSummary:
|
||||
crawl_run_id: int
|
||||
status: RunStatus
|
||||
scholar_count: int
|
||||
succeeded_count: int
|
||||
failed_count: int
|
||||
partial_count: int
|
||||
new_publication_count: int
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PagedParseResult:
|
||||
fetch_result: FetchResult
|
||||
parsed_page: ParsedProfilePage
|
||||
publications: list[PublicationCandidate]
|
||||
attempt_log: list[dict[str, Any]]
|
||||
page_logs: list[dict[str, Any]]
|
||||
pages_fetched: int
|
||||
pages_attempted: int
|
||||
has_more_remaining: bool
|
||||
pagination_truncated_reason: str | None
|
||||
continuation_cstart: int | None
|
||||
|
||||
|
||||
class RunAlreadyInProgressError(RuntimeError):
|
||||
"""Raised when a run lock for a user is already held by another process."""
|
||||
|
||||
|
||||
class ScholarIngestionService:
|
||||
def __init__(self, *, source: ScholarSource) -> None:
|
||||
self._source = source
|
||||
|
||||
async def run_for_user(
|
||||
self,
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
trigger_type: RunTriggerType,
|
||||
request_delay_seconds: int,
|
||||
network_error_retries: int = 1,
|
||||
retry_backoff_seconds: float = 1.0,
|
||||
max_pages_per_scholar: int = 30,
|
||||
page_size: int = 100,
|
||||
scholar_profile_ids: set[int] | None = None,
|
||||
start_cstart_by_scholar_id: dict[int, int] | None = None,
|
||||
auto_queue_continuations: bool = True,
|
||||
queue_delay_seconds: int = 60,
|
||||
) -> RunExecutionSummary:
|
||||
lock_acquired = await self._try_acquire_user_lock(
|
||||
db_session,
|
||||
user_id=user_id,
|
||||
)
|
||||
if not lock_acquired:
|
||||
raise RunAlreadyInProgressError(f"Run already in progress for user_id={user_id}.")
|
||||
|
||||
filtered_scholar_ids = (
|
||||
{int(value) for value in scholar_profile_ids}
|
||||
if scholar_profile_ids is not None
|
||||
else None
|
||||
)
|
||||
start_cstart_map = {
|
||||
int(key): max(0, int(value))
|
||||
for key, value in (start_cstart_by_scholar_id or {}).items()
|
||||
}
|
||||
|
||||
scholars_stmt = (
|
||||
select(ScholarProfile)
|
||||
.where(
|
||||
ScholarProfile.user_id == user_id,
|
||||
ScholarProfile.is_enabled.is_(True),
|
||||
)
|
||||
.order_by(ScholarProfile.created_at.asc(), ScholarProfile.id.asc())
|
||||
)
|
||||
if filtered_scholar_ids is not None:
|
||||
scholars_stmt = scholars_stmt.where(
|
||||
ScholarProfile.id.in_(filtered_scholar_ids)
|
||||
)
|
||||
|
||||
scholars_result = await db_session.execute(
|
||||
scholars_stmt
|
||||
)
|
||||
scholars = list(scholars_result.scalars().all())
|
||||
if filtered_scholar_ids is not None:
|
||||
found_ids = {int(scholar.id) for scholar in scholars}
|
||||
missing_ids = filtered_scholar_ids - found_ids
|
||||
for scholar_profile_id in missing_ids:
|
||||
await queue_service.clear_job_for_scholar(
|
||||
db_session,
|
||||
user_id=user_id,
|
||||
scholar_profile_id=scholar_profile_id,
|
||||
)
|
||||
logger.info(
|
||||
"ingestion.run_started",
|
||||
extra={
|
||||
"event": "ingestion.run_started",
|
||||
"user_id": user_id,
|
||||
"trigger_type": trigger_type.value,
|
||||
"scholar_count": len(scholars),
|
||||
"is_filtered_run": filtered_scholar_ids is not None,
|
||||
"request_delay_seconds": request_delay_seconds,
|
||||
"network_error_retries": network_error_retries,
|
||||
"retry_backoff_seconds": retry_backoff_seconds,
|
||||
"max_pages_per_scholar": max_pages_per_scholar,
|
||||
"page_size": page_size,
|
||||
},
|
||||
)
|
||||
|
||||
run = CrawlRun(
|
||||
user_id=user_id,
|
||||
trigger_type=trigger_type,
|
||||
status=RunStatus.RUNNING,
|
||||
scholar_count=len(scholars),
|
||||
new_pub_count=0,
|
||||
error_log={},
|
||||
)
|
||||
db_session.add(run)
|
||||
await db_session.flush()
|
||||
|
||||
succeeded_count = 0
|
||||
failed_count = 0
|
||||
partial_count = 0
|
||||
scholar_results: list[dict[str, Any]] = []
|
||||
|
||||
for index, scholar in enumerate(scholars):
|
||||
if index > 0 and request_delay_seconds > 0:
|
||||
await asyncio.sleep(float(request_delay_seconds))
|
||||
|
||||
run_dt = datetime.now(timezone.utc)
|
||||
start_cstart = int(start_cstart_map.get(int(scholar.id), 0))
|
||||
|
||||
paged_parse_result = await self._fetch_and_parse_all_pages_with_retry(
|
||||
scholar_id=scholar.scholar_id,
|
||||
start_cstart=start_cstart,
|
||||
request_delay_seconds=request_delay_seconds,
|
||||
network_error_retries=network_error_retries,
|
||||
retry_backoff_seconds=retry_backoff_seconds,
|
||||
max_pages=max_pages_per_scholar,
|
||||
page_size=page_size,
|
||||
)
|
||||
fetch_result = paged_parse_result.fetch_result
|
||||
parsed_page = paged_parse_result.parsed_page
|
||||
publications = paged_parse_result.publications
|
||||
attempt_log = paged_parse_result.attempt_log
|
||||
page_logs = paged_parse_result.page_logs
|
||||
|
||||
logger.info(
|
||||
"ingestion.scholar_parsed",
|
||||
extra={
|
||||
"event": "ingestion.scholar_parsed",
|
||||
"user_id": user_id,
|
||||
"crawl_run_id": run.id,
|
||||
"scholar_profile_id": scholar.id,
|
||||
"scholar_id": scholar.scholar_id,
|
||||
"state": parsed_page.state.value,
|
||||
"publication_count": len(publications),
|
||||
"has_show_more_button": parsed_page.has_show_more_button,
|
||||
"pages_fetched": paged_parse_result.pages_fetched,
|
||||
"pages_attempted": paged_parse_result.pages_attempted,
|
||||
"has_more_remaining": paged_parse_result.has_more_remaining,
|
||||
"pagination_truncated_reason": paged_parse_result.pagination_truncated_reason,
|
||||
"warning_count": len(parsed_page.warnings),
|
||||
},
|
||||
)
|
||||
|
||||
result_entry = {
|
||||
"scholar_profile_id": scholar.id,
|
||||
"scholar_id": scholar.scholar_id,
|
||||
"state": parsed_page.state.value,
|
||||
"state_reason": parsed_page.state_reason,
|
||||
"outcome": "failed",
|
||||
"attempt_count": len(attempt_log),
|
||||
"publication_count": len(publications),
|
||||
"start_cstart": start_cstart,
|
||||
"articles_range": parsed_page.articles_range,
|
||||
"warnings": parsed_page.warnings,
|
||||
"has_show_more_button": parsed_page.has_show_more_button,
|
||||
"pages_fetched": paged_parse_result.pages_fetched,
|
||||
"pages_attempted": paged_parse_result.pages_attempted,
|
||||
"has_more_remaining": paged_parse_result.has_more_remaining,
|
||||
"pagination_truncated_reason": paged_parse_result.pagination_truncated_reason,
|
||||
"continuation_cstart": paged_parse_result.continuation_cstart,
|
||||
}
|
||||
had_page_failure = parsed_page.state not in {ParseState.OK, ParseState.NO_RESULTS}
|
||||
has_partial_publication_set = len(publications) > 0 and had_page_failure
|
||||
is_partial_due_to_pagination = (
|
||||
paged_parse_result.has_more_remaining
|
||||
or paged_parse_result.pagination_truncated_reason is not None
|
||||
)
|
||||
|
||||
if (not had_page_failure) or has_partial_publication_set:
|
||||
try:
|
||||
discovered_publication_count = await self._upsert_profile_publications(
|
||||
db_session,
|
||||
run=run,
|
||||
scholar=scholar,
|
||||
publications=publications,
|
||||
)
|
||||
run.new_pub_count = int(run.new_pub_count or 0) + discovered_publication_count
|
||||
|
||||
is_partial_scholar = is_partial_due_to_pagination or has_partial_publication_set
|
||||
scholar.last_run_status = (
|
||||
RunStatus.PARTIAL_FAILURE if is_partial_scholar else RunStatus.SUCCESS
|
||||
)
|
||||
scholar.last_run_dt = run_dt
|
||||
succeeded_count += 1
|
||||
result_entry["outcome"] = "partial" if is_partial_scholar else "success"
|
||||
|
||||
if is_partial_scholar:
|
||||
partial_count += 1
|
||||
result_entry["debug"] = self._build_failure_debug_context(
|
||||
fetch_result=fetch_result,
|
||||
parsed_page=parsed_page,
|
||||
attempt_log=attempt_log,
|
||||
page_logs=page_logs,
|
||||
)
|
||||
except Exception as exc:
|
||||
failed_count += 1
|
||||
scholar.last_run_status = RunStatus.FAILED
|
||||
scholar.last_run_dt = run_dt
|
||||
result_entry["state"] = "ingestion_error"
|
||||
result_entry["state_reason"] = "publication_upsert_exception"
|
||||
result_entry["outcome"] = "failed"
|
||||
result_entry["error"] = str(exc)
|
||||
result_entry["debug"] = self._build_failure_debug_context(
|
||||
fetch_result=fetch_result,
|
||||
parsed_page=parsed_page,
|
||||
attempt_log=attempt_log,
|
||||
page_logs=page_logs,
|
||||
exception=exc,
|
||||
)
|
||||
logger.exception(
|
||||
"ingestion.scholar_failed",
|
||||
extra={
|
||||
"event": "ingestion.scholar_failed",
|
||||
"user_id": user_id,
|
||||
"crawl_run_id": run.id,
|
||||
"scholar_profile_id": scholar.id,
|
||||
"scholar_id": scholar.scholar_id,
|
||||
},
|
||||
)
|
||||
else:
|
||||
failed_count += 1
|
||||
scholar.last_run_status = RunStatus.FAILED
|
||||
scholar.last_run_dt = run_dt
|
||||
result_entry["debug"] = self._build_failure_debug_context(
|
||||
fetch_result=fetch_result,
|
||||
parsed_page=parsed_page,
|
||||
attempt_log=attempt_log,
|
||||
page_logs=page_logs,
|
||||
)
|
||||
logger.warning(
|
||||
"ingestion.scholar_parse_failed",
|
||||
extra={
|
||||
"event": "ingestion.scholar_parse_failed",
|
||||
"user_id": user_id,
|
||||
"crawl_run_id": run.id,
|
||||
"scholar_profile_id": scholar.id,
|
||||
"scholar_id": scholar.scholar_id,
|
||||
"state": parsed_page.state.value,
|
||||
"state_reason": parsed_page.state_reason,
|
||||
"status_code": fetch_result.status_code,
|
||||
},
|
||||
)
|
||||
|
||||
queue_reason, queue_cstart = self._resolve_continuation_queue_target(
|
||||
outcome=str(result_entry.get("outcome", "")),
|
||||
state=str(result_entry.get("state", "")),
|
||||
pagination_truncated_reason=paged_parse_result.pagination_truncated_reason,
|
||||
continuation_cstart=paged_parse_result.continuation_cstart,
|
||||
fallback_cstart=start_cstart,
|
||||
)
|
||||
if auto_queue_continuations and queue_reason is not None:
|
||||
await queue_service.upsert_job(
|
||||
db_session,
|
||||
user_id=user_id,
|
||||
scholar_profile_id=scholar.id,
|
||||
resume_cstart=queue_cstart,
|
||||
reason=queue_reason,
|
||||
run_id=run.id,
|
||||
delay_seconds=queue_delay_seconds,
|
||||
)
|
||||
result_entry["continuation_enqueued"] = True
|
||||
result_entry["continuation_reason"] = queue_reason
|
||||
result_entry["continuation_cstart"] = queue_cstart
|
||||
else:
|
||||
cleared = await queue_service.clear_job_for_scholar(
|
||||
db_session,
|
||||
user_id=user_id,
|
||||
scholar_profile_id=scholar.id,
|
||||
)
|
||||
if cleared:
|
||||
result_entry["continuation_cleared"] = True
|
||||
|
||||
scholar_results.append(result_entry)
|
||||
|
||||
failed_state_counts: dict[str, int] = {}
|
||||
failed_reason_counts: dict[str, int] = {}
|
||||
for entry in scholar_results:
|
||||
if str(entry.get("outcome", "")) != "failed":
|
||||
continue
|
||||
state = str(entry.get("state", ""))
|
||||
if state not in FAILED_STATES:
|
||||
continue
|
||||
failed_state_counts[state] = failed_state_counts.get(state, 0) + 1
|
||||
reason = str(entry.get("state_reason", "")).strip()
|
||||
if reason:
|
||||
failed_reason_counts[reason] = failed_reason_counts.get(reason, 0) + 1
|
||||
|
||||
run.end_dt = datetime.now(timezone.utc)
|
||||
run.status = self._resolve_run_status(
|
||||
scholar_count=len(scholars),
|
||||
succeeded_count=succeeded_count,
|
||||
failed_count=failed_count,
|
||||
partial_count=partial_count,
|
||||
)
|
||||
run.error_log = {
|
||||
"scholar_results": scholar_results,
|
||||
"summary": {
|
||||
"succeeded_count": succeeded_count,
|
||||
"failed_count": failed_count,
|
||||
"partial_count": partial_count,
|
||||
"failed_state_counts": failed_state_counts,
|
||||
"failed_reason_counts": failed_reason_counts,
|
||||
},
|
||||
}
|
||||
|
||||
await db_session.commit()
|
||||
logger.info(
|
||||
"ingestion.run_completed",
|
||||
extra={
|
||||
"event": "ingestion.run_completed",
|
||||
"user_id": user_id,
|
||||
"crawl_run_id": run.id,
|
||||
"status": run.status.value,
|
||||
"scholar_count": len(scholars),
|
||||
"succeeded_count": succeeded_count,
|
||||
"failed_count": failed_count,
|
||||
"partial_count": partial_count,
|
||||
"new_publication_count": run.new_pub_count,
|
||||
},
|
||||
)
|
||||
|
||||
return RunExecutionSummary(
|
||||
crawl_run_id=run.id,
|
||||
status=run.status,
|
||||
scholar_count=len(scholars),
|
||||
succeeded_count=succeeded_count,
|
||||
failed_count=failed_count,
|
||||
partial_count=partial_count,
|
||||
new_publication_count=run.new_pub_count,
|
||||
)
|
||||
|
||||
async def _fetch_profile_page(
|
||||
self,
|
||||
*,
|
||||
scholar_id: str,
|
||||
cstart: int,
|
||||
page_size: int,
|
||||
) -> FetchResult:
|
||||
try:
|
||||
page_fetcher = getattr(self._source, "fetch_profile_page_html", None)
|
||||
if callable(page_fetcher):
|
||||
return await page_fetcher(
|
||||
scholar_id,
|
||||
cstart=cstart,
|
||||
pagesize=page_size,
|
||||
)
|
||||
if cstart <= 0:
|
||||
return await self._source.fetch_profile_html(scholar_id)
|
||||
return FetchResult(
|
||||
requested_url=(
|
||||
"https://scholar.google.com/citations"
|
||||
f"?hl=en&user={scholar_id}&cstart={cstart}&pagesize={page_size}"
|
||||
),
|
||||
status_code=None,
|
||||
final_url=None,
|
||||
body="",
|
||||
error="source_does_not_support_pagination",
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.exception(
|
||||
"ingestion.fetch_unexpected_error",
|
||||
extra={
|
||||
"event": "ingestion.fetch_unexpected_error",
|
||||
"scholar_id": scholar_id,
|
||||
"cstart": cstart,
|
||||
"page_size": page_size,
|
||||
},
|
||||
)
|
||||
return FetchResult(
|
||||
requested_url=(
|
||||
"https://scholar.google.com/citations"
|
||||
f"?hl=en&user={scholar_id}&cstart={cstart}&pagesize={page_size}"
|
||||
),
|
||||
status_code=None,
|
||||
final_url=None,
|
||||
body="",
|
||||
error=str(exc),
|
||||
)
|
||||
|
||||
async def _fetch_and_parse_page_with_retry(
|
||||
self,
|
||||
*,
|
||||
scholar_id: str,
|
||||
cstart: int,
|
||||
page_size: int,
|
||||
network_error_retries: int,
|
||||
retry_backoff_seconds: float,
|
||||
) -> tuple[FetchResult, ParsedProfilePage, list[dict[str, Any]]]:
|
||||
max_attempts = max(1, int(network_error_retries) + 1)
|
||||
backoff = max(float(retry_backoff_seconds), 0.0)
|
||||
attempt_log: list[dict[str, Any]] = []
|
||||
fetch_result: FetchResult | None = None
|
||||
parsed_page: ParsedProfilePage | None = None
|
||||
|
||||
for attempt_index in range(max_attempts):
|
||||
fetch_result = await self._fetch_profile_page(
|
||||
scholar_id=scholar_id,
|
||||
cstart=cstart,
|
||||
page_size=page_size,
|
||||
)
|
||||
parsed_page = parse_profile_page(fetch_result)
|
||||
attempt_log.append(
|
||||
{
|
||||
"attempt": attempt_index + 1,
|
||||
"cstart": cstart,
|
||||
"state": parsed_page.state.value,
|
||||
"state_reason": parsed_page.state_reason,
|
||||
"status_code": fetch_result.status_code,
|
||||
"fetch_error": fetch_result.error,
|
||||
}
|
||||
)
|
||||
|
||||
should_retry = (
|
||||
parsed_page.state == ParseState.NETWORK_ERROR
|
||||
and attempt_index < max_attempts - 1
|
||||
)
|
||||
if not should_retry:
|
||||
break
|
||||
|
||||
sleep_seconds = backoff * (2**attempt_index)
|
||||
logger.warning(
|
||||
"ingestion.scholar_retry_scheduled",
|
||||
extra={
|
||||
"event": "ingestion.scholar_retry_scheduled",
|
||||
"scholar_id": scholar_id,
|
||||
"cstart": cstart,
|
||||
"attempt": attempt_index + 1,
|
||||
"next_attempt": attempt_index + 2,
|
||||
"sleep_seconds": sleep_seconds,
|
||||
"state_reason": parsed_page.state_reason,
|
||||
},
|
||||
)
|
||||
if sleep_seconds > 0:
|
||||
await asyncio.sleep(sleep_seconds)
|
||||
|
||||
if fetch_result is None or parsed_page is None:
|
||||
raise RuntimeError("Fetch-and-parse retry loop produced no result.")
|
||||
return fetch_result, parsed_page, attempt_log
|
||||
|
||||
async def _fetch_and_parse_all_pages_with_retry(
|
||||
self,
|
||||
*,
|
||||
scholar_id: str,
|
||||
start_cstart: int,
|
||||
request_delay_seconds: int,
|
||||
network_error_retries: int,
|
||||
retry_backoff_seconds: float,
|
||||
max_pages: int,
|
||||
page_size: int,
|
||||
) -> PagedParseResult:
|
||||
bounded_max_pages = max(1, int(max_pages))
|
||||
bounded_page_size = max(1, int(page_size))
|
||||
|
||||
(
|
||||
fetch_result,
|
||||
parsed_page,
|
||||
first_attempt_log,
|
||||
) = await self._fetch_and_parse_page_with_retry(
|
||||
scholar_id=scholar_id,
|
||||
cstart=start_cstart,
|
||||
page_size=bounded_page_size,
|
||||
network_error_retries=network_error_retries,
|
||||
retry_backoff_seconds=retry_backoff_seconds,
|
||||
)
|
||||
|
||||
attempt_log: list[dict[str, Any]] = list(first_attempt_log)
|
||||
page_logs: list[dict[str, Any]] = []
|
||||
page_logs.append(
|
||||
{
|
||||
"page": 1,
|
||||
"cstart": start_cstart,
|
||||
"state": parsed_page.state.value,
|
||||
"state_reason": parsed_page.state_reason,
|
||||
"status_code": fetch_result.status_code,
|
||||
"publication_count": len(parsed_page.publications),
|
||||
"articles_range": parsed_page.articles_range,
|
||||
"has_show_more_button": parsed_page.has_show_more_button,
|
||||
"warning_codes": parsed_page.warnings,
|
||||
"attempt_count": len(first_attempt_log),
|
||||
}
|
||||
)
|
||||
pages_attempted = 1
|
||||
|
||||
# Immediate hard failure: nothing to salvage.
|
||||
if parsed_page.state not in {ParseState.OK, ParseState.NO_RESULTS}:
|
||||
return PagedParseResult(
|
||||
fetch_result=fetch_result,
|
||||
parsed_page=parsed_page,
|
||||
publications=[],
|
||||
attempt_log=attempt_log,
|
||||
page_logs=page_logs,
|
||||
pages_fetched=0,
|
||||
pages_attempted=pages_attempted,
|
||||
has_more_remaining=False,
|
||||
pagination_truncated_reason=None,
|
||||
continuation_cstart=(
|
||||
start_cstart if parsed_page.state == ParseState.NETWORK_ERROR else None
|
||||
),
|
||||
)
|
||||
|
||||
publications = list(parsed_page.publications)
|
||||
pages_fetched = 1
|
||||
has_more_remaining = False
|
||||
pagination_truncated_reason: str | None = None
|
||||
continuation_cstart: int | None = None
|
||||
current_cstart = start_cstart
|
||||
next_cstart = _next_cstart_value(
|
||||
articles_range=parsed_page.articles_range,
|
||||
fallback=current_cstart + len(parsed_page.publications),
|
||||
)
|
||||
|
||||
while parsed_page.has_show_more_button:
|
||||
if pages_fetched >= bounded_max_pages:
|
||||
has_more_remaining = True
|
||||
pagination_truncated_reason = "max_pages_reached"
|
||||
continuation_cstart = next_cstart if next_cstart > current_cstart else current_cstart
|
||||
break
|
||||
if next_cstart <= current_cstart:
|
||||
has_more_remaining = True
|
||||
pagination_truncated_reason = "pagination_cursor_stalled"
|
||||
continuation_cstart = current_cstart
|
||||
break
|
||||
if request_delay_seconds > 0:
|
||||
await asyncio.sleep(float(request_delay_seconds))
|
||||
|
||||
current_cstart = next_cstart
|
||||
(
|
||||
page_fetch_result,
|
||||
page_parsed,
|
||||
page_attempt_log,
|
||||
) = await self._fetch_and_parse_page_with_retry(
|
||||
scholar_id=scholar_id,
|
||||
cstart=current_cstart,
|
||||
page_size=bounded_page_size,
|
||||
network_error_retries=network_error_retries,
|
||||
retry_backoff_seconds=retry_backoff_seconds,
|
||||
)
|
||||
|
||||
pages_attempted += 1
|
||||
attempt_log.extend(page_attempt_log)
|
||||
page_logs.append(
|
||||
{
|
||||
"page": pages_attempted,
|
||||
"cstart": current_cstart,
|
||||
"state": page_parsed.state.value,
|
||||
"state_reason": page_parsed.state_reason,
|
||||
"status_code": page_fetch_result.status_code,
|
||||
"publication_count": len(page_parsed.publications),
|
||||
"articles_range": page_parsed.articles_range,
|
||||
"has_show_more_button": page_parsed.has_show_more_button,
|
||||
"warning_codes": page_parsed.warnings,
|
||||
"attempt_count": len(page_attempt_log),
|
||||
}
|
||||
)
|
||||
|
||||
fetch_result = page_fetch_result
|
||||
parsed_page = page_parsed
|
||||
if parsed_page.state not in {ParseState.OK, ParseState.NO_RESULTS}:
|
||||
has_more_remaining = True
|
||||
pagination_truncated_reason = f"page_state_{parsed_page.state.value}"
|
||||
continuation_cstart = current_cstart
|
||||
break
|
||||
|
||||
# Google may keep a stale/disabled "show more" marker while returning an empty tail page.
|
||||
# Treat this as a terminal page to avoid false cursor-stalled partial runs.
|
||||
if parsed_page.state == ParseState.NO_RESULTS and len(parsed_page.publications) == 0:
|
||||
pages_fetched += 1
|
||||
break
|
||||
|
||||
pages_fetched += 1
|
||||
publications.extend(parsed_page.publications)
|
||||
next_cstart = _next_cstart_value(
|
||||
articles_range=parsed_page.articles_range,
|
||||
fallback=current_cstart + len(parsed_page.publications),
|
||||
)
|
||||
|
||||
return PagedParseResult(
|
||||
fetch_result=fetch_result,
|
||||
parsed_page=parsed_page,
|
||||
publications=_dedupe_publication_candidates(publications),
|
||||
attempt_log=attempt_log,
|
||||
page_logs=page_logs,
|
||||
pages_fetched=pages_fetched,
|
||||
pages_attempted=pages_attempted,
|
||||
has_more_remaining=has_more_remaining,
|
||||
pagination_truncated_reason=pagination_truncated_reason,
|
||||
continuation_cstart=continuation_cstart,
|
||||
)
|
||||
|
||||
async def _upsert_profile_publications(
|
||||
self,
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
run: CrawlRun,
|
||||
scholar: ScholarProfile,
|
||||
publications: list[PublicationCandidate],
|
||||
) -> int:
|
||||
seen_publication_ids: set[int] = set()
|
||||
discovered_count = 0
|
||||
|
||||
for candidate in publications:
|
||||
publication = await self._resolve_publication(db_session, candidate)
|
||||
if publication.id in seen_publication_ids:
|
||||
continue
|
||||
seen_publication_ids.add(publication.id)
|
||||
|
||||
link_result = await db_session.execute(
|
||||
select(ScholarPublication).where(
|
||||
ScholarPublication.scholar_profile_id == scholar.id,
|
||||
ScholarPublication.publication_id == publication.id,
|
||||
)
|
||||
)
|
||||
link = link_result.scalar_one_or_none()
|
||||
if link is not None:
|
||||
continue
|
||||
|
||||
link = ScholarPublication(
|
||||
scholar_profile_id=scholar.id,
|
||||
publication_id=publication.id,
|
||||
is_read=False,
|
||||
first_seen_run_id=run.id,
|
||||
)
|
||||
db_session.add(link)
|
||||
discovered_count += 1
|
||||
|
||||
logger.debug(
|
||||
"ingestion.publication_discovered",
|
||||
extra={
|
||||
"event": "ingestion.publication_discovered",
|
||||
"scholar_profile_id": scholar.id,
|
||||
"publication_id": publication.id,
|
||||
"crawl_run_id": run.id,
|
||||
},
|
||||
)
|
||||
|
||||
if not scholar.baseline_completed:
|
||||
scholar.baseline_completed = True
|
||||
|
||||
return discovered_count
|
||||
|
||||
async def _resolve_publication(
|
||||
self,
|
||||
db_session: AsyncSession,
|
||||
candidate: PublicationCandidate,
|
||||
) -> Publication:
|
||||
fingerprint = build_publication_fingerprint(candidate)
|
||||
|
||||
publication: Publication | None = None
|
||||
cluster_publication: Publication | None = None
|
||||
|
||||
if candidate.cluster_id:
|
||||
cluster_result = await db_session.execute(
|
||||
select(Publication).where(Publication.cluster_id == candidate.cluster_id)
|
||||
)
|
||||
cluster_publication = cluster_result.scalar_one_or_none()
|
||||
publication = cluster_publication
|
||||
|
||||
if publication is None:
|
||||
fingerprint_result = await db_session.execute(
|
||||
select(Publication).where(Publication.fingerprint_sha256 == fingerprint)
|
||||
)
|
||||
publication = fingerprint_result.scalar_one_or_none()
|
||||
|
||||
if publication is not None and cluster_publication is not None and publication.id != cluster_publication.id:
|
||||
publication = cluster_publication
|
||||
|
||||
if publication is None:
|
||||
publication = Publication(
|
||||
cluster_id=candidate.cluster_id,
|
||||
fingerprint_sha256=fingerprint,
|
||||
title_raw=candidate.title,
|
||||
title_normalized=normalize_title(candidate.title),
|
||||
year=candidate.year,
|
||||
citation_count=int(candidate.citation_count or 0),
|
||||
author_text=candidate.authors_text,
|
||||
venue_text=candidate.venue_text,
|
||||
pub_url=build_publication_url(candidate.title_url),
|
||||
pdf_url=None,
|
||||
)
|
||||
db_session.add(publication)
|
||||
await db_session.flush()
|
||||
logger.debug(
|
||||
"ingestion.publication_created",
|
||||
extra={
|
||||
"event": "ingestion.publication_created",
|
||||
"publication_id": publication.id,
|
||||
"cluster_id": publication.cluster_id,
|
||||
},
|
||||
)
|
||||
return publication
|
||||
|
||||
if candidate.cluster_id and publication.cluster_id is None:
|
||||
publication.cluster_id = candidate.cluster_id
|
||||
publication.title_raw = candidate.title
|
||||
publication.title_normalized = normalize_title(candidate.title)
|
||||
if candidate.year is not None:
|
||||
publication.year = candidate.year
|
||||
if candidate.citation_count is not None:
|
||||
publication.citation_count = int(candidate.citation_count)
|
||||
if candidate.authors_text:
|
||||
publication.author_text = candidate.authors_text
|
||||
if candidate.venue_text:
|
||||
publication.venue_text = candidate.venue_text
|
||||
if candidate.title_url:
|
||||
publication.pub_url = build_publication_url(candidate.title_url)
|
||||
|
||||
return publication
|
||||
|
||||
def _resolve_run_status(
|
||||
self,
|
||||
*,
|
||||
scholar_count: int,
|
||||
succeeded_count: int,
|
||||
failed_count: int,
|
||||
partial_count: int,
|
||||
) -> RunStatus:
|
||||
if scholar_count == 0:
|
||||
return RunStatus.SUCCESS
|
||||
if failed_count == scholar_count:
|
||||
return RunStatus.FAILED
|
||||
if failed_count > 0 or partial_count > 0:
|
||||
return RunStatus.PARTIAL_FAILURE
|
||||
if succeeded_count > 0:
|
||||
return RunStatus.SUCCESS
|
||||
return RunStatus.FAILED
|
||||
|
||||
def _resolve_continuation_queue_target(
|
||||
self,
|
||||
*,
|
||||
outcome: str,
|
||||
state: str,
|
||||
pagination_truncated_reason: str | None,
|
||||
continuation_cstart: int | None,
|
||||
fallback_cstart: int,
|
||||
) -> tuple[str | None, int]:
|
||||
if outcome == "partial":
|
||||
reason = (pagination_truncated_reason or "").strip()
|
||||
if reason in RESUMABLE_PARTIAL_REASONS or reason.startswith(
|
||||
RESUMABLE_PARTIAL_REASON_PREFIXES
|
||||
):
|
||||
return reason, queue_service.normalize_cstart(
|
||||
continuation_cstart if continuation_cstart is not None else fallback_cstart
|
||||
)
|
||||
return None, queue_service.normalize_cstart(fallback_cstart)
|
||||
|
||||
if outcome == "failed" and state == ParseState.NETWORK_ERROR.value:
|
||||
return "network_error_retry", queue_service.normalize_cstart(
|
||||
continuation_cstart if continuation_cstart is not None else fallback_cstart
|
||||
)
|
||||
|
||||
return None, queue_service.normalize_cstart(fallback_cstart)
|
||||
|
||||
def _build_failure_debug_context(
|
||||
self,
|
||||
*,
|
||||
fetch_result: FetchResult,
|
||||
parsed_page: ParsedProfilePage,
|
||||
attempt_log: list[dict[str, Any]],
|
||||
page_logs: list[dict[str, Any]] | None = None,
|
||||
exception: Exception | None = None,
|
||||
) -> dict[str, Any]:
|
||||
context: dict[str, Any] = {
|
||||
"requested_url": fetch_result.requested_url,
|
||||
"final_url": fetch_result.final_url,
|
||||
"status_code": fetch_result.status_code,
|
||||
"fetch_error": fetch_result.error,
|
||||
"state_reason": parsed_page.state_reason,
|
||||
"profile_name": parsed_page.profile_name,
|
||||
"articles_range": parsed_page.articles_range,
|
||||
"has_show_more_button": parsed_page.has_show_more_button,
|
||||
"has_operation_error_banner": parsed_page.has_operation_error_banner,
|
||||
"warning_codes": parsed_page.warnings,
|
||||
"marker_counts_nonzero": {
|
||||
key: value for key, value in parsed_page.marker_counts.items() if value > 0
|
||||
},
|
||||
"body_length": len(fetch_result.body),
|
||||
"body_sha256": hashlib.sha256(fetch_result.body.encode("utf-8")).hexdigest()
|
||||
if fetch_result.body
|
||||
else None,
|
||||
"body_excerpt": _build_body_excerpt(fetch_result.body),
|
||||
"attempt_log": attempt_log,
|
||||
}
|
||||
if page_logs:
|
||||
context["page_logs"] = page_logs
|
||||
if exception is not None:
|
||||
context["exception_type"] = type(exception).__name__
|
||||
context["exception_message"] = str(exception)
|
||||
return context
|
||||
|
||||
async def _try_acquire_user_lock(
|
||||
self,
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
) -> bool:
|
||||
result = await db_session.execute(
|
||||
text(
|
||||
"SELECT pg_try_advisory_xact_lock(:namespace, :user_key)"
|
||||
),
|
||||
{
|
||||
"namespace": RUN_LOCK_NAMESPACE,
|
||||
"user_key": int(user_id),
|
||||
},
|
||||
)
|
||||
return bool(result.scalar_one())
|
||||
|
||||
|
||||
def normalize_title(value: str) -> str:
|
||||
lowered = value.lower()
|
||||
cleaned = TITLE_ALNUM_RE.sub("", lowered)
|
||||
return cleaned
|
||||
|
||||
|
||||
def _first_author_last_name(authors_text: str | None) -> str:
|
||||
if not authors_text:
|
||||
return ""
|
||||
first_author = authors_text.split(",", maxsplit=1)[0].strip().lower()
|
||||
words = WORD_RE.findall(first_author)
|
||||
if not words:
|
||||
return ""
|
||||
return words[-1]
|
||||
|
||||
|
||||
def _first_venue_word(venue_text: str | None) -> str:
|
||||
if not venue_text:
|
||||
return ""
|
||||
words = WORD_RE.findall(venue_text.lower())
|
||||
if not words:
|
||||
return ""
|
||||
return words[0]
|
||||
|
||||
|
||||
def build_publication_fingerprint(candidate: PublicationCandidate) -> str:
|
||||
canonical = "|".join(
|
||||
[
|
||||
normalize_title(candidate.title),
|
||||
str(candidate.year) if candidate.year is not None else "",
|
||||
_first_author_last_name(candidate.authors_text),
|
||||
_first_venue_word(candidate.venue_text),
|
||||
]
|
||||
)
|
||||
return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def build_publication_url(path_or_url: str | None) -> str | None:
|
||||
if not path_or_url:
|
||||
return None
|
||||
return urljoin("https://scholar.google.com", path_or_url)
|
||||
|
||||
|
||||
def _next_cstart_value(*, articles_range: str | None, fallback: int) -> int:
|
||||
if articles_range:
|
||||
numbers = re.findall(r"\d+", articles_range)
|
||||
if len(numbers) >= 2:
|
||||
try:
|
||||
return int(numbers[1])
|
||||
except ValueError:
|
||||
pass
|
||||
return int(fallback)
|
||||
|
||||
|
||||
def _dedupe_publication_candidates(
|
||||
publications: list[PublicationCandidate],
|
||||
) -> list[PublicationCandidate]:
|
||||
deduped: list[PublicationCandidate] = []
|
||||
seen: set[str] = set()
|
||||
for publication in publications:
|
||||
if publication.cluster_id:
|
||||
identity = f"cluster:{publication.cluster_id}"
|
||||
else:
|
||||
identity = "|".join(
|
||||
[
|
||||
"fallback",
|
||||
normalize_title(publication.title),
|
||||
str(publication.year) if publication.year is not None else "",
|
||||
publication.authors_text or "",
|
||||
publication.venue_text or "",
|
||||
]
|
||||
)
|
||||
if identity in seen:
|
||||
continue
|
||||
seen.add(identity)
|
||||
deduped.append(publication)
|
||||
return deduped
|
||||
|
||||
|
||||
def _build_body_excerpt(body: str, *, max_chars: int = 220) -> str | None:
|
||||
if not body:
|
||||
return None
|
||||
flattened = SPACE_RE.sub(" ", HTML_TAG_RE.sub(" ", body)).strip()
|
||||
if not flattened:
|
||||
return None
|
||||
if len(flattened) <= max_chars:
|
||||
return flattened
|
||||
return f"{flattened[:max_chars - 1]}..."
|
||||
297
app/services/publications.py
Normal file
297
app/services/publications.py
Normal file
|
|
@ -0,0 +1,297 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import Select, func, select, update
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.models import (
|
||||
CrawlRun,
|
||||
Publication,
|
||||
RunStatus,
|
||||
ScholarProfile,
|
||||
ScholarPublication,
|
||||
)
|
||||
|
||||
MODE_NEW = "new"
|
||||
MODE_ALL = "all"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PublicationListItem:
|
||||
publication_id: int
|
||||
scholar_profile_id: int
|
||||
scholar_label: str
|
||||
title: str
|
||||
year: int | None
|
||||
citation_count: int
|
||||
venue_text: str | None
|
||||
pub_url: str | None
|
||||
is_read: bool
|
||||
first_seen_at: datetime
|
||||
is_new_in_latest_run: bool
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class UnreadPublicationItem:
|
||||
publication_id: int
|
||||
scholar_profile_id: int
|
||||
scholar_label: str
|
||||
title: str
|
||||
year: int | None
|
||||
citation_count: int
|
||||
venue_text: str | None
|
||||
pub_url: str | None
|
||||
|
||||
|
||||
def resolve_mode(value: str | None) -> str:
|
||||
if value == MODE_ALL:
|
||||
return MODE_ALL
|
||||
return MODE_NEW
|
||||
|
||||
|
||||
async def get_latest_completed_run_id_for_user(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
) -> int | None:
|
||||
result = await db_session.execute(
|
||||
select(func.max(CrawlRun.id)).where(
|
||||
CrawlRun.user_id == user_id,
|
||||
CrawlRun.status != RunStatus.RUNNING,
|
||||
)
|
||||
)
|
||||
latest_run_id = result.scalar_one_or_none()
|
||||
return int(latest_run_id) if latest_run_id is not None else None
|
||||
|
||||
|
||||
def publications_query(
|
||||
*,
|
||||
user_id: int,
|
||||
mode: str,
|
||||
latest_run_id: int | None,
|
||||
scholar_profile_id: int | None,
|
||||
limit: int,
|
||||
) -> Select[tuple]:
|
||||
scholar_label = ScholarProfile.display_name
|
||||
|
||||
stmt = (
|
||||
select(
|
||||
Publication.id,
|
||||
ScholarProfile.id,
|
||||
scholar_label,
|
||||
ScholarProfile.scholar_id,
|
||||
Publication.title_raw,
|
||||
Publication.year,
|
||||
Publication.citation_count,
|
||||
Publication.venue_text,
|
||||
Publication.pub_url,
|
||||
ScholarPublication.is_read,
|
||||
ScholarPublication.first_seen_run_id,
|
||||
ScholarPublication.created_at,
|
||||
)
|
||||
.join(ScholarPublication, ScholarPublication.publication_id == Publication.id)
|
||||
.join(ScholarProfile, ScholarProfile.id == ScholarPublication.scholar_profile_id)
|
||||
.where(ScholarProfile.user_id == user_id)
|
||||
.order_by(ScholarPublication.created_at.desc(), Publication.id.desc())
|
||||
.limit(limit)
|
||||
)
|
||||
if scholar_profile_id is not None:
|
||||
stmt = stmt.where(ScholarProfile.id == scholar_profile_id)
|
||||
if mode == MODE_NEW:
|
||||
# "New" means discovered in the latest completed run.
|
||||
if latest_run_id is None:
|
||||
stmt = stmt.where(False)
|
||||
else:
|
||||
stmt = stmt.where(ScholarPublication.first_seen_run_id == latest_run_id)
|
||||
return stmt
|
||||
|
||||
|
||||
async def list_for_user(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
mode: str = MODE_ALL,
|
||||
scholar_profile_id: int | None = None,
|
||||
limit: int = 300,
|
||||
) -> list[PublicationListItem]:
|
||||
resolved_mode = resolve_mode(mode)
|
||||
latest_run_id = await get_latest_completed_run_id_for_user(
|
||||
db_session,
|
||||
user_id=user_id,
|
||||
)
|
||||
result = await db_session.execute(
|
||||
publications_query(
|
||||
user_id=user_id,
|
||||
mode=resolved_mode,
|
||||
latest_run_id=latest_run_id,
|
||||
scholar_profile_id=scholar_profile_id,
|
||||
limit=limit,
|
||||
)
|
||||
)
|
||||
|
||||
rows = result.all()
|
||||
items: list[PublicationListItem] = []
|
||||
for row in rows:
|
||||
(
|
||||
publication_id,
|
||||
scholar_profile_id,
|
||||
display_name,
|
||||
scholar_id,
|
||||
title_raw,
|
||||
year,
|
||||
citation_count,
|
||||
venue_text,
|
||||
pub_url,
|
||||
is_read,
|
||||
first_seen_run_id,
|
||||
created_at,
|
||||
) = row
|
||||
items.append(
|
||||
PublicationListItem(
|
||||
publication_id=int(publication_id),
|
||||
scholar_profile_id=int(scholar_profile_id),
|
||||
scholar_label=(display_name or scholar_id),
|
||||
title=title_raw,
|
||||
year=year,
|
||||
citation_count=int(citation_count or 0),
|
||||
venue_text=venue_text,
|
||||
pub_url=pub_url,
|
||||
is_read=bool(is_read),
|
||||
first_seen_at=created_at,
|
||||
is_new_in_latest_run=(
|
||||
latest_run_id is not None and int(first_seen_run_id or 0) == latest_run_id
|
||||
),
|
||||
)
|
||||
)
|
||||
return items
|
||||
|
||||
|
||||
async def list_unread_for_user(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
limit: int = 100,
|
||||
) -> list[UnreadPublicationItem]:
|
||||
result = await db_session.execute(
|
||||
publications_query(
|
||||
user_id=user_id,
|
||||
mode=MODE_ALL,
|
||||
latest_run_id=None,
|
||||
scholar_profile_id=None,
|
||||
limit=limit,
|
||||
).where(ScholarPublication.is_read.is_(False))
|
||||
)
|
||||
rows = result.all()
|
||||
items: list[UnreadPublicationItem] = []
|
||||
for row in rows:
|
||||
(
|
||||
publication_id,
|
||||
scholar_profile_id,
|
||||
display_name,
|
||||
scholar_id,
|
||||
title_raw,
|
||||
year,
|
||||
citation_count,
|
||||
venue_text,
|
||||
pub_url,
|
||||
_is_read,
|
||||
_first_seen_run_id,
|
||||
_created_at,
|
||||
) = row
|
||||
items.append(
|
||||
UnreadPublicationItem(
|
||||
publication_id=int(publication_id),
|
||||
scholar_profile_id=int(scholar_profile_id),
|
||||
scholar_label=(display_name or scholar_id),
|
||||
title=title_raw,
|
||||
year=year,
|
||||
citation_count=int(citation_count or 0),
|
||||
venue_text=venue_text,
|
||||
pub_url=pub_url,
|
||||
)
|
||||
)
|
||||
return items
|
||||
|
||||
|
||||
async def list_new_for_latest_run_for_user(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
limit: int = 100,
|
||||
) -> list[UnreadPublicationItem]:
|
||||
rows = await list_for_user(
|
||||
db_session,
|
||||
user_id=user_id,
|
||||
mode=MODE_NEW,
|
||||
scholar_profile_id=None,
|
||||
limit=limit,
|
||||
)
|
||||
return [
|
||||
UnreadPublicationItem(
|
||||
publication_id=row.publication_id,
|
||||
scholar_profile_id=row.scholar_profile_id,
|
||||
scholar_label=row.scholar_label,
|
||||
title=row.title,
|
||||
year=row.year,
|
||||
citation_count=row.citation_count,
|
||||
venue_text=row.venue_text,
|
||||
pub_url=row.pub_url,
|
||||
)
|
||||
for row in rows
|
||||
]
|
||||
|
||||
|
||||
async def count_for_user(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
mode: str = MODE_ALL,
|
||||
scholar_profile_id: int | None = None,
|
||||
) -> int:
|
||||
resolved_mode = resolve_mode(mode)
|
||||
latest_run_id = await get_latest_completed_run_id_for_user(
|
||||
db_session,
|
||||
user_id=user_id,
|
||||
)
|
||||
stmt = (
|
||||
select(func.count())
|
||||
.select_from(ScholarPublication)
|
||||
.join(ScholarProfile, ScholarProfile.id == ScholarPublication.scholar_profile_id)
|
||||
.where(ScholarProfile.user_id == user_id)
|
||||
)
|
||||
if scholar_profile_id is not None:
|
||||
stmt = stmt.where(ScholarProfile.id == scholar_profile_id)
|
||||
if resolved_mode == MODE_NEW:
|
||||
if latest_run_id is None:
|
||||
return 0
|
||||
stmt = stmt.where(ScholarPublication.first_seen_run_id == latest_run_id)
|
||||
result = await db_session.execute(stmt)
|
||||
return int(result.scalar_one() or 0)
|
||||
|
||||
|
||||
async def mark_all_unread_as_read_for_user(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
) -> int:
|
||||
scholar_ids = (
|
||||
select(ScholarProfile.id)
|
||||
.where(ScholarProfile.user_id == user_id)
|
||||
.scalar_subquery()
|
||||
)
|
||||
|
||||
stmt = (
|
||||
update(ScholarPublication)
|
||||
.where(
|
||||
ScholarPublication.scholar_profile_id.in_(scholar_ids),
|
||||
ScholarPublication.is_read.is_(False),
|
||||
)
|
||||
.values(is_read=True)
|
||||
)
|
||||
result = await db_session.execute(stmt)
|
||||
await db_session.commit()
|
||||
|
||||
rowcount = result.rowcount
|
||||
return int(rowcount or 0)
|
||||
464
app/services/runs.py
Normal file
464
app/services/runs.py
Normal file
|
|
@ -0,0 +1,464 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import and_, case, func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.models import (
|
||||
CrawlRun,
|
||||
IngestionQueueItem,
|
||||
RunStatus,
|
||||
RunTriggerType,
|
||||
ScholarProfile,
|
||||
)
|
||||
from app.services import continuation_queue as queue_service
|
||||
|
||||
QUEUE_STATUS_QUEUED = "queued"
|
||||
QUEUE_STATUS_RETRYING = "retrying"
|
||||
QUEUE_STATUS_DROPPED = "dropped"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class QueueListItem:
|
||||
id: int
|
||||
scholar_profile_id: int
|
||||
scholar_label: str
|
||||
status: str
|
||||
reason: str
|
||||
dropped_reason: str | None
|
||||
attempt_count: int
|
||||
resume_cstart: int
|
||||
next_attempt_dt: datetime | None
|
||||
updated_at: datetime
|
||||
last_error: str | None
|
||||
last_run_id: int | None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class QueueClearResult:
|
||||
queue_item_id: int
|
||||
previous_status: str
|
||||
|
||||
|
||||
class QueueTransitionError(RuntimeError):
|
||||
def __init__(self, *, code: str, message: str, current_status: str) -> None:
|
||||
super().__init__(message)
|
||||
self.code = code
|
||||
self.message = message
|
||||
self.current_status = current_status
|
||||
|
||||
|
||||
def _safe_int(value: object, default: int = 0) -> int:
|
||||
try:
|
||||
return int(value)
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
|
||||
|
||||
def _summary_dict(error_log: object) -> dict[str, Any]:
|
||||
if not isinstance(error_log, dict):
|
||||
return {}
|
||||
summary = error_log.get("summary")
|
||||
if not isinstance(summary, dict):
|
||||
return {}
|
||||
return summary
|
||||
|
||||
|
||||
def extract_run_summary(error_log: object) -> dict[str, Any]:
|
||||
summary = _summary_dict(error_log)
|
||||
return {
|
||||
"succeeded_count": _safe_int(summary.get("succeeded_count", 0)),
|
||||
"failed_count": _safe_int(summary.get("failed_count", 0)),
|
||||
"partial_count": _safe_int(summary.get("partial_count", 0)),
|
||||
"failed_state_counts": {
|
||||
str(key): _safe_int(value, 0)
|
||||
for key, value in summary.get("failed_state_counts", {}).items()
|
||||
if isinstance(key, str)
|
||||
}
|
||||
if isinstance(summary.get("failed_state_counts"), dict)
|
||||
else {},
|
||||
"failed_reason_counts": {
|
||||
str(key): _safe_int(value, 0)
|
||||
for key, value in summary.get("failed_reason_counts", {}).items()
|
||||
if isinstance(key, str)
|
||||
}
|
||||
if isinstance(summary.get("failed_reason_counts"), dict)
|
||||
else {},
|
||||
}
|
||||
|
||||
|
||||
async def list_recent_runs_for_user(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
limit: int = 20,
|
||||
) -> list[CrawlRun]:
|
||||
result = await db_session.execute(
|
||||
select(CrawlRun)
|
||||
.where(CrawlRun.user_id == user_id)
|
||||
.order_by(CrawlRun.start_dt.desc(), CrawlRun.id.desc())
|
||||
.limit(limit)
|
||||
)
|
||||
return list(result.scalars().all())
|
||||
|
||||
|
||||
async def list_runs_for_user(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
limit: int = 100,
|
||||
failed_only: bool = False,
|
||||
) -> list[CrawlRun]:
|
||||
stmt = (
|
||||
select(CrawlRun)
|
||||
.where(CrawlRun.user_id == user_id)
|
||||
.order_by(CrawlRun.start_dt.desc(), CrawlRun.id.desc())
|
||||
.limit(limit)
|
||||
)
|
||||
if failed_only:
|
||||
stmt = stmt.where(
|
||||
CrawlRun.status.in_([RunStatus.FAILED, RunStatus.PARTIAL_FAILURE])
|
||||
)
|
||||
result = await db_session.execute(stmt)
|
||||
return list(result.scalars().all())
|
||||
|
||||
|
||||
async def get_run_for_user(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
run_id: int,
|
||||
) -> CrawlRun | None:
|
||||
result = await db_session.execute(
|
||||
select(CrawlRun).where(
|
||||
CrawlRun.user_id == user_id,
|
||||
CrawlRun.id == run_id,
|
||||
)
|
||||
)
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
|
||||
async def get_manual_run_by_idempotency_key(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
idempotency_key: str,
|
||||
) -> CrawlRun | None:
|
||||
result = await db_session.execute(
|
||||
select(CrawlRun)
|
||||
.where(
|
||||
CrawlRun.user_id == user_id,
|
||||
CrawlRun.trigger_type == RunTriggerType.MANUAL,
|
||||
CrawlRun.error_log["meta"]["idempotency_key"].astext == idempotency_key,
|
||||
)
|
||||
.order_by(CrawlRun.start_dt.desc(), CrawlRun.id.desc())
|
||||
.limit(1)
|
||||
)
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
|
||||
async def set_manual_run_idempotency_key(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
run_id: int,
|
||||
idempotency_key: str,
|
||||
) -> bool:
|
||||
run = await get_run_for_user(
|
||||
db_session,
|
||||
user_id=user_id,
|
||||
run_id=run_id,
|
||||
)
|
||||
if run is None:
|
||||
return False
|
||||
if run.trigger_type != RunTriggerType.MANUAL:
|
||||
return False
|
||||
|
||||
payload = dict(run.error_log) if isinstance(run.error_log, dict) else {}
|
||||
meta = payload.get("meta")
|
||||
meta_dict = dict(meta) if isinstance(meta, dict) else {}
|
||||
meta_dict["idempotency_key"] = idempotency_key
|
||||
payload["meta"] = meta_dict
|
||||
run.error_log = payload
|
||||
await db_session.commit()
|
||||
return True
|
||||
|
||||
|
||||
async def list_queue_items_for_user(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
limit: int = 200,
|
||||
) -> list[QueueListItem]:
|
||||
result = await db_session.execute(
|
||||
select(
|
||||
IngestionQueueItem.id,
|
||||
IngestionQueueItem.scholar_profile_id,
|
||||
ScholarProfile.display_name,
|
||||
ScholarProfile.scholar_id,
|
||||
IngestionQueueItem.status,
|
||||
IngestionQueueItem.reason,
|
||||
IngestionQueueItem.dropped_reason,
|
||||
IngestionQueueItem.attempt_count,
|
||||
IngestionQueueItem.resume_cstart,
|
||||
IngestionQueueItem.next_attempt_dt,
|
||||
IngestionQueueItem.updated_at,
|
||||
IngestionQueueItem.last_error,
|
||||
IngestionQueueItem.last_run_id,
|
||||
)
|
||||
.join(
|
||||
ScholarProfile,
|
||||
and_(
|
||||
ScholarProfile.id == IngestionQueueItem.scholar_profile_id,
|
||||
ScholarProfile.user_id == IngestionQueueItem.user_id,
|
||||
),
|
||||
)
|
||||
.where(IngestionQueueItem.user_id == user_id)
|
||||
.order_by(
|
||||
case((IngestionQueueItem.status == "dropped", 1), else_=0).asc(),
|
||||
IngestionQueueItem.next_attempt_dt.asc(),
|
||||
IngestionQueueItem.id.asc(),
|
||||
)
|
||||
.limit(limit)
|
||||
)
|
||||
|
||||
items: list[QueueListItem] = []
|
||||
for row in result.all():
|
||||
(
|
||||
item_id,
|
||||
scholar_profile_id,
|
||||
display_name,
|
||||
scholar_id,
|
||||
status,
|
||||
reason,
|
||||
dropped_reason,
|
||||
attempt_count,
|
||||
resume_cstart,
|
||||
next_attempt_dt,
|
||||
updated_at,
|
||||
last_error,
|
||||
last_run_id,
|
||||
) = row
|
||||
items.append(
|
||||
QueueListItem(
|
||||
id=int(item_id),
|
||||
scholar_profile_id=int(scholar_profile_id),
|
||||
scholar_label=(display_name or scholar_id),
|
||||
status=str(status),
|
||||
reason=str(reason),
|
||||
dropped_reason=dropped_reason,
|
||||
attempt_count=int(attempt_count or 0),
|
||||
resume_cstart=int(resume_cstart or 0),
|
||||
next_attempt_dt=next_attempt_dt,
|
||||
updated_at=updated_at,
|
||||
last_error=last_error,
|
||||
last_run_id=int(last_run_id) if last_run_id is not None else None,
|
||||
)
|
||||
)
|
||||
return items
|
||||
|
||||
|
||||
async def get_queue_item_for_user(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
queue_item_id: int,
|
||||
) -> QueueListItem | None:
|
||||
result = await db_session.execute(
|
||||
select(
|
||||
IngestionQueueItem.id,
|
||||
IngestionQueueItem.scholar_profile_id,
|
||||
ScholarProfile.display_name,
|
||||
ScholarProfile.scholar_id,
|
||||
IngestionQueueItem.status,
|
||||
IngestionQueueItem.reason,
|
||||
IngestionQueueItem.dropped_reason,
|
||||
IngestionQueueItem.attempt_count,
|
||||
IngestionQueueItem.resume_cstart,
|
||||
IngestionQueueItem.next_attempt_dt,
|
||||
IngestionQueueItem.updated_at,
|
||||
IngestionQueueItem.last_error,
|
||||
IngestionQueueItem.last_run_id,
|
||||
)
|
||||
.join(
|
||||
ScholarProfile,
|
||||
and_(
|
||||
ScholarProfile.id == IngestionQueueItem.scholar_profile_id,
|
||||
ScholarProfile.user_id == IngestionQueueItem.user_id,
|
||||
),
|
||||
)
|
||||
.where(
|
||||
IngestionQueueItem.user_id == user_id,
|
||||
IngestionQueueItem.id == queue_item_id,
|
||||
)
|
||||
.limit(1)
|
||||
)
|
||||
row = result.one_or_none()
|
||||
if row is None:
|
||||
return None
|
||||
(
|
||||
item_id,
|
||||
scholar_profile_id,
|
||||
display_name,
|
||||
scholar_id,
|
||||
status,
|
||||
reason,
|
||||
dropped_reason,
|
||||
attempt_count,
|
||||
resume_cstart,
|
||||
next_attempt_dt,
|
||||
updated_at,
|
||||
last_error,
|
||||
last_run_id,
|
||||
) = row
|
||||
return QueueListItem(
|
||||
id=int(item_id),
|
||||
scholar_profile_id=int(scholar_profile_id),
|
||||
scholar_label=(display_name or scholar_id),
|
||||
status=str(status),
|
||||
reason=str(reason),
|
||||
dropped_reason=dropped_reason,
|
||||
attempt_count=int(attempt_count or 0),
|
||||
resume_cstart=int(resume_cstart or 0),
|
||||
next_attempt_dt=next_attempt_dt,
|
||||
updated_at=updated_at,
|
||||
last_error=last_error,
|
||||
last_run_id=int(last_run_id) if last_run_id is not None else None,
|
||||
)
|
||||
|
||||
|
||||
async def retry_queue_item_for_user(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
queue_item_id: int,
|
||||
) -> QueueListItem | None:
|
||||
result = await db_session.execute(
|
||||
select(IngestionQueueItem).where(
|
||||
IngestionQueueItem.id == queue_item_id,
|
||||
IngestionQueueItem.user_id == user_id,
|
||||
)
|
||||
)
|
||||
item = result.scalar_one_or_none()
|
||||
if item is None:
|
||||
return None
|
||||
if item.status == QUEUE_STATUS_QUEUED:
|
||||
raise QueueTransitionError(
|
||||
code="queue_item_already_queued",
|
||||
message="Queue item is already queued.",
|
||||
current_status=item.status,
|
||||
)
|
||||
if item.status == QUEUE_STATUS_RETRYING:
|
||||
raise QueueTransitionError(
|
||||
code="queue_item_retrying",
|
||||
message="Queue item is currently retrying.",
|
||||
current_status=item.status,
|
||||
)
|
||||
|
||||
await queue_service.mark_queued_now(
|
||||
db_session,
|
||||
job_id=item.id,
|
||||
reason="manual_retry",
|
||||
reset_attempt_count=(item.status == QUEUE_STATUS_DROPPED),
|
||||
)
|
||||
await db_session.commit()
|
||||
|
||||
return await get_queue_item_for_user(
|
||||
db_session,
|
||||
user_id=user_id,
|
||||
queue_item_id=queue_item_id,
|
||||
)
|
||||
|
||||
|
||||
async def drop_queue_item_for_user(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
queue_item_id: int,
|
||||
) -> QueueListItem | None:
|
||||
result = await db_session.execute(
|
||||
select(IngestionQueueItem).where(
|
||||
IngestionQueueItem.id == queue_item_id,
|
||||
IngestionQueueItem.user_id == user_id,
|
||||
)
|
||||
)
|
||||
item = result.scalar_one_or_none()
|
||||
if item is None:
|
||||
return None
|
||||
if item.status == QUEUE_STATUS_DROPPED:
|
||||
raise QueueTransitionError(
|
||||
code="queue_item_already_dropped",
|
||||
message="Queue item is already dropped.",
|
||||
current_status=item.status,
|
||||
)
|
||||
|
||||
await queue_service.mark_dropped(
|
||||
db_session,
|
||||
job_id=int(item.id),
|
||||
reason="manual_drop",
|
||||
)
|
||||
await db_session.commit()
|
||||
return await get_queue_item_for_user(
|
||||
db_session,
|
||||
user_id=user_id,
|
||||
queue_item_id=queue_item_id,
|
||||
)
|
||||
|
||||
|
||||
async def clear_queue_item_for_user(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
queue_item_id: int,
|
||||
) -> QueueClearResult | None:
|
||||
result = await db_session.execute(
|
||||
select(IngestionQueueItem).where(
|
||||
IngestionQueueItem.id == queue_item_id,
|
||||
IngestionQueueItem.user_id == user_id,
|
||||
)
|
||||
)
|
||||
item = result.scalar_one_or_none()
|
||||
if item is None:
|
||||
return None
|
||||
if item.status != QUEUE_STATUS_DROPPED:
|
||||
raise QueueTransitionError(
|
||||
code="queue_item_not_dropped",
|
||||
message="Queue item can only be cleared after it is dropped.",
|
||||
current_status=item.status,
|
||||
)
|
||||
|
||||
item_id = int(item.id)
|
||||
previous_status = str(item.status)
|
||||
deleted = await queue_service.delete_job_by_id(
|
||||
db_session,
|
||||
job_id=item_id,
|
||||
)
|
||||
await db_session.commit()
|
||||
if not deleted:
|
||||
return None
|
||||
return QueueClearResult(
|
||||
queue_item_id=item_id,
|
||||
previous_status=previous_status,
|
||||
)
|
||||
|
||||
|
||||
async def queue_status_counts_for_user(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
) -> dict[str, int]:
|
||||
result = await db_session.execute(
|
||||
select(
|
||||
IngestionQueueItem.status,
|
||||
func.count(IngestionQueueItem.id),
|
||||
)
|
||||
.where(IngestionQueueItem.user_id == user_id)
|
||||
.group_by(IngestionQueueItem.status)
|
||||
)
|
||||
counts: dict[str, int] = {"queued": 0, "retrying": 0, "dropped": 0}
|
||||
for status, count in result.all():
|
||||
counts[str(status)] = int(count or 0)
|
||||
return counts
|
||||
478
app/services/scheduler.py
Normal file
478
app/services/scheduler.py
Normal file
|
|
@ -0,0 +1,478 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta, timezone
|
||||
import logging
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.models import (
|
||||
CrawlRun,
|
||||
QueueItemStatus,
|
||||
RunTriggerType,
|
||||
ScholarProfile,
|
||||
User,
|
||||
UserSetting,
|
||||
)
|
||||
from app.db.session import get_session_factory
|
||||
from app.services import continuation_queue as queue_service
|
||||
from app.services.ingestion import RunAlreadyInProgressError, ScholarIngestionService
|
||||
from app.services.scholar_source import LiveScholarSource
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _AutoRunCandidate:
|
||||
user_id: int
|
||||
run_interval_minutes: int
|
||||
request_delay_seconds: int
|
||||
|
||||
|
||||
class SchedulerService:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
enabled: bool,
|
||||
tick_seconds: int,
|
||||
network_error_retries: int,
|
||||
retry_backoff_seconds: float,
|
||||
max_pages_per_scholar: int,
|
||||
page_size: int,
|
||||
continuation_queue_enabled: bool,
|
||||
continuation_base_delay_seconds: int,
|
||||
continuation_max_delay_seconds: int,
|
||||
continuation_max_attempts: int,
|
||||
queue_batch_size: int,
|
||||
) -> None:
|
||||
self._enabled = enabled
|
||||
self._tick_seconds = max(5, int(tick_seconds))
|
||||
self._network_error_retries = max(0, int(network_error_retries))
|
||||
self._retry_backoff_seconds = max(0.0, float(retry_backoff_seconds))
|
||||
self._max_pages_per_scholar = max(1, int(max_pages_per_scholar))
|
||||
self._page_size = max(1, int(page_size))
|
||||
self._continuation_queue_enabled = bool(continuation_queue_enabled)
|
||||
self._continuation_base_delay_seconds = max(1, int(continuation_base_delay_seconds))
|
||||
self._continuation_max_delay_seconds = max(
|
||||
self._continuation_base_delay_seconds,
|
||||
int(continuation_max_delay_seconds),
|
||||
)
|
||||
self._continuation_max_attempts = max(1, int(continuation_max_attempts))
|
||||
self._queue_batch_size = max(1, int(queue_batch_size))
|
||||
self._task: asyncio.Task[None] | None = None
|
||||
self._source = LiveScholarSource()
|
||||
|
||||
async def start(self) -> None:
|
||||
if not self._enabled:
|
||||
logger.info(
|
||||
"scheduler.disabled",
|
||||
extra={
|
||||
"event": "scheduler.disabled",
|
||||
},
|
||||
)
|
||||
return
|
||||
if self._task is not None:
|
||||
return
|
||||
self._task = asyncio.create_task(self._run_loop(), name="scholarr-scheduler")
|
||||
logger.info(
|
||||
"scheduler.started",
|
||||
extra={
|
||||
"event": "scheduler.started",
|
||||
"tick_seconds": self._tick_seconds,
|
||||
"network_error_retries": self._network_error_retries,
|
||||
"retry_backoff_seconds": self._retry_backoff_seconds,
|
||||
"max_pages_per_scholar": self._max_pages_per_scholar,
|
||||
"page_size": self._page_size,
|
||||
"continuation_queue_enabled": self._continuation_queue_enabled,
|
||||
"continuation_base_delay_seconds": self._continuation_base_delay_seconds,
|
||||
"continuation_max_delay_seconds": self._continuation_max_delay_seconds,
|
||||
"continuation_max_attempts": self._continuation_max_attempts,
|
||||
"queue_batch_size": self._queue_batch_size,
|
||||
},
|
||||
)
|
||||
|
||||
async def stop(self) -> None:
|
||||
if self._task is None:
|
||||
return
|
||||
self._task.cancel()
|
||||
try:
|
||||
await self._task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
finally:
|
||||
self._task = None
|
||||
logger.info("scheduler.stopped", extra={"event": "scheduler.stopped"})
|
||||
|
||||
async def _run_loop(self) -> None:
|
||||
while True:
|
||||
try:
|
||||
await self._tick_once()
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"scheduler.tick_failed",
|
||||
extra={
|
||||
"event": "scheduler.tick_failed",
|
||||
},
|
||||
)
|
||||
await asyncio.sleep(float(self._tick_seconds))
|
||||
|
||||
async def _tick_once(self) -> None:
|
||||
if self._continuation_queue_enabled:
|
||||
await self._drain_continuation_queue()
|
||||
candidates = await self._load_candidates()
|
||||
if not candidates:
|
||||
return
|
||||
now = datetime.now(timezone.utc)
|
||||
for candidate in candidates:
|
||||
if not await self._is_due(candidate, now=now):
|
||||
continue
|
||||
await self._run_candidate(candidate)
|
||||
|
||||
async def _load_candidates(self) -> list[_AutoRunCandidate]:
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as session:
|
||||
result = await session.execute(
|
||||
select(
|
||||
UserSetting.user_id,
|
||||
UserSetting.run_interval_minutes,
|
||||
UserSetting.request_delay_seconds,
|
||||
)
|
||||
.join(User, User.id == UserSetting.user_id)
|
||||
.where(
|
||||
User.is_active.is_(True),
|
||||
UserSetting.auto_run_enabled.is_(True),
|
||||
)
|
||||
.order_by(UserSetting.user_id.asc())
|
||||
)
|
||||
rows = result.all()
|
||||
return [
|
||||
_AutoRunCandidate(
|
||||
user_id=int(user_id),
|
||||
run_interval_minutes=int(run_interval_minutes),
|
||||
request_delay_seconds=int(request_delay_seconds),
|
||||
)
|
||||
for user_id, run_interval_minutes, request_delay_seconds in rows
|
||||
]
|
||||
|
||||
async def _is_due(self, candidate: _AutoRunCandidate, *, now: datetime) -> bool:
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as session:
|
||||
result = await session.execute(
|
||||
select(CrawlRun.start_dt)
|
||||
.where(
|
||||
CrawlRun.user_id == candidate.user_id,
|
||||
)
|
||||
.order_by(CrawlRun.start_dt.desc(), CrawlRun.id.desc())
|
||||
.limit(1)
|
||||
)
|
||||
last_run = result.scalar_one_or_none()
|
||||
|
||||
if last_run is None:
|
||||
return True
|
||||
|
||||
next_due_dt = last_run + timedelta(
|
||||
minutes=candidate.run_interval_minutes
|
||||
)
|
||||
return now >= next_due_dt
|
||||
|
||||
async def _run_candidate(self, candidate: _AutoRunCandidate) -> None:
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as session:
|
||||
ingestion = ScholarIngestionService(source=self._source)
|
||||
try:
|
||||
run_summary = await ingestion.run_for_user(
|
||||
session,
|
||||
user_id=candidate.user_id,
|
||||
trigger_type=RunTriggerType.SCHEDULED,
|
||||
request_delay_seconds=candidate.request_delay_seconds,
|
||||
network_error_retries=self._network_error_retries,
|
||||
retry_backoff_seconds=self._retry_backoff_seconds,
|
||||
max_pages_per_scholar=self._max_pages_per_scholar,
|
||||
page_size=self._page_size,
|
||||
auto_queue_continuations=self._continuation_queue_enabled,
|
||||
queue_delay_seconds=self._continuation_base_delay_seconds,
|
||||
)
|
||||
except RunAlreadyInProgressError:
|
||||
await session.rollback()
|
||||
logger.info(
|
||||
"scheduler.run_skipped_locked",
|
||||
extra={
|
||||
"event": "scheduler.run_skipped_locked",
|
||||
"user_id": candidate.user_id,
|
||||
},
|
||||
)
|
||||
return
|
||||
except Exception:
|
||||
await session.rollback()
|
||||
logger.exception(
|
||||
"scheduler.run_failed",
|
||||
extra={
|
||||
"event": "scheduler.run_failed",
|
||||
"user_id": candidate.user_id,
|
||||
},
|
||||
)
|
||||
return
|
||||
|
||||
logger.info(
|
||||
"scheduler.run_completed",
|
||||
extra={
|
||||
"event": "scheduler.run_completed",
|
||||
"user_id": candidate.user_id,
|
||||
"run_id": run_summary.crawl_run_id,
|
||||
"status": run_summary.status.value,
|
||||
"scholar_count": run_summary.scholar_count,
|
||||
"new_publication_count": run_summary.new_publication_count,
|
||||
},
|
||||
)
|
||||
|
||||
async def _drain_continuation_queue(self) -> None:
|
||||
now = datetime.now(timezone.utc)
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as session:
|
||||
jobs = await queue_service.list_due_jobs(
|
||||
session,
|
||||
now=now,
|
||||
limit=self._queue_batch_size,
|
||||
)
|
||||
for job in jobs:
|
||||
await self._run_queue_job(job)
|
||||
|
||||
async def _run_queue_job(self, job: queue_service.ContinuationQueueJob) -> None:
|
||||
if job.attempt_count >= self._continuation_max_attempts:
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as session:
|
||||
dropped = await queue_service.mark_dropped(
|
||||
session,
|
||||
job_id=job.id,
|
||||
reason="max_attempts_reached",
|
||||
)
|
||||
await session.commit()
|
||||
if dropped is not None:
|
||||
logger.warning(
|
||||
"scheduler.queue_item_dropped_max_attempts",
|
||||
extra={
|
||||
"event": "scheduler.queue_item_dropped_max_attempts",
|
||||
"queue_item_id": job.id,
|
||||
"user_id": job.user_id,
|
||||
"scholar_profile_id": job.scholar_profile_id,
|
||||
"attempt_count": job.attempt_count,
|
||||
"max_attempts": self._continuation_max_attempts,
|
||||
},
|
||||
)
|
||||
return
|
||||
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as session:
|
||||
queue_item = await queue_service.mark_retrying(session, job_id=job.id)
|
||||
if queue_item is None:
|
||||
await session.commit()
|
||||
return
|
||||
if queue_item.status == QueueItemStatus.DROPPED.value:
|
||||
await session.commit()
|
||||
return
|
||||
await session.commit()
|
||||
|
||||
async with session_factory() as session:
|
||||
scholar_result = await session.execute(
|
||||
select(ScholarProfile.id).where(
|
||||
ScholarProfile.user_id == job.user_id,
|
||||
ScholarProfile.id == job.scholar_profile_id,
|
||||
ScholarProfile.is_enabled.is_(True),
|
||||
)
|
||||
)
|
||||
scholar_id = scholar_result.scalar_one_or_none()
|
||||
if scholar_id is None:
|
||||
dropped = await queue_service.mark_dropped(
|
||||
session,
|
||||
job_id=job.id,
|
||||
reason="scholar_unavailable",
|
||||
)
|
||||
await session.commit()
|
||||
if dropped is not None:
|
||||
logger.info(
|
||||
"scheduler.queue_item_dropped_scholar_unavailable",
|
||||
extra={
|
||||
"event": "scheduler.queue_item_dropped_scholar_unavailable",
|
||||
"queue_item_id": job.id,
|
||||
"user_id": job.user_id,
|
||||
"scholar_profile_id": job.scholar_profile_id,
|
||||
},
|
||||
)
|
||||
return
|
||||
|
||||
async with session_factory() as session:
|
||||
request_delay_seconds = await self._load_request_delay_for_user(
|
||||
session,
|
||||
user_id=job.user_id,
|
||||
)
|
||||
ingestion = ScholarIngestionService(source=self._source)
|
||||
try:
|
||||
run_summary = await ingestion.run_for_user(
|
||||
session,
|
||||
user_id=job.user_id,
|
||||
trigger_type=RunTriggerType.SCHEDULED,
|
||||
request_delay_seconds=request_delay_seconds,
|
||||
network_error_retries=self._network_error_retries,
|
||||
retry_backoff_seconds=self._retry_backoff_seconds,
|
||||
max_pages_per_scholar=self._max_pages_per_scholar,
|
||||
page_size=self._page_size,
|
||||
scholar_profile_ids={job.scholar_profile_id},
|
||||
start_cstart_by_scholar_id={
|
||||
job.scholar_profile_id: job.resume_cstart,
|
||||
},
|
||||
auto_queue_continuations=self._continuation_queue_enabled,
|
||||
queue_delay_seconds=self._continuation_base_delay_seconds,
|
||||
)
|
||||
except RunAlreadyInProgressError:
|
||||
await session.rollback()
|
||||
async with session_factory() as recovery_session:
|
||||
await queue_service.reschedule_job(
|
||||
recovery_session,
|
||||
job_id=job.id,
|
||||
delay_seconds=max(self._tick_seconds, 15),
|
||||
reason="user_run_lock_active",
|
||||
error="run_already_in_progress",
|
||||
)
|
||||
await recovery_session.commit()
|
||||
logger.info(
|
||||
"scheduler.queue_item_deferred_lock",
|
||||
extra={
|
||||
"event": "scheduler.queue_item_deferred_lock",
|
||||
"queue_item_id": job.id,
|
||||
"user_id": job.user_id,
|
||||
},
|
||||
)
|
||||
return
|
||||
except Exception as exc:
|
||||
await session.rollback()
|
||||
async with session_factory() as recovery_session:
|
||||
queue_item = await queue_service.increment_attempt_count(
|
||||
recovery_session,
|
||||
job_id=job.id,
|
||||
)
|
||||
if queue_item is None:
|
||||
await recovery_session.commit()
|
||||
return
|
||||
if int(queue_item.attempt_count) >= self._continuation_max_attempts:
|
||||
await queue_service.mark_dropped(
|
||||
recovery_session,
|
||||
job_id=job.id,
|
||||
reason="scheduler_exception_max_attempts",
|
||||
error=str(exc),
|
||||
)
|
||||
await recovery_session.commit()
|
||||
logger.warning(
|
||||
"scheduler.queue_item_dropped_after_exception",
|
||||
extra={
|
||||
"event": "scheduler.queue_item_dropped_after_exception",
|
||||
"queue_item_id": job.id,
|
||||
"user_id": job.user_id,
|
||||
"attempt_count": queue_item.attempt_count,
|
||||
},
|
||||
)
|
||||
return
|
||||
delay_seconds = queue_service.compute_backoff_seconds(
|
||||
base_seconds=self._continuation_base_delay_seconds,
|
||||
attempt_count=int(queue_item.attempt_count),
|
||||
max_seconds=self._continuation_max_delay_seconds,
|
||||
)
|
||||
await queue_service.reschedule_job(
|
||||
recovery_session,
|
||||
job_id=job.id,
|
||||
delay_seconds=delay_seconds,
|
||||
reason="scheduler_exception",
|
||||
error=str(exc),
|
||||
)
|
||||
await recovery_session.commit()
|
||||
logger.exception(
|
||||
"scheduler.queue_item_run_failed",
|
||||
extra={
|
||||
"event": "scheduler.queue_item_run_failed",
|
||||
"queue_item_id": job.id,
|
||||
"user_id": job.user_id,
|
||||
},
|
||||
)
|
||||
return
|
||||
|
||||
async with session_factory() as session:
|
||||
queue_item = await queue_service.increment_attempt_count(
|
||||
session,
|
||||
job_id=job.id,
|
||||
)
|
||||
if queue_item is None:
|
||||
await session.commit()
|
||||
logger.info(
|
||||
"scheduler.queue_item_resolved",
|
||||
extra={
|
||||
"event": "scheduler.queue_item_resolved",
|
||||
"queue_item_id": job.id,
|
||||
"user_id": job.user_id,
|
||||
"run_id": run_summary.crawl_run_id,
|
||||
"status": run_summary.status.value,
|
||||
},
|
||||
)
|
||||
return
|
||||
|
||||
if int(queue_item.attempt_count) >= self._continuation_max_attempts:
|
||||
await queue_service.mark_dropped(
|
||||
session,
|
||||
job_id=job.id,
|
||||
reason="max_attempts_after_run",
|
||||
)
|
||||
await session.commit()
|
||||
logger.warning(
|
||||
"scheduler.queue_item_dropped_max_attempts_after_run",
|
||||
extra={
|
||||
"event": "scheduler.queue_item_dropped_max_attempts_after_run",
|
||||
"queue_item_id": job.id,
|
||||
"user_id": job.user_id,
|
||||
"attempt_count": queue_item.attempt_count,
|
||||
"run_id": run_summary.crawl_run_id,
|
||||
"status": run_summary.status.value,
|
||||
},
|
||||
)
|
||||
return
|
||||
|
||||
delay_seconds = queue_service.compute_backoff_seconds(
|
||||
base_seconds=self._continuation_base_delay_seconds,
|
||||
attempt_count=int(queue_item.attempt_count),
|
||||
max_seconds=self._continuation_max_delay_seconds,
|
||||
)
|
||||
await queue_service.reschedule_job(
|
||||
session,
|
||||
job_id=job.id,
|
||||
delay_seconds=delay_seconds,
|
||||
reason=queue_item.reason,
|
||||
error=queue_item.last_error,
|
||||
)
|
||||
await session.commit()
|
||||
logger.info(
|
||||
"scheduler.queue_item_rescheduled",
|
||||
extra={
|
||||
"event": "scheduler.queue_item_rescheduled",
|
||||
"queue_item_id": job.id,
|
||||
"user_id": job.user_id,
|
||||
"attempt_count": queue_item.attempt_count,
|
||||
"delay_seconds": delay_seconds,
|
||||
"run_id": run_summary.crawl_run_id,
|
||||
"status": run_summary.status.value,
|
||||
},
|
||||
)
|
||||
|
||||
async def _load_request_delay_for_user(
|
||||
self,
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
) -> int:
|
||||
result = await db_session.execute(
|
||||
select(UserSetting.request_delay_seconds).where(UserSetting.user_id == user_id)
|
||||
)
|
||||
delay = result.scalar_one_or_none()
|
||||
if delay is None:
|
||||
return 10
|
||||
return max(1, int(delay))
|
||||
370
app/services/scholar_parser.py
Normal file
370
app/services/scholar_parser.py
Normal file
|
|
@ -0,0 +1,370 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from enum import StrEnum
|
||||
from html import unescape
|
||||
from html.parser import HTMLParser
|
||||
from typing import Any
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
|
||||
from app.services.scholar_source import FetchResult
|
||||
|
||||
BLOCKED_KEYWORDS = [
|
||||
"unusual traffic",
|
||||
"sorry/index",
|
||||
"not a robot",
|
||||
"our systems have detected",
|
||||
"automated queries",
|
||||
]
|
||||
|
||||
NO_RESULTS_KEYWORDS = [
|
||||
"didn't match any articles",
|
||||
"did not match any articles",
|
||||
"no articles",
|
||||
"no documents",
|
||||
]
|
||||
|
||||
MARKER_KEYS = [
|
||||
"gsc_a_tr",
|
||||
"gsc_a_at",
|
||||
"gsc_a_ac",
|
||||
"gsc_a_h",
|
||||
"gsc_a_y",
|
||||
"gs_gray",
|
||||
"gsc_prf_in",
|
||||
"gsc_rsb_st",
|
||||
]
|
||||
|
||||
TAG_RE = re.compile(r"<[^>]+>", re.S)
|
||||
SCRIPT_STYLE_RE = re.compile(r"<(script|style)\b[^>]*>.*?</\1>", re.I | re.S)
|
||||
SHOW_MORE_BUTTON_RE = re.compile(
|
||||
r"<button\b[^>]*\bid\s*=\s*['\"]gsc_bpf_more['\"][^>]*>",
|
||||
re.I | re.S,
|
||||
)
|
||||
|
||||
|
||||
class ParseState(StrEnum):
|
||||
OK = "ok"
|
||||
NO_RESULTS = "no_results"
|
||||
BLOCKED_OR_CAPTCHA = "blocked_or_captcha"
|
||||
LAYOUT_CHANGED = "layout_changed"
|
||||
NETWORK_ERROR = "network_error"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PublicationCandidate:
|
||||
title: str
|
||||
title_url: str | None
|
||||
cluster_id: str | None
|
||||
year: int | None
|
||||
citation_count: int | None
|
||||
authors_text: str | None
|
||||
venue_text: str | None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ParsedProfilePage:
|
||||
state: ParseState
|
||||
state_reason: str
|
||||
profile_name: str | None
|
||||
publications: list[PublicationCandidate]
|
||||
marker_counts: dict[str, int]
|
||||
warnings: list[str]
|
||||
has_show_more_button: bool
|
||||
has_operation_error_banner: bool
|
||||
articles_range: str | None
|
||||
|
||||
|
||||
def normalize_space(value: str) -> str:
|
||||
return " ".join(unescape(value).split())
|
||||
|
||||
|
||||
def strip_tags(value: str) -> str:
|
||||
return normalize_space(TAG_RE.sub(" ", value))
|
||||
|
||||
|
||||
def attr_class(attrs: list[tuple[str, str | None]]) -> str:
|
||||
for name, raw_value in attrs:
|
||||
if name.lower() == "class":
|
||||
return raw_value or ""
|
||||
return ""
|
||||
|
||||
|
||||
def attr_href(attrs: list[tuple[str, str | None]]) -> str | None:
|
||||
for name, raw_value in attrs:
|
||||
if name.lower() == "href":
|
||||
return raw_value
|
||||
return None
|
||||
|
||||
|
||||
class ScholarRowParser(HTMLParser):
|
||||
def __init__(self) -> None:
|
||||
super().__init__(convert_charrefs=True)
|
||||
self.title_href: str | None = None
|
||||
self.title_parts: list[str] = []
|
||||
self.citation_parts: list[str] = []
|
||||
self.year_parts: list[str] = []
|
||||
self.gray_texts: list[str] = []
|
||||
|
||||
self._title_depth = 0
|
||||
self._citation_depth = 0
|
||||
self._year_depth = 0
|
||||
self._gray_stack: list[dict[str, Any]] = []
|
||||
|
||||
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
||||
if self._title_depth > 0:
|
||||
self._title_depth += 1
|
||||
if self._citation_depth > 0:
|
||||
self._citation_depth += 1
|
||||
if self._year_depth > 0:
|
||||
self._year_depth += 1
|
||||
if self._gray_stack:
|
||||
self._gray_stack[-1]["depth"] += 1
|
||||
|
||||
classes = attr_class(attrs)
|
||||
|
||||
if tag == "a" and "gsc_a_at" in classes:
|
||||
self._title_depth = 1
|
||||
self.title_href = attr_href(attrs)
|
||||
return
|
||||
|
||||
if tag == "a" and "gsc_a_ac" in classes:
|
||||
self._citation_depth = 1
|
||||
return
|
||||
|
||||
if tag in {"span", "a"} and ("gsc_a_h" in classes or "gsc_a_y" in classes):
|
||||
self._year_depth = 1
|
||||
return
|
||||
|
||||
if tag == "div" and "gs_gray" in classes:
|
||||
self._gray_stack.append({"depth": 1, "parts": []})
|
||||
return
|
||||
|
||||
def handle_data(self, data: str) -> None:
|
||||
if self._title_depth > 0:
|
||||
self.title_parts.append(data)
|
||||
if self._citation_depth > 0:
|
||||
self.citation_parts.append(data)
|
||||
if self._year_depth > 0:
|
||||
self.year_parts.append(data)
|
||||
if self._gray_stack:
|
||||
self._gray_stack[-1]["parts"].append(data)
|
||||
|
||||
def handle_endtag(self, _tag: str) -> None:
|
||||
if self._title_depth > 0:
|
||||
self._title_depth -= 1
|
||||
if self._citation_depth > 0:
|
||||
self._citation_depth -= 1
|
||||
if self._year_depth > 0:
|
||||
self._year_depth -= 1
|
||||
if self._gray_stack:
|
||||
self._gray_stack[-1]["depth"] -= 1
|
||||
if self._gray_stack[-1]["depth"] == 0:
|
||||
text = normalize_space("".join(self._gray_stack[-1]["parts"]))
|
||||
if text:
|
||||
self.gray_texts.append(text)
|
||||
self._gray_stack.pop()
|
||||
|
||||
|
||||
def extract_rows(html: str) -> list[str]:
|
||||
pattern = re.compile(
|
||||
r"<tr\b(?=[^>]*\bclass\s*=\s*['\"][^'\"]*\bgsc_a_tr\b[^'\"]*['\"])[^>]*>(.*?)</tr>",
|
||||
re.I | re.S,
|
||||
)
|
||||
return [match.group(1) for match in pattern.finditer(html)]
|
||||
|
||||
|
||||
def parse_cluster_id_from_href(href: str | None) -> str | None:
|
||||
if not href:
|
||||
return None
|
||||
parsed = urlparse(href)
|
||||
query = parse_qs(parsed.query)
|
||||
|
||||
citation_for_view = query.get("citation_for_view")
|
||||
if citation_for_view:
|
||||
token = citation_for_view[0].strip()
|
||||
if token:
|
||||
if ":" in token:
|
||||
return token.rsplit(":", 1)[-1] or None
|
||||
return token
|
||||
|
||||
cluster = query.get("cluster")
|
||||
if cluster:
|
||||
token = cluster[0].strip()
|
||||
if token:
|
||||
return token
|
||||
return None
|
||||
|
||||
|
||||
def parse_year(parts: list[str]) -> int | None:
|
||||
text = normalize_space(" ".join(parts))
|
||||
match = re.search(r"\b(19|20)\d{2}\b", text)
|
||||
if not match:
|
||||
return None
|
||||
try:
|
||||
return int(match.group(0))
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def parse_citation_count(parts: list[str]) -> int | None:
|
||||
text = normalize_space(" ".join(parts))
|
||||
if not text:
|
||||
return 0
|
||||
match = re.search(r"\d+", text)
|
||||
if not match:
|
||||
return None
|
||||
return int(match.group(0))
|
||||
|
||||
|
||||
def parse_publications(html: str) -> tuple[list[PublicationCandidate], list[str]]:
|
||||
rows = extract_rows(html)
|
||||
warnings: list[str] = []
|
||||
publications: list[PublicationCandidate] = []
|
||||
|
||||
for row_html in rows:
|
||||
parser = ScholarRowParser()
|
||||
parser.feed(row_html)
|
||||
|
||||
title = normalize_space("".join(parser.title_parts))
|
||||
if not title:
|
||||
warnings.append("row_missing_title")
|
||||
continue
|
||||
|
||||
authors_text = parser.gray_texts[0] if len(parser.gray_texts) > 0 else None
|
||||
venue_text = parser.gray_texts[1] if len(parser.gray_texts) > 1 else None
|
||||
|
||||
publications.append(
|
||||
PublicationCandidate(
|
||||
title=title,
|
||||
title_url=parser.title_href,
|
||||
cluster_id=parse_cluster_id_from_href(parser.title_href),
|
||||
year=parse_year(parser.year_parts),
|
||||
citation_count=parse_citation_count(parser.citation_parts),
|
||||
authors_text=authors_text,
|
||||
venue_text=venue_text,
|
||||
)
|
||||
)
|
||||
|
||||
if not rows:
|
||||
warnings.append("no_rows_detected")
|
||||
|
||||
return publications, sorted(set(warnings))
|
||||
|
||||
|
||||
def extract_profile_name(html: str) -> str | None:
|
||||
pattern = re.compile(
|
||||
r"<[^>]*\bid\s*=\s*['\"]gsc_prf_in['\"][^>]*>(.*?)</[^>]+>",
|
||||
re.I | re.S,
|
||||
)
|
||||
match = pattern.search(html)
|
||||
if not match:
|
||||
return None
|
||||
value = strip_tags(match.group(1))
|
||||
return value or None
|
||||
|
||||
|
||||
def extract_articles_range(html: str) -> str | None:
|
||||
pattern = re.compile(
|
||||
r"<[^>]*\bid\s*=\s*['\"]gsc_a_nn['\"][^>]*>(.*?)</[^>]+>",
|
||||
re.I | re.S,
|
||||
)
|
||||
match = pattern.search(html)
|
||||
if not match:
|
||||
return None
|
||||
value = strip_tags(match.group(1))
|
||||
return value or None
|
||||
|
||||
|
||||
def has_show_more_button(html: str) -> bool:
|
||||
match = SHOW_MORE_BUTTON_RE.search(html)
|
||||
if match is None:
|
||||
return False
|
||||
|
||||
button_tag = match.group(0).lower()
|
||||
if "disabled" in button_tag:
|
||||
return False
|
||||
if 'aria-disabled="true"' in button_tag or "aria-disabled='true'" in button_tag:
|
||||
return False
|
||||
if "gs_dis" in button_tag:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def has_operation_error_banner(html: str) -> bool:
|
||||
lowered = html.lower()
|
||||
if "id=\"gsc_a_err\"" not in lowered and "id='gsc_a_err'" not in lowered:
|
||||
return False
|
||||
return "can't perform the operation now" in lowered or "cannot perform the operation now" in lowered
|
||||
|
||||
|
||||
def count_markers(html: str) -> dict[str, int]:
|
||||
lowered = html.lower()
|
||||
return {key: lowered.count(key.lower()) for key in MARKER_KEYS}
|
||||
|
||||
|
||||
def detect_state(
|
||||
fetch_result: FetchResult,
|
||||
publications: list[PublicationCandidate],
|
||||
marker_counts: dict[str, int],
|
||||
*,
|
||||
visible_text: str,
|
||||
) -> tuple[ParseState, str]:
|
||||
if fetch_result.status_code is None:
|
||||
return ParseState.NETWORK_ERROR, "network_error_missing_status_code"
|
||||
|
||||
lowered = fetch_result.body.lower()
|
||||
final = (fetch_result.final_url or "").lower()
|
||||
|
||||
if "accounts.google.com" in final and ("signin" in final or "servicelogin" in final):
|
||||
return ParseState.BLOCKED_OR_CAPTCHA, "blocked_accounts_redirect"
|
||||
if any(keyword in lowered for keyword in BLOCKED_KEYWORDS) or "sorry/index" in final:
|
||||
return ParseState.BLOCKED_OR_CAPTCHA, "blocked_keyword_detected"
|
||||
|
||||
if not publications and any(keyword in visible_text for keyword in NO_RESULTS_KEYWORDS):
|
||||
return ParseState.NO_RESULTS, "no_results_keyword_detected"
|
||||
|
||||
if not publications:
|
||||
has_profile_markers = marker_counts.get("gsc_prf_in", 0) > 0
|
||||
has_table_markers = marker_counts.get("gsc_a_tr", 0) > 0 or marker_counts.get("gsc_a_at", 0) > 0
|
||||
if not has_profile_markers and not has_table_markers:
|
||||
return ParseState.LAYOUT_CHANGED, "layout_markers_missing"
|
||||
return ParseState.OK, "no_rows_with_known_markers"
|
||||
|
||||
return ParseState.OK, "publications_extracted"
|
||||
|
||||
|
||||
def parse_profile_page(fetch_result: FetchResult) -> ParsedProfilePage:
|
||||
publications, warnings = parse_publications(fetch_result.body)
|
||||
marker_counts = count_markers(fetch_result.body)
|
||||
visible_text = strip_tags(SCRIPT_STYLE_RE.sub(" ", fetch_result.body)).lower()
|
||||
|
||||
show_more = has_show_more_button(fetch_result.body)
|
||||
operation_error_banner = has_operation_error_banner(fetch_result.body)
|
||||
|
||||
if show_more:
|
||||
warnings.append("possible_partial_page_show_more_present")
|
||||
if operation_error_banner:
|
||||
warnings.append("operation_error_banner_present")
|
||||
|
||||
warnings = sorted(set(warnings))
|
||||
|
||||
state, state_reason = detect_state(
|
||||
fetch_result,
|
||||
publications,
|
||||
marker_counts,
|
||||
visible_text=visible_text,
|
||||
)
|
||||
|
||||
return ParsedProfilePage(
|
||||
state=state,
|
||||
state_reason=state_reason,
|
||||
profile_name=extract_profile_name(fetch_result.body),
|
||||
publications=publications,
|
||||
marker_counts=marker_counts,
|
||||
warnings=warnings,
|
||||
has_show_more_button=show_more,
|
||||
has_operation_error_banner=operation_error_banner,
|
||||
articles_range=extract_articles_range(fetch_result.body),
|
||||
)
|
||||
171
app/services/scholar_source.py
Normal file
171
app/services/scholar_source.py
Normal file
|
|
@ -0,0 +1,171 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import random
|
||||
from dataclasses import dataclass
|
||||
from typing import Protocol
|
||||
from urllib.error import HTTPError, URLError
|
||||
from urllib.parse import urlencode
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
SCHOLAR_PROFILE_URL = "https://scholar.google.com/citations"
|
||||
DEFAULT_PAGE_SIZE = 100
|
||||
|
||||
DEFAULT_USER_AGENTS = [
|
||||
(
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
|
||||
),
|
||||
(
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 14_7) AppleWebKit/605.1.15 "
|
||||
"(KHTML, like Gecko) Version/18.1 Safari/605.1.15"
|
||||
),
|
||||
(
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:131.0) "
|
||||
"Gecko/20100101 Firefox/131.0"
|
||||
),
|
||||
]
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FetchResult:
|
||||
requested_url: str
|
||||
status_code: int | None
|
||||
final_url: str | None
|
||||
body: str
|
||||
error: str | None
|
||||
|
||||
|
||||
class ScholarSource(Protocol):
|
||||
async def fetch_profile_html(self, scholar_id: str) -> FetchResult:
|
||||
...
|
||||
|
||||
async def fetch_profile_page_html(
|
||||
self,
|
||||
scholar_id: str,
|
||||
*,
|
||||
cstart: int,
|
||||
pagesize: int,
|
||||
) -> FetchResult:
|
||||
...
|
||||
|
||||
|
||||
class LiveScholarSource:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
timeout_seconds: float = 25.0,
|
||||
user_agents: list[str] | None = None,
|
||||
) -> None:
|
||||
self._timeout_seconds = timeout_seconds
|
||||
self._user_agents = user_agents or DEFAULT_USER_AGENTS
|
||||
|
||||
async def fetch_profile_html(self, scholar_id: str) -> FetchResult:
|
||||
return await self.fetch_profile_page_html(
|
||||
scholar_id,
|
||||
cstart=0,
|
||||
pagesize=DEFAULT_PAGE_SIZE,
|
||||
)
|
||||
|
||||
async def fetch_profile_page_html(
|
||||
self,
|
||||
scholar_id: str,
|
||||
*,
|
||||
cstart: int,
|
||||
pagesize: int = DEFAULT_PAGE_SIZE,
|
||||
) -> FetchResult:
|
||||
requested_url = _build_profile_url(
|
||||
scholar_id=scholar_id,
|
||||
cstart=cstart,
|
||||
pagesize=pagesize,
|
||||
)
|
||||
logger.debug(
|
||||
"scholar_source.fetch_started",
|
||||
extra={
|
||||
"event": "scholar_source.fetch_started",
|
||||
"scholar_id": scholar_id,
|
||||
"requested_url": requested_url,
|
||||
"cstart": cstart,
|
||||
"pagesize": pagesize,
|
||||
},
|
||||
)
|
||||
return await asyncio.to_thread(self._fetch_sync, requested_url)
|
||||
|
||||
def _fetch_sync(self, requested_url: str) -> FetchResult:
|
||||
request = Request(
|
||||
requested_url,
|
||||
headers={
|
||||
"User-Agent": random.choice(self._user_agents),
|
||||
"Accept": "text/html,application/xhtml+xml",
|
||||
"Accept-Language": "en-US,en;q=0.9",
|
||||
"Connection": "close",
|
||||
},
|
||||
)
|
||||
|
||||
try:
|
||||
with urlopen(request, timeout=self._timeout_seconds) as response:
|
||||
body = response.read().decode("utf-8", errors="replace")
|
||||
status_code = getattr(response, "status", 200)
|
||||
logger.debug(
|
||||
"scholar_source.fetch_succeeded",
|
||||
extra={
|
||||
"event": "scholar_source.fetch_succeeded",
|
||||
"requested_url": requested_url,
|
||||
"status_code": status_code,
|
||||
},
|
||||
)
|
||||
return FetchResult(
|
||||
requested_url=requested_url,
|
||||
status_code=status_code,
|
||||
final_url=response.geturl(),
|
||||
body=body,
|
||||
error=None,
|
||||
)
|
||||
except HTTPError as exc:
|
||||
body = ""
|
||||
try:
|
||||
body = exc.read().decode("utf-8", errors="replace")
|
||||
except Exception:
|
||||
body = ""
|
||||
logger.warning(
|
||||
"scholar_source.fetch_http_error",
|
||||
extra={
|
||||
"event": "scholar_source.fetch_http_error",
|
||||
"requested_url": requested_url,
|
||||
"status_code": exc.code,
|
||||
},
|
||||
)
|
||||
return FetchResult(
|
||||
requested_url=requested_url,
|
||||
status_code=exc.code,
|
||||
final_url=exc.geturl(),
|
||||
body=body,
|
||||
error=str(exc),
|
||||
)
|
||||
except URLError as exc:
|
||||
logger.warning(
|
||||
"scholar_source.fetch_network_error",
|
||||
extra={
|
||||
"event": "scholar_source.fetch_network_error",
|
||||
"requested_url": requested_url,
|
||||
},
|
||||
)
|
||||
return FetchResult(
|
||||
requested_url=requested_url,
|
||||
status_code=None,
|
||||
final_url=None,
|
||||
body="",
|
||||
error=str(exc),
|
||||
)
|
||||
|
||||
|
||||
def _build_profile_url(*, scholar_id: str, cstart: int, pagesize: int) -> str:
|
||||
query: dict[str, int | str] = {"hl": "en", "user": scholar_id}
|
||||
if cstart > 0:
|
||||
query["cstart"] = int(cstart)
|
||||
if pagesize > 0 and cstart > 0:
|
||||
query["pagesize"] = int(pagesize)
|
||||
return f"{SCHOLAR_PROFILE_URL}?{urlencode(query)}"
|
||||
98
app/services/scholars.py
Normal file
98
app/services/scholars.py
Normal file
|
|
@ -0,0 +1,98 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.models import ScholarProfile
|
||||
|
||||
SCHOLAR_ID_PATTERN = re.compile(r"^[a-zA-Z0-9_-]{12}$")
|
||||
|
||||
|
||||
class ScholarServiceError(ValueError):
|
||||
"""Raised for expected scholar-management validation failures."""
|
||||
|
||||
|
||||
def validate_scholar_id(value: str) -> str:
|
||||
scholar_id = value.strip()
|
||||
if not SCHOLAR_ID_PATTERN.fullmatch(scholar_id):
|
||||
raise ScholarServiceError("Scholar ID must match [a-zA-Z0-9_-]{12}.")
|
||||
return scholar_id
|
||||
|
||||
|
||||
def normalize_display_name(value: str) -> str | None:
|
||||
normalized = value.strip()
|
||||
return normalized if normalized else None
|
||||
|
||||
|
||||
async def list_scholars_for_user(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
) -> list[ScholarProfile]:
|
||||
result = await db_session.execute(
|
||||
select(ScholarProfile)
|
||||
.where(ScholarProfile.user_id == user_id)
|
||||
.order_by(ScholarProfile.created_at.desc(), ScholarProfile.id.desc())
|
||||
)
|
||||
return list(result.scalars().all())
|
||||
|
||||
|
||||
async def create_scholar_for_user(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
scholar_id: str,
|
||||
display_name: str,
|
||||
) -> ScholarProfile:
|
||||
profile = ScholarProfile(
|
||||
user_id=user_id,
|
||||
scholar_id=validate_scholar_id(scholar_id),
|
||||
display_name=normalize_display_name(display_name),
|
||||
)
|
||||
db_session.add(profile)
|
||||
try:
|
||||
await db_session.commit()
|
||||
except IntegrityError as exc:
|
||||
await db_session.rollback()
|
||||
raise ScholarServiceError("That scholar is already tracked for this account.") from exc
|
||||
await db_session.refresh(profile)
|
||||
return profile
|
||||
|
||||
|
||||
async def get_user_scholar_by_id(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
scholar_profile_id: int,
|
||||
) -> ScholarProfile | None:
|
||||
result = await db_session.execute(
|
||||
select(ScholarProfile).where(
|
||||
ScholarProfile.id == scholar_profile_id,
|
||||
ScholarProfile.user_id == user_id,
|
||||
)
|
||||
)
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
|
||||
async def toggle_scholar_enabled(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
profile: ScholarProfile,
|
||||
) -> ScholarProfile:
|
||||
profile.is_enabled = not profile.is_enabled
|
||||
await db_session.commit()
|
||||
await db_session.refresh(profile)
|
||||
return profile
|
||||
|
||||
|
||||
async def delete_scholar(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
profile: ScholarProfile,
|
||||
) -> None:
|
||||
await db_session.delete(profile)
|
||||
await db_session.commit()
|
||||
|
||||
66
app/services/user_settings.py
Normal file
66
app/services/user_settings.py
Normal file
|
|
@ -0,0 +1,66 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.models import UserSetting
|
||||
|
||||
|
||||
class UserSettingsServiceError(ValueError):
|
||||
"""Raised for expected settings-validation failures."""
|
||||
|
||||
|
||||
def parse_run_interval_minutes(value: str) -> int:
|
||||
try:
|
||||
parsed = int(value)
|
||||
except ValueError as exc:
|
||||
raise UserSettingsServiceError("Run interval must be a whole number.") from exc
|
||||
if parsed < 15:
|
||||
raise UserSettingsServiceError("Run interval must be at least 15 minutes.")
|
||||
return parsed
|
||||
|
||||
|
||||
def parse_request_delay_seconds(value: str) -> int:
|
||||
try:
|
||||
parsed = int(value)
|
||||
except ValueError as exc:
|
||||
raise UserSettingsServiceError("Request delay must be a whole number.") from exc
|
||||
if parsed < 1:
|
||||
raise UserSettingsServiceError("Request delay must be at least 1 second.")
|
||||
return parsed
|
||||
|
||||
|
||||
async def get_or_create_settings(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
) -> UserSetting:
|
||||
result = await db_session.execute(
|
||||
select(UserSetting).where(UserSetting.user_id == user_id)
|
||||
)
|
||||
settings = result.scalar_one_or_none()
|
||||
if settings is not None:
|
||||
return settings
|
||||
|
||||
settings = UserSetting(user_id=user_id)
|
||||
db_session.add(settings)
|
||||
await db_session.commit()
|
||||
await db_session.refresh(settings)
|
||||
return settings
|
||||
|
||||
|
||||
async def update_settings(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
settings: UserSetting,
|
||||
auto_run_enabled: bool,
|
||||
run_interval_minutes: int,
|
||||
request_delay_seconds: int,
|
||||
) -> UserSetting:
|
||||
settings.auto_run_enabled = auto_run_enabled
|
||||
settings.run_interval_minutes = run_interval_minutes
|
||||
settings.request_delay_seconds = request_delay_seconds
|
||||
await db_session.commit()
|
||||
await db_session.refresh(settings)
|
||||
return settings
|
||||
|
||||
98
app/services/users.py
Normal file
98
app/services/users.py
Normal file
|
|
@ -0,0 +1,98 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.models import User
|
||||
|
||||
EMAIL_PATTERN = re.compile(r"^[^@\s]+@[^@\s]+\.[^@\s]+$")
|
||||
|
||||
|
||||
class UserServiceError(ValueError):
|
||||
"""Raised for expected user-management validation failures."""
|
||||
|
||||
|
||||
def normalize_email(value: str) -> str:
|
||||
return value.strip().lower()
|
||||
|
||||
|
||||
def validate_email(value: str) -> str:
|
||||
email = normalize_email(value)
|
||||
if not EMAIL_PATTERN.fullmatch(email):
|
||||
raise UserServiceError("Enter a valid email address.")
|
||||
return email
|
||||
|
||||
|
||||
def validate_password(value: str) -> str:
|
||||
password = value.strip()
|
||||
if len(password) < 8:
|
||||
raise UserServiceError("Password must be at least 8 characters.")
|
||||
return password
|
||||
|
||||
|
||||
async def get_user_by_id(db_session: AsyncSession, user_id: int) -> User | None:
|
||||
result = await db_session.execute(select(User).where(User.id == user_id))
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
|
||||
async def get_user_by_email(db_session: AsyncSession, email: str) -> User | None:
|
||||
result = await db_session.execute(
|
||||
select(User).where(User.email == normalize_email(email))
|
||||
)
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
|
||||
async def list_users(db_session: AsyncSession) -> list[User]:
|
||||
result = await db_session.execute(select(User).order_by(User.email.asc()))
|
||||
return list(result.scalars().all())
|
||||
|
||||
|
||||
async def create_user(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
email: str,
|
||||
password_hash: str,
|
||||
is_admin: bool,
|
||||
) -> User:
|
||||
user = User(
|
||||
email=validate_email(email),
|
||||
password_hash=password_hash,
|
||||
is_admin=is_admin,
|
||||
is_active=True,
|
||||
)
|
||||
db_session.add(user)
|
||||
try:
|
||||
await db_session.commit()
|
||||
except IntegrityError as exc:
|
||||
await db_session.rollback()
|
||||
raise UserServiceError("A user with that email already exists.") from exc
|
||||
await db_session.refresh(user)
|
||||
return user
|
||||
|
||||
|
||||
async def set_user_active(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user: User,
|
||||
is_active: bool,
|
||||
) -> User:
|
||||
user.is_active = is_active
|
||||
await db_session.commit()
|
||||
await db_session.refresh(user)
|
||||
return user
|
||||
|
||||
|
||||
async def set_user_password_hash(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user: User,
|
||||
password_hash: str,
|
||||
) -> User:
|
||||
user.password_hash = password_hash
|
||||
await db_session.commit()
|
||||
await db_session.refresh(user)
|
||||
return user
|
||||
|
||||
Loading…
Add table
Add a link
Reference in a new issue