modularize service layer, add mobile nav drawer, and sync docs
This commit is contained in:
parent
7f7a8ce2b0
commit
6b6ed91b92
72 changed files with 8122 additions and 7501 deletions
1
app/services/domains/ingestion/__init__.py
Normal file
1
app/services/domains/ingestion/__init__.py
Normal file
|
|
@ -0,0 +1 @@
|
|||
from __future__ import annotations
|
||||
2110
app/services/domains/ingestion/application.py
Normal file
2110
app/services/domains/ingestion/application.py
Normal file
File diff suppressed because it is too large
Load diff
31
app/services/domains/ingestion/constants.py
Normal file
31
app/services/domains/ingestion/constants.py
Normal file
|
|
@ -0,0 +1,31 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from app.services.domains.scholar.parser import ParseState
|
||||
|
||||
TITLE_ALNUM_RE = re.compile(r"[^a-z0-9]+")
|
||||
WORD_RE = re.compile(r"[a-z0-9]+")
|
||||
HTML_TAG_RE = re.compile(r"<[^>]+>", re.S)
|
||||
SPACE_RE = re.compile(r"\s+")
|
||||
|
||||
FAILED_STATES = {
|
||||
ParseState.BLOCKED_OR_CAPTCHA.value,
|
||||
ParseState.LAYOUT_CHANGED.value,
|
||||
ParseState.NETWORK_ERROR.value,
|
||||
"ingestion_error",
|
||||
}
|
||||
|
||||
FAILURE_BUCKET_BLOCKED = "blocked_or_captcha"
|
||||
FAILURE_BUCKET_NETWORK = "network_error"
|
||||
FAILURE_BUCKET_LAYOUT = "layout_changed"
|
||||
FAILURE_BUCKET_INGESTION = "ingestion_error"
|
||||
FAILURE_BUCKET_OTHER = "other_failure"
|
||||
|
||||
RUN_LOCK_NAMESPACE = 8217
|
||||
RESUMABLE_PARTIAL_REASONS = {
|
||||
"max_pages_reached",
|
||||
"pagination_cursor_stalled",
|
||||
}
|
||||
RESUMABLE_PARTIAL_REASON_PREFIXES = ("page_state_network_error",)
|
||||
INITIAL_PAGE_FINGERPRINT_MAX_PUBLICATIONS = 30
|
||||
136
app/services/domains/ingestion/fingerprints.py
Normal file
136
app/services/domains/ingestion/fingerprints.py
Normal file
|
|
@ -0,0 +1,136 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from typing import Any
|
||||
from urllib.parse import urljoin
|
||||
|
||||
from app.services.domains.ingestion.constants import (
|
||||
HTML_TAG_RE,
|
||||
INITIAL_PAGE_FINGERPRINT_MAX_PUBLICATIONS,
|
||||
SPACE_RE,
|
||||
TITLE_ALNUM_RE,
|
||||
WORD_RE,
|
||||
)
|
||||
from app.services.domains.scholar.parser import ParseState, ParsedProfilePage, PublicationCandidate
|
||||
|
||||
|
||||
def normalize_title(value: str) -> str:
|
||||
lowered = value.lower()
|
||||
return TITLE_ALNUM_RE.sub("", lowered)
|
||||
|
||||
|
||||
def _first_author_last_name(authors_text: str | None) -> str:
|
||||
if not authors_text:
|
||||
return ""
|
||||
first_author = authors_text.split(",", maxsplit=1)[0].strip().lower()
|
||||
words = WORD_RE.findall(first_author)
|
||||
if not words:
|
||||
return ""
|
||||
return words[-1]
|
||||
|
||||
|
||||
def _first_venue_word(venue_text: str | None) -> str:
|
||||
if not venue_text:
|
||||
return ""
|
||||
words = WORD_RE.findall(venue_text.lower())
|
||||
if not words:
|
||||
return ""
|
||||
return words[0]
|
||||
|
||||
|
||||
def build_publication_fingerprint(candidate: PublicationCandidate) -> str:
|
||||
canonical = "|".join(
|
||||
[
|
||||
normalize_title(candidate.title),
|
||||
str(candidate.year) if candidate.year is not None else "",
|
||||
_first_author_last_name(candidate.authors_text),
|
||||
_first_venue_word(candidate.venue_text),
|
||||
]
|
||||
)
|
||||
return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def build_initial_page_fingerprint(parsed_page: ParsedProfilePage) -> str | None:
|
||||
if parsed_page.state not in {ParseState.OK, ParseState.NO_RESULTS}:
|
||||
return None
|
||||
|
||||
normalized_rows: list[dict[str, Any]] = []
|
||||
for publication in parsed_page.publications[:INITIAL_PAGE_FINGERPRINT_MAX_PUBLICATIONS]:
|
||||
normalized_rows.append(
|
||||
{
|
||||
"cluster_id": publication.cluster_id or "",
|
||||
"title_normalized": normalize_title(publication.title),
|
||||
"year": publication.year,
|
||||
"citation_count": publication.citation_count,
|
||||
}
|
||||
)
|
||||
|
||||
payload = {
|
||||
"state": parsed_page.state.value,
|
||||
"articles_range": parsed_page.articles_range or "",
|
||||
"has_show_more_button": parsed_page.has_show_more_button,
|
||||
"profile_name": parsed_page.profile_name or "",
|
||||
"publications": normalized_rows,
|
||||
}
|
||||
canonical = json.dumps(
|
||||
payload,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
ensure_ascii=True,
|
||||
)
|
||||
return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def build_publication_url(path_or_url: str | None) -> str | None:
|
||||
if not path_or_url:
|
||||
return None
|
||||
return urljoin("https://scholar.google.com", path_or_url)
|
||||
|
||||
|
||||
def _next_cstart_value(*, articles_range: str | None, fallback: int) -> int:
|
||||
if articles_range:
|
||||
numbers = re.findall(r"\d+", articles_range)
|
||||
if len(numbers) >= 2:
|
||||
try:
|
||||
return int(numbers[1])
|
||||
except ValueError:
|
||||
pass
|
||||
return int(fallback)
|
||||
|
||||
|
||||
def _dedupe_publication_candidates(
|
||||
publications: list[PublicationCandidate],
|
||||
) -> list[PublicationCandidate]:
|
||||
deduped: list[PublicationCandidate] = []
|
||||
seen: set[str] = set()
|
||||
for publication in publications:
|
||||
if publication.cluster_id:
|
||||
identity = f"cluster:{publication.cluster_id}"
|
||||
else:
|
||||
identity = "|".join(
|
||||
[
|
||||
"fallback",
|
||||
normalize_title(publication.title),
|
||||
str(publication.year) if publication.year is not None else "",
|
||||
publication.authors_text or "",
|
||||
publication.venue_text or "",
|
||||
]
|
||||
)
|
||||
if identity in seen:
|
||||
continue
|
||||
seen.add(identity)
|
||||
deduped.append(publication)
|
||||
return deduped
|
||||
|
||||
|
||||
def _build_body_excerpt(body: str, *, max_chars: int = 220) -> str | None:
|
||||
if not body:
|
||||
return None
|
||||
flattened = SPACE_RE.sub(" ", HTML_TAG_RE.sub(" ", body)).strip()
|
||||
if not flattened:
|
||||
return None
|
||||
if len(flattened) <= max_chars:
|
||||
return flattened
|
||||
return f"{flattened[:max_chars - 1]}..."
|
||||
348
app/services/domains/ingestion/queue.py
Normal file
348
app/services/domains/ingestion/queue.py
Normal file
|
|
@ -0,0 +1,348 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.models import IngestionQueueItem, QueueItemStatus
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ContinuationQueueJob:
|
||||
id: int
|
||||
user_id: int
|
||||
scholar_profile_id: int
|
||||
resume_cstart: int
|
||||
reason: str
|
||||
status: str
|
||||
attempt_count: int
|
||||
next_attempt_dt: datetime
|
||||
|
||||
|
||||
ACTIVE_QUEUE_STATUSES: tuple[str, ...] = (
|
||||
QueueItemStatus.QUEUED.value,
|
||||
QueueItemStatus.RETRYING.value,
|
||||
)
|
||||
|
||||
|
||||
def normalize_cstart(value: int | None) -> int:
|
||||
if value is None:
|
||||
return 0
|
||||
return max(0, int(value))
|
||||
|
||||
|
||||
def compute_backoff_seconds(*, base_seconds: int, attempt_count: int, max_seconds: int) -> int:
|
||||
base = max(1, int(base_seconds))
|
||||
attempts = max(1, int(attempt_count))
|
||||
maximum = max(base, int(max_seconds))
|
||||
seconds = base * (2 ** max(0, attempts - 1))
|
||||
return min(seconds, maximum)
|
||||
|
||||
|
||||
async def _get_item_for_user_scholar(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
scholar_profile_id: int,
|
||||
) -> IngestionQueueItem | None:
|
||||
result = await db_session.execute(
|
||||
select(IngestionQueueItem).where(
|
||||
IngestionQueueItem.user_id == user_id,
|
||||
IngestionQueueItem.scholar_profile_id == scholar_profile_id,
|
||||
)
|
||||
)
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
|
||||
def _build_queue_item(
|
||||
*,
|
||||
now: datetime,
|
||||
user_id: int,
|
||||
scholar_profile_id: int,
|
||||
normalized_cstart: int,
|
||||
reason: str,
|
||||
run_id: int | None,
|
||||
next_attempt_dt: datetime,
|
||||
) -> IngestionQueueItem:
|
||||
return IngestionQueueItem(
|
||||
user_id=user_id,
|
||||
scholar_profile_id=scholar_profile_id,
|
||||
resume_cstart=normalized_cstart,
|
||||
reason=reason,
|
||||
status=QueueItemStatus.QUEUED.value,
|
||||
attempt_count=0,
|
||||
next_attempt_dt=next_attempt_dt,
|
||||
last_run_id=run_id,
|
||||
last_error=None,
|
||||
dropped_reason=None,
|
||||
dropped_at=None,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
|
||||
|
||||
def _update_existing_queue_item(
|
||||
*,
|
||||
item: IngestionQueueItem,
|
||||
now: datetime,
|
||||
normalized_cstart: int,
|
||||
reason: str,
|
||||
run_id: int | None,
|
||||
next_attempt_dt: datetime,
|
||||
) -> None:
|
||||
item.resume_cstart = normalized_cstart
|
||||
item.reason = reason
|
||||
if item.status == QueueItemStatus.DROPPED.value:
|
||||
item.attempt_count = 0
|
||||
item.status = QueueItemStatus.QUEUED.value
|
||||
item.next_attempt_dt = next_attempt_dt
|
||||
item.last_run_id = run_id
|
||||
item.last_error = None
|
||||
item.dropped_reason = None
|
||||
item.dropped_at = None
|
||||
item.updated_at = now
|
||||
|
||||
|
||||
async def upsert_job(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
scholar_profile_id: int,
|
||||
resume_cstart: int,
|
||||
reason: str,
|
||||
run_id: int | None,
|
||||
delay_seconds: int,
|
||||
) -> IngestionQueueItem:
|
||||
now = datetime.now(timezone.utc)
|
||||
next_attempt_dt = now + timedelta(seconds=max(0, int(delay_seconds)))
|
||||
item = await _get_item_for_user_scholar(
|
||||
db_session,
|
||||
user_id=user_id,
|
||||
scholar_profile_id=scholar_profile_id,
|
||||
)
|
||||
normalized_cstart = normalize_cstart(resume_cstart)
|
||||
if item is None:
|
||||
item = _build_queue_item(
|
||||
now=now,
|
||||
user_id=user_id,
|
||||
scholar_profile_id=scholar_profile_id,
|
||||
normalized_cstart=normalized_cstart,
|
||||
reason=reason,
|
||||
run_id=run_id,
|
||||
next_attempt_dt=next_attempt_dt,
|
||||
)
|
||||
db_session.add(item)
|
||||
return item
|
||||
|
||||
_update_existing_queue_item(
|
||||
item=item,
|
||||
now=now,
|
||||
normalized_cstart=normalized_cstart,
|
||||
reason=reason,
|
||||
run_id=run_id,
|
||||
next_attempt_dt=next_attempt_dt,
|
||||
)
|
||||
return item
|
||||
|
||||
|
||||
async def clear_job_for_scholar(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
scholar_profile_id: int,
|
||||
) -> bool:
|
||||
result = await db_session.execute(
|
||||
select(IngestionQueueItem).where(
|
||||
IngestionQueueItem.user_id == user_id,
|
||||
IngestionQueueItem.scholar_profile_id == scholar_profile_id,
|
||||
)
|
||||
)
|
||||
item = result.scalar_one_or_none()
|
||||
if item is None:
|
||||
return False
|
||||
await db_session.delete(item)
|
||||
return True
|
||||
|
||||
|
||||
async def delete_job_by_id(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
job_id: int,
|
||||
) -> bool:
|
||||
result = await db_session.execute(
|
||||
select(IngestionQueueItem).where(IngestionQueueItem.id == job_id)
|
||||
)
|
||||
item = result.scalar_one_or_none()
|
||||
if item is None:
|
||||
return False
|
||||
await db_session.delete(item)
|
||||
return True
|
||||
|
||||
|
||||
async def list_due_jobs(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
now: datetime,
|
||||
limit: int,
|
||||
) -> list[ContinuationQueueJob]:
|
||||
result = await db_session.execute(
|
||||
select(IngestionQueueItem)
|
||||
.where(
|
||||
IngestionQueueItem.next_attempt_dt <= now,
|
||||
IngestionQueueItem.status.in_(ACTIVE_QUEUE_STATUSES),
|
||||
)
|
||||
.order_by(
|
||||
IngestionQueueItem.next_attempt_dt.asc(),
|
||||
IngestionQueueItem.id.asc(),
|
||||
)
|
||||
.limit(limit)
|
||||
)
|
||||
rows = list(result.scalars().all())
|
||||
jobs: list[ContinuationQueueJob] = []
|
||||
for row in rows:
|
||||
jobs.append(
|
||||
ContinuationQueueJob(
|
||||
id=int(row.id),
|
||||
user_id=int(row.user_id),
|
||||
scholar_profile_id=int(row.scholar_profile_id),
|
||||
resume_cstart=normalize_cstart(row.resume_cstart),
|
||||
reason=row.reason,
|
||||
status=row.status,
|
||||
attempt_count=int(row.attempt_count),
|
||||
next_attempt_dt=row.next_attempt_dt,
|
||||
)
|
||||
)
|
||||
return jobs
|
||||
|
||||
|
||||
async def increment_attempt_count(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
job_id: int,
|
||||
) -> IngestionQueueItem | None:
|
||||
now = datetime.now(timezone.utc)
|
||||
result = await db_session.execute(
|
||||
select(IngestionQueueItem).where(IngestionQueueItem.id == job_id)
|
||||
)
|
||||
item = result.scalar_one_or_none()
|
||||
if item is None:
|
||||
return None
|
||||
item.attempt_count = int(item.attempt_count or 0) + 1
|
||||
item.updated_at = now
|
||||
return item
|
||||
|
||||
|
||||
async def reset_attempt_count(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
job_id: int,
|
||||
) -> IngestionQueueItem | None:
|
||||
now = datetime.now(timezone.utc)
|
||||
result = await db_session.execute(
|
||||
select(IngestionQueueItem).where(IngestionQueueItem.id == job_id)
|
||||
)
|
||||
item = result.scalar_one_or_none()
|
||||
if item is None:
|
||||
return None
|
||||
item.attempt_count = 0
|
||||
item.updated_at = now
|
||||
return item
|
||||
|
||||
|
||||
async def reschedule_job(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
job_id: int,
|
||||
delay_seconds: int,
|
||||
reason: str,
|
||||
error: str | None = None,
|
||||
) -> IngestionQueueItem | None:
|
||||
now = datetime.now(timezone.utc)
|
||||
result = await db_session.execute(
|
||||
select(IngestionQueueItem).where(IngestionQueueItem.id == job_id)
|
||||
)
|
||||
item = result.scalar_one_or_none()
|
||||
if item is None:
|
||||
return None
|
||||
item.next_attempt_dt = now + timedelta(seconds=max(1, int(delay_seconds)))
|
||||
item.status = QueueItemStatus.QUEUED.value
|
||||
item.reason = reason
|
||||
item.last_error = error
|
||||
item.dropped_reason = None
|
||||
item.dropped_at = None
|
||||
item.updated_at = now
|
||||
return item
|
||||
|
||||
|
||||
async def mark_retrying(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
job_id: int,
|
||||
reason: str | None = None,
|
||||
) -> IngestionQueueItem | None:
|
||||
now = datetime.now(timezone.utc)
|
||||
result = await db_session.execute(
|
||||
select(IngestionQueueItem).where(IngestionQueueItem.id == job_id)
|
||||
)
|
||||
item = result.scalar_one_or_none()
|
||||
if item is None:
|
||||
return None
|
||||
if item.status == QueueItemStatus.DROPPED.value:
|
||||
return item
|
||||
item.status = QueueItemStatus.RETRYING.value
|
||||
if reason:
|
||||
item.reason = reason
|
||||
item.updated_at = now
|
||||
return item
|
||||
|
||||
|
||||
async def mark_dropped(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
job_id: int,
|
||||
reason: str,
|
||||
error: str | None = None,
|
||||
) -> IngestionQueueItem | None:
|
||||
now = datetime.now(timezone.utc)
|
||||
result = await db_session.execute(
|
||||
select(IngestionQueueItem).where(IngestionQueueItem.id == job_id)
|
||||
)
|
||||
item = result.scalar_one_or_none()
|
||||
if item is None:
|
||||
return None
|
||||
item.status = QueueItemStatus.DROPPED.value
|
||||
item.reason = "dropped"
|
||||
item.dropped_reason = reason
|
||||
item.dropped_at = now
|
||||
if error is not None:
|
||||
item.last_error = error
|
||||
item.updated_at = now
|
||||
return item
|
||||
|
||||
|
||||
async def mark_queued_now(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
job_id: int,
|
||||
reason: str,
|
||||
reset_attempt_count: bool = False,
|
||||
) -> IngestionQueueItem | None:
|
||||
now = datetime.now(timezone.utc)
|
||||
result = await db_session.execute(
|
||||
select(IngestionQueueItem).where(IngestionQueueItem.id == job_id)
|
||||
)
|
||||
item = result.scalar_one_or_none()
|
||||
if item is None:
|
||||
return None
|
||||
item.status = QueueItemStatus.QUEUED.value
|
||||
item.reason = reason
|
||||
item.next_attempt_dt = now
|
||||
if reset_attempt_count:
|
||||
item.attempt_count = 0
|
||||
item.last_error = None
|
||||
item.dropped_reason = None
|
||||
item.dropped_at = None
|
||||
item.updated_at = now
|
||||
return item
|
||||
282
app/services/domains/ingestion/safety.py
Normal file
282
app/services/domains/ingestion/safety.py
Normal file
|
|
@ -0,0 +1,282 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any
|
||||
|
||||
from app.db.models import UserSetting
|
||||
|
||||
COOLDOWN_REASON_BLOCKED_FAILURE_THRESHOLD = "blocked_failure_threshold_exceeded"
|
||||
COOLDOWN_REASON_NETWORK_FAILURE_THRESHOLD = "network_failure_threshold_exceeded"
|
||||
|
||||
_COUNTER_CONSECUTIVE_BLOCKED_RUNS = "consecutive_blocked_runs"
|
||||
_COUNTER_CONSECUTIVE_NETWORK_RUNS = "consecutive_network_runs"
|
||||
_COUNTER_COOLDOWN_ENTRY_COUNT = "cooldown_entry_count"
|
||||
_COUNTER_BLOCKED_START_COUNT = "blocked_start_count"
|
||||
_COUNTER_LAST_BLOCKED_FAILURE_COUNT = "last_blocked_failure_count"
|
||||
_COUNTER_LAST_NETWORK_FAILURE_COUNT = "last_network_failure_count"
|
||||
_COUNTER_LAST_EVALUATED_RUN_ID = "last_evaluated_run_id"
|
||||
|
||||
|
||||
def _utcnow() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def _safe_int(value: Any, default: int = 0) -> int:
|
||||
try:
|
||||
return int(value)
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
|
||||
|
||||
def _safe_optional_int(value: Any) -> int | None:
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
return int(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _normalize_datetime(value: datetime | None) -> datetime | None:
|
||||
if value is None:
|
||||
return None
|
||||
if value.tzinfo is None:
|
||||
return value.replace(tzinfo=timezone.utc)
|
||||
return value.astimezone(timezone.utc)
|
||||
|
||||
|
||||
def _state_dict(settings: UserSetting) -> dict[str, Any]:
|
||||
state = settings.scrape_safety_state
|
||||
if isinstance(state, dict):
|
||||
return state
|
||||
return {}
|
||||
|
||||
|
||||
def _counters_from_state(settings: UserSetting) -> dict[str, Any]:
|
||||
state = _state_dict(settings)
|
||||
return {
|
||||
_COUNTER_CONSECUTIVE_BLOCKED_RUNS: max(
|
||||
0,
|
||||
_safe_int(state.get(_COUNTER_CONSECUTIVE_BLOCKED_RUNS), 0),
|
||||
),
|
||||
_COUNTER_CONSECUTIVE_NETWORK_RUNS: max(
|
||||
0,
|
||||
_safe_int(state.get(_COUNTER_CONSECUTIVE_NETWORK_RUNS), 0),
|
||||
),
|
||||
_COUNTER_COOLDOWN_ENTRY_COUNT: max(
|
||||
0,
|
||||
_safe_int(state.get(_COUNTER_COOLDOWN_ENTRY_COUNT), 0),
|
||||
),
|
||||
_COUNTER_BLOCKED_START_COUNT: max(
|
||||
0,
|
||||
_safe_int(state.get(_COUNTER_BLOCKED_START_COUNT), 0),
|
||||
),
|
||||
_COUNTER_LAST_BLOCKED_FAILURE_COUNT: max(
|
||||
0,
|
||||
_safe_int(state.get(_COUNTER_LAST_BLOCKED_FAILURE_COUNT), 0),
|
||||
),
|
||||
_COUNTER_LAST_NETWORK_FAILURE_COUNT: max(
|
||||
0,
|
||||
_safe_int(state.get(_COUNTER_LAST_NETWORK_FAILURE_COUNT), 0),
|
||||
),
|
||||
_COUNTER_LAST_EVALUATED_RUN_ID: _safe_optional_int(
|
||||
state.get(_COUNTER_LAST_EVALUATED_RUN_ID),
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def _cooldown_reason_label(reason: str | None) -> str | None:
|
||||
if reason == COOLDOWN_REASON_BLOCKED_FAILURE_THRESHOLD:
|
||||
return "Blocked responses exceeded safety threshold"
|
||||
if reason == COOLDOWN_REASON_NETWORK_FAILURE_THRESHOLD:
|
||||
return "Network failures exceeded safety threshold"
|
||||
return None
|
||||
|
||||
|
||||
def _recommended_action(reason: str | None) -> str | None:
|
||||
if reason == COOLDOWN_REASON_BLOCKED_FAILURE_THRESHOLD:
|
||||
return (
|
||||
"Google Scholar appears to be blocking requests. Wait for cooldown to expire, "
|
||||
"increase request delay, and avoid repeated manual retries."
|
||||
)
|
||||
if reason == COOLDOWN_REASON_NETWORK_FAILURE_THRESHOLD:
|
||||
return (
|
||||
"Network failures crossed the threshold. Verify connectivity and retry after cooldown."
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def is_cooldown_active(
|
||||
settings: UserSetting,
|
||||
*,
|
||||
now_utc: datetime | None = None,
|
||||
) -> bool:
|
||||
now = now_utc or _utcnow()
|
||||
cooldown_until = _normalize_datetime(settings.scrape_cooldown_until)
|
||||
if cooldown_until is None:
|
||||
return False
|
||||
return cooldown_until > now
|
||||
|
||||
|
||||
def clear_expired_cooldown(
|
||||
settings: UserSetting,
|
||||
*,
|
||||
now_utc: datetime | None = None,
|
||||
) -> bool:
|
||||
now = now_utc or _utcnow()
|
||||
cooldown_until = _normalize_datetime(settings.scrape_cooldown_until)
|
||||
if cooldown_until is None:
|
||||
return False
|
||||
if cooldown_until > now:
|
||||
return False
|
||||
settings.scrape_cooldown_until = None
|
||||
settings.scrape_cooldown_reason = None
|
||||
return True
|
||||
|
||||
|
||||
def register_cooldown_blocked_start(
|
||||
settings: UserSetting,
|
||||
*,
|
||||
now_utc: datetime | None = None,
|
||||
) -> dict[str, Any]:
|
||||
now = now_utc or _utcnow()
|
||||
counters = _counters_from_state(settings)
|
||||
counters[_COUNTER_BLOCKED_START_COUNT] = int(counters[_COUNTER_BLOCKED_START_COUNT]) + 1
|
||||
settings.scrape_safety_state = counters
|
||||
return get_safety_state_payload(settings, now_utc=now)
|
||||
|
||||
|
||||
def _update_run_counters(
|
||||
*,
|
||||
counters: dict[str, Any],
|
||||
run_id: int,
|
||||
blocked_failure_count: int,
|
||||
network_failure_count: int,
|
||||
) -> tuple[int, int]:
|
||||
bounded_blocked_failures = max(0, int(blocked_failure_count))
|
||||
bounded_network_failures = max(0, int(network_failure_count))
|
||||
counters[_COUNTER_LAST_BLOCKED_FAILURE_COUNT] = bounded_blocked_failures
|
||||
counters[_COUNTER_LAST_NETWORK_FAILURE_COUNT] = bounded_network_failures
|
||||
counters[_COUNTER_LAST_EVALUATED_RUN_ID] = int(run_id)
|
||||
counters[_COUNTER_CONSECUTIVE_BLOCKED_RUNS] = (
|
||||
int(counters[_COUNTER_CONSECUTIVE_BLOCKED_RUNS]) + 1
|
||||
if bounded_blocked_failures > 0
|
||||
else 0
|
||||
)
|
||||
counters[_COUNTER_CONSECUTIVE_NETWORK_RUNS] = (
|
||||
int(counters[_COUNTER_CONSECUTIVE_NETWORK_RUNS]) + 1
|
||||
if bounded_network_failures > 0
|
||||
else 0
|
||||
)
|
||||
return bounded_blocked_failures, bounded_network_failures
|
||||
|
||||
|
||||
def _resolve_cooldown_trigger(
|
||||
*,
|
||||
blocked_failures: int,
|
||||
network_failures: int,
|
||||
blocked_failure_threshold: int,
|
||||
network_failure_threshold: int,
|
||||
blocked_cooldown_seconds: int,
|
||||
network_cooldown_seconds: int,
|
||||
) -> tuple[str | None, int]:
|
||||
if blocked_failures >= max(1, int(blocked_failure_threshold)):
|
||||
return COOLDOWN_REASON_BLOCKED_FAILURE_THRESHOLD, max(60, int(blocked_cooldown_seconds))
|
||||
if network_failures >= max(1, int(network_failure_threshold)):
|
||||
return COOLDOWN_REASON_NETWORK_FAILURE_THRESHOLD, max(60, int(network_cooldown_seconds))
|
||||
return None, 0
|
||||
|
||||
|
||||
def _apply_cooldown_decision(
|
||||
*,
|
||||
settings: UserSetting,
|
||||
counters: dict[str, Any],
|
||||
now: datetime,
|
||||
reason: str | None,
|
||||
cooldown_seconds: int,
|
||||
) -> None:
|
||||
if reason is None:
|
||||
clear_expired_cooldown(settings, now_utc=now)
|
||||
return
|
||||
settings.scrape_cooldown_reason = reason
|
||||
settings.scrape_cooldown_until = now + timedelta(seconds=max(60, int(cooldown_seconds)))
|
||||
counters[_COUNTER_COOLDOWN_ENTRY_COUNT] = int(counters[_COUNTER_COOLDOWN_ENTRY_COUNT]) + 1
|
||||
|
||||
|
||||
def apply_run_safety_outcome(
|
||||
settings: UserSetting,
|
||||
*,
|
||||
run_id: int,
|
||||
blocked_failure_count: int,
|
||||
network_failure_count: int,
|
||||
blocked_failure_threshold: int,
|
||||
network_failure_threshold: int,
|
||||
blocked_cooldown_seconds: int,
|
||||
network_cooldown_seconds: int,
|
||||
now_utc: datetime | None = None,
|
||||
) -> tuple[dict[str, Any], str | None]:
|
||||
now = now_utc or _utcnow()
|
||||
counters = _counters_from_state(settings)
|
||||
blocked_failures, network_failures = _update_run_counters(
|
||||
counters=counters,
|
||||
run_id=run_id,
|
||||
blocked_failure_count=blocked_failure_count,
|
||||
network_failure_count=network_failure_count,
|
||||
)
|
||||
reason, cooldown_seconds = _resolve_cooldown_trigger(
|
||||
blocked_failures=blocked_failures,
|
||||
network_failures=network_failures,
|
||||
blocked_failure_threshold=blocked_failure_threshold,
|
||||
network_failure_threshold=network_failure_threshold,
|
||||
blocked_cooldown_seconds=blocked_cooldown_seconds,
|
||||
network_cooldown_seconds=network_cooldown_seconds,
|
||||
)
|
||||
_apply_cooldown_decision(
|
||||
settings=settings,
|
||||
counters=counters,
|
||||
now=now,
|
||||
reason=reason,
|
||||
cooldown_seconds=cooldown_seconds,
|
||||
)
|
||||
settings.scrape_safety_state = counters
|
||||
return get_safety_state_payload(settings, now_utc=now), reason
|
||||
|
||||
|
||||
def get_safety_state_payload(
|
||||
settings: UserSetting,
|
||||
*,
|
||||
now_utc: datetime | None = None,
|
||||
) -> dict[str, Any]:
|
||||
now = now_utc or _utcnow()
|
||||
cooldown_until = _normalize_datetime(settings.scrape_cooldown_until)
|
||||
cooldown_active = bool(cooldown_until is not None and cooldown_until > now)
|
||||
cooldown_remaining_seconds = 0
|
||||
if cooldown_active and cooldown_until is not None:
|
||||
cooldown_remaining_seconds = max(0, int((cooldown_until - now).total_seconds()))
|
||||
|
||||
reason = settings.scrape_cooldown_reason if cooldown_active else None
|
||||
|
||||
return {
|
||||
"cooldown_active": cooldown_active,
|
||||
"cooldown_reason": reason,
|
||||
"cooldown_reason_label": _cooldown_reason_label(reason),
|
||||
"cooldown_until": cooldown_until,
|
||||
"cooldown_remaining_seconds": cooldown_remaining_seconds,
|
||||
"recommended_action": _recommended_action(reason),
|
||||
"counters": _counters_from_state(settings),
|
||||
}
|
||||
|
||||
|
||||
def get_safety_event_context(
|
||||
settings: UserSetting,
|
||||
*,
|
||||
now_utc: datetime | None = None,
|
||||
) -> dict[str, Any]:
|
||||
payload = get_safety_state_payload(settings, now_utc=now_utc)
|
||||
return {
|
||||
"cooldown_active": bool(payload.get("cooldown_active")),
|
||||
"cooldown_reason": payload.get("cooldown_reason"),
|
||||
"cooldown_until": payload.get("cooldown_until"),
|
||||
"cooldown_remaining_seconds": int(payload.get("cooldown_remaining_seconds") or 0),
|
||||
"safety_counters": payload.get("counters", {}),
|
||||
}
|
||||
582
app/services/domains/ingestion/scheduler.py
Normal file
582
app/services/domains/ingestion/scheduler.py
Normal file
|
|
@ -0,0 +1,582 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta, timezone
|
||||
import logging
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.models import (
|
||||
CrawlRun,
|
||||
QueueItemStatus,
|
||||
RunTriggerType,
|
||||
ScholarProfile,
|
||||
User,
|
||||
UserSetting,
|
||||
)
|
||||
from app.db.session import get_session_factory
|
||||
from app.services.domains.ingestion import queue as queue_service
|
||||
from app.services.domains.ingestion.application import (
|
||||
RunAlreadyInProgressError,
|
||||
RunBlockedBySafetyPolicyError,
|
||||
ScholarIngestionService,
|
||||
)
|
||||
from app.services.domains.scholar.source import LiveScholarSource
|
||||
from app.settings import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _AutoRunCandidate:
|
||||
user_id: int
|
||||
run_interval_minutes: int
|
||||
request_delay_seconds: int
|
||||
cooldown_until: datetime | None
|
||||
cooldown_reason: str | None
|
||||
|
||||
|
||||
class SchedulerService:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
enabled: bool,
|
||||
tick_seconds: int,
|
||||
network_error_retries: int,
|
||||
retry_backoff_seconds: float,
|
||||
max_pages_per_scholar: int,
|
||||
page_size: int,
|
||||
continuation_queue_enabled: bool,
|
||||
continuation_base_delay_seconds: int,
|
||||
continuation_max_delay_seconds: int,
|
||||
continuation_max_attempts: int,
|
||||
queue_batch_size: int,
|
||||
) -> None:
|
||||
self._enabled = enabled
|
||||
self._tick_seconds = max(5, int(tick_seconds))
|
||||
self._network_error_retries = max(0, int(network_error_retries))
|
||||
self._retry_backoff_seconds = max(0.0, float(retry_backoff_seconds))
|
||||
self._max_pages_per_scholar = max(1, int(max_pages_per_scholar))
|
||||
self._page_size = max(1, int(page_size))
|
||||
self._continuation_queue_enabled = bool(continuation_queue_enabled)
|
||||
self._continuation_base_delay_seconds = max(1, int(continuation_base_delay_seconds))
|
||||
self._continuation_max_delay_seconds = max(
|
||||
self._continuation_base_delay_seconds,
|
||||
int(continuation_max_delay_seconds),
|
||||
)
|
||||
self._continuation_max_attempts = max(1, int(continuation_max_attempts))
|
||||
self._queue_batch_size = max(1, int(queue_batch_size))
|
||||
self._task: asyncio.Task[None] | None = None
|
||||
self._source = LiveScholarSource()
|
||||
|
||||
async def start(self) -> None:
|
||||
if not self._enabled:
|
||||
logger.info(
|
||||
"scheduler.disabled",
|
||||
extra={
|
||||
"event": "scheduler.disabled",
|
||||
},
|
||||
)
|
||||
return
|
||||
if self._task is not None:
|
||||
return
|
||||
self._task = asyncio.create_task(self._run_loop(), name="scholarr-scheduler")
|
||||
logger.info(
|
||||
"scheduler.started",
|
||||
extra={
|
||||
"event": "scheduler.started",
|
||||
"tick_seconds": self._tick_seconds,
|
||||
"network_error_retries": self._network_error_retries,
|
||||
"retry_backoff_seconds": self._retry_backoff_seconds,
|
||||
"max_pages_per_scholar": self._max_pages_per_scholar,
|
||||
"page_size": self._page_size,
|
||||
"continuation_queue_enabled": self._continuation_queue_enabled,
|
||||
"continuation_base_delay_seconds": self._continuation_base_delay_seconds,
|
||||
"continuation_max_delay_seconds": self._continuation_max_delay_seconds,
|
||||
"continuation_max_attempts": self._continuation_max_attempts,
|
||||
"queue_batch_size": self._queue_batch_size,
|
||||
},
|
||||
)
|
||||
|
||||
async def stop(self) -> None:
|
||||
if self._task is None:
|
||||
return
|
||||
self._task.cancel()
|
||||
try:
|
||||
await self._task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
finally:
|
||||
self._task = None
|
||||
logger.info("scheduler.stopped", extra={"event": "scheduler.stopped"})
|
||||
|
||||
async def _run_loop(self) -> None:
|
||||
while True:
|
||||
try:
|
||||
await self._tick_once()
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"scheduler.tick_failed",
|
||||
extra={
|
||||
"event": "scheduler.tick_failed",
|
||||
},
|
||||
)
|
||||
await asyncio.sleep(float(self._tick_seconds))
|
||||
|
||||
async def _tick_once(self) -> None:
|
||||
if self._continuation_queue_enabled:
|
||||
await self._drain_continuation_queue()
|
||||
candidates = await self._load_candidates()
|
||||
if not candidates:
|
||||
return
|
||||
now = datetime.now(timezone.utc)
|
||||
for candidate in candidates:
|
||||
if not await self._is_due(candidate, now=now):
|
||||
continue
|
||||
await self._run_candidate(candidate)
|
||||
|
||||
async def _load_candidate_rows(self) -> list[tuple]:
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as session:
|
||||
result = await session.execute(
|
||||
select(
|
||||
UserSetting.user_id,
|
||||
UserSetting.run_interval_minutes,
|
||||
UserSetting.request_delay_seconds,
|
||||
UserSetting.scrape_cooldown_until,
|
||||
UserSetting.scrape_cooldown_reason,
|
||||
)
|
||||
.join(User, User.id == UserSetting.user_id)
|
||||
.where(User.is_active.is_(True), UserSetting.auto_run_enabled.is_(True))
|
||||
.order_by(UserSetting.user_id.asc())
|
||||
)
|
||||
return result.all()
|
||||
|
||||
@staticmethod
|
||||
def _candidate_from_row(row: tuple, *, now_utc: datetime) -> _AutoRunCandidate | None:
|
||||
user_id, run_interval_minutes, request_delay_seconds, cooldown_until, cooldown_reason = row
|
||||
if cooldown_until is not None and cooldown_until.tzinfo is None:
|
||||
cooldown_until = cooldown_until.replace(tzinfo=timezone.utc)
|
||||
if cooldown_until is not None and cooldown_until > now_utc:
|
||||
logger.info(
|
||||
"scheduler.run_skipped_safety_cooldown_precheck",
|
||||
extra={
|
||||
"event": "scheduler.run_skipped_safety_cooldown_precheck",
|
||||
"user_id": int(user_id),
|
||||
"reason": cooldown_reason,
|
||||
"cooldown_until": cooldown_until,
|
||||
"cooldown_remaining_seconds": int((cooldown_until - now_utc).total_seconds()),
|
||||
"metric_name": "scheduler_run_skipped_safety_cooldown_total",
|
||||
"metric_value": 1,
|
||||
},
|
||||
)
|
||||
return None
|
||||
return _AutoRunCandidate(
|
||||
user_id=int(user_id),
|
||||
run_interval_minutes=int(run_interval_minutes),
|
||||
request_delay_seconds=int(request_delay_seconds),
|
||||
cooldown_until=cooldown_until,
|
||||
cooldown_reason=(str(cooldown_reason).strip() if cooldown_reason else None),
|
||||
)
|
||||
|
||||
async def _load_candidates(self) -> list[_AutoRunCandidate]:
|
||||
if not settings.ingestion_automation_allowed:
|
||||
return []
|
||||
rows = await self._load_candidate_rows()
|
||||
now_utc = datetime.now(timezone.utc)
|
||||
candidates: list[_AutoRunCandidate] = []
|
||||
for row in rows:
|
||||
candidate = self._candidate_from_row(row, now_utc=now_utc)
|
||||
if candidate is not None:
|
||||
candidates.append(candidate)
|
||||
return candidates
|
||||
|
||||
async def _is_due(self, candidate: _AutoRunCandidate, *, now: datetime) -> bool:
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as session:
|
||||
result = await session.execute(
|
||||
select(CrawlRun.start_dt)
|
||||
.where(
|
||||
CrawlRun.user_id == candidate.user_id,
|
||||
)
|
||||
.order_by(CrawlRun.start_dt.desc(), CrawlRun.id.desc())
|
||||
.limit(1)
|
||||
)
|
||||
last_run = result.scalar_one_or_none()
|
||||
|
||||
if last_run is None:
|
||||
return True
|
||||
|
||||
next_due_dt = last_run + timedelta(
|
||||
minutes=candidate.run_interval_minutes
|
||||
)
|
||||
return now >= next_due_dt
|
||||
|
||||
async def _run_candidate_ingestion(
|
||||
self,
|
||||
*,
|
||||
candidate: _AutoRunCandidate,
|
||||
):
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as session:
|
||||
ingestion = ScholarIngestionService(source=self._source)
|
||||
try:
|
||||
return await ingestion.run_for_user(
|
||||
session,
|
||||
user_id=candidate.user_id,
|
||||
trigger_type=RunTriggerType.SCHEDULED,
|
||||
request_delay_seconds=candidate.request_delay_seconds,
|
||||
network_error_retries=self._network_error_retries,
|
||||
retry_backoff_seconds=self._retry_backoff_seconds,
|
||||
max_pages_per_scholar=self._max_pages_per_scholar,
|
||||
page_size=self._page_size,
|
||||
auto_queue_continuations=self._continuation_queue_enabled,
|
||||
queue_delay_seconds=self._continuation_base_delay_seconds,
|
||||
alert_blocked_failure_threshold=settings.ingestion_alert_blocked_failure_threshold,
|
||||
alert_network_failure_threshold=settings.ingestion_alert_network_failure_threshold,
|
||||
alert_retry_scheduled_threshold=settings.ingestion_alert_retry_scheduled_threshold,
|
||||
)
|
||||
except RunAlreadyInProgressError:
|
||||
await session.rollback()
|
||||
logger.info("scheduler.run_skipped_locked", extra={"event": "scheduler.run_skipped_locked", "user_id": candidate.user_id})
|
||||
return None
|
||||
except RunBlockedBySafetyPolicyError as exc:
|
||||
await session.rollback()
|
||||
logger.info(
|
||||
"scheduler.run_skipped_safety_cooldown",
|
||||
extra={
|
||||
"event": "scheduler.run_skipped_safety_cooldown",
|
||||
"user_id": candidate.user_id,
|
||||
"reason": exc.safety_state.get("cooldown_reason"),
|
||||
"cooldown_until": exc.safety_state.get("cooldown_until"),
|
||||
"cooldown_remaining_seconds": exc.safety_state.get("cooldown_remaining_seconds"),
|
||||
"metric_name": "scheduler_run_skipped_safety_cooldown_total",
|
||||
"metric_value": 1,
|
||||
},
|
||||
)
|
||||
return None
|
||||
except Exception:
|
||||
await session.rollback()
|
||||
logger.exception("scheduler.run_failed", extra={"event": "scheduler.run_failed", "user_id": candidate.user_id})
|
||||
return None
|
||||
|
||||
async def _run_candidate(self, candidate: _AutoRunCandidate) -> None:
|
||||
run_summary = await self._run_candidate_ingestion(candidate=candidate)
|
||||
if run_summary is None:
|
||||
return
|
||||
logger.info(
|
||||
"scheduler.run_completed",
|
||||
extra={
|
||||
"event": "scheduler.run_completed",
|
||||
"user_id": candidate.user_id,
|
||||
"run_id": run_summary.crawl_run_id,
|
||||
"status": run_summary.status.value,
|
||||
"scholar_count": run_summary.scholar_count,
|
||||
"new_publication_count": run_summary.new_publication_count,
|
||||
},
|
||||
)
|
||||
|
||||
async def _drain_continuation_queue(self) -> None:
|
||||
now = datetime.now(timezone.utc)
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as session:
|
||||
jobs = await queue_service.list_due_jobs(
|
||||
session,
|
||||
now=now,
|
||||
limit=self._queue_batch_size,
|
||||
)
|
||||
for job in jobs:
|
||||
await self._run_queue_job(job)
|
||||
|
||||
async def _drop_queue_job_if_max_attempts(
|
||||
self,
|
||||
job: queue_service.ContinuationQueueJob,
|
||||
) -> bool:
|
||||
if job.attempt_count < self._continuation_max_attempts:
|
||||
return False
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as session:
|
||||
dropped = await queue_service.mark_dropped(
|
||||
session,
|
||||
job_id=job.id,
|
||||
reason="max_attempts_reached",
|
||||
)
|
||||
await session.commit()
|
||||
if dropped is not None:
|
||||
logger.warning(
|
||||
"scheduler.queue_item_dropped_max_attempts",
|
||||
extra={
|
||||
"event": "scheduler.queue_item_dropped_max_attempts",
|
||||
"queue_item_id": job.id,
|
||||
"user_id": job.user_id,
|
||||
"scholar_profile_id": job.scholar_profile_id,
|
||||
"attempt_count": job.attempt_count,
|
||||
"max_attempts": self._continuation_max_attempts,
|
||||
},
|
||||
)
|
||||
return True
|
||||
|
||||
async def _mark_queue_job_retrying(
|
||||
self,
|
||||
job: queue_service.ContinuationQueueJob,
|
||||
) -> bool:
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as session:
|
||||
queue_item = await queue_service.mark_retrying(session, job_id=job.id)
|
||||
await session.commit()
|
||||
if queue_item is None:
|
||||
return False
|
||||
if queue_item.status == QueueItemStatus.DROPPED.value:
|
||||
return False
|
||||
return True
|
||||
|
||||
async def _queue_job_has_available_scholar(
|
||||
self,
|
||||
job: queue_service.ContinuationQueueJob,
|
||||
) -> bool:
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as session:
|
||||
scholar_result = await session.execute(
|
||||
select(ScholarProfile.id).where(
|
||||
ScholarProfile.user_id == job.user_id,
|
||||
ScholarProfile.id == job.scholar_profile_id,
|
||||
ScholarProfile.is_enabled.is_(True),
|
||||
)
|
||||
)
|
||||
scholar_id = scholar_result.scalar_one_or_none()
|
||||
if scholar_id is not None:
|
||||
return True
|
||||
dropped = await queue_service.mark_dropped(
|
||||
session,
|
||||
job_id=job.id,
|
||||
reason="scholar_unavailable",
|
||||
)
|
||||
await session.commit()
|
||||
if dropped is not None:
|
||||
logger.info(
|
||||
"scheduler.queue_item_dropped_scholar_unavailable",
|
||||
extra={
|
||||
"event": "scheduler.queue_item_dropped_scholar_unavailable",
|
||||
"queue_item_id": job.id,
|
||||
"user_id": job.user_id,
|
||||
"scholar_profile_id": job.scholar_profile_id,
|
||||
},
|
||||
)
|
||||
return False
|
||||
|
||||
async def _reschedule_queue_job_lock_active(self, job: queue_service.ContinuationQueueJob) -> None:
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as recovery_session:
|
||||
await queue_service.reschedule_job(
|
||||
recovery_session,
|
||||
job_id=job.id,
|
||||
delay_seconds=max(self._tick_seconds, 15),
|
||||
reason="user_run_lock_active",
|
||||
error="run_already_in_progress",
|
||||
)
|
||||
await recovery_session.commit()
|
||||
logger.info(
|
||||
"scheduler.queue_item_deferred_lock",
|
||||
extra={"event": "scheduler.queue_item_deferred_lock", "queue_item_id": job.id, "user_id": job.user_id},
|
||||
)
|
||||
|
||||
async def _reschedule_queue_job_safety_cooldown(
|
||||
self,
|
||||
job: queue_service.ContinuationQueueJob,
|
||||
exc: RunBlockedBySafetyPolicyError,
|
||||
) -> None:
|
||||
cooldown_remaining_seconds = max(
|
||||
self._tick_seconds,
|
||||
int(exc.safety_state.get("cooldown_remaining_seconds") or 0),
|
||||
)
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as recovery_session:
|
||||
await queue_service.reschedule_job(
|
||||
recovery_session,
|
||||
job_id=job.id,
|
||||
delay_seconds=max(self._tick_seconds, cooldown_remaining_seconds),
|
||||
reason="scrape_safety_cooldown",
|
||||
error=str(exc.message),
|
||||
)
|
||||
await recovery_session.commit()
|
||||
logger.info(
|
||||
"scheduler.queue_item_deferred_safety_cooldown",
|
||||
extra={
|
||||
"event": "scheduler.queue_item_deferred_safety_cooldown",
|
||||
"queue_item_id": job.id,
|
||||
"user_id": job.user_id,
|
||||
"reason": exc.safety_state.get("cooldown_reason"),
|
||||
"cooldown_remaining_seconds": cooldown_remaining_seconds,
|
||||
"metric_name": "scheduler_queue_item_deferred_safety_cooldown_total",
|
||||
"metric_value": 1,
|
||||
},
|
||||
)
|
||||
|
||||
async def _reschedule_queue_job_after_exception(
|
||||
self,
|
||||
job: queue_service.ContinuationQueueJob,
|
||||
*,
|
||||
exc: Exception,
|
||||
) -> None:
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as recovery_session:
|
||||
queue_item = await queue_service.increment_attempt_count(recovery_session, job_id=job.id)
|
||||
if queue_item is None:
|
||||
await recovery_session.commit()
|
||||
return
|
||||
if int(queue_item.attempt_count) >= self._continuation_max_attempts:
|
||||
await queue_service.mark_dropped(
|
||||
recovery_session,
|
||||
job_id=job.id,
|
||||
reason="scheduler_exception_max_attempts",
|
||||
error=str(exc),
|
||||
)
|
||||
await recovery_session.commit()
|
||||
logger.warning(
|
||||
"scheduler.queue_item_dropped_after_exception",
|
||||
extra={"event": "scheduler.queue_item_dropped_after_exception", "queue_item_id": job.id, "user_id": job.user_id, "attempt_count": queue_item.attempt_count},
|
||||
)
|
||||
return
|
||||
delay_seconds = queue_service.compute_backoff_seconds(
|
||||
base_seconds=self._continuation_base_delay_seconds,
|
||||
attempt_count=int(queue_item.attempt_count),
|
||||
max_seconds=self._continuation_max_delay_seconds,
|
||||
)
|
||||
await queue_service.reschedule_job(
|
||||
recovery_session,
|
||||
job_id=job.id,
|
||||
delay_seconds=delay_seconds,
|
||||
reason="scheduler_exception",
|
||||
error=str(exc),
|
||||
)
|
||||
await recovery_session.commit()
|
||||
logger.exception("scheduler.queue_item_run_failed", extra={"event": "scheduler.queue_item_run_failed", "queue_item_id": job.id, "user_id": job.user_id})
|
||||
|
||||
async def _run_ingestion_for_queue_job(
|
||||
self,
|
||||
job: queue_service.ContinuationQueueJob,
|
||||
):
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as session:
|
||||
request_delay_seconds = await self._load_request_delay_for_user(session, user_id=job.user_id)
|
||||
ingestion = ScholarIngestionService(source=self._source)
|
||||
try:
|
||||
return await ingestion.run_for_user(
|
||||
session,
|
||||
user_id=job.user_id,
|
||||
trigger_type=RunTriggerType.SCHEDULED,
|
||||
request_delay_seconds=request_delay_seconds,
|
||||
network_error_retries=self._network_error_retries,
|
||||
retry_backoff_seconds=self._retry_backoff_seconds,
|
||||
max_pages_per_scholar=self._max_pages_per_scholar,
|
||||
page_size=self._page_size,
|
||||
scholar_profile_ids={job.scholar_profile_id},
|
||||
start_cstart_by_scholar_id={job.scholar_profile_id: job.resume_cstart},
|
||||
auto_queue_continuations=self._continuation_queue_enabled,
|
||||
queue_delay_seconds=self._continuation_base_delay_seconds,
|
||||
alert_blocked_failure_threshold=settings.ingestion_alert_blocked_failure_threshold,
|
||||
alert_network_failure_threshold=settings.ingestion_alert_network_failure_threshold,
|
||||
alert_retry_scheduled_threshold=settings.ingestion_alert_retry_scheduled_threshold,
|
||||
)
|
||||
except RunAlreadyInProgressError:
|
||||
await session.rollback()
|
||||
await self._reschedule_queue_job_lock_active(job)
|
||||
except RunBlockedBySafetyPolicyError as exc:
|
||||
await session.rollback()
|
||||
await self._reschedule_queue_job_safety_cooldown(job, exc)
|
||||
except Exception as exc:
|
||||
await session.rollback()
|
||||
await self._reschedule_queue_job_after_exception(job, exc=exc)
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _log_queue_item_resolved(
|
||||
*,
|
||||
event_name: str,
|
||||
job: queue_service.ContinuationQueueJob,
|
||||
run_summary,
|
||||
attempt_count: int | None = None,
|
||||
delay_seconds: int | None = None,
|
||||
) -> None:
|
||||
payload = {
|
||||
"event": event_name,
|
||||
"queue_item_id": job.id,
|
||||
"user_id": job.user_id,
|
||||
"run_id": run_summary.crawl_run_id,
|
||||
"status": run_summary.status.value,
|
||||
}
|
||||
if attempt_count is not None:
|
||||
payload["attempt_count"] = attempt_count
|
||||
if delay_seconds is not None:
|
||||
payload["delay_seconds"] = delay_seconds
|
||||
logger.info(event_name, extra=payload)
|
||||
|
||||
async def _finalize_queue_job_after_run(self, job: queue_service.ContinuationQueueJob, run_summary) -> None:
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as session:
|
||||
if int(run_summary.failed_count) <= 0:
|
||||
queue_item = await queue_service.reset_attempt_count(session, job_id=job.id)
|
||||
await session.commit()
|
||||
if queue_item is None:
|
||||
self._log_queue_item_resolved(event_name="scheduler.queue_item_resolved", job=job, run_summary=run_summary)
|
||||
return
|
||||
self._log_queue_item_resolved(
|
||||
event_name="scheduler.queue_item_progressed",
|
||||
job=job,
|
||||
run_summary=run_summary,
|
||||
attempt_count=int(queue_item.attempt_count),
|
||||
)
|
||||
return
|
||||
queue_item = await queue_service.increment_attempt_count(session, job_id=job.id)
|
||||
if queue_item is None:
|
||||
await session.commit()
|
||||
self._log_queue_item_resolved(event_name="scheduler.queue_item_resolved", job=job, run_summary=run_summary)
|
||||
return
|
||||
if int(queue_item.attempt_count) >= self._continuation_max_attempts:
|
||||
await queue_service.mark_dropped(session, job_id=job.id, reason="max_attempts_after_run")
|
||||
await session.commit()
|
||||
logger.warning(
|
||||
"scheduler.queue_item_dropped_max_attempts_after_run",
|
||||
extra={"event": "scheduler.queue_item_dropped_max_attempts_after_run", "queue_item_id": job.id, "user_id": job.user_id, "attempt_count": queue_item.attempt_count, "run_id": run_summary.crawl_run_id, "status": run_summary.status.value},
|
||||
)
|
||||
return
|
||||
delay_seconds = queue_service.compute_backoff_seconds(base_seconds=self._continuation_base_delay_seconds, attempt_count=int(queue_item.attempt_count), max_seconds=self._continuation_max_delay_seconds)
|
||||
await queue_service.reschedule_job(session, job_id=job.id, delay_seconds=delay_seconds, reason=queue_item.reason, error=queue_item.last_error)
|
||||
await session.commit()
|
||||
self._log_queue_item_resolved(
|
||||
event_name="scheduler.queue_item_rescheduled_failed",
|
||||
job=job,
|
||||
run_summary=run_summary,
|
||||
attempt_count=int(queue_item.attempt_count),
|
||||
delay_seconds=delay_seconds,
|
||||
)
|
||||
|
||||
async def _run_queue_job(self, job: queue_service.ContinuationQueueJob) -> None:
|
||||
if await self._drop_queue_job_if_max_attempts(job):
|
||||
return
|
||||
if not await self._mark_queue_job_retrying(job):
|
||||
return
|
||||
if not await self._queue_job_has_available_scholar(job):
|
||||
return
|
||||
run_summary = await self._run_ingestion_for_queue_job(job)
|
||||
if run_summary is None:
|
||||
return
|
||||
await self._finalize_queue_job_after_run(job, run_summary)
|
||||
|
||||
async def _load_request_delay_for_user(
|
||||
self,
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
) -> int:
|
||||
result = await db_session.execute(
|
||||
select(UserSetting.request_delay_seconds).where(UserSetting.user_id == user_id)
|
||||
)
|
||||
delay = result.scalar_one_or_none()
|
||||
if delay is None:
|
||||
return 10
|
||||
return max(1, int(delay))
|
||||
108
app/services/domains/ingestion/types.py
Normal file
108
app/services/domains/ingestion/types.py
Normal file
|
|
@ -0,0 +1,108 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from app.db.models import RunStatus
|
||||
from app.services.domains.scholar.parser import ParsedProfilePage, PublicationCandidate
|
||||
from app.services.domains.scholar.source import FetchResult
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RunExecutionSummary:
|
||||
crawl_run_id: int
|
||||
status: RunStatus
|
||||
scholar_count: int
|
||||
succeeded_count: int
|
||||
failed_count: int
|
||||
partial_count: int
|
||||
new_publication_count: int
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PagedParseResult:
|
||||
fetch_result: FetchResult
|
||||
parsed_page: ParsedProfilePage
|
||||
first_page_fetch_result: FetchResult
|
||||
first_page_parsed_page: ParsedProfilePage
|
||||
first_page_fingerprint_sha256: str | None
|
||||
publications: list[PublicationCandidate]
|
||||
attempt_log: list[dict[str, Any]]
|
||||
page_logs: list[dict[str, Any]]
|
||||
pages_fetched: int
|
||||
pages_attempted: int
|
||||
has_more_remaining: bool
|
||||
pagination_truncated_reason: str | None
|
||||
continuation_cstart: int | None
|
||||
skipped_no_change: bool
|
||||
|
||||
|
||||
@dataclass
|
||||
class RunProgress:
|
||||
succeeded_count: int = 0
|
||||
failed_count: int = 0
|
||||
partial_count: int = 0
|
||||
scholar_results: list[dict[str, Any]] = field(default_factory=list)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ScholarProcessingOutcome:
|
||||
result_entry: dict[str, Any]
|
||||
succeeded_count_delta: int
|
||||
failed_count_delta: int
|
||||
partial_count_delta: int
|
||||
discovered_publication_count: int
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RunFailureSummary:
|
||||
failed_state_counts: dict[str, int]
|
||||
failed_reason_counts: dict[str, int]
|
||||
scrape_failure_counts: dict[str, int]
|
||||
retries_scheduled_count: int
|
||||
scholars_with_retries_count: int
|
||||
retry_exhausted_count: int
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RunAlertSummary:
|
||||
blocked_failure_count: int
|
||||
network_failure_count: int
|
||||
blocked_failure_threshold: int
|
||||
network_failure_threshold: int
|
||||
retry_scheduled_threshold: int
|
||||
alert_flags: dict[str, bool]
|
||||
|
||||
|
||||
@dataclass
|
||||
class PagedLoopState:
|
||||
fetch_result: FetchResult
|
||||
parsed_page: ParsedProfilePage
|
||||
attempt_log: list[dict[str, Any]]
|
||||
page_logs: list[dict[str, Any]]
|
||||
publications: list[PublicationCandidate]
|
||||
pages_fetched: int
|
||||
pages_attempted: int
|
||||
current_cstart: int
|
||||
next_cstart: int
|
||||
has_more_remaining: bool = False
|
||||
pagination_truncated_reason: str | None = None
|
||||
continuation_cstart: int | None = None
|
||||
|
||||
|
||||
class RunAlreadyInProgressError(RuntimeError):
|
||||
"""Raised when a run lock for a user is already held by another process."""
|
||||
|
||||
|
||||
class RunBlockedBySafetyPolicyError(RuntimeError):
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
code: str,
|
||||
message: str,
|
||||
safety_state: dict[str, Any],
|
||||
) -> None:
|
||||
super().__init__(message)
|
||||
self.code = code
|
||||
self.message = message
|
||||
self.safety_state = safety_state
|
||||
Loading…
Add table
Add a link
Reference in a new issue