justinzeus/scholarr
1
0
Fork 0
Code Issues 1 Pull requests 18 Projects Releases Packages Wiki Activity Actions

ci: add CodeQL security scanning and Dependabot

Browse source 
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Justin Visser 2026-02-27 00:05:17 +01:00
parent ac002131d6
commit 3866c6d6f0
90 changed files with 40 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

1
app/services/ingestion/__init__.py Normal file
View file

@ -0,0 +1 @@
from __future__ import annotations

2955
app/services/ingestion/application.py Normal file
View file

File diff suppressed because it is too large Load diff

31
app/services/ingestion/constants.py Normal file
View file

@ -0,0 +1,31 @@
from __future__ import annotations
import re
from app.services.domains.scholar.parser import ParseState
TITLE_ALNUM_RE = re.compile(r"[^a-z0-9]+")
WORD_RE = re.compile(r"[a-z0-9]+")
HTML_TAG_RE = re.compile(r"<[^>]+>", re.S)
SPACE_RE = re.compile(r"\s+")
FAILED_STATES = {
ParseState.BLOCKED_OR_CAPTCHA.value,
ParseState.LAYOUT_CHANGED.value,
ParseState.NETWORK_ERROR.value,
"ingestion_error",
}
FAILURE_BUCKET_BLOCKED = "blocked_or_captcha"
FAILURE_BUCKET_NETWORK = "network_error"
FAILURE_BUCKET_LAYOUT = "layout_changed"
FAILURE_BUCKET_INGESTION = "ingestion_error"
FAILURE_BUCKET_OTHER = "other_failure"
RUN_LOCK_NAMESPACE = 8217
RESUMABLE_PARTIAL_REASONS = {
"max_pages_reached",
"pagination_cursor_stalled",
}
RESUMABLE_PARTIAL_REASON_PREFIXES = ("page_state_network_error",)
INITIAL_PAGE_FINGERPRINT_MAX_PUBLICATIONS = 30

381
app/services/ingestion/fingerprints.py Normal file
View file

@ -0,0 +1,381 @@
from __future__ import annotations
import hashlib
import json
import re
import unicodedata
from typing import Any
from urllib.parse import urljoin
from app.services.domains.ingestion.constants import (
HTML_TAG_RE,
INITIAL_PAGE_FINGERPRINT_MAX_PUBLICATIONS,
SPACE_RE,
TITLE_ALNUM_RE,
WORD_RE,
)
from app.services.domains.scholar.parser import ParsedProfilePage, ParseState, PublicationCandidate
# Scholar-specific noise patterns stripped before canonical comparison.
# Applied in order; each targets a different Scholar metadata injection style.
_NOISE_DOI_RE = re.compile(r"[,.\s]+doi\s*:\s*\S+.*$", re.IGNORECASE)
_NOISE_ARXIV_RE = re.compile(r"[,.\s]+arxiv\b.*$", re.IGNORECASE)
_NOISE_PREPRINT_RE = re.compile(
r"[,\s]+(?:preprint|extended\s+version|technical\s+report|working\s+paper)\b.*$",
re.IGNORECASE,
)
_NOISE_TRAILING_YEAR_RE = re.compile(r"\s*[,(]\s*\d{4}\s*[),]?\s*$")
_NOISE_TRAILING_MONTH_YEAR_RE = re.compile(
r"\s*[,(]\s*(?:jan|feb|mar|apr|may|jun|jul|aug|sep|sept|oct|nov|dec)[a-z]*\.?\s+\d{4}\s*[),]?\s*$",
re.IGNORECASE,
)
_NOISE_TRAILING_PUBLICATION_TYPE_RE = re.compile(
r"[,.\s]+(?:conference\s+paper|journal\s+article)\s*$",
re.IGNORECASE,
)
_NOISE_IN_PROCEEDINGS_SUFFIX_RE = re.compile(r"\s+in:\s+proceedings\b.*$", re.IGNORECASE)
# Strips ". Capitalised sentence" appended as venue: ". Comput. Sci…", ". Journal of…"
_NOISE_VENUE_SENTENCE_RE = re.compile(r"(?<=\w{3})\.\s+[A-Z][a-z].*$")
_MOJIBAKE_HINT_RE = re.compile(r"[ÃÂâ]")
_MOJIBAKE_CHAR_RE = re.compile(r"[Ó”€™]")
_METADATA_ORDINAL_RE = re.compile(r"^\d+(st|nd|rd|th)$")
_NOISE_LEADING_DATE_PREFIX_RE = re.compile(
r"^(?:jan|feb|mar|apr|may|jun|jul|aug|sep|sept|oct|nov|dec)[a-z]*\s+\d{1,2}(?:\s*[-–]\s*\d{1,2})?\)?[,\.\s:;-]+",
re.IGNORECASE,
)
_NOISE_LEADING_AUTHOR_FRAGMENT_RE = re.compile(r"^(?:and|&)\s+[a-z.\s]{1,40}:\s*", re.IGNORECASE)
_METADATA_SEPARATORS = (" - ", " — ", ",", ";", ". ")
_VENUE_HINT_TOKENS = {
"aaai",
"conference",
"conf",
"cvpr",
"eccv",
"iclr",
"icml",
"journal",
"nips",
"neurips",
"proceedings",
"proc",
"symposium",
"workshop",
}
_PUBLICATION_TYPE_TOKENS = {"conference", "paper", "journal", "article"}
_MIN_METADATA_HINT_TOKENS = 2
_MIN_METADATA_CONTEXT_TOKENS = 4
_CANONICAL_DEDUP_THRESHOLD = 0.82
def normalize_title(value: str) -> str:
lowered = _normalized_text(value).lower()
return TITLE_ALNUM_RE.sub("", lowered)
def canonical_title_for_dedup(title: str) -> str:
"""Strip Scholar-specific noise suffixes then normalize for dedup comparison."""
return normalize_title(_canonical_title_text(title))
def canonical_title_text_for_dedup(title: str) -> str:
"""Noise-stripped lowercase title with spaces preserved for token-level matching."""
return _stripped_title_for_canonical(title)
def canonical_title_tokens_for_dedup(title: str) -> set[str]:
"""Word tokens of the noise-stripped title."""
return _canonical_title_tokens(title)
def _stripped_title_for_canonical(title: str) -> str:
"""Apply noise-stripping and lowercase but PRESERVE spaces (for later tokenization)."""
t = _canonical_title_text(title)
return t.lower().strip()
def _canonical_title_text(title: str) -> str:
t = _normalized_text(title)
t = _strip_noise_suffixes(t)
t = _strip_venue_metadata_suffixes(t)
return _NOISE_VENUE_SENTENCE_RE.sub("", t).strip()
def _strip_noise_suffixes(value: str) -> str:
t = _strip_leading_noise_prefixes(value.strip())
t = _NOISE_DOI_RE.sub("", t)
t = _NOISE_ARXIV_RE.sub("", t)
t = _NOISE_PREPRINT_RE.sub("", t)
t = _NOISE_TRAILING_YEAR_RE.sub("", t)
t = _NOISE_TRAILING_MONTH_YEAR_RE.sub("", t)
t = _NOISE_TRAILING_PUBLICATION_TYPE_RE.sub("", t)
t = _NOISE_IN_PROCEEDINGS_SUFFIX_RE.sub("", t)
return t.strip()
def _strip_venue_metadata_suffixes(value: str) -> str:
stripped = value.strip()
while True:
cut_index = _metadata_cut_index(stripped)
if cut_index is None:
return stripped
stripped = stripped[:cut_index].strip()
def _metadata_cut_index(value: str) -> int | None:
candidates: list[int] = []
for candidate in _METADATA_SEPARATORS:
start = 0
while True:
index = value.find(candidate, start)
if index <= 0:
break
suffix = value[index + len(candidate) :].strip()
if suffix and _looks_like_venue_metadata(suffix):
candidates.append(index)
start = index + len(candidate)
if not candidates:
return None
return min(candidates)
def _looks_like_venue_metadata(value: str) -> bool:
tokens = WORD_RE.findall(value.lower())
if len(tokens) < _MIN_METADATA_HINT_TOKENS:
return False
has_hint = any(_is_venue_hint_token(token) for token in tokens)
if not has_hint:
return False
has_year = any(_is_year_token(token) for token in tokens)
has_ordinal = any(_METADATA_ORDINAL_RE.match(token) for token in tokens)
publication_type_only = all(token in _PUBLICATION_TYPE_TOKENS for token in tokens)
return has_year or has_ordinal or publication_type_only or len(tokens) >= _MIN_METADATA_CONTEXT_TOKENS
def _strip_leading_noise_prefixes(value: str) -> str:
stripped = value
while True:
next_value = _NOISE_LEADING_DATE_PREFIX_RE.sub("", stripped).strip()
next_value = _NOISE_LEADING_AUTHOR_FRAGMENT_RE.sub("", next_value).strip()
if next_value == stripped:
return stripped
stripped = next_value
def _is_venue_hint_token(token: str) -> bool:
if token in _VENUE_HINT_TOKENS:
return True
return token.startswith("conf") or token.startswith("proceed")
def _is_year_token(token: str) -> bool:
if len(token) != 4 or not token.isdigit():
return False
year = int(token)
return 1900 <= year <= 2100
def _normalized_text(value: str) -> str:
repaired = _repair_mojibake(value.strip())
normalized = unicodedata.normalize("NFKC", repaired)
cleaned = _MOJIBAKE_CHAR_RE.sub(" ", normalized)
return SPACE_RE.sub(" ", cleaned).strip()
def _repair_mojibake(value: str) -> str:
if not value or not _MOJIBAKE_HINT_RE.search(value):
return value
try:
repaired = value.encode("latin1").decode("utf-8")
except UnicodeError:
return value
if _mojibake_score(repaired) < _mojibake_score(value):
return repaired
return value
def _mojibake_score(value: str) -> int:
return len(_MOJIBAKE_HINT_RE.findall(value))
def _canonical_title_tokens(title: str) -> set[str]:
"""Word tokens of the noise-stripped title (preserves token boundaries)."""
return set(WORD_RE.findall(_stripped_title_for_canonical(title)))
def _jaccard(a: set[str], b: set[str]) -> float:
if not a or not b:
return 0.0
return len(a & b) / len(a | b)
def _first_author_last_name(authors_text: str | None) -> str:
if not authors_text:
return ""
first_author = authors_text.split(",", maxsplit=1)[0].strip().lower()
words = WORD_RE.findall(first_author)
if not words:
return ""
return words[-1]
def _first_venue_word(venue_text: str | None) -> str:
if not venue_text:
return ""
words = WORD_RE.findall(venue_text.lower())
if not words:
return ""
return words[0]
def build_publication_fingerprint(candidate: PublicationCandidate) -> str:
canonical = "|".join(
[
normalize_title(candidate.title),
str(candidate.year) if candidate.year is not None else "",
_first_author_last_name(candidate.authors_text),
_first_venue_word(candidate.venue_text),
]
)
return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
def build_initial_page_fingerprint(parsed_page: ParsedProfilePage) -> str | None:
if parsed_page.state not in {ParseState.OK, ParseState.NO_RESULTS}:
return None
normalized_rows: list[dict[str, Any]] = []
for publication in parsed_page.publications[:INITIAL_PAGE_FINGERPRINT_MAX_PUBLICATIONS]:
normalized_rows.append(
{
"cluster_id": publication.cluster_id or "",
"title_normalized": normalize_title(publication.title),
"year": publication.year,
"citation_count": publication.citation_count,
}
)
payload = {
"state": parsed_page.state.value,
"articles_range": parsed_page.articles_range or "",
"has_show_more_button": parsed_page.has_show_more_button,
"profile_name": parsed_page.profile_name or "",
"publications": normalized_rows,
}
canonical = json.dumps(
payload,
sort_keys=True,
separators=(",", ":"),
ensure_ascii=True,
)
return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
def build_publication_url(path_or_url: str | None) -> str | None:
if not path_or_url:
return None
return urljoin("https://scholar.google.com", path_or_url)
def _next_cstart_value(*, articles_range: str | None, fallback: int) -> int:
if articles_range:
numbers = re.findall(r"\d+", articles_range)
if len(numbers) >= 2:
try:
return int(numbers[1])
except ValueError:
pass
return int(fallback)
def _title_tokens(value: str) -> set[str]:
"""Extract normalized word tokens for fuzzy title comparison."""
return set(WORD_RE.findall(value.lower()))
def fuzzy_titles_match(
title_a: str,
title_b: str,
*,
threshold: float = 0.85,
) -> bool:
"""Return True if two titles are near-duplicates by token-level Jaccard similarity.
A threshold of 0.85 catches common academic duplicate patterns:
differences in punctuation, minor word variations, subtitle changes.
"""
tokens_a = _title_tokens(title_a)
tokens_b = _title_tokens(title_b)
return _jaccard(tokens_a, tokens_b) >= threshold
def _dedupe_publication_candidates(
publications: list[PublicationCandidate],
*,
seen_canonical: set[str] | None = None,
) -> list[PublicationCandidate]:
"""Deduplicate candidates using canonical title matching.
Args:
publications: candidates to filter
seen_canonical: optional mutable set shared across pages. Stores the
noise-stripped *lowercased* (but space-preserved) canonical string
so it can be tokenized on the next page for cross-page fuzzy dedup.
Accepted canonicals are added; existing entries are consulted.
"""
deduped: list[PublicationCandidate] = []
seen_exact: set[str] = set()
# Token sets for fuzzy comparison; seeded from cross-page state.
seen_tokens: list[set[str]] = []
if seen_canonical:
for stripped in seen_canonical:
seen_tokens.append(set(WORD_RE.findall(stripped)))
for pub in publications:
identity = _publication_identity(pub)
if identity in seen_exact:
continue
# Use space-preserving stripped form for token-level fuzzy match.
tokens = _canonical_title_tokens(pub.title)
if _is_fuzzy_dup(tokens, seen_tokens):
continue
seen_exact.add(identity)
seen_tokens.append(tokens)
if seen_canonical is not None:
# Store the noise-stripped lowercased (space-preserved) form.
seen_canonical.add(_stripped_title_for_canonical(pub.title))
deduped.append(pub)
return deduped
def _publication_identity(pub: PublicationCandidate) -> str:
if pub.cluster_id:
return f"cluster:{pub.cluster_id}"
canonical = canonical_title_for_dedup(pub.title)
return "|".join(
[
"fallback",
canonical,
str(pub.year) if pub.year is not None else "",
_first_author_last_name(pub.authors_text),
]
)
def _is_fuzzy_dup(tokens: set[str], seen: list[set[str]]) -> bool:
return any(_jaccard(tokens, existing) >= _CANONICAL_DEDUP_THRESHOLD for existing in seen)
def _build_body_excerpt(body: str, *, max_chars: int = 220) -> str | None:
if not body:
return None
flattened = SPACE_RE.sub(" ", HTML_TAG_RE.sub(" ", body)).strip()
if not flattened:
return None
if len(flattened) <= max_chars:
return flattened
return f"{flattened[: max_chars - 1]}..."

334
app/services/ingestion/queue.py Normal file
View file

@ -0,0 +1,334 @@
from __future__ import annotations
from dataclasses import dataclass
from datetime import UTC, datetime, timedelta
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.db.models import IngestionQueueItem, QueueItemStatus
@dataclass(frozen=True)
class ContinuationQueueJob:
id: int
user_id: int
scholar_profile_id: int
resume_cstart: int
reason: str
status: str
attempt_count: int
next_attempt_dt: datetime
ACTIVE_QUEUE_STATUSES: tuple[str, ...] = (
QueueItemStatus.QUEUED.value,
QueueItemStatus.RETRYING.value,
)
def normalize_cstart(value: int | None) -> int:
if value is None:
return 0
return max(0, int(value))
def compute_backoff_seconds(*, base_seconds: int, attempt_count: int, max_seconds: int) -> int:
base = max(1, int(base_seconds))
attempts = max(1, int(attempt_count))
maximum = max(base, int(max_seconds))
seconds = base * (2 ** max(0, attempts - 1))
return min(seconds, maximum)
async def _get_item_for_user_scholar(
db_session: AsyncSession,
*,
user_id: int,
scholar_profile_id: int,
) -> IngestionQueueItem | None:
result = await db_session.execute(
select(IngestionQueueItem).where(
IngestionQueueItem.user_id == user_id,
IngestionQueueItem.scholar_profile_id == scholar_profile_id,
)
)
return result.scalar_one_or_none()
def _build_queue_item(
*,
now: datetime,
user_id: int,
scholar_profile_id: int,
normalized_cstart: int,
reason: str,
run_id: int | None,
next_attempt_dt: datetime,
) -> IngestionQueueItem:
return IngestionQueueItem(
user_id=user_id,
scholar_profile_id=scholar_profile_id,
resume_cstart=normalized_cstart,
reason=reason,
status=QueueItemStatus.QUEUED.value,
attempt_count=0,
next_attempt_dt=next_attempt_dt,
last_run_id=run_id,
last_error=None,
dropped_reason=None,
dropped_at=None,
created_at=now,
updated_at=now,
)
def _update_existing_queue_item(
*,
item: IngestionQueueItem,
now: datetime,
normalized_cstart: int,
reason: str,
run_id: int | None,
next_attempt_dt: datetime,
) -> None:
item.resume_cstart = normalized_cstart
item.reason = reason
if item.status == QueueItemStatus.DROPPED.value:
item.attempt_count = 0
item.status = QueueItemStatus.QUEUED.value
item.next_attempt_dt = next_attempt_dt
item.last_run_id = run_id
item.last_error = None
item.dropped_reason = None
item.dropped_at = None
item.updated_at = now
async def upsert_job(
db_session: AsyncSession,
*,
user_id: int,
scholar_profile_id: int,
resume_cstart: int,
reason: str,
run_id: int | None,
delay_seconds: int,
) -> IngestionQueueItem:
now = datetime.now(UTC)
next_attempt_dt = now + timedelta(seconds=max(0, int(delay_seconds)))
item = await _get_item_for_user_scholar(
db_session,
user_id=user_id,
scholar_profile_id=scholar_profile_id,
)
normalized_cstart = normalize_cstart(resume_cstart)
if item is None:
item = _build_queue_item(
now=now,
user_id=user_id,
scholar_profile_id=scholar_profile_id,
normalized_cstart=normalized_cstart,
reason=reason,
run_id=run_id,
next_attempt_dt=next_attempt_dt,
)
db_session.add(item)
return item
_update_existing_queue_item(
item=item,
now=now,
normalized_cstart=normalized_cstart,
reason=reason,
run_id=run_id,
next_attempt_dt=next_attempt_dt,
)
return item
async def clear_job_for_scholar(
db_session: AsyncSession,
*,
user_id: int,
scholar_profile_id: int,
) -> bool:
result = await db_session.execute(
select(IngestionQueueItem).where(
IngestionQueueItem.user_id == user_id,
IngestionQueueItem.scholar_profile_id == scholar_profile_id,
)
)
item = result.scalar_one_or_none()
if item is None:
return False
await db_session.delete(item)
return True
async def delete_job_by_id(
db_session: AsyncSession,
*,
job_id: int,
) -> bool:
result = await db_session.execute(select(IngestionQueueItem).where(IngestionQueueItem.id == job_id))
item = result.scalar_one_or_none()
if item is None:
return False
await db_session.delete(item)
return True
async def list_due_jobs(
db_session: AsyncSession,
*,
now: datetime,
limit: int,
) -> list[ContinuationQueueJob]:
result = await db_session.execute(
select(IngestionQueueItem)
.where(
IngestionQueueItem.next_attempt_dt <= now,
IngestionQueueItem.status.in_(ACTIVE_QUEUE_STATUSES),
)
.order_by(
IngestionQueueItem.next_attempt_dt.asc(),
IngestionQueueItem.id.asc(),
)
.limit(limit)
)
rows = list(result.scalars().all())
jobs: list[ContinuationQueueJob] = []
for row in rows:
jobs.append(
ContinuationQueueJob(
id=int(row.id),
user_id=int(row.user_id),
scholar_profile_id=int(row.scholar_profile_id),
resume_cstart=normalize_cstart(row.resume_cstart),
reason=row.reason,
status=row.status,
attempt_count=int(row.attempt_count),
next_attempt_dt=row.next_attempt_dt,
)
)
return jobs
async def increment_attempt_count(
db_session: AsyncSession,
*,
job_id: int,
) -> IngestionQueueItem | None:
now = datetime.now(UTC)
result = await db_session.execute(select(IngestionQueueItem).where(IngestionQueueItem.id == job_id))
item = result.scalar_one_or_none()
if item is None:
return None
item.attempt_count = int(item.attempt_count or 0) + 1
item.updated_at = now
return item
async def reset_attempt_count(
db_session: AsyncSession,
*,
job_id: int,
) -> IngestionQueueItem | None:
now = datetime.now(UTC)
result = await db_session.execute(select(IngestionQueueItem).where(IngestionQueueItem.id == job_id))
item = result.scalar_one_or_none()
if item is None:
return None
item.attempt_count = 0
item.updated_at = now
return item
async def reschedule_job(
db_session: AsyncSession,
*,
job_id: int,
delay_seconds: int,
reason: str,
error: str | None = None,
) -> IngestionQueueItem | None:
now = datetime.now(UTC)
result = await db_session.execute(select(IngestionQueueItem).where(IngestionQueueItem.id == job_id))
item = result.scalar_one_or_none()
if item is None:
return None
item.next_attempt_dt = now + timedelta(seconds=max(1, int(delay_seconds)))
item.status = QueueItemStatus.QUEUED.value
item.reason = reason
item.last_error = error
item.dropped_reason = None
item.dropped_at = None
item.updated_at = now
return item
async def mark_retrying(
db_session: AsyncSession,
*,
job_id: int,
reason: str | None = None,
) -> IngestionQueueItem | None:
now = datetime.now(UTC)
result = await db_session.execute(select(IngestionQueueItem).where(IngestionQueueItem.id == job_id))
item = result.scalar_one_or_none()
if item is None:
return None
if item.status == QueueItemStatus.DROPPED.value:
return item
item.status = QueueItemStatus.RETRYING.value
if reason:
item.reason = reason
item.updated_at = now
return item
async def mark_dropped(
db_session: AsyncSession,
*,
job_id: int,
reason: str,
error: str | None = None,
) -> IngestionQueueItem | None:
now = datetime.now(UTC)
result = await db_session.execute(select(IngestionQueueItem).where(IngestionQueueItem.id == job_id))
item = result.scalar_one_or_none()
if item is None:
return None
item.status = QueueItemStatus.DROPPED.value
item.reason = "dropped"
item.dropped_reason = reason
item.dropped_at = now
if error is not None:
item.last_error = error
item.updated_at = now
return item
async def mark_queued_now(
db_session: AsyncSession,
*,
job_id: int,
reason: str,
reset_attempt_count: bool = False,
) -> IngestionQueueItem | None:
now = datetime.now(UTC)
result = await db_session.execute(select(IngestionQueueItem).where(IngestionQueueItem.id == job_id))
item = result.scalar_one_or_none()
if item is None:
return None
item.status = QueueItemStatus.QUEUED.value
item.reason = reason
item.next_attempt_dt = now
if reset_attempt_count:
item.attempt_count = 0
item.last_error = None
item.dropped_reason = None
item.dropped_at = None
item.updated_at = now
return item

276
app/services/ingestion/safety.py Normal file
View file

@ -0,0 +1,276 @@
from __future__ import annotations
from datetime import UTC, datetime, timedelta
from typing import Any
from app.db.models import UserSetting
COOLDOWN_REASON_BLOCKED_FAILURE_THRESHOLD = "blocked_failure_threshold_exceeded"
COOLDOWN_REASON_NETWORK_FAILURE_THRESHOLD = "network_failure_threshold_exceeded"
_COUNTER_CONSECUTIVE_BLOCKED_RUNS = "consecutive_blocked_runs"
_COUNTER_CONSECUTIVE_NETWORK_RUNS = "consecutive_network_runs"
_COUNTER_COOLDOWN_ENTRY_COUNT = "cooldown_entry_count"
_COUNTER_BLOCKED_START_COUNT = "blocked_start_count"
_COUNTER_LAST_BLOCKED_FAILURE_COUNT = "last_blocked_failure_count"
_COUNTER_LAST_NETWORK_FAILURE_COUNT = "last_network_failure_count"
_COUNTER_LAST_EVALUATED_RUN_ID = "last_evaluated_run_id"
def _utcnow() -> datetime:
return datetime.now(UTC)
def _safe_int(value: Any, default: int = 0) -> int:
try:
return int(value)
except (TypeError, ValueError):
return default
def _safe_optional_int(value: Any) -> int | None:
if value is None:
return None
try:
return int(value)
except (TypeError, ValueError):
return None
def _normalize_datetime(value: datetime | None) -> datetime | None:
if value is None:
return None
if value.tzinfo is None:
return value.replace(tzinfo=UTC)
return value.astimezone(UTC)
def _state_dict(settings: UserSetting) -> dict[str, Any]:
state = settings.scrape_safety_state
if isinstance(state, dict):
return state
return {}
def _counters_from_state(settings: UserSetting) -> dict[str, Any]:
state = _state_dict(settings)
return {
_COUNTER_CONSECUTIVE_BLOCKED_RUNS: max(
0,
_safe_int(state.get(_COUNTER_CONSECUTIVE_BLOCKED_RUNS), 0),
),
_COUNTER_CONSECUTIVE_NETWORK_RUNS: max(
0,
_safe_int(state.get(_COUNTER_CONSECUTIVE_NETWORK_RUNS), 0),
),
_COUNTER_COOLDOWN_ENTRY_COUNT: max(
0,
_safe_int(state.get(_COUNTER_COOLDOWN_ENTRY_COUNT), 0),
),
_COUNTER_BLOCKED_START_COUNT: max(
0,
_safe_int(state.get(_COUNTER_BLOCKED_START_COUNT), 0),
),
_COUNTER_LAST_BLOCKED_FAILURE_COUNT: max(
0,
_safe_int(state.get(_COUNTER_LAST_BLOCKED_FAILURE_COUNT), 0),
),
_COUNTER_LAST_NETWORK_FAILURE_COUNT: max(
0,
_safe_int(state.get(_COUNTER_LAST_NETWORK_FAILURE_COUNT), 0),
),
_COUNTER_LAST_EVALUATED_RUN_ID: _safe_optional_int(
state.get(_COUNTER_LAST_EVALUATED_RUN_ID),
),
}
def _cooldown_reason_label(reason: str | None) -> str | None:
if reason == COOLDOWN_REASON_BLOCKED_FAILURE_THRESHOLD:
return "Blocked responses exceeded safety threshold"
if reason == COOLDOWN_REASON_NETWORK_FAILURE_THRESHOLD:
return "Network failures exceeded safety threshold"
return None
def _recommended_action(reason: str | None) -> str | None:
if reason == COOLDOWN_REASON_BLOCKED_FAILURE_THRESHOLD:
return (
"Google Scholar appears to be blocking requests. Wait for cooldown to expire, "
"increase request delay, and avoid repeated manual retries."
)
if reason == COOLDOWN_REASON_NETWORK_FAILURE_THRESHOLD:
return "Network failures crossed the threshold. Verify connectivity and retry after cooldown."
return None
def is_cooldown_active(
settings: UserSetting,
*,
now_utc: datetime | None = None,
) -> bool:
now = now_utc or _utcnow()
cooldown_until = _normalize_datetime(settings.scrape_cooldown_until)
if cooldown_until is None:
return False
return cooldown_until > now
def clear_expired_cooldown(
settings: UserSetting,
*,
now_utc: datetime | None = None,
) -> bool:
now = now_utc or _utcnow()
cooldown_until = _normalize_datetime(settings.scrape_cooldown_until)
if cooldown_until is None:
return False
if cooldown_until > now:
return False
settings.scrape_cooldown_until = None
settings.scrape_cooldown_reason = None
return True
def register_cooldown_blocked_start(
settings: UserSetting,
*,
now_utc: datetime | None = None,
) -> dict[str, Any]:
now = now_utc or _utcnow()
counters = _counters_from_state(settings)
counters[_COUNTER_BLOCKED_START_COUNT] = int(counters[_COUNTER_BLOCKED_START_COUNT]) + 1
settings.scrape_safety_state = counters
return get_safety_state_payload(settings, now_utc=now)
def _update_run_counters(
*,
counters: dict[str, Any],
run_id: int,
blocked_failure_count: int,
network_failure_count: int,
) -> tuple[int, int]:
bounded_blocked_failures = max(0, int(blocked_failure_count))
bounded_network_failures = max(0, int(network_failure_count))
counters[_COUNTER_LAST_BLOCKED_FAILURE_COUNT] = bounded_blocked_failures
counters[_COUNTER_LAST_NETWORK_FAILURE_COUNT] = bounded_network_failures
counters[_COUNTER_LAST_EVALUATED_RUN_ID] = int(run_id)
counters[_COUNTER_CONSECUTIVE_BLOCKED_RUNS] = (
int(counters[_COUNTER_CONSECUTIVE_BLOCKED_RUNS]) + 1 if bounded_blocked_failures > 0 else 0
)
counters[_COUNTER_CONSECUTIVE_NETWORK_RUNS] = (
int(counters[_COUNTER_CONSECUTIVE_NETWORK_RUNS]) + 1 if bounded_network_failures > 0 else 0
)
return bounded_blocked_failures, bounded_network_failures
def _resolve_cooldown_trigger(
*,
blocked_failures: int,
network_failures: int,
blocked_failure_threshold: int,
network_failure_threshold: int,
blocked_cooldown_seconds: int,
network_cooldown_seconds: int,
) -> tuple[str | None, int]:
if blocked_failures >= max(1, int(blocked_failure_threshold)):
return COOLDOWN_REASON_BLOCKED_FAILURE_THRESHOLD, max(60, int(blocked_cooldown_seconds))
if network_failures >= max(1, int(network_failure_threshold)):
return COOLDOWN_REASON_NETWORK_FAILURE_THRESHOLD, max(60, int(network_cooldown_seconds))
return None, 0
def _apply_cooldown_decision(
*,
settings: UserSetting,
counters: dict[str, Any],
now: datetime,
reason: str | None,
cooldown_seconds: int,
) -> None:
if reason is None:
clear_expired_cooldown(settings, now_utc=now)
return
settings.scrape_cooldown_reason = reason
settings.scrape_cooldown_until = now + timedelta(seconds=max(60, int(cooldown_seconds)))
counters[_COUNTER_COOLDOWN_ENTRY_COUNT] = int(counters[_COUNTER_COOLDOWN_ENTRY_COUNT]) + 1
def apply_run_safety_outcome(
settings: UserSetting,
*,
run_id: int,
blocked_failure_count: int,
network_failure_count: int,
blocked_failure_threshold: int,
network_failure_threshold: int,
blocked_cooldown_seconds: int,
network_cooldown_seconds: int,
now_utc: datetime | None = None,
) -> tuple[dict[str, Any], str | None]:
now = now_utc or _utcnow()
counters = _counters_from_state(settings)
blocked_failures, network_failures = _update_run_counters(
counters=counters,
run_id=run_id,
blocked_failure_count=blocked_failure_count,
network_failure_count=network_failure_count,
)
reason, cooldown_seconds = _resolve_cooldown_trigger(
blocked_failures=blocked_failures,
network_failures=network_failures,
blocked_failure_threshold=blocked_failure_threshold,
network_failure_threshold=network_failure_threshold,
blocked_cooldown_seconds=blocked_cooldown_seconds,
network_cooldown_seconds=network_cooldown_seconds,
)
_apply_cooldown_decision(
settings=settings,
counters=counters,
now=now,
reason=reason,
cooldown_seconds=cooldown_seconds,
)
settings.scrape_safety_state = counters
return get_safety_state_payload(settings, now_utc=now), reason
def get_safety_state_payload(
settings: UserSetting,
*,
now_utc: datetime | None = None,
) -> dict[str, Any]:
now = now_utc or _utcnow()
cooldown_until = _normalize_datetime(settings.scrape_cooldown_until)
cooldown_active = bool(cooldown_until is not None and cooldown_until > now)
cooldown_remaining_seconds = 0
if cooldown_active and cooldown_until is not None:
cooldown_remaining_seconds = max(0, int((cooldown_until - now).total_seconds()))
reason = settings.scrape_cooldown_reason if cooldown_active else None
return {
"cooldown_active": cooldown_active,
"cooldown_reason": reason,
"cooldown_reason_label": _cooldown_reason_label(reason),
"cooldown_until": cooldown_until,
"cooldown_remaining_seconds": cooldown_remaining_seconds,
"recommended_action": _recommended_action(reason),
"counters": _counters_from_state(settings),
}
def get_safety_event_context(
settings: UserSetting,
*,
now_utc: datetime | None = None,
) -> dict[str, Any]:
payload = get_safety_state_payload(settings, now_utc=now_utc)
return {
"cooldown_active": bool(payload.get("cooldown_active")),
"cooldown_reason": payload.get("cooldown_reason"),
"cooldown_until": payload.get("cooldown_until"),
"cooldown_remaining_seconds": int(payload.get("cooldown_remaining_seconds") or 0),
"safety_counters": payload.get("counters", {}),
}

622
app/services/ingestion/scheduler.py Normal file
View file

@ -0,0 +1,622 @@
from __future__ import annotations
import asyncio
import logging
from dataclasses import dataclass
from datetime import UTC, datetime, timedelta
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.db.models import (
CrawlRun,
QueueItemStatus,
RunTriggerType,
ScholarProfile,
User,
UserSetting,
)
from app.db.session import get_session_factory
from app.logging_utils import structured_log
from app.services.domains.ingestion import queue as queue_service
from app.services.domains.ingestion.application import (
RunAlreadyInProgressError,
RunBlockedBySafetyPolicyError,
ScholarIngestionService,
)
from app.services.domains.scholar.source import LiveScholarSource
from app.services.domains.settings import application as user_settings_service
from app.settings import settings
logger = logging.getLogger(__name__)
def _request_delay_floor_seconds() -> int:
return user_settings_service.resolve_request_delay_minimum(settings.ingestion_min_request_delay_seconds)
def _effective_request_delay_seconds(value: int | None) -> int:
floor = _request_delay_floor_seconds()
try:
parsed = int(value) if value is not None else floor
except (TypeError, ValueError):
parsed = floor
return max(floor, parsed)
@dataclass(frozen=True)
class _AutoRunCandidate:
user_id: int
run_interval_minutes: int
request_delay_seconds: int
cooldown_until: datetime | None
cooldown_reason: str | None
class SchedulerService:
def __init__(
self,
*,
enabled: bool,
tick_seconds: int,
network_error_retries: int,
retry_backoff_seconds: float,
max_pages_per_scholar: int,
page_size: int,
continuation_queue_enabled: bool,
continuation_base_delay_seconds: int,
continuation_max_delay_seconds: int,
continuation_max_attempts: int,
queue_batch_size: int,
) -> None:
self._enabled = enabled
self._tick_seconds = max(5, int(tick_seconds))
self._network_error_retries = max(0, int(network_error_retries))
self._retry_backoff_seconds = max(0.0, float(retry_backoff_seconds))
self._max_pages_per_scholar = max(1, int(max_pages_per_scholar))
self._page_size = max(1, int(page_size))
self._continuation_queue_enabled = bool(continuation_queue_enabled)
self._continuation_base_delay_seconds = max(1, int(continuation_base_delay_seconds))
self._continuation_max_delay_seconds = max(
self._continuation_base_delay_seconds,
int(continuation_max_delay_seconds),
)
self._continuation_max_attempts = max(1, int(continuation_max_attempts))
self._queue_batch_size = max(1, int(queue_batch_size))
self._task: asyncio.Task[None] | None = None
self._source = LiveScholarSource()
async def start(self) -> None:
if not self._enabled:
structured_log(logger, "info", "scheduler.disabled")
return
if self._task is not None:
return
self._task = asyncio.create_task(self._run_loop(), name="scholarr-scheduler")
structured_log(
logger,
"info",
"scheduler.started",
tick_seconds=self._tick_seconds,
network_error_retries=self._network_error_retries,
retry_backoff_seconds=self._retry_backoff_seconds,
max_pages_per_scholar=self._max_pages_per_scholar,
page_size=self._page_size,
continuation_queue_enabled=self._continuation_queue_enabled,
continuation_base_delay_seconds=self._continuation_base_delay_seconds,
continuation_max_delay_seconds=self._continuation_max_delay_seconds,
continuation_max_attempts=self._continuation_max_attempts,
queue_batch_size=self._queue_batch_size,
)
async def stop(self) -> None:
if self._task is None:
return
self._task.cancel()
try:
await self._task
except asyncio.CancelledError:
pass
finally:
self._task = None
structured_log(logger, "info", "scheduler.stopped")
async def _run_loop(self) -> None:
while True:
try:
await self._tick_once()
except asyncio.CancelledError:
raise
except Exception:
logger.exception(
"scheduler.tick_failed",
extra={},
)
await asyncio.sleep(float(self._tick_seconds))
async def _tick_once(self) -> None:
if self._continuation_queue_enabled:
await self._drain_continuation_queue()
await self._drain_pdf_queue()
candidates = await self._load_candidates()
if not candidates:
return
now = datetime.now(UTC)
for candidate in candidates:
if not await self._is_due(candidate, now=now):
continue
await self._run_candidate(candidate)
async def _load_candidate_rows(self) -> list[tuple]:
session_factory = get_session_factory()
async with session_factory() as session:
result = await session.execute(
select(
UserSetting.user_id,
UserSetting.run_interval_minutes,
UserSetting.request_delay_seconds,
UserSetting.scrape_cooldown_until,
UserSetting.scrape_cooldown_reason,
)
.join(User, User.id == UserSetting.user_id)
.where(User.is_active.is_(True), UserSetting.auto_run_enabled.is_(True))
.order_by(UserSetting.user_id.asc())
)
return result.all()
@staticmethod
def _candidate_from_row(row: tuple, *, now_utc: datetime) -> _AutoRunCandidate | None:
user_id, run_interval_minutes, request_delay_seconds, cooldown_until, cooldown_reason = row
if cooldown_until is not None and cooldown_until.tzinfo is None:
cooldown_until = cooldown_until.replace(tzinfo=UTC)
if cooldown_until is not None and cooldown_until > now_utc:
structured_log(
logger,
"info",
"scheduler.run_skipped_safety_cooldown_precheck",
user_id=int(user_id),
reason=cooldown_reason,
cooldown_until=cooldown_until,
cooldown_remaining_seconds=int((cooldown_until - now_utc).total_seconds()),
)
return None
return _AutoRunCandidate(
user_id=int(user_id),
run_interval_minutes=int(run_interval_minutes),
request_delay_seconds=_effective_request_delay_seconds(request_delay_seconds),
cooldown_until=cooldown_until,
cooldown_reason=(str(cooldown_reason).strip() if cooldown_reason else None),
)
async def _load_candidates(self) -> list[_AutoRunCandidate]:
if not settings.ingestion_automation_allowed:
return []
rows = await self._load_candidate_rows()
now_utc = datetime.now(UTC)
candidates: list[_AutoRunCandidate] = []
for row in rows:
candidate = self._candidate_from_row(row, now_utc=now_utc)
if candidate is not None:
candidates.append(candidate)
return candidates
async def _is_due(self, candidate: _AutoRunCandidate, *, now: datetime) -> bool:
session_factory = get_session_factory()
async with session_factory() as session:
result = await session.execute(
select(CrawlRun.start_dt)
.where(
CrawlRun.user_id == candidate.user_id,
)
.order_by(CrawlRun.start_dt.desc(), CrawlRun.id.desc())
.limit(1)
)
last_run = result.scalar_one_or_none()
if last_run is None:
return True
next_due_dt = last_run + timedelta(minutes=candidate.run_interval_minutes)
return now >= next_due_dt
async def _run_candidate_ingestion(
self,
*,
candidate: _AutoRunCandidate,
):
session_factory = get_session_factory()
async with session_factory() as session:
ingestion = ScholarIngestionService(source=self._source)
try:
return await ingestion.run_for_user(
session,
user_id=candidate.user_id,
trigger_type=RunTriggerType.SCHEDULED,
request_delay_seconds=candidate.request_delay_seconds,
network_error_retries=self._network_error_retries,
retry_backoff_seconds=self._retry_backoff_seconds,
max_pages_per_scholar=self._max_pages_per_scholar,
page_size=self._page_size,
auto_queue_continuations=self._continuation_queue_enabled,
queue_delay_seconds=self._continuation_base_delay_seconds,
alert_blocked_failure_threshold=settings.ingestion_alert_blocked_failure_threshold,
alert_network_failure_threshold=settings.ingestion_alert_network_failure_threshold,
alert_retry_scheduled_threshold=settings.ingestion_alert_retry_scheduled_threshold,
)
except RunAlreadyInProgressError:
await session.rollback()
structured_log(logger, "info", "scheduler.run_skipped_locked", user_id=candidate.user_id)
return None
except RunBlockedBySafetyPolicyError as exc:
await session.rollback()
structured_log(
logger,
"info",
"scheduler.run_skipped_safety_cooldown",
user_id=candidate.user_id,
reason=exc.safety_state.get("cooldown_reason"),
cooldown_until=exc.safety_state.get("cooldown_until"),
cooldown_remaining_seconds=exc.safety_state.get("cooldown_remaining_seconds"),
)
return None
except Exception:
await session.rollback()
logger.exception("scheduler.run_failed", extra={"user_id": candidate.user_id})
return None
async def _run_candidate(self, candidate: _AutoRunCandidate) -> None:
run_summary = await self._run_candidate_ingestion(candidate=candidate)
if run_summary is None:
return
structured_log(
logger,
"info",
"scheduler.run_completed",
user_id=candidate.user_id,
run_id=run_summary.crawl_run_id,
status=run_summary.status.value,
scholar_count=run_summary.scholar_count,
new_publication_count=run_summary.new_publication_count,
)
async def _drain_continuation_queue(self) -> None:
now = datetime.now(UTC)
session_factory = get_session_factory()
async with session_factory() as session:
jobs = await queue_service.list_due_jobs(
session,
now=now,
limit=self._queue_batch_size,
)
for job in jobs:
await self._run_queue_job(job)
async def _drop_queue_job_if_max_attempts(
self,
job: queue_service.ContinuationQueueJob,
) -> bool:
if job.attempt_count < self._continuation_max_attempts:
return False
session_factory = get_session_factory()
async with session_factory() as session:
dropped = await queue_service.mark_dropped(
session,
job_id=job.id,
reason="max_attempts_reached",
)
await session.commit()
if dropped is not None:
structured_log(
logger,
"warning",
"scheduler.queue_item_dropped_max_attempts",
queue_item_id=job.id,
user_id=job.user_id,
scholar_profile_id=job.scholar_profile_id,
attempt_count=job.attempt_count,
max_attempts=self._continuation_max_attempts,
)
return True
async def _mark_queue_job_retrying(
self,
job: queue_service.ContinuationQueueJob,
) -> bool:
session_factory = get_session_factory()
async with session_factory() as session:
queue_item = await queue_service.mark_retrying(session, job_id=job.id)
await session.commit()
if queue_item is None:
return False
return queue_item.status != QueueItemStatus.DROPPED.value
async def _queue_job_has_available_scholar(
self,
job: queue_service.ContinuationQueueJob,
) -> bool:
session_factory = get_session_factory()
async with session_factory() as session:
scholar_result = await session.execute(
select(ScholarProfile.id).where(
ScholarProfile.user_id == job.user_id,
ScholarProfile.id == job.scholar_profile_id,
ScholarProfile.is_enabled.is_(True),
)
)
scholar_id = scholar_result.scalar_one_or_none()
if scholar_id is not None:
return True
dropped = await queue_service.mark_dropped(
session,
job_id=job.id,
reason="scholar_unavailable",
)
await session.commit()
if dropped is not None:
structured_log(
logger,
"info",
"scheduler.queue_item_dropped_scholar_unavailable",
queue_item_id=job.id,
user_id=job.user_id,
scholar_profile_id=job.scholar_profile_id,
)
return False
async def _reschedule_queue_job_lock_active(self, job: queue_service.ContinuationQueueJob) -> None:
session_factory = get_session_factory()
async with session_factory() as recovery_session:
await queue_service.reschedule_job(
recovery_session,
job_id=job.id,
delay_seconds=max(self._tick_seconds, 15),
reason="user_run_lock_active",
error="run_already_in_progress",
)
await recovery_session.commit()
structured_log(
logger,
"info",
"scheduler.queue_item_deferred_lock",
queue_item_id=job.id,
user_id=job.user_id,
)
async def _reschedule_queue_job_safety_cooldown(
self,
job: queue_service.ContinuationQueueJob,
exc: RunBlockedBySafetyPolicyError,
) -> None:
cooldown_remaining_seconds = max(
self._tick_seconds,
int(exc.safety_state.get("cooldown_remaining_seconds") or 0),
)
session_factory = get_session_factory()
async with session_factory() as recovery_session:
await queue_service.reschedule_job(
recovery_session,
job_id=job.id,
delay_seconds=max(self._tick_seconds, cooldown_remaining_seconds),
reason="scrape_safety_cooldown",
error=str(exc.message),
)
await recovery_session.commit()
structured_log(
logger,
"info",
"scheduler.queue_item_deferred_safety_cooldown",
queue_item_id=job.id,
user_id=job.user_id,
reason=exc.safety_state.get("cooldown_reason"),
cooldown_remaining_seconds=cooldown_remaining_seconds,
)
async def _reschedule_queue_job_after_exception(
self,
job: queue_service.ContinuationQueueJob,
*,
exc: Exception,
) -> None:
session_factory = get_session_factory()
async with session_factory() as recovery_session:
queue_item = await queue_service.increment_attempt_count(recovery_session, job_id=job.id)
if queue_item is None:
await recovery_session.commit()
return
if int(queue_item.attempt_count) >= self._continuation_max_attempts:
await queue_service.mark_dropped(
recovery_session,
job_id=job.id,
reason="scheduler_exception_max_attempts",
error=str(exc),
)
await recovery_session.commit()
structured_log(
logger,
"warning",
"scheduler.queue_item_dropped_after_exception",
queue_item_id=job.id,
user_id=job.user_id,
attempt_count=queue_item.attempt_count,
)
return
delay_seconds = queue_service.compute_backoff_seconds(
base_seconds=self._continuation_base_delay_seconds,
attempt_count=int(queue_item.attempt_count),
max_seconds=self._continuation_max_delay_seconds,
)
await queue_service.reschedule_job(
recovery_session,
job_id=job.id,
delay_seconds=delay_seconds,
reason="scheduler_exception",
error=str(exc),
)
await recovery_session.commit()
logger.exception("scheduler.queue_item_run_failed", extra={"queue_item_id": job.id, "user_id": job.user_id})
async def _run_ingestion_for_queue_job(
self,
job: queue_service.ContinuationQueueJob,
):
session_factory = get_session_factory()
async with session_factory() as session:
request_delay_seconds = await self._load_request_delay_for_user(session, user_id=job.user_id)
ingestion = ScholarIngestionService(source=self._source)
try:
return await ingestion.run_for_user(
session,
user_id=job.user_id,
trigger_type=RunTriggerType.SCHEDULED,
request_delay_seconds=request_delay_seconds,
network_error_retries=self._network_error_retries,
retry_backoff_seconds=self._retry_backoff_seconds,
rate_limit_retries=settings.ingestion_rate_limit_retries,
rate_limit_backoff_seconds=settings.ingestion_rate_limit_backoff_seconds,
max_pages_per_scholar=self._max_pages_per_scholar,
page_size=self._page_size,
scholar_profile_ids={job.scholar_profile_id},
start_cstart_by_scholar_id={job.scholar_profile_id: job.resume_cstart},
auto_queue_continuations=self._continuation_queue_enabled,
queue_delay_seconds=self._continuation_base_delay_seconds,
alert_blocked_failure_threshold=settings.ingestion_alert_blocked_failure_threshold,
alert_network_failure_threshold=settings.ingestion_alert_network_failure_threshold,
alert_retry_scheduled_threshold=settings.ingestion_alert_retry_scheduled_threshold,
)
except RunAlreadyInProgressError:
await session.rollback()
await self._reschedule_queue_job_lock_active(job)
except RunBlockedBySafetyPolicyError as exc:
await session.rollback()
await self._reschedule_queue_job_safety_cooldown(job, exc)
except Exception as exc:
await session.rollback()
await self._reschedule_queue_job_after_exception(job, exc=exc)
return None
async def _finalize_queue_job_after_run(self, job: queue_service.ContinuationQueueJob, run_summary) -> None:
session_factory = get_session_factory()
async with session_factory() as session:
if int(run_summary.failed_count) <= 0:
queue_item = await queue_service.reset_attempt_count(session, job_id=job.id)
await session.commit()
if queue_item is None:
structured_log(
logger,
"info",
"scheduler.queue_item_resolved",
queue_item_id=job.id,
user_id=job.user_id,
run_id=run_summary.crawl_run_id,
status=run_summary.status.value,
)
return
structured_log(
logger,
"info",
"scheduler.queue_item_progressed",
queue_item_id=job.id,
user_id=job.user_id,
run_id=run_summary.crawl_run_id,
status=run_summary.status.value,
attempt_count=int(queue_item.attempt_count),
)
return
queue_item = await queue_service.increment_attempt_count(session, job_id=job.id)
if queue_item is None:
await session.commit()
structured_log(
logger,
"info",
"scheduler.queue_item_resolved",
queue_item_id=job.id,
user_id=job.user_id,
run_id=run_summary.crawl_run_id,
status=run_summary.status.value,
)
return
if int(queue_item.attempt_count) >= self._continuation_max_attempts:
await queue_service.mark_dropped(session, job_id=job.id, reason="max_attempts_after_run")
await session.commit()
structured_log(
logger,
"warning",
"scheduler.queue_item_dropped_max_attempts_after_run",
queue_item_id=job.id,
user_id=job.user_id,
attempt_count=queue_item.attempt_count,
run_id=run_summary.crawl_run_id,
status=run_summary.status.value,
)
return
delay_seconds = queue_service.compute_backoff_seconds(
base_seconds=self._continuation_base_delay_seconds,
attempt_count=int(queue_item.attempt_count),
max_seconds=self._continuation_max_delay_seconds,
)
await queue_service.reschedule_job(
session,
job_id=job.id,
delay_seconds=delay_seconds,
reason=queue_item.reason,
error=queue_item.last_error,
)
await session.commit()
structured_log(
logger,
"info",
"scheduler.queue_item_rescheduled_failed",
queue_item_id=job.id,
user_id=job.user_id,
run_id=run_summary.crawl_run_id,
status=run_summary.status.value,
attempt_count=int(queue_item.attempt_count),
delay_seconds=delay_seconds,
)
async def _run_queue_job(self, job: queue_service.ContinuationQueueJob) -> None:
if await self._drop_queue_job_if_max_attempts(job):
return
if not await self._mark_queue_job_retrying(job):
return
if not await self._queue_job_has_available_scholar(job):
return
run_summary = await self._run_ingestion_for_queue_job(job)
if run_summary is None:
return
await self._finalize_queue_job_after_run(job, run_summary)
async def _drain_pdf_queue(self) -> None:
from app.services.domains.publications.pdf_queue import drain_ready_jobs
session_factory = get_session_factory()
async with session_factory() as session:
try:
processed = await drain_ready_jobs(
session,
limit=settings.scheduler_pdf_queue_batch_size,
max_attempts=settings.pdf_auto_retry_max_attempts,
)
if processed > 0:
structured_log(
logger,
"info",
"scheduler.pdf_queue_drain_completed",
processed_count=processed,
)
except Exception:
logger.exception("scheduler.pdf_queue_drain_failed", extra={})
async def _load_request_delay_for_user(
self,
db_session: AsyncSession,
*,
user_id: int,
) -> int:
result = await db_session.execute(
select(UserSetting.request_delay_seconds).where(UserSetting.user_id == user_id)
)
delay = result.scalar_one_or_none()
return _effective_request_delay_seconds(delay)

110
app/services/ingestion/types.py Normal file
View file

@ -0,0 +1,110 @@
from __future__ import annotations
from dataclasses import dataclass, field
from typing import Any
from app.db.models import RunStatus
from app.services.domains.scholar.parser import ParsedProfilePage, PublicationCandidate
from app.services.domains.scholar.source import FetchResult
@dataclass(frozen=True)
class RunExecutionSummary:
crawl_run_id: int
status: RunStatus
scholar_count: int
succeeded_count: int
failed_count: int
partial_count: int
new_publication_count: int
@dataclass(frozen=True)
class PagedParseResult:
fetch_result: FetchResult
parsed_page: ParsedProfilePage
first_page_fetch_result: FetchResult
first_page_parsed_page: ParsedProfilePage
first_page_fingerprint_sha256: str | None
publications: list[PublicationCandidate]
attempt_log: list[dict[str, Any]]
page_logs: list[dict[str, Any]]
pages_fetched: int
pages_attempted: int
has_more_remaining: bool
pagination_truncated_reason: str | None
continuation_cstart: int | None
skipped_no_change: bool
discovered_publication_count: int
@dataclass
class RunProgress:
succeeded_count: int = 0
failed_count: int = 0
partial_count: int = 0
scholar_results: list[dict[str, Any]] = field(default_factory=list)
@dataclass(frozen=True)
class ScholarProcessingOutcome:
result_entry: dict[str, Any]
succeeded_count_delta: int
failed_count_delta: int
partial_count_delta: int
discovered_publication_count: int
@dataclass(frozen=True)
class RunFailureSummary:
failed_state_counts: dict[str, int]
failed_reason_counts: dict[str, int]
scrape_failure_counts: dict[str, int]
retries_scheduled_count: int
scholars_with_retries_count: int
retry_exhausted_count: int
@dataclass(frozen=True)
class RunAlertSummary:
blocked_failure_count: int
network_failure_count: int
blocked_failure_threshold: int
network_failure_threshold: int
retry_scheduled_threshold: int
alert_flags: dict[str, bool]
@dataclass
class PagedLoopState:
fetch_result: FetchResult
parsed_page: ParsedProfilePage
attempt_log: list[dict[str, Any]]
page_logs: list[dict[str, Any]]
publications: list[PublicationCandidate]
pages_fetched: int
pages_attempted: int
current_cstart: int
next_cstart: int
has_more_remaining: bool = False
pagination_truncated_reason: str | None = None
continuation_cstart: int | None = None
discovered_publication_count: int = 0
class RunAlreadyInProgressError(RuntimeError):
"""Raised when a run lock for a user is already held by another process."""
class RunBlockedBySafetyPolicyError(RuntimeError):
def __init__(
self,
*,
code: str,
message: str,
safety_state: dict[str, Any],
) -> None:
super().__init__(message)
self.code = code
self.message = message
self.safety_state = safety_state