justinzeus/scholarr
1
0
Fork 0
Code Issues 1 Pull requests 18 Projects Releases Packages Wiki Activity Actions

refactor: extract pagination engine and publication upsert from ingestion service

Browse source 
Split app/services/ingestion/application.py (2,955 lines) into three
focused modules:

- page_fetch.py (219 lines): PageFetcher class for single-page fetch,
  parse-or-layout-error handling, and retry with backoff
- pagination.py (486 lines): PaginationEngine class for multi-page
  orchestration, loop state management, and short-circuit detection
- publication_upsert.py (239 lines): module-level functions for
  resolve_publication, upsert_profile_publications, and all
  find/create/update helpers

Also fixes two external import paths in portability/ that incorrectly
sourced normalize_title and build_publication_url from application.py
instead of fingerprints.py.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Justin Visser 2026-02-27 13:47:23 +01:00
parent 4741a2da50
commit 9a7fa5004e
8 changed files with 1169 additions and 885 deletions
Download patch file Download diff file Expand all files Collapse all files

889
app/services/ingestion/application.py
View file

@ -21,7 +21,6 @@ from app.db.models import (
)
from app.logging_utils import structured_log
from app.services.arxiv.errors import ArxivRateLimitError
from app.services.doi.normalize import first_doi_from_texts
from app.services.ingestion import queue as queue_service
from app.services.ingestion import safety as run_safety_service
from app.services.ingestion.constants import (
@ -37,16 +36,10 @@ from app.services.ingestion.constants import (
)
from app.services.ingestion.fingerprints import (
_build_body_excerpt,
_dedupe_publication_candidates,
_next_cstart_value,
build_initial_page_fingerprint,
build_publication_fingerprint,
build_publication_url,
canonical_title_for_dedup,
normalize_title,
)
from app.services.ingestion.pagination import PaginationEngine
from app.services.ingestion.publication_upsert import upsert_profile_publications
from app.services.ingestion.types import (
PagedLoopState,
PagedParseResult,
RunAlertSummary,
RunAlreadyInProgressError,
@ -62,8 +55,6 @@ from app.services.scholar.parser import (
ParsedProfilePage,
ParseState,
PublicationCandidate,
ScholarParserError,
parse_profile_page,
)
from app.services.scholar.source import FetchResult, ScholarSource
from app.services.settings import application as user_settings_service
@ -115,6 +106,7 @@ _background_tasks: set[asyncio.Task[Any]] = set()
class ScholarIngestionService:
def __init__(self, *, source: ScholarSource) -> None:
self._source = source
self._pagination = PaginationEngine(source=source)
@staticmethod
def _effective_request_delay_seconds(value: int) -> int:
@ -290,7 +282,9 @@ class ScholarIngestionService:
paged_parse_result: PagedParseResult,
) -> None:
parsed_page = paged_parse_result.parsed_page
if parsed_page.state in {ParseState.OK, ParseState.NO_RESULTS} and any(code.startswith("layout_") for code in parsed_page.warnings):
if parsed_page.state in {ParseState.OK, ParseState.NO_RESULTS} and any(
code.startswith("layout_") for code in parsed_page.warnings
):
raise RuntimeError(f"Layout warning marked as terminal for scholar_id={scholar_id}.")
for publication in paged_parse_result.publications:
if not publication.title.strip():
@ -445,7 +439,7 @@ class ScholarIngestionService:
has_partial_publication_set: bool,
) -> ScholarProcessingOutcome:
# We no longer aggregate and upsert the publications here.
# Eager DB insertions have already saved and committed them inside `_fetch_and_parse_all_pages_with_retry` and `_paginate_loop`.
# Eager DB insertions have already saved and committed them inside `fetch_and_parse_all_pages` and `_paginate_loop`.
discovered_count = paged_parse_result.discovered_publication_count
is_partial = (
paged_parse_result.has_more_remaining
@ -683,7 +677,7 @@ class ScholarIngestionService:
page_size: int,
) -> tuple[datetime, PagedParseResult, dict[str, Any]]:
run_dt = datetime.now(UTC)
paged_parse_result = await self._fetch_and_parse_all_pages_with_retry(
paged_parse_result = await self._pagination.fetch_and_parse_all_pages(
scholar=scholar,
run=run,
db_session=db_session,
@ -696,6 +690,7 @@ class ScholarIngestionService:
max_pages=max_pages_per_scholar,
page_size=page_size,
previous_initial_page_fingerprint_sha256=scholar.last_initial_page_fingerprint_sha256,
upsert_publications_fn=upsert_profile_publications,
)
self._assert_valid_paged_parse_result(scholar_id=scholar.scholar_id, paged_parse_result=paged_parse_result)
self._apply_first_page_profile_metadata(scholar=scholar, paged_parse_result=paged_parse_result, run_dt=run_dt)
@ -1701,650 +1696,6 @@ class ScholarIngestionService:
)
return self._run_execution_summary(run=run, scholars=scholars, progress=progress)
async def _fetch_profile_page(
self,
*,
scholar_id: str,
cstart: int,
page_size: int,
) -> FetchResult:
try:
page_fetcher = getattr(self._source, "fetch_profile_page_html", None)
if callable(page_fetcher):
return await page_fetcher(
scholar_id,
cstart=cstart,
pagesize=page_size,
)
if cstart <= 0:
return await self._source.fetch_profile_html(scholar_id)
return FetchResult(
requested_url=(
f"https://scholar.google.com/citations?hl=en&user={scholar_id}&cstart={cstart}&pagesize={page_size}"
),
status_code=None,
final_url=None,
body="",
error="source_does_not_support_pagination",
)
except Exception as exc:
logger.exception(
"ingestion.fetch_unexpected_error",
extra={
"scholar_id": scholar_id,
"cstart": cstart,
"page_size": page_size,
},
)
return FetchResult(
requested_url=(
f"https://scholar.google.com/citations?hl=en&user={scholar_id}&cstart={cstart}&pagesize={page_size}"
),
status_code=None,
final_url=None,
body="",
error=str(exc),
)
@staticmethod
def _should_retry_page_fetch(
*,
parsed_page: ParsedProfilePage,
network_attempt_count: int,
rate_limit_attempt_count: int,
network_error_retries: int,
rate_limit_retries: int,
) -> bool:
if parsed_page.state == ParseState.NETWORK_ERROR:
return network_attempt_count <= network_error_retries
if (
parsed_page.state == ParseState.BLOCKED_OR_CAPTCHA
and parsed_page.state_reason == "blocked_http_429_rate_limited"
):
return rate_limit_attempt_count <= rate_limit_retries
return False
@staticmethod
async def _sleep_retry_backoff(
*,
scholar_id: str,
cstart: int,
network_attempt_count: int,
rate_limit_attempt_count: int,
retry_backoff_seconds: float,
rate_limit_backoff_seconds: float,
state_reason: str,
) -> None:
if state_reason == "blocked_http_429_rate_limited":
sleep_seconds = rate_limit_backoff_seconds * rate_limit_attempt_count
attempt_label = rate_limit_attempt_count
else:
sleep_seconds = retry_backoff_seconds * (2 ** (network_attempt_count - 1))
attempt_label = network_attempt_count
structured_log(
logger,
"warning",
"ingestion.scholar_retry_scheduled",
scholar_id=scholar_id,
cstart=cstart,
attempt_count=attempt_label,
sleep_seconds=sleep_seconds,
state_reason=state_reason,
)
if sleep_seconds > 0:
await asyncio.sleep(sleep_seconds)
async def _fetch_and_parse_page_with_retry(
self,
*,
scholar_id: str,
cstart: int,
page_size: int,
network_error_retries: int,
retry_backoff_seconds: float,
rate_limit_retries: int,
rate_limit_backoff_seconds: float,
) -> tuple[FetchResult, ParsedProfilePage, list[dict[str, Any]]]:
network_attempts = 0
rate_limit_attempts = 0
attempt_log: list[dict[str, Any]] = []
fetch_result: FetchResult | None = None
parsed_page: ParsedProfilePage | None = None
while True:
fetch_result = await self._fetch_profile_page(
scholar_id=scholar_id,
cstart=cstart,
page_size=page_size,
)
parsed_page = self._parse_profile_page_or_layout_error(fetch_result=fetch_result)
if parsed_page.state == ParseState.NETWORK_ERROR:
network_attempts += 1
total_attempts = network_attempts
elif (
parsed_page.state == ParseState.BLOCKED_OR_CAPTCHA
and parsed_page.state_reason == "blocked_http_429_rate_limited"
):
rate_limit_attempts += 1
total_attempts = rate_limit_attempts
else:
total_attempts = network_attempts + rate_limit_attempts + 1
attempt_log.append(
{
"attempt": total_attempts,
"cstart": cstart,
"state": parsed_page.state.value,
"state_reason": parsed_page.state_reason,
"status_code": fetch_result.status_code,
"fetch_error": fetch_result.error,
}
)
if not self._should_retry_page_fetch(
parsed_page=parsed_page,
network_attempt_count=network_attempts,
rate_limit_attempt_count=rate_limit_attempts,
network_error_retries=network_error_retries,
rate_limit_retries=rate_limit_retries,
):
break
await self._sleep_retry_backoff(
scholar_id=scholar_id,
cstart=cstart,
network_attempt_count=network_attempts,
rate_limit_attempt_count=rate_limit_attempts,
retry_backoff_seconds=max(float(retry_backoff_seconds), 0.0),
rate_limit_backoff_seconds=max(float(rate_limit_backoff_seconds), 0.0),
state_reason=parsed_page.state_reason,
)
if fetch_result is None or parsed_page is None:
raise RuntimeError("Fetch-and-parse retry loop produced no result.")
return fetch_result, parsed_page, attempt_log
def _parse_profile_page_or_layout_error(
self,
*,
fetch_result: FetchResult,
) -> ParsedProfilePage:
try:
return parse_profile_page(fetch_result)
except ScholarParserError as exc:
return self._parsed_page_from_parser_error(
fetch_result=fetch_result,
code=exc.code,
)
@staticmethod
def _parsed_page_from_parser_error(
*,
fetch_result: FetchResult,
code: str,
) -> ParsedProfilePage:
return ParsedProfilePage(
state=ParseState.LAYOUT_CHANGED,
state_reason=code,
profile_name=None,
profile_image_url=None,
publications=[],
marker_counts={},
warnings=[code],
has_show_more_button=False,
has_operation_error_banner=False,
articles_range=None,
)
@staticmethod
def _should_skip_no_change(
*,
start_cstart: int,
first_page_fingerprint_sha256: str | None,
previous_initial_page_fingerprint_sha256: str | None,
parsed_page: ParsedProfilePage,
) -> bool:
return (
start_cstart <= 0
and first_page_fingerprint_sha256 is not None
and previous_initial_page_fingerprint_sha256 is not None
and first_page_fingerprint_sha256 == previous_initial_page_fingerprint_sha256
and parsed_page.state in {ParseState.OK, ParseState.NO_RESULTS}
)
@staticmethod
def _skip_no_change_result(
*,
fetch_result: FetchResult,
parsed_page: ParsedProfilePage,
first_page_fingerprint_sha256: str | None,
attempt_log: list[dict[str, Any]],
page_logs: list[dict[str, Any]],
) -> PagedParseResult:
return PagedParseResult(
fetch_result=fetch_result,
parsed_page=parsed_page,
first_page_fetch_result=fetch_result,
first_page_parsed_page=parsed_page,
first_page_fingerprint_sha256=first_page_fingerprint_sha256,
publications=[],
attempt_log=attempt_log,
page_logs=page_logs,
pages_fetched=1,
pages_attempted=1,
has_more_remaining=False,
pagination_truncated_reason=None,
continuation_cstart=None,
skipped_no_change=True,
discovered_publication_count=0,
)
@staticmethod
def _initial_failure_result(
*,
fetch_result: FetchResult,
parsed_page: ParsedProfilePage,
first_page_fingerprint_sha256: str | None,
start_cstart: int,
attempt_log: list[dict[str, Any]],
page_logs: list[dict[str, Any]],
) -> PagedParseResult:
continuation_cstart = start_cstart if parsed_page.state == ParseState.NETWORK_ERROR else None
return PagedParseResult(
fetch_result=fetch_result,
parsed_page=parsed_page,
first_page_fetch_result=fetch_result,
first_page_parsed_page=parsed_page,
first_page_fingerprint_sha256=first_page_fingerprint_sha256,
publications=[],
attempt_log=attempt_log,
page_logs=page_logs,
pages_fetched=0,
pages_attempted=1,
has_more_remaining=False,
pagination_truncated_reason=None,
continuation_cstart=continuation_cstart,
skipped_no_change=False,
discovered_publication_count=0,
)
@staticmethod
def _build_loop_state(
*,
start_cstart: int,
fetch_result: FetchResult,
parsed_page: ParsedProfilePage,
attempt_log: list[dict[str, Any]],
page_logs: list[dict[str, Any]],
) -> PagedLoopState:
next_cstart = _next_cstart_value(
articles_range=parsed_page.articles_range,
fallback=start_cstart + len(parsed_page.publications),
)
return PagedLoopState(
fetch_result=fetch_result,
parsed_page=parsed_page,
attempt_log=attempt_log,
page_logs=page_logs,
publications=list(parsed_page.publications),
pages_fetched=1,
pages_attempted=1,
current_cstart=start_cstart,
next_cstart=next_cstart,
)
@staticmethod
def _set_truncated_state(
*,
state: PagedLoopState,
reason: str,
continuation_cstart: int,
) -> None:
state.has_more_remaining = True
state.pagination_truncated_reason = reason
state.continuation_cstart = continuation_cstart
def _should_stop_pagination(self, *, state: PagedLoopState, bounded_max_pages: int) -> bool:
if state.pages_fetched >= bounded_max_pages:
self._set_truncated_state(
state=state,
reason="max_pages_reached",
continuation_cstart=(
state.next_cstart if state.next_cstart > state.current_cstart else state.current_cstart
),
)
return True
if state.next_cstart <= state.current_cstart:
self._set_truncated_state(
state=state,
reason="pagination_cursor_stalled",
continuation_cstart=state.current_cstart,
)
return True
return False
async def _fetch_next_page(
self,
*,
scholar_id: str,
state: PagedLoopState,
request_delay_seconds: int,
bounded_page_size: int,
network_error_retries: int,
retry_backoff_seconds: float,
rate_limit_retries: int,
rate_limit_backoff_seconds: float,
) -> tuple[FetchResult, ParsedProfilePage, list[dict[str, Any]]]:
if request_delay_seconds > 0:
await asyncio.sleep(float(request_delay_seconds))
state.current_cstart = state.next_cstart
return await self._fetch_and_parse_page_with_retry(
scholar_id=scholar_id,
cstart=state.current_cstart,
page_size=bounded_page_size,
network_error_retries=network_error_retries,
retry_backoff_seconds=retry_backoff_seconds,
rate_limit_retries=rate_limit_retries,
rate_limit_backoff_seconds=rate_limit_backoff_seconds,
)
@staticmethod
def _record_next_page(
*,
state: PagedLoopState,
fetch_result: FetchResult,
parsed_page: ParsedProfilePage,
page_attempt_log: list[dict[str, Any]],
) -> None:
state.pages_attempted += 1
state.attempt_log.extend(page_attempt_log)
state.page_logs.append(
{
"page": state.pages_attempted,
"cstart": state.current_cstart,
"state": parsed_page.state.value,
"state_reason": parsed_page.state_reason,
"status_code": fetch_result.status_code,
"publication_count": len(parsed_page.publications),
"articles_range": parsed_page.articles_range,
"has_show_more_button": parsed_page.has_show_more_button,
"warning_codes": parsed_page.warnings,
"attempt_count": len(page_attempt_log),
}
)
state.fetch_result = fetch_result
state.parsed_page = parsed_page
@staticmethod
def _handle_page_state_transition(*, state: PagedLoopState) -> bool:
if state.parsed_page.state not in {ParseState.OK, ParseState.NO_RESULTS}:
ScholarIngestionService._set_truncated_state(
state=state,
reason=f"page_state_{state.parsed_page.state.value}",
continuation_cstart=state.current_cstart,
)
return True
if state.parsed_page.state == ParseState.NO_RESULTS and len(state.parsed_page.publications) == 0:
state.pages_fetched += 1
return True
state.pages_fetched += 1
state.publications.extend(state.parsed_page.publications)
state.next_cstart = _next_cstart_value(
articles_range=state.parsed_page.articles_range,
fallback=state.current_cstart + len(state.parsed_page.publications),
)
return False
async def _fetch_initial_page_context(
self,
*,
scholar_id: str,
start_cstart: int,
bounded_page_size: int,
network_error_retries: int,
retry_backoff_seconds: float,
rate_limit_retries: int,
rate_limit_backoff_seconds: float,
) -> tuple[FetchResult, ParsedProfilePage, str | None, list[dict[str, Any]], list[dict[str, Any]]]:
fetch_result, parsed_page, first_attempt_log = await self._fetch_and_parse_page_with_retry(
scholar_id=scholar_id,
cstart=start_cstart,
page_size=bounded_page_size,
network_error_retries=network_error_retries,
retry_backoff_seconds=retry_backoff_seconds,
rate_limit_retries=rate_limit_retries,
rate_limit_backoff_seconds=rate_limit_backoff_seconds,
)
first_page_fingerprint_sha256 = build_initial_page_fingerprint(parsed_page)
attempt_log = list(first_attempt_log)
page_logs = [
{
"page": 1,
"cstart": start_cstart,
"state": parsed_page.state.value,
"state_reason": parsed_page.state_reason,
"status_code": fetch_result.status_code,
"publication_count": len(parsed_page.publications),
"articles_range": parsed_page.articles_range,
"has_show_more_button": parsed_page.has_show_more_button,
"warning_codes": parsed_page.warnings,
"attempt_count": len(first_attempt_log),
}
]
return fetch_result, parsed_page, first_page_fingerprint_sha256, attempt_log, page_logs
async def _paginate_loop(
self,
*,
scholar: ScholarProfile,
run: CrawlRun,
db_session: AsyncSession,
state: PagedLoopState,
bounded_max_pages: int,
request_delay_seconds: int,
bounded_page_size: int,
network_error_retries: int,
retry_backoff_seconds: float,
rate_limit_retries: int,
rate_limit_backoff_seconds: float,
) -> None:
# Cross-page canonical dedup state; grows across all pages of this scholar.
seen_canonical: set[str] = set()
if state.parsed_page.publications:
deduped_first = _dedupe_publication_candidates(
list(state.parsed_page.publications), seen_canonical=seen_canonical
)
if deduped_first:
discovered_count = await self._upsert_profile_publications(
db_session, run=run, scholar=scholar, publications=deduped_first
)
state.discovered_publication_count += discovered_count
while state.parsed_page.has_show_more_button:
await db_session.refresh(run)
if run.status == RunStatus.CANCELED:
structured_log(
logger,
"info",
"ingestion.pagination_canceled",
run_id=run.id,
)
self._set_truncated_state(
state=state,
reason="run_canceled",
continuation_cstart=state.current_cstart,
)
return
if self._should_stop_pagination(state=state, bounded_max_pages=bounded_max_pages):
return
next_fetch_result, next_parsed_page, next_attempt_log = await self._fetch_next_page(
scholar_id=scholar.scholar_id,
state=state,
request_delay_seconds=request_delay_seconds,
bounded_page_size=bounded_page_size,
network_error_retries=network_error_retries,
retry_backoff_seconds=retry_backoff_seconds,
rate_limit_retries=rate_limit_retries,
rate_limit_backoff_seconds=rate_limit_backoff_seconds,
)
self._record_next_page(
state=state,
fetch_result=next_fetch_result,
parsed_page=next_parsed_page,
page_attempt_log=next_attempt_log,
)
# Deduplicate against all publications already committed this run,
# then immediately commit the surviving candidates to the DB.
if next_parsed_page.publications:
deduped_next = _dedupe_publication_candidates(
list(next_parsed_page.publications), seen_canonical=seen_canonical
)
if deduped_next:
discovered_count = await self._upsert_profile_publications(
db_session, run=run, scholar=scholar, publications=deduped_next
)
state.discovered_publication_count += discovered_count
if self._handle_page_state_transition(state=state):
return
@staticmethod
def _result_from_pagination_state(
*,
state: PagedLoopState,
first_page_fetch_result: FetchResult,
first_page_parsed_page: ParsedProfilePage,
first_page_fingerprint_sha256: str | None,
) -> PagedParseResult:
return PagedParseResult(
fetch_result=state.fetch_result,
parsed_page=state.parsed_page,
first_page_fetch_result=first_page_fetch_result,
first_page_parsed_page=first_page_parsed_page,
first_page_fingerprint_sha256=first_page_fingerprint_sha256,
publications=_dedupe_publication_candidates(state.publications),
attempt_log=state.attempt_log,
page_logs=state.page_logs,
pages_fetched=state.pages_fetched,
pages_attempted=state.pages_attempted,
has_more_remaining=state.has_more_remaining,
pagination_truncated_reason=state.pagination_truncated_reason,
continuation_cstart=state.continuation_cstart,
skipped_no_change=False,
discovered_publication_count=state.discovered_publication_count,
)
def _short_circuit_initial_page(
self,
*,
start_cstart: int,
previous_initial_page_fingerprint_sha256: str | None,
fetch_result: FetchResult,
parsed_page: ParsedProfilePage,
first_page_fingerprint_sha256: str | None,
attempt_log: list[dict[str, Any]],
page_logs: list[dict[str, Any]],
) -> PagedParseResult | None:
if self._should_skip_no_change(
start_cstart=start_cstart,
first_page_fingerprint_sha256=first_page_fingerprint_sha256,
previous_initial_page_fingerprint_sha256=previous_initial_page_fingerprint_sha256,
parsed_page=parsed_page,
):
return self._skip_no_change_result(
fetch_result=fetch_result,
parsed_page=parsed_page,
first_page_fingerprint_sha256=first_page_fingerprint_sha256,
attempt_log=attempt_log,
page_logs=page_logs,
)
if parsed_page.state not in {ParseState.OK, ParseState.NO_RESULTS}:
return self._initial_failure_result(
fetch_result=fetch_result,
parsed_page=parsed_page,
first_page_fingerprint_sha256=first_page_fingerprint_sha256,
start_cstart=start_cstart,
attempt_log=attempt_log,
page_logs=page_logs,
)
return None
async def _fetch_and_parse_all_pages_with_retry(
self,
*,
scholar: ScholarProfile,
run: CrawlRun,
db_session: AsyncSession,
start_cstart: int,
request_delay_seconds: int,
network_error_retries: int,
retry_backoff_seconds: float,
rate_limit_retries: int,
rate_limit_backoff_seconds: float,
max_pages: int,
page_size: int,
previous_initial_page_fingerprint_sha256: str | None = None,
) -> PagedParseResult:
bounded_max_pages = max(1, int(max_pages))
bounded_page_size = max(1, int(page_size))
(
fetch_result,
parsed_page,
first_page_fingerprint_sha256,
attempt_log,
page_logs,
) = await self._fetch_initial_page_context(
scholar_id=scholar.scholar_id,
start_cstart=start_cstart,
bounded_page_size=bounded_page_size,
network_error_retries=network_error_retries,
retry_backoff_seconds=retry_backoff_seconds,
rate_limit_retries=rate_limit_retries,
rate_limit_backoff_seconds=rate_limit_backoff_seconds,
)
shortcut_result = self._short_circuit_initial_page(
start_cstart=start_cstart,
previous_initial_page_fingerprint_sha256=previous_initial_page_fingerprint_sha256,
fetch_result=fetch_result,
parsed_page=parsed_page,
first_page_fingerprint_sha256=first_page_fingerprint_sha256,
attempt_log=attempt_log,
page_logs=page_logs,
)
if shortcut_result is not None:
return shortcut_result
state = self._build_loop_state(
start_cstart=start_cstart,
fetch_result=fetch_result,
parsed_page=parsed_page,
attempt_log=attempt_log,
page_logs=page_logs,
)
await self._paginate_loop(
scholar=scholar,
run=run,
db_session=db_session,
state=state,
bounded_max_pages=bounded_max_pages,
request_delay_seconds=request_delay_seconds,
bounded_page_size=bounded_page_size,
network_error_retries=network_error_retries,
retry_backoff_seconds=retry_backoff_seconds,
rate_limit_retries=rate_limit_retries,
rate_limit_backoff_seconds=rate_limit_backoff_seconds,
)
return self._result_from_pagination_state(
state=state,
first_page_fetch_result=fetch_result,
first_page_parsed_page=parsed_page,
first_page_fingerprint_sha256=first_page_fingerprint_sha256,
)
async def _run_is_canceled(
self,
db_session: AsyncSession,
@ -2641,228 +1992,6 @@ class ScholarIngestionService:
enriched.append(p)
return enriched
async def _upsert_profile_publications(
self,
db_session: AsyncSession,
*,
run: CrawlRun,
scholar: ScholarProfile,
publications: list[PublicationCandidate],
) -> int:
# We no longer enrich inline here. Enrichment is now deferred to the end of the run.
seen_publication_ids: set[int] = set()
discovered_count = 0
for candidate in publications:
publication = await self._resolve_publication(db_session, candidate)
if publication.id in seen_publication_ids:
continue
seen_publication_ids.add(publication.id)
link_result = await db_session.execute(
select(ScholarPublication).where(
ScholarPublication.scholar_profile_id == scholar.id,
ScholarPublication.publication_id == publication.id,
)
)
link = link_result.scalar_one_or_none()
if link is not None:
continue
link = ScholarPublication(
scholar_profile_id=scholar.id,
publication_id=publication.id,
is_read=False,
first_seen_run_id=run.id,
)
db_session.add(link)
discovered_count += 1
await self._commit_discovered_publication(
db_session,
run=run,
scholar=scholar,
publication=publication,
)
if not scholar.baseline_completed:
scholar.baseline_completed = True
return discovered_count
async def _commit_discovered_publication(
self,
db_session: AsyncSession,
*,
run: CrawlRun,
scholar: ScholarProfile,
publication: Publication,
) -> None:
run.new_pub_count = int(run.new_pub_count or 0) + 1
await db_session.commit()
await run_events.publish(
run_id=run.id,
event_type="publication_discovered",
data={
"publication_id": publication.id,
"title": publication.title_raw,
"pub_url": publication.pub_url,
"scholar_profile_id": scholar.id,
"scholar_label": scholar.display_name or scholar.scholar_id,
"first_seen_at": datetime.now(UTC).isoformat(),
"new_publication_count": int(run.new_pub_count or 0),
},
)
@staticmethod
def _validate_publication_candidate(candidate: PublicationCandidate) -> None:
if not candidate.title.strip():
raise RuntimeError("Publication candidate is missing title.")
if candidate.citation_count is not None and int(candidate.citation_count) < 0:
raise RuntimeError("Publication candidate has negative citation_count.")
async def _find_publication_by_cluster(
self,
db_session: AsyncSession,
*,
cluster_id: str | None,
) -> Publication | None:
if not cluster_id:
return None
result = await db_session.execute(select(Publication).where(Publication.cluster_id == cluster_id))
return result.scalar_one_or_none()
async def _find_publication_by_fingerprint(
self,
db_session: AsyncSession,
*,
fingerprint: str,
) -> Publication | None:
result = await db_session.execute(select(Publication).where(Publication.fingerprint_sha256 == fingerprint))
return result.scalar_one_or_none()
@staticmethod
def _select_existing_publication(
*,
cluster_publication: Publication | None,
fingerprint_publication: Publication | None,
) -> Publication | None:
if cluster_publication is not None:
return cluster_publication
return fingerprint_publication
@staticmethod
def _compute_canonical_title_hash(title: str) -> str:
canonical = canonical_title_for_dedup(title)
return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
async def _find_publication_by_canonical_title_hash(
self,
db_session: AsyncSession,
*,
canonical_title_hash: str,
) -> Publication | None:
result = await db_session.execute(
select(Publication).where(Publication.canonical_title_hash == canonical_title_hash)
)
return result.scalar_one_or_none()
async def _create_publication(
self,
db_session: AsyncSession,
*,
candidate: PublicationCandidate,
fingerprint: str,
) -> Publication:
publication = Publication(
cluster_id=candidate.cluster_id,
fingerprint_sha256=fingerprint,
title_raw=candidate.title,
title_normalized=normalize_title(candidate.title),
canonical_title_hash=self._compute_canonical_title_hash(candidate.title),
year=candidate.year,
citation_count=int(candidate.citation_count or 0),
author_text=candidate.authors_text,
venue_text=candidate.venue_text,
pub_url=build_publication_url(candidate.title_url),
pdf_url=None,
)
db_session.add(publication)
await db_session.flush()
return publication
@staticmethod
def _update_existing_publication(
*,
publication: Publication,
candidate: PublicationCandidate,
) -> None:
if candidate.cluster_id and publication.cluster_id is None:
publication.cluster_id = candidate.cluster_id
publication.title_raw = candidate.title
publication.title_normalized = normalize_title(candidate.title)
if candidate.year is not None:
publication.year = candidate.year
if candidate.citation_count is not None:
publication.citation_count = int(candidate.citation_count)
if candidate.authors_text:
publication.author_text = candidate.authors_text
if candidate.venue_text:
publication.venue_text = candidate.venue_text
if candidate.title_url:
publication.pub_url = build_publication_url(candidate.title_url)
first_doi_from_texts(candidate.title_url, candidate.venue_text, candidate.title)
async def _resolve_publication(
self,
db_session: AsyncSession,
candidate: PublicationCandidate,
) -> Publication:
self._validate_publication_candidate(candidate)
fingerprint = build_publication_fingerprint(candidate)
cluster_publication = await self._find_publication_by_cluster(
db_session,
cluster_id=candidate.cluster_id,
)
fingerprint_publication = await self._find_publication_by_fingerprint(
db_session,
fingerprint=fingerprint,
)
publication = self._select_existing_publication(
cluster_publication=cluster_publication,
fingerprint_publication=fingerprint_publication,
)
if publication is None:
# Fallback: canonical title hash — catches cross-scholar noise variants
# (e.g. "Adam preprint (2014)" vs "Adam arXiv 2017" → same normalized form)
canonical_hash = self._compute_canonical_title_hash(candidate.title)
publication = await self._find_publication_by_canonical_title_hash(
db_session,
canonical_title_hash=canonical_hash,
)
if publication is None:
created = await self._create_publication(
db_session,
candidate=candidate,
fingerprint=fingerprint,
)
# Sync identifiers from local fields only for fast UI response
await identifier_service.sync_identifiers_for_publication_fields(
db_session,
publication=created,
)
return created
self._update_existing_publication(
publication=publication,
candidate=candidate,
)
# Sync identifiers from local fields only for fast UI response
await identifier_service.sync_identifiers_for_publication_fields(
db_session,
publication=publication,
)
return publication
def _resolve_run_status(
self,
*,