Intermediate commit

app/services/domains/ingestion/application.py

@@ -30,6 +30,7 @@ from app.services.domains.ingestion.constants import (
     RESUMABLE_PARTIAL_REASONS,
     RUN_LOCK_NAMESPACE,
 )
+from app.services.domains.arxiv.errors import ArxivRateLimitError
 from app.services.domains.doi.normalize import first_doi_from_texts
 from app.services.domains.publication_identifiers import application as identifier_service
 from app.services.domains.ingestion.fingerprints import (
@@ -830,17 +831,79 @@ class ScholarIngestionService:
             result_entry=result_entry,
         )
 
+    @staticmethod
+    def _result_counters(result_entry: dict[str, Any]) -> tuple[int, int, int]:
+        outcome = str(result_entry.get("outcome", "")).strip().lower()
+        if outcome == "success":
+            return 1, 0, 0
+        if outcome == "partial":
+            return 1, 0, 1
+        if outcome == "failed":
+            return 0, 1, 0
+        raise RuntimeError(f"Unexpected scholar outcome label: {outcome!r}")
+
+    @staticmethod
+    def _find_scholar_result_index(
+        *,
+        scholar_results: list[dict[str, Any]],
+        scholar_profile_id: int,
+    ) -> int | None:
+        for index, result_entry in enumerate(scholar_results):
+            current_scholar_id = _int_or_default(result_entry.get("scholar_profile_id"), 0)
+            if current_scholar_id == scholar_profile_id:
+                return index
+        return None
+
+    @staticmethod
+    def _adjust_progress_counts(
+        *,
+        progress: RunProgress,
+        succeeded_delta: int,
+        failed_delta: int,
+        partial_delta: int,
+    ) -> None:
+        progress.succeeded_count += succeeded_delta
+        progress.failed_count += failed_delta
+        progress.partial_count += partial_delta
+        if progress.succeeded_count < 0 or progress.failed_count < 0 or progress.partial_count < 0:
+            raise RuntimeError("RunProgress counters entered invalid negative state.")
+
     @staticmethod
     def _apply_outcome_to_progress(
         *,
         progress: RunProgress,
-        run: CrawlRun,
         outcome: ScholarProcessingOutcome,
     ) -> None:
-        progress.succeeded_count += outcome.succeeded_count_delta
-        progress.failed_count += outcome.failed_count_delta
-        progress.partial_count += outcome.partial_count_delta
-        progress.scholar_results.append(outcome.result_entry)
+        scholar_profile_id = _int_or_default(outcome.result_entry.get("scholar_profile_id"), 0)
+        if scholar_profile_id <= 0:
+            raise RuntimeError("Scholar outcome missing valid scholar_profile_id.")
+        prior_index = ScholarIngestionService._find_scholar_result_index(
+            scholar_results=progress.scholar_results,
+            scholar_profile_id=scholar_profile_id,
+        )
+        next_succeeded, next_failed, next_partial = ScholarIngestionService._result_counters(
+            outcome.result_entry
+        )
+        if prior_index is None:
+            progress.scholar_results.append(outcome.result_entry)
+            ScholarIngestionService._adjust_progress_counts(
+                progress=progress,
+                succeeded_delta=next_succeeded,
+                failed_delta=next_failed,
+                partial_delta=next_partial,
+            )
+            return
+        previous_entry = progress.scholar_results[prior_index]
+        prev_succeeded, prev_failed, prev_partial = ScholarIngestionService._result_counters(
+            previous_entry
+        )
+        progress.scholar_results[prior_index] = outcome.result_entry
+        ScholarIngestionService._adjust_progress_counts(
+            progress=progress,
+            succeeded_delta=next_succeeded - prev_succeeded,
+            failed_delta=next_failed - prev_failed,
+            partial_delta=next_partial - prev_partial,
+        )
 
     def _unexpected_scholar_exception_outcome(
         self,
@@ -1327,7 +1390,7 @@ class ScholarIngestionService:
                 auto_queue_continuations=False,
                 queue_delay_seconds=queue_delay_seconds,
             )
-            self._apply_outcome_to_progress(progress=progress, run=run, outcome=outcome)
+            self._apply_outcome_to_progress(progress=progress, outcome=outcome)
             # Track where to resume from for the deep pass
             resume_cstart = outcome.result_entry.get("continuation_cstart")
             if resume_cstart is not None and int(resume_cstart) > start_cstart:
@@ -1366,7 +1429,7 @@ class ScholarIngestionService:
                 auto_queue_continuations=auto_queue_continuations,
                 queue_delay_seconds=queue_delay_seconds,
             )
-            self._apply_outcome_to_progress(progress=progress, run=run, outcome=outcome)
+            self._apply_outcome_to_progress(progress=progress, outcome=outcome)
         return progress
 
     def _complete_run_for_user(
@@ -2448,6 +2511,7 @@ class ScholarIngestionService:
         resolved_key = openalex_api_key or settings.openalex_api_key
         client = OpenAlexClient(api_key=resolved_key, mailto=settings.crossref_api_mailto)
         batch_size = 25
+        arxiv_lookup_allowed = True
 
         for i in range(0, len(publications), batch_size):
             if await self._run_is_canceled(db_session, run_id=run_id):
@@ -2494,12 +2558,11 @@ class ScholarIngestionService:
                     return
 
                 p.openalex_last_attempt_at = now
-                
-                # Perform identifier discovery (moved from synchronous ingest loop)
-                await identifier_service.discover_and_sync_identifiers_for_publication(
+                arxiv_lookup_allowed = await self._discover_identifiers_for_enrichment(
                     db_session,
                     publication=p,
-                    scholar_label=p.author_text or "",
+                    run_id=run_id,
+                    allow_arxiv_lookup=arxiv_lookup_allowed,
                 )
 
                 match = find_best_match(
@@ -2529,6 +2592,86 @@ class ScholarIngestionService:
                 },
             )
 
+    async def _discover_identifiers_for_enrichment(
+        self,
+        db_session: AsyncSession,
+        *,
+        publication: Publication,
+        run_id: int,
+        allow_arxiv_lookup: bool,
+    ) -> bool:
+        if not allow_arxiv_lookup:
+            await identifier_service.sync_identifiers_for_publication_fields(
+                db_session,
+                publication=publication,
+            )
+            await self._publish_identifier_update_event(
+                db_session,
+                run_id=run_id,
+                publication_id=int(publication.id),
+            )
+            return False
+        try:
+            await identifier_service.discover_and_sync_identifiers_for_publication(
+                db_session,
+                publication=publication,
+                scholar_label=publication.author_text or "",
+            )
+            await self._publish_identifier_update_event(
+                db_session,
+                run_id=run_id,
+                publication_id=int(publication.id),
+            )
+            return True
+        except ArxivRateLimitError:
+            logger.warning(
+                "ingestion.arxiv_rate_limited",
+                extra={
+                    "event": "ingestion.arxiv_rate_limited",
+                    "run_id": run_id,
+                    "publication_id": int(publication.id),
+                    "detail": "arXiv temporarily disabled for remaining enrichment pass",
+                },
+            )
+            await identifier_service.sync_identifiers_for_publication_fields(
+                db_session,
+                publication=publication,
+            )
+            await self._publish_identifier_update_event(
+                db_session,
+                run_id=run_id,
+                publication_id=int(publication.id),
+            )
+            return False
+
+    async def _publish_identifier_update_event(
+        self,
+        db_session: AsyncSession,
+        *,
+        run_id: int,
+        publication_id: int,
+    ) -> None:
+        display = await identifier_service.display_identifier_for_publication_id(
+            db_session,
+            publication_id=publication_id,
+        )
+        if display is None:
+            return
+        await run_events.publish(
+            run_id=run_id,
+            event_type="identifier_updated",
+            data={
+                "publication_id": int(publication_id),
+                "display_identifier": {
+                    "kind": display.kind,
+                    "value": display.value,
+                    "label": display.label,
+                    "url": display.url,
+                    "confidence_score": float(display.confidence_score),
+                },
+            },
+        )
+
     async def _enrich_publications_with_openalex(
         self,
         scholar: ScholarProfile,

app/services/domains/ingestion/fingerprints.py

@@ -4,6 +4,7 @@ import hashlib
 import json
 import re
 from typing import Any
+import unicodedata
 from urllib.parse import urljoin
 
 from app.services.domains.ingestion.constants import (
@@ -24,37 +25,177 @@ _NOISE_PREPRINT_RE = re.compile(
     re.IGNORECASE,
 )
 _NOISE_TRAILING_YEAR_RE = re.compile(r"\s*[,(]\s*\d{4}\s*[),]?\s*$")
+_NOISE_TRAILING_MONTH_YEAR_RE = re.compile(
+    r"\s*[,(]\s*(?:jan|feb|mar|apr|may|jun|jul|aug|sep|sept|oct|nov|dec)[a-z]*\.?\s+\d{4}\s*[),]?\s*$",
+    re.IGNORECASE,
+)
+_NOISE_TRAILING_PUBLICATION_TYPE_RE = re.compile(
+    r"[,.\s]+(?:conference\s+paper|journal\s+article)\s*$",
+    re.IGNORECASE,
+)
+_NOISE_IN_PROCEEDINGS_SUFFIX_RE = re.compile(r"\s+in:\s+proceedings\b.*$", re.IGNORECASE)
 # Strips ". Capitalised sentence" appended as venue: ". Comput. Sci…", ". Journal of…"
 _NOISE_VENUE_SENTENCE_RE = re.compile(r"(?<=\w{3})\.\s+[A-Z][a-z].*$")
+_MOJIBAKE_HINT_RE = re.compile(r"[ÃÂâ]")
+_MOJIBAKE_CHAR_RE = re.compile(r"[Ó”€™]")
+_METADATA_ORDINAL_RE = re.compile(r"^\d+(st|nd|rd|th)$")
+_NOISE_LEADING_DATE_PREFIX_RE = re.compile(
+    r"^(?:jan|feb|mar|apr|may|jun|jul|aug|sep|sept|oct|nov|dec)[a-z]*\s+\d{1,2}(?:\s*[-–]\s*\d{1,2})?\)?[,\.\s:;-]+",
+    re.IGNORECASE,
+)
+_NOISE_LEADING_AUTHOR_FRAGMENT_RE = re.compile(r"^(?:and|&)\s+[a-z.\s]{1,40}:\s*", re.IGNORECASE)
+_METADATA_SEPARATORS = (" - ", " — ", ",", ";", ". ")
+_VENUE_HINT_TOKENS = {
+    "aaai",
+    "conference",
+    "conf",
+    "cvpr",
+    "eccv",
+    "iclr",
+    "icml",
+    "journal",
+    "nips",
+    "neurips",
+    "proceedings",
+    "proc",
+    "symposium",
+    "workshop",
+}
+_PUBLICATION_TYPE_TOKENS = {"conference", "paper", "journal", "article"}
+_MIN_METADATA_HINT_TOKENS = 2
+_MIN_METADATA_CONTEXT_TOKENS = 4
 
 _CANONICAL_DEDUP_THRESHOLD = 0.82
 
 
 def normalize_title(value: str) -> str:
-    lowered = value.lower()
+    lowered = _normalized_text(value).lower()
     return TITLE_ALNUM_RE.sub("", lowered)
 
 
 def canonical_title_for_dedup(title: str) -> str:
     """Strip Scholar-specific noise suffixes then normalize for dedup comparison."""
-    t = title.strip()
-    t = _NOISE_DOI_RE.sub("", t)
-    t = _NOISE_ARXIV_RE.sub("", t)
-    t = _NOISE_PREPRINT_RE.sub("", t)
-    t = _NOISE_TRAILING_YEAR_RE.sub("", t)
-    t = _NOISE_VENUE_SENTENCE_RE.sub("", t)
-    return normalize_title(t.strip())
+    return normalize_title(_canonical_title_text(title))
+
+
+def canonical_title_text_for_dedup(title: str) -> str:
+    """Noise-stripped lowercase title with spaces preserved for token-level matching."""
+    return _stripped_title_for_canonical(title)
+
+
+def canonical_title_tokens_for_dedup(title: str) -> set[str]:
+    """Word tokens of the noise-stripped title."""
+    return _canonical_title_tokens(title)
 
 
 def _stripped_title_for_canonical(title: str) -> str:
     """Apply noise-stripping and lowercase but PRESERVE spaces (for later tokenization)."""
-    t = title.strip()
+    t = _canonical_title_text(title)
+    return t.lower().strip()
+
+
+def _canonical_title_text(title: str) -> str:
+    t = _normalized_text(title)
+    t = _strip_noise_suffixes(t)
+    t = _strip_venue_metadata_suffixes(t)
+    return _NOISE_VENUE_SENTENCE_RE.sub("", t).strip()
+
+
+def _strip_noise_suffixes(value: str) -> str:
+    t = _strip_leading_noise_prefixes(value.strip())
     t = _NOISE_DOI_RE.sub("", t)
     t = _NOISE_ARXIV_RE.sub("", t)
     t = _NOISE_PREPRINT_RE.sub("", t)
     t = _NOISE_TRAILING_YEAR_RE.sub("", t)
-    t = _NOISE_VENUE_SENTENCE_RE.sub("", t)
-    return t.lower().strip()
+    t = _NOISE_TRAILING_MONTH_YEAR_RE.sub("", t)
+    t = _NOISE_TRAILING_PUBLICATION_TYPE_RE.sub("", t)
+    t = _NOISE_IN_PROCEEDINGS_SUFFIX_RE.sub("", t)
+    return t.strip()
+
+
+def _strip_venue_metadata_suffixes(value: str) -> str:
+    stripped = value.strip()
+    while True:
+        cut_index = _metadata_cut_index(stripped)
+        if cut_index is None:
+            return stripped
+        stripped = stripped[:cut_index].strip()
+
+
+def _metadata_cut_index(value: str) -> int | None:
+    candidates: list[int] = []
+    for candidate in _METADATA_SEPARATORS:
+        start = 0
+        while True:
+            index = value.find(candidate, start)
+            if index <= 0:
+                break
+            suffix = value[index + len(candidate) :].strip()
+            if suffix and _looks_like_venue_metadata(suffix):
+                candidates.append(index)
+            start = index + len(candidate)
+    if not candidates:
+        return None
+    return min(candidates)
+
+
+def _looks_like_venue_metadata(value: str) -> bool:
+    tokens = WORD_RE.findall(value.lower())
+    if len(tokens) < _MIN_METADATA_HINT_TOKENS:
+        return False
+    has_hint = any(_is_venue_hint_token(token) for token in tokens)
+    if not has_hint:
+        return False
+    has_year = any(_is_year_token(token) for token in tokens)
+    has_ordinal = any(_METADATA_ORDINAL_RE.match(token) for token in tokens)
+    publication_type_only = all(token in _PUBLICATION_TYPE_TOKENS for token in tokens)
+    return has_year or has_ordinal or publication_type_only or len(tokens) >= _MIN_METADATA_CONTEXT_TOKENS
+
+
+def _strip_leading_noise_prefixes(value: str) -> str:
+    stripped = value
+    while True:
+        next_value = _NOISE_LEADING_DATE_PREFIX_RE.sub("", stripped).strip()
+        next_value = _NOISE_LEADING_AUTHOR_FRAGMENT_RE.sub("", next_value).strip()
+        if next_value == stripped:
+            return stripped
+        stripped = next_value
+
+
+def _is_venue_hint_token(token: str) -> bool:
+    if token in _VENUE_HINT_TOKENS:
+        return True
+    return token.startswith("conf") or token.startswith("proceed")
+
+
+def _is_year_token(token: str) -> bool:
+    if len(token) != 4 or not token.isdigit():
+        return False
+    year = int(token)
+    return 1900 <= year <= 2100
+
+
+def _normalized_text(value: str) -> str:
+    repaired = _repair_mojibake(value.strip())
+    normalized = unicodedata.normalize("NFKC", repaired)
+    cleaned = _MOJIBAKE_CHAR_RE.sub(" ", normalized)
+    return SPACE_RE.sub(" ", cleaned).strip()
+
+
+def _repair_mojibake(value: str) -> str:
+    if not value or not _MOJIBAKE_HINT_RE.search(value):
+        return value
+    try:
+        repaired = value.encode("latin1").decode("utf-8")
+    except UnicodeError:
+        return value
+    if _mojibake_score(repaired) < _mojibake_score(value):
+        return repaired
+    return value
+
+
+def _mojibake_score(value: str) -> int:
+    return len(_MOJIBAKE_HINT_RE.findall(value))
 
 
 def _canonical_title_tokens(title: str) -> set[str]: