Initial release prep

This commit is contained in:
Justin Visser 2026-02-27 10:46:05 +01:00
parent c9b9d892f4
commit 92395b2b4b
100 changed files with 405 additions and 401 deletions

View file

@ -20,11 +20,11 @@ from app.db.models import (
ScholarPublication,
)
from app.logging_utils import structured_log
from app.services.domains.arxiv.errors import ArxivRateLimitError
from app.services.domains.doi.normalize import first_doi_from_texts
from app.services.domains.ingestion import queue as queue_service
from app.services.domains.ingestion import safety as run_safety_service
from app.services.domains.ingestion.constants import (
from app.services.arxiv.errors import ArxivRateLimitError
from app.services.doi.normalize import first_doi_from_texts
from app.services.ingestion import queue as queue_service
from app.services.ingestion import safety as run_safety_service
from app.services.ingestion.constants import (
FAILED_STATES,
FAILURE_BUCKET_BLOCKED,
FAILURE_BUCKET_INGESTION,
@ -35,7 +35,7 @@ from app.services.domains.ingestion.constants import (
RESUMABLE_PARTIAL_REASONS,
RUN_LOCK_NAMESPACE,
)
from app.services.domains.ingestion.fingerprints import (
from app.services.ingestion.fingerprints import (
_build_body_excerpt,
_dedupe_publication_candidates,
_next_cstart_value,
@ -45,7 +45,7 @@ from app.services.domains.ingestion.fingerprints import (
canonical_title_for_dedup,
normalize_title,
)
from app.services.domains.ingestion.types import (
from app.services.ingestion.types import (
PagedLoopState,
PagedParseResult,
RunAlertSummary,
@ -56,17 +56,17 @@ from app.services.domains.ingestion.types import (
RunProgress,
ScholarProcessingOutcome,
)
from app.services.domains.publication_identifiers import application as identifier_service
from app.services.domains.runs.events import run_events
from app.services.domains.scholar.parser import (
from app.services.publication_identifiers import application as identifier_service
from app.services.runs.events import run_events
from app.services.scholar.parser import (
ParsedProfilePage,
ParseState,
PublicationCandidate,
ScholarParserError,
parse_profile_page,
)
from app.services.domains.scholar.source import FetchResult, ScholarSource
from app.services.domains.settings import application as user_settings_service
from app.services.scholar.source import FetchResult, ScholarSource
from app.services.settings import application as user_settings_service
from app.settings import settings
logger = logging.getLogger(__name__)
@ -2369,12 +2369,12 @@ class ScholarIngestionService:
Stops immediately on budget exhaustion (429 with $0 remaining).
Sleeps 60s and continues on transient rate limits.
"""
from app.services.domains.openalex.client import (
from app.services.openalex.client import (
OpenAlexBudgetExhaustedError,
OpenAlexClient,
OpenAlexRateLimitError,
)
from app.services.domains.openalex.matching import find_best_match
from app.services.openalex.matching import find_best_match
run_result = await db_session.execute(select(CrawlRun.user_id).where(CrawlRun.id == run_id))
user_id = run_result.scalar_one()
@ -2480,7 +2480,7 @@ class ScholarIngestionService:
await db_session.flush()
from app.services.domains.publications.dedup import sweep_identifier_duplicates
from app.services.publications.dedup import sweep_identifier_duplicates
merge_count = await sweep_identifier_duplicates(db_session)
if merge_count:
@ -2579,8 +2579,8 @@ class ScholarIngestionService:
if not publications:
return publications
from app.services.domains.openalex.client import OpenAlexClient
from app.services.domains.openalex.matching import find_best_match
from app.services.openalex.client import OpenAlexClient
from app.services.openalex.matching import find_best_match
client = OpenAlexClient(api_key=settings.openalex_api_key, mailto=settings.crossref_api_mailto)

View file

@ -2,7 +2,7 @@ from __future__ import annotations
import re
from app.services.domains.scholar.parser import ParseState
from app.services.scholar.parser import ParseState
TITLE_ALNUM_RE = re.compile(r"[^a-z0-9]+")
WORD_RE = re.compile(r"[a-z0-9]+")

View file

@ -7,14 +7,14 @@ import unicodedata
from typing import Any
from urllib.parse import urljoin
from app.services.domains.ingestion.constants import (
from app.services.ingestion.constants import (
HTML_TAG_RE,
INITIAL_PAGE_FINGERPRINT_MAX_PUBLICATIONS,
SPACE_RE,
TITLE_ALNUM_RE,
WORD_RE,
)
from app.services.domains.scholar.parser import ParsedProfilePage, ParseState, PublicationCandidate
from app.services.scholar.parser import ParsedProfilePage, ParseState, PublicationCandidate
# Scholar-specific noise patterns stripped before canonical comparison.
# Applied in order; each targets a different Scholar metadata injection style.

View file

@ -18,14 +18,14 @@ from app.db.models import (
)
from app.db.session import get_session_factory
from app.logging_utils import structured_log
from app.services.domains.ingestion import queue as queue_service
from app.services.domains.ingestion.application import (
from app.services.ingestion import queue as queue_service
from app.services.ingestion.application import (
RunAlreadyInProgressError,
RunBlockedBySafetyPolicyError,
ScholarIngestionService,
)
from app.services.domains.scholar.source import LiveScholarSource
from app.services.domains.settings import application as user_settings_service
from app.services.scholar.source import LiveScholarSource
from app.services.settings import application as user_settings_service
from app.settings import settings
logger = logging.getLogger(__name__)
@ -589,7 +589,7 @@ class SchedulerService:
await self._finalize_queue_job_after_run(job, run_summary)
async def _drain_pdf_queue(self) -> None:
from app.services.domains.publications.pdf_queue import drain_ready_jobs
from app.services.publications.pdf_queue import drain_ready_jobs
session_factory = get_session_factory()
async with session_factory() as session:

View file

@ -4,8 +4,8 @@ from dataclasses import dataclass, field
from typing import Any
from app.db.models import RunStatus
from app.services.domains.scholar.parser import ParsedProfilePage, PublicationCandidate
from app.services.domains.scholar.source import FetchResult
from app.services.scholar.parser import ParsedProfilePage, PublicationCandidate
from app.services.scholar.source import FetchResult
@dataclass(frozen=True)