justinzeus/scholarr
1
0
Fork 0
Code Issues 1 Pull requests 18 Projects Releases Packages Wiki Activity Actions

Initial release prep

Browse source
This commit is contained in:
Justin Visser 2026-02-27 10:46:05 +01:00
parent c9b9d892f4
commit 92395b2b4b
100 changed files with 405 additions and 401 deletions
Download patch file Download diff file Expand all files Collapse all files

4
app/services/arxiv/application.py
View file

@ -2,13 +2,13 @@ from __future__ import annotations
from typing import TYPE_CHECKING
from app.services.domains.arxiv.gateway import (
from app.services.arxiv.gateway import (
build_arxiv_query,
get_arxiv_gateway,
)
if TYPE_CHECKING:
from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem
from app.services.publications.types import PublicationListItem, UnreadPublicationItem
def _build_arxiv_query(title: str, author_surname: str | None) -> str | None:

4
app/services/arxiv/cache.py
View file

@ -12,8 +12,8 @@ from sqlalchemy import delete, func, select
from app.db.models import ArxivQueryCacheEntry
from app.db.session import get_session_factory
from app.services.domains.arxiv.constants import ARXIV_CACHE_FINGERPRINT_VERSION
from app.services.domains.arxiv.types import ArxivEntry, ArxivFeed, ArxivOpenSearchMeta
from app.services.arxiv.constants import ARXIV_CACHE_FINGERPRINT_VERSION
from app.services.arxiv.types import ArxivEntry, ArxivFeed, ArxivOpenSearchMeta
_INFLIGHT_LOCK = asyncio.Lock()
_INFLIGHT_FEEDS: dict[str, asyncio.Future[ArxivFeed]] = {}

12
app/services/arxiv/client.py
View file

@ -6,21 +6,21 @@ from collections.abc import Awaitable, Callable
import httpx
from app.logging_utils import structured_log
from app.services.domains.arxiv.cache import (
from app.services.arxiv.cache import (
build_query_fingerprint,
get_cached_feed,
run_with_inflight_dedupe,
set_cached_feed,
)
from app.services.domains.arxiv.constants import (
from app.services.arxiv.constants import (
ARXIV_SOURCE_PATH_LOOKUP_IDS,
ARXIV_SOURCE_PATH_SEARCH,
ARXIV_SOURCE_PATH_UNKNOWN,
)
from app.services.domains.arxiv.errors import ArxivClientValidationError, ArxivRateLimitError
from app.services.domains.arxiv.parser import parse_arxiv_feed
from app.services.domains.arxiv.rate_limit import get_arxiv_cooldown_status, run_with_global_arxiv_limit
from app.services.domains.arxiv.types import ArxivFeed
from app.services.arxiv.errors import ArxivClientValidationError, ArxivRateLimitError
from app.services.arxiv.parser import parse_arxiv_feed
from app.services.arxiv.rate_limit import get_arxiv_cooldown_status, run_with_global_arxiv_limit
from app.services.arxiv.types import ArxivFeed
from app.settings import settings
_ARXIV_API_URL = "https://export.arxiv.org/api/query"

8
app/services/arxiv/gateway.py
View file

@ -6,13 +6,13 @@ import unicodedata
from typing import TYPE_CHECKING, Protocol
from app.logging_utils import structured_log
from app.services.domains.arxiv.client import ArxivClient
from app.services.domains.arxiv.errors import ArxivRateLimitError
from app.services.domains.arxiv.types import ArxivFeed
from app.services.arxiv.client import ArxivClient
from app.services.arxiv.errors import ArxivRateLimitError
from app.services.arxiv.types import ArxivFeed
from app.settings import settings
if TYPE_CHECKING:
from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem
from app.services.publications.types import PublicationListItem, UnreadPublicationItem
logger = logging.getLogger(__name__)

8
app/services/arxiv/guards.py
View file

@ -3,17 +3,17 @@ from __future__ import annotations
import re
from typing import TYPE_CHECKING
from app.services.domains.arxiv.constants import (
from app.services.arxiv.constants import (
ARXIV_STRONG_IDENTIFIER_CONFIDENCE,
ARXIV_TITLE_MIN_ALPHA_TOKENS,
ARXIV_TITLE_MIN_TOKENS,
ARXIV_TITLE_TOKEN_MIN_LENGTH,
)
from app.services.domains.doi.normalize import normalize_doi
from app.services.domains.publication_identifiers.normalize import normalize_arxiv_id
from app.services.doi.normalize import normalize_doi
from app.services.publication_identifiers.normalize import normalize_arxiv_id
if TYPE_CHECKING:
from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem
from app.services.publications.types import PublicationListItem, UnreadPublicationItem
_TITLE_TOKEN_RE = re.compile(r"[a-z0-9]+")

6
app/services/arxiv/parser.py
View file

@ -2,9 +2,9 @@ from __future__ import annotations
import xml.etree.ElementTree as ET
from app.services.domains.arxiv.errors import ArxivParseError
from app.services.domains.arxiv.types import ArxivEntry, ArxivFeed, ArxivOpenSearchMeta
from app.services.domains.publication_identifiers.normalize import normalize_arxiv_id
from app.services.arxiv.errors import ArxivParseError
from app.services.arxiv.types import ArxivEntry, ArxivFeed, ArxivOpenSearchMeta
from app.services.publication_identifiers.normalize import normalize_arxiv_id
_NAMESPACES = {
"atom": "http://www.w3.org/2005/Atom",

4
app/services/arxiv/rate_limit.py
View file

@ -13,13 +13,13 @@ from sqlalchemy.ext.asyncio import AsyncSession
from app.db.models import ArxivRuntimeState
from app.db.session import get_session_factory
from app.logging_utils import structured_log
from app.services.domains.arxiv.constants import (
from app.services.arxiv.constants import (
ARXIV_RATE_LIMIT_LOCK_KEY,
ARXIV_RATE_LIMIT_LOCK_NAMESPACE,
ARXIV_RUNTIME_STATE_KEY,
ARXIV_SOURCE_PATH_UNKNOWN,
)
from app.services.domains.arxiv.errors import ArxivRateLimitError
from app.services.arxiv.errors import ArxivRateLimitError
from app.settings import settings
logger = logging.getLogger(__name__)

2
app/services/crossref/__init__.py
View file

@ -1,5 +1,5 @@
from __future__ import annotations
from app.services.domains.crossref.application import discover_doi_for_publication
from app.services.crossref.application import discover_doi_for_publication
__all__ = ["discover_doi_for_publication"]

8
app/services/dbops/__init__.py
View file

@ -1,9 +1,9 @@
from app.services.domains.dbops.application import run_publication_link_repair
from app.services.domains.dbops.integrity import collect_integrity_report
from app.services.domains.dbops.near_duplicate_repair import (
from app.services.dbops.application import run_publication_link_repair
from app.services.dbops.integrity import collect_integrity_report
from app.services.dbops.near_duplicate_repair import (
run_publication_near_duplicate_repair,
)
from app.services.domains.dbops.query import list_repair_jobs
from app.services.dbops.query import list_repair_jobs
__all__ = [
"collect_integrity_report",

2
app/services/dbops/near_duplicate_repair.py
View file

@ -6,7 +6,7 @@ from typing import Any
from sqlalchemy.ext.asyncio import AsyncSession
from app.db.models import DataRepairJob
from app.services.domains.publications import dedup as dedup_service
from app.services.publications import dedup as dedup_service
REPAIR_STATUS_PLANNED = "planned"
REPAIR_STATUS_RUNNING = "running"

34
app/services/ingestion/application.py
View file

@ -20,11 +20,11 @@ from app.db.models import (
ScholarPublication,
)
from app.logging_utils import structured_log
from app.services.domains.arxiv.errors import ArxivRateLimitError
from app.services.domains.doi.normalize import first_doi_from_texts
from app.services.domains.ingestion import queue as queue_service
from app.services.domains.ingestion import safety as run_safety_service
from app.services.domains.ingestion.constants import (
from app.services.arxiv.errors import ArxivRateLimitError
from app.services.doi.normalize import first_doi_from_texts
from app.services.ingestion import queue as queue_service
from app.services.ingestion import safety as run_safety_service
from app.services.ingestion.constants import (
FAILED_STATES,
FAILURE_BUCKET_BLOCKED,
FAILURE_BUCKET_INGESTION,
@ -35,7 +35,7 @@ from app.services.domains.ingestion.constants import (
RESUMABLE_PARTIAL_REASONS,
RUN_LOCK_NAMESPACE,
)
from app.services.domains.ingestion.fingerprints import (
from app.services.ingestion.fingerprints import (
_build_body_excerpt,
_dedupe_publication_candidates,
_next_cstart_value,
@ -45,7 +45,7 @@ from app.services.domains.ingestion.fingerprints import (
canonical_title_for_dedup,
normalize_title,
)
from app.services.domains.ingestion.types import (
from app.services.ingestion.types import (
PagedLoopState,
PagedParseResult,
RunAlertSummary,
@ -56,17 +56,17 @@ from app.services.domains.ingestion.types import (
RunProgress,
ScholarProcessingOutcome,
)
from app.services.domains.publication_identifiers import application as identifier_service
from app.services.domains.runs.events import run_events
from app.services.domains.scholar.parser import (
from app.services.publication_identifiers import application as identifier_service
from app.services.runs.events import run_events
from app.services.scholar.parser import (
ParsedProfilePage,
ParseState,
PublicationCandidate,
ScholarParserError,
parse_profile_page,
)
from app.services.domains.scholar.source import FetchResult, ScholarSource
from app.services.domains.settings import application as user_settings_service
from app.services.scholar.source import FetchResult, ScholarSource
from app.services.settings import application as user_settings_service
from app.settings import settings
logger = logging.getLogger(__name__)
@ -2369,12 +2369,12 @@ class ScholarIngestionService:
Stops immediately on budget exhaustion (429 with $0 remaining).
Sleeps 60s and continues on transient rate limits.
"""
from app.services.domains.openalex.client import (
from app.services.openalex.client import (
OpenAlexBudgetExhaustedError,
OpenAlexClient,
OpenAlexRateLimitError,
)
from app.services.domains.openalex.matching import find_best_match
from app.services.openalex.matching import find_best_match
run_result = await db_session.execute(select(CrawlRun.user_id).where(CrawlRun.id == run_id))
user_id = run_result.scalar_one()
@ -2480,7 +2480,7 @@ class ScholarIngestionService:
await db_session.flush()
from app.services.domains.publications.dedup import sweep_identifier_duplicates
from app.services.publications.dedup import sweep_identifier_duplicates
merge_count = await sweep_identifier_duplicates(db_session)
if merge_count:
@ -2579,8 +2579,8 @@ class ScholarIngestionService:
if not publications:
return publications
from app.services.domains.openalex.client import OpenAlexClient
from app.services.domains.openalex.matching import find_best_match
from app.services.openalex.client import OpenAlexClient
from app.services.openalex.matching import find_best_match
client = OpenAlexClient(api_key=settings.openalex_api_key, mailto=settings.crossref_api_mailto)

2
app/services/ingestion/constants.py
View file

@ -2,7 +2,7 @@ from __future__ import annotations
import re
from app.services.domains.scholar.parser import ParseState
from app.services.scholar.parser import ParseState
TITLE_ALNUM_RE = re.compile(r"[^a-z0-9]+")
WORD_RE = re.compile(r"[a-z0-9]+")

4
app/services/ingestion/fingerprints.py
View file

@ -7,14 +7,14 @@ import unicodedata
from typing import Any
from urllib.parse import urljoin
from app.services.domains.ingestion.constants import (
from app.services.ingestion.constants import (
HTML_TAG_RE,
INITIAL_PAGE_FINGERPRINT_MAX_PUBLICATIONS,
SPACE_RE,
TITLE_ALNUM_RE,
WORD_RE,
)
from app.services.domains.scholar.parser import ParsedProfilePage, ParseState, PublicationCandidate
from app.services.scholar.parser import ParsedProfilePage, ParseState, PublicationCandidate
# Scholar-specific noise patterns stripped before canonical comparison.
# Applied in order; each targets a different Scholar metadata injection style.

10
app/services/ingestion/scheduler.py
View file

@ -18,14 +18,14 @@ from app.db.models import (
)
from app.db.session import get_session_factory
from app.logging_utils import structured_log
from app.services.domains.ingestion import queue as queue_service
from app.services.domains.ingestion.application import (
from app.services.ingestion import queue as queue_service
from app.services.ingestion.application import (
RunAlreadyInProgressError,
RunBlockedBySafetyPolicyError,
ScholarIngestionService,
)
from app.services.domains.scholar.source import LiveScholarSource
from app.services.domains.settings import application as user_settings_service
from app.services.scholar.source import LiveScholarSource
from app.services.settings import application as user_settings_service
from app.settings import settings
logger = logging.getLogger(__name__)
@ -589,7 +589,7 @@ class SchedulerService:
await self._finalize_queue_job_after_run(job, run_summary)
async def _drain_pdf_queue(self) -> None:
from app.services.domains.publications.pdf_queue import drain_ready_jobs
from app.services.publications.pdf_queue import drain_ready_jobs
session_factory = get_session_factory()
async with session_factory() as session:

4
app/services/ingestion/types.py
View file

@ -4,8 +4,8 @@ from dataclasses import dataclass, field
from typing import Any
from app.db.models import RunStatus
from app.services.domains.scholar.parser import ParsedProfilePage, PublicationCandidate
from app.services.domains.scholar.source import FetchResult
from app.services.scholar.parser import ParsedProfilePage, PublicationCandidate
from app.services.scholar.source import FetchResult
@dataclass(frozen=True)

2
app/services/openalex/client.py
View file

@ -8,7 +8,7 @@ from tenacity import (
wait_exponential,
)
from app.services.domains.openalex.types import OpenAlexWork
from app.services.openalex.types import OpenAlexWork
logger = logging.getLogger(__name__)

2
app/services/openalex/matching.py
View file

@ -3,7 +3,7 @@ import re
from rapidfuzz import fuzz
from app.services.domains.openalex.types import OpenAlexWork
from app.services.openalex.types import OpenAlexWork
logger = logging.getLogger(__name__)

14
app/services/portability/__init__.py
View file

@ -1,21 +1,21 @@
from app.services.domains.portability.application import (
from app.services.portability.application import (
EXPORT_SCHEMA_VERSION as EXPORT_SCHEMA_VERSION,
)
from app.services.domains.portability.application import (
from app.services.portability.application import (
MAX_IMPORT_PUBLICATIONS as MAX_IMPORT_PUBLICATIONS,
)
from app.services.domains.portability.application import (
from app.services.portability.application import (
MAX_IMPORT_SCHOLARS as MAX_IMPORT_SCHOLARS,
)
from app.services.domains.portability.application import (
from app.services.portability.application import (
ImportedPublicationInput as ImportedPublicationInput,
)
from app.services.domains.portability.application import (
from app.services.portability.application import (
ImportExportError as ImportExportError,
)
from app.services.domains.portability.application import (
from app.services.portability.application import (
export_user_data as export_user_data,
)
from app.services.domains.portability.application import (
from app.services.portability.application import (
import_user_data as import_user_data,
)

12
app/services/portability/application.py
View file

@ -5,20 +5,20 @@ from typing import Any
from sqlalchemy.ext.asyncio import AsyncSession
from app.db.models import Publication
from app.services.domains.portability.constants import (
from app.services.portability.constants import (
EXPORT_SCHEMA_VERSION,
MAX_IMPORT_PUBLICATIONS,
MAX_IMPORT_SCHOLARS,
)
from app.services.domains.portability.exporting import export_user_data
from app.services.domains.portability.normalize import _validate_import_sizes
from app.services.domains.portability.publication_import import (
from app.services.portability.exporting import export_user_data
from app.services.portability.normalize import _validate_import_sizes
from app.services.portability.publication_import import (
_build_imported_publication_input,
_initialize_import_counters,
_upsert_imported_publication,
)
from app.services.domains.portability.scholar_import import _upsert_imported_scholars
from app.services.domains.portability.types import ImportedPublicationInput, ImportExportError
from app.services.portability.scholar_import import _upsert_imported_scholars
from app.services.portability.types import ImportedPublicationInput, ImportExportError
async def import_user_data(

2
app/services/portability/exporting.py
View file

@ -7,7 +7,7 @@ from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.db.models import Publication, ScholarProfile, ScholarPublication
from app.services.domains.portability.constants import EXPORT_SCHEMA_VERSION
from app.services.portability.constants import EXPORT_SCHEMA_VERSION
def _exported_at_iso() -> str:

6
app/services/portability/normalize.py
View file

@ -3,14 +3,14 @@ from __future__ import annotations
import hashlib
from typing import Any
from app.services.domains.ingestion.application import normalize_title
from app.services.domains.portability.constants import (
from app.services.ingestion.application import normalize_title
from app.services.portability.constants import (
MAX_IMPORT_PUBLICATIONS,
MAX_IMPORT_SCHOLARS,
SHA256_RE,
WORD_RE,
)
from app.services.domains.portability.types import ImportExportError
from app.services.portability.types import ImportExportError
def _normalize_optional_text(value: Any) -> str | None:

8
app/services/portability/publication_import.py
View file

@ -6,15 +6,15 @@ from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.db.models import Publication, ScholarProfile, ScholarPublication
from app.services.domains.doi.normalize import normalize_doi
from app.services.domains.ingestion.application import build_publication_url, normalize_title
from app.services.domains.portability.normalize import (
from app.services.doi.normalize import normalize_doi
from app.services.ingestion.application import build_publication_url, normalize_title
from app.services.portability.normalize import (
_normalize_citation_count,
_normalize_optional_text,
_normalize_optional_year,
_resolve_fingerprint,
)
from app.services.domains.portability.types import ImportedPublicationInput
from app.services.portability.types import ImportedPublicationInput
async def _find_publication_by_cluster(

4
app/services/portability/scholar_import.py
View file

@ -6,8 +6,8 @@ from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.db.models import ScholarProfile
from app.services.domains.portability.normalize import _normalize_optional_text
from app.services.domains.scholars import application as scholar_service
from app.services.portability.normalize import _normalize_optional_text
from app.services.scholars import application as scholar_service
async def _load_user_scholar_map(

2
app/services/publication_identifiers/__init__.py
View file

@ -1,4 +1,4 @@
from app.services.domains.publication_identifiers.application import (
from app.services.publication_identifiers.application import (
DisplayIdentifier,
derive_display_identifier_from_values,
display_identifier_for_publication_id,

18
app/services/publication_identifiers/application.py
View file

@ -7,22 +7,22 @@ from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.db.models import Publication, PublicationIdentifier
from app.services.domains.arxiv.guards import arxiv_skip_reason_for_item
from app.services.domains.doi.normalize import normalize_doi
from app.services.domains.publication_identifiers.normalize import (
from app.services.arxiv.guards import arxiv_skip_reason_for_item
from app.services.doi.normalize import normalize_doi
from app.services.publication_identifiers.normalize import (
normalize_arxiv_id,
normalize_pmcid,
normalize_pmid,
)
from app.services.domains.publication_identifiers.types import (
from app.services.publication_identifiers.types import (
DisplayIdentifier,
IdentifierCandidate,
IdentifierKind,
)
if TYPE_CHECKING:
from app.services.domains.publications.pdf_queue import PdfQueueListItem
from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem
from app.services.publications.pdf_queue import PdfQueueListItem
from app.services.publications.types import PublicationListItem, UnreadPublicationItem
CONFIDENCE_HIGH = 0.98
CONFIDENCE_MEDIUM = 0.9
@ -163,7 +163,7 @@ def _identifier_lookup_item(
publication: Publication,
scholar_label: str,
) -> UnreadPublicationItem:
from app.services.domains.publications.types import UnreadPublicationItem
from app.services.publications.types import UnreadPublicationItem
return UnreadPublicationItem(
publication_id=int(publication.id),
@ -184,7 +184,7 @@ async def _discover_crossref_doi(
publication_id: int,
item: UnreadPublicationItem,
) -> bool:
from app.services.domains.crossref import application as crossref_service
from app.services.crossref import application as crossref_service
discovered_doi = await crossref_service.discover_doi_for_publication(item=item)
normalized_doi = normalize_doi(discovered_doi)
@ -212,7 +212,7 @@ async def _discover_arxiv_identifier(
publication_id: int,
item: UnreadPublicationItem,
) -> None:
from app.services.domains.arxiv import application as arxiv_service
from app.services.arxiv import application as arxiv_service
discovered_arxiv = await arxiv_service.discover_arxiv_id_for_publication(item=item)
normalized_arxiv = normalize_arxiv_id(discovered_arxiv)

4
app/services/publication_identifiers/normalize.py
View file

@ -3,8 +3,8 @@ from __future__ import annotations
import re
from urllib.parse import urlparse
from app.services.domains.doi.normalize import normalize_doi
from app.services.domains.publication_identifiers.types import IdentifierKind
from app.services.doi.normalize import normalize_doi
from app.services.publication_identifiers.types import IdentifierKind
ARXIV_ABS_RE = re.compile(r"\barxiv:\s*([a-z-]+/\d{7}|\d{4}\.\d{4,5})(v\d+)?\b", re.I)
ARXIV_RAW_RE = re.compile(r"^([a-z-]+/\d{7}|\d{4}\.\d{4,5})(v\d+)?$", re.I)

56
app/services/publications/__init__.py
View file

@ -1,84 +1,84 @@
from app.services.domains.publications.application import (
from app.services.publications.application import (
MODE_ALL as MODE_ALL,
)
from app.services.domains.publications.application import (
from app.services.publications.application import (
MODE_LATEST as MODE_LATEST,
)
from app.services.domains.publications.application import (
from app.services.publications.application import (
MODE_NEW as MODE_NEW,
)
from app.services.domains.publications.application import (
from app.services.publications.application import (
MODE_UNREAD as MODE_UNREAD,
)
from app.services.domains.publications.application import (
from app.services.publications.application import (
PublicationListItem as PublicationListItem,
)
from app.services.domains.publications.application import (
from app.services.publications.application import (
UnreadPublicationItem as UnreadPublicationItem,
)
from app.services.domains.publications.application import (
from app.services.publications.application import (
count_favorite_for_user as count_favorite_for_user,
)
from app.services.domains.publications.application import (
from app.services.publications.application import (
count_for_user as count_for_user,
)
from app.services.domains.publications.application import (
from app.services.publications.application import (
count_latest_for_user as count_latest_for_user,
)
from app.services.domains.publications.application import (
from app.services.publications.application import (
count_pdf_queue_items as count_pdf_queue_items,
)
from app.services.domains.publications.application import (
from app.services.publications.application import (
count_unread_for_user as count_unread_for_user,
)
from app.services.domains.publications.application import (
from app.services.publications.application import (
enqueue_all_missing_pdf_jobs as enqueue_all_missing_pdf_jobs,
)
from app.services.domains.publications.application import (
from app.services.publications.application import (
enqueue_retry_pdf_job_for_publication_id as enqueue_retry_pdf_job_for_publication_id,
)
from app.services.domains.publications.application import (
from app.services.publications.application import (
get_latest_run_id_for_user as get_latest_run_id_for_user,
)
from app.services.domains.publications.application import (
from app.services.publications.application import (
get_publication_item_for_user as get_publication_item_for_user,
)
from app.services.domains.publications.application import (
from app.services.publications.application import (
hydrate_pdf_enrichment_state as hydrate_pdf_enrichment_state,
)
from app.services.domains.publications.application import (
from app.services.publications.application import (
list_for_user as list_for_user,
)
from app.services.domains.publications.application import (
from app.services.publications.application import (
list_pdf_queue_items as list_pdf_queue_items,
)
from app.services.domains.publications.application import (
from app.services.publications.application import (
list_pdf_queue_page as list_pdf_queue_page,
)
from app.services.domains.publications.application import (
from app.services.publications.application import (
list_unread_for_user as list_unread_for_user,
)
from app.services.domains.publications.application import (
from app.services.publications.application import (
mark_all_unread_as_read_for_user as mark_all_unread_as_read_for_user,
)
from app.services.domains.publications.application import (
from app.services.publications.application import (
mark_selected_as_read_for_user as mark_selected_as_read_for_user,
)
from app.services.domains.publications.application import (
from app.services.publications.application import (
publications_query as publications_query,
)
from app.services.domains.publications.application import (
from app.services.publications.application import (
resolve_publication_view_mode as resolve_publication_view_mode,
)
from app.services.domains.publications.application import (
from app.services.publications.application import (
retry_pdf_for_user as retry_pdf_for_user,
)
from app.services.domains.publications.application import (
from app.services.publications.application import (
schedule_missing_pdf_enrichment_for_user as schedule_missing_pdf_enrichment_for_user,
)
from app.services.domains.publications.application import (
from app.services.publications.application import (
schedule_retry_pdf_enrichment_for_row as schedule_retry_pdf_enrichment_for_row,
)
from app.services.domains.publications.application import (
from app.services.publications.application import (
set_publication_favorite_for_user as set_publication_favorite_for_user,
)

16
app/services/publications/application.py
View file

@ -1,46 +1,46 @@
from __future__ import annotations
from app.services.domains.publications.counts import (
from app.services.publications.counts import (
count_favorite_for_user,
count_for_user,
count_latest_for_user,
count_unread_for_user,
)
from app.services.domains.publications.enrichment import (
from app.services.publications.enrichment import (
hydrate_pdf_enrichment_state,
schedule_missing_pdf_enrichment_for_user,
schedule_retry_pdf_enrichment_for_row,
)
from app.services.domains.publications.listing import (
from app.services.publications.listing import (
list_for_user,
list_unread_for_user,
retry_pdf_for_user,
)
from app.services.domains.publications.modes import (
from app.services.publications.modes import (
MODE_ALL,
MODE_LATEST,
MODE_NEW,
MODE_UNREAD,
resolve_publication_view_mode,
)
from app.services.domains.publications.pdf_queue import (
from app.services.publications.pdf_queue import (
count_pdf_queue_items,
enqueue_all_missing_pdf_jobs,
enqueue_retry_pdf_job_for_publication_id,
list_pdf_queue_items,
list_pdf_queue_page,
)
from app.services.domains.publications.queries import (
from app.services.publications.queries import (
get_latest_run_id_for_user,
get_publication_item_for_user,
publications_query,
)
from app.services.domains.publications.read_state import (
from app.services.publications.read_state import (
mark_all_unread_as_read_for_user,
mark_selected_as_read_for_user,
set_publication_favorite_for_user,
)
from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem
from app.services.publications.types import PublicationListItem, UnreadPublicationItem
__all__ = [
"MODE_ALL",

4
app/services/publications/counts.py
View file

@ -6,13 +6,13 @@ from sqlalchemy import distinct, func, select
from sqlalchemy.ext.asyncio import AsyncSession
from app.db.models import Publication, ScholarProfile, ScholarPublication
from app.services.domains.publications.modes import (
from app.services.publications.modes import (
MODE_ALL,
MODE_LATEST,
MODE_UNREAD,
resolve_publication_view_mode,
)
from app.services.domains.publications.queries import get_latest_run_id_for_user
from app.services.publications.queries import get_latest_run_id_for_user
async def count_for_user(

2
app/services/publications/dedup.py
View file

@ -11,7 +11,7 @@ from sqlalchemy.orm import aliased
from app.db.models import Publication, PublicationIdentifier, ScholarPublication
from app.logging_utils import structured_log
from app.services.domains.ingestion.fingerprints import (
from app.services.ingestion.fingerprints import (
canonical_title_text_for_dedup,
canonical_title_tokens_for_dedup,
normalize_title,

4
app/services/publications/enrichment.py
View file

@ -5,12 +5,12 @@ import logging
from sqlalchemy.ext.asyncio import AsyncSession
from app.logging_utils import structured_log
from app.services.domains.publications.pdf_queue import (
from app.services.publications.pdf_queue import (
enqueue_missing_pdf_jobs,
enqueue_retry_pdf_job,
overlay_pdf_job_state,
)
from app.services.domains.publications.types import PublicationListItem
from app.services.publications.types import PublicationListItem
logger = logging.getLogger(__name__)

8
app/services/publications/listing.py
View file

@ -4,20 +4,20 @@ from datetime import datetime
from sqlalchemy.ext.asyncio import AsyncSession
from app.services.domains.publication_identifiers import application as identifier_service
from app.services.domains.publications.modes import (
from app.services.publication_identifiers import application as identifier_service
from app.services.publications.modes import (
MODE_ALL,
MODE_UNREAD,
resolve_publication_view_mode,
)
from app.services.domains.publications.queries import (
from app.services.publications.queries import (
get_latest_run_id_for_user,
get_publication_item_for_user,
publication_list_item_from_row,
publications_query,
unread_item_from_row,
)
from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem
from app.services.publications.types import PublicationListItem, UnreadPublicationItem
async def list_for_user(

16
app/services/publications/pdf_queue.py
View file

@ -18,13 +18,13 @@ from app.db.models import (
)
from app.db.session import get_session_factory
from app.logging_utils import structured_log
from app.services.domains.publication_identifiers import application as identifier_service
from app.services.domains.publication_identifiers.types import DisplayIdentifier
from app.services.domains.publications.pdf_resolution_pipeline import (
from app.services.publication_identifiers import application as identifier_service
from app.services.publication_identifiers.types import DisplayIdentifier
from app.services.publications.pdf_resolution_pipeline import (
resolve_publication_pdf_outcome_for_row,
)
from app.services.domains.publications.types import PublicationListItem
from app.services.domains.unpaywall.application import (
from app.services.publications.types import PublicationListItem
from app.services.unpaywall.application import (
FAILURE_RESOLUTION_EXCEPTION,
OaResolutionOutcome,
)
@ -450,7 +450,7 @@ async def _resolve_publication_row(
openalex_api_key: str | None = None,
allow_arxiv_lookup: bool = True,
) -> bool:
from app.services.domains.openalex.client import OpenAlexBudgetExhaustedError
from app.services.openalex.client import OpenAlexBudgetExhaustedError
await _mark_attempt_started(publication_id=row.publication_id, user_id=user_id)
try:
@ -493,8 +493,8 @@ async def _run_resolution_task(
request_email: str | None,
rows: list[PublicationListItem],
) -> None:
from app.services.domains.openalex.client import OpenAlexBudgetExhaustedError
from app.services.domains.settings import application as user_settings_service
from app.services.openalex.client import OpenAlexBudgetExhaustedError
from app.services.settings import application as user_settings_service
# Resolve the best available API key: per-user setting → env var fallback.
openalex_api_key: str | None = None

16
app/services/publications/pdf_resolution_pipeline.py
View file

@ -5,11 +5,11 @@ from dataclasses import dataclass
from typing import Any
from app.logging_utils import structured_log
from app.services.domains.arxiv.errors import ArxivRateLimitError
from app.services.domains.arxiv.guards import arxiv_skip_reason_for_item
from app.services.domains.openalex.client import OpenAlexBudgetExhaustedError
from app.services.domains.publications.types import PublicationListItem
from app.services.domains.unpaywall.application import OaResolutionOutcome, resolve_publication_oa_outcomes
from app.services.arxiv.errors import ArxivRateLimitError
from app.services.arxiv.guards import arxiv_skip_reason_for_item
from app.services.openalex.client import OpenAlexBudgetExhaustedError
from app.services.publications.types import PublicationListItem
from app.services.unpaywall.application import OaResolutionOutcome, resolve_publication_oa_outcomes
from app.settings import settings
logger = logging.getLogger(__name__)
@ -59,8 +59,8 @@ async def _openalex_outcome(
request_email: str | None,
openalex_api_key: str | None = None,
) -> OaResolutionOutcome | None:
from app.services.domains.openalex.client import OpenAlexClient
from app.services.domains.openalex.matching import find_best_match
from app.services.openalex.client import OpenAlexClient
from app.services.openalex.matching import find_best_match
if not row.title:
return None
@ -105,7 +105,7 @@ async def _arxiv_outcome(
request_email: str | None,
allow_lookup: bool = True,
) -> OaResolutionOutcome | None:
from app.services.domains.arxiv.application import discover_arxiv_id_for_publication
from app.services.arxiv.application import discover_arxiv_id_for_publication
if not allow_lookup:
structured_log(

4
app/services/publications/queries.py
View file

@ -13,8 +13,8 @@ from app.db.models import (
ScholarProfile,
ScholarPublication,
)
from app.services.domains.publications.modes import MODE_LATEST, MODE_UNREAD
from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem
from app.services.publications.modes import MODE_LATEST, MODE_UNREAD
from app.services.publications.types import PublicationListItem, UnreadPublicationItem
def _normalized_citation_count(value: object) -> int:

2
app/services/publications/types.py
View file

@ -3,7 +3,7 @@ from __future__ import annotations
from dataclasses import dataclass
from datetime import datetime
from app.services.domains.publication_identifiers.types import DisplayIdentifier
from app.services.publication_identifiers.types import DisplayIdentifier
@dataclass(frozen=True)

34
app/services/runs/__init__.py
View file

@ -1,51 +1,51 @@
from app.services.domains.runs.application import (
from app.services.runs.application import (
QUEUE_STATUS_DROPPED as QUEUE_STATUS_DROPPED,
)
from app.services.domains.runs.application import (
from app.services.runs.application import (
QUEUE_STATUS_QUEUED as QUEUE_STATUS_QUEUED,
)
from app.services.domains.runs.application import (
from app.services.runs.application import (
QUEUE_STATUS_RETRYING as QUEUE_STATUS_RETRYING,
)
from app.services.domains.runs.application import (
from app.services.runs.application import (
QueueClearResult as QueueClearResult,
)
from app.services.domains.runs.application import (
from app.services.runs.application import (
QueueListItem as QueueListItem,
)
from app.services.domains.runs.application import (
from app.services.runs.application import (
QueueTransitionError as QueueTransitionError,
)
from app.services.domains.runs.application import (
from app.services.runs.application import (
clear_queue_item_for_user as clear_queue_item_for_user,
)
from app.services.domains.runs.application import (
from app.services.runs.application import (
drop_queue_item_for_user as drop_queue_item_for_user,
)
from app.services.domains.runs.application import (
from app.services.runs.application import (
extract_run_summary as extract_run_summary,
)
from app.services.domains.runs.application import (
from app.services.runs.application import (
get_manual_run_by_idempotency_key as get_manual_run_by_idempotency_key,
)
from app.services.domains.runs.application import (
from app.services.runs.application import (
get_queue_item_for_user as get_queue_item_for_user,
)
from app.services.domains.runs.application import (
from app.services.runs.application import (
get_run_for_user as get_run_for_user,
)
from app.services.domains.runs.application import (
from app.services.runs.application import (
list_queue_items_for_user as list_queue_items_for_user,
)
from app.services.domains.runs.application import (
from app.services.runs.application import (
list_recent_runs_for_user as list_recent_runs_for_user,
)
from app.services.domains.runs.application import (
from app.services.runs.application import (
list_runs_for_user as list_runs_for_user,
)
from app.services.domains.runs.application import (
from app.services.runs.application import (
queue_status_counts_for_user as queue_status_counts_for_user,
)
from app.services.domains.runs.application import (
from app.services.runs.application import (
retry_queue_item_for_user as retry_queue_item_for_user,
)

8
app/services/runs/application.py
View file

@ -1,6 +1,6 @@
from __future__ import annotations
from app.services.domains.runs.queue_service import (
from app.services.runs.queue_service import (
clear_queue_item_for_user,
drop_queue_item_for_user,
get_queue_item_for_user,
@ -8,14 +8,14 @@ from app.services.domains.runs.queue_service import (
queue_status_counts_for_user,
retry_queue_item_for_user,
)
from app.services.domains.runs.runs_service import (
from app.services.runs.runs_service import (
get_manual_run_by_idempotency_key,
get_run_for_user,
list_recent_runs_for_user,
list_runs_for_user,
)
from app.services.domains.runs.summary import extract_run_summary
from app.services.domains.runs.types import (
from app.services.runs.summary import extract_run_summary
from app.services.runs.types import (
QUEUE_STATUS_DROPPED,
QUEUE_STATUS_QUEUED,
QUEUE_STATUS_RETRYING,

2
app/services/runs/queue_queries.py
View file

@ -3,7 +3,7 @@ from __future__ import annotations
from sqlalchemy import and_, select
from app.db.models import IngestionQueueItem, ScholarProfile
from app.services.domains.runs.types import QueueListItem
from app.services.runs.types import QueueListItem
def queue_item_columns() -> tuple:

6
app/services/runs/queue_service.py
View file

@ -4,9 +4,9 @@ from sqlalchemy import case, func, select
from sqlalchemy.ext.asyncio import AsyncSession
from app.db.models import IngestionQueueItem
from app.services.domains.ingestion import queue as queue_mutations
from app.services.domains.runs.queue_queries import queue_item_select, queue_list_item_from_row
from app.services.domains.runs.types import (
from app.services.ingestion import queue as queue_mutations
from app.services.runs.queue_queries import queue_item_select, queue_list_item_from_row
from app.services.runs.types import (
QUEUE_STATUS_DROPPED,
QUEUE_STATUS_QUEUED,
QUEUE_STATUS_RETRYING,

6
app/services/scholar/author_rows.py
View file

@ -5,9 +5,9 @@ from html.parser import HTMLParser
from typing import Any
from urllib.parse import parse_qs, urlparse
from app.services.domains.scholar.parser_constants import AUTHOR_SEARCH_MARKER_KEYS
from app.services.domains.scholar.parser_types import ScholarSearchCandidate
from app.services.domains.scholar.parser_utils import (
from app.services.scholar.parser_constants import AUTHOR_SEARCH_MARKER_KEYS
from app.services.scholar.parser_types import ScholarSearchCandidate
from app.services.scholar.parser_utils import (
attr_class,
attr_href,
attr_src,

20
app/services/scholar/parser.py
View file

@ -1,30 +1,30 @@
from __future__ import annotations
from app.services.domains.scholar.author_rows import (
from app.services.scholar.author_rows import (
ScholarAuthorSearchParser,
count_author_search_markers,
)
from app.services.domains.scholar.parser_constants import SCRIPT_STYLE_RE
from app.services.domains.scholar.parser_types import (
from app.services.scholar.parser_constants import SCRIPT_STYLE_RE
from app.services.scholar.parser_types import (
ParsedAuthorSearchPage,
ParsedProfilePage,
PublicationCandidate,
ScholarDomInvariantError,
ScholarMalformedDataError,
)
from app.services.domains.scholar.parser_types import (
from app.services.scholar.parser_types import (
ParseState as ParseState,
)
from app.services.domains.scholar.parser_types import (
from app.services.scholar.parser_types import (
ScholarParserError as ScholarParserError,
)
from app.services.domains.scholar.parser_types import (
from app.services.scholar.parser_types import (
ScholarSearchCandidate as ScholarSearchCandidate,
)
from app.services.domains.scholar.parser_utils import (
from app.services.scholar.parser_utils import (
strip_tags,
)
from app.services.domains.scholar.profile_rows import (
from app.services.scholar.profile_rows import (
count_markers,
extract_articles_range,
extract_profile_image_url,
@ -33,8 +33,8 @@ from app.services.domains.scholar.profile_rows import (
has_show_more_button,
parse_publications,
)
from app.services.domains.scholar.source import FetchResult
from app.services.domains.scholar.state_detection import (
from app.services.scholar.source import FetchResult
from app.services.scholar.state_detection import (
detect_author_search_state,
detect_state,
)

2
app/services/scholar/parser_utils.py
View file

@ -2,7 +2,7 @@ from __future__ import annotations
from html import unescape
from app.services.domains.scholar.parser_constants import TAG_RE
from app.services.scholar.parser_constants import TAG_RE
def normalize_space(value: str) -> str:

6
app/services/scholar/profile_rows.py
View file

@ -5,12 +5,12 @@ from html.parser import HTMLParser
from typing import Any
from urllib.parse import parse_qs, urlparse
from app.services.domains.scholar.parser_constants import (
from app.services.scholar.parser_constants import (
MARKER_KEYS,
SHOW_MORE_BUTTON_RE,
)
from app.services.domains.scholar.parser_types import PublicationCandidate
from app.services.domains.scholar.parser_utils import (
from app.services.scholar.parser_types import PublicationCandidate
from app.services.scholar.parser_utils import (
attr_class,
attr_href,
build_absolute_scholar_url,

2
app/services/scholar/source.py
View file

@ -10,7 +10,7 @@ from urllib.parse import urlencode
from urllib.request import Request, urlopen
from app.logging_utils import structured_log
from app.services.domains.scholar import rate_limit as scholar_rate_limit
from app.services.scholar import rate_limit as scholar_rate_limit
from app.settings import settings
SCHOLAR_PROFILE_URL = "https://scholar.google.com/citations"

6
app/services/scholar/state_detection.py
View file

@ -1,6 +1,6 @@
from __future__ import annotations
from app.services.domains.scholar.parser_constants import (
from app.services.scholar.parser_constants import (
BLOCKED_KEYWORDS,
NETWORK_DNS_ERROR_KEYWORDS,
NETWORK_TIMEOUT_KEYWORDS,
@ -8,8 +8,8 @@ from app.services.domains.scholar.parser_constants import (
NO_AUTHOR_RESULTS_KEYWORDS,
NO_RESULTS_KEYWORDS,
)
from app.services.domains.scholar.parser_types import ParseState, PublicationCandidate, ScholarSearchCandidate
from app.services.domains.scholar.source import FetchResult
from app.services.scholar.parser_types import ParseState, PublicationCandidate, ScholarSearchCandidate
from app.services.scholar.source import FetchResult
def classify_network_error_reason(fetch_error: str | None) -> str:

16
app/services/scholars/application.py
View file

@ -15,15 +15,15 @@ from sqlalchemy.ext.asyncio import AsyncSession
from app.db.models import AuthorSearchCacheEntry, AuthorSearchRuntimeState, ScholarProfile
from app.logging_utils import structured_log
from app.services.domains.scholar.parser import (
from app.services.scholar.parser import (
ParsedAuthorSearchPage,
ParseState,
ScholarParserError,
parse_author_search_page,
parse_profile_page,
)
from app.services.domains.scholar.source import ScholarSource
from app.services.domains.scholars.constants import (
from app.services.scholar.source import ScholarSource
from app.services.scholars.constants import (
ALLOWED_IMAGE_UPLOAD_CONTENT_TYPES,
AUTHOR_SEARCH_LOCK_KEY,
AUTHOR_SEARCH_LOCK_NAMESPACE,
@ -41,18 +41,18 @@ from app.services.domains.scholars.constants import (
SEARCH_COOLDOWN_REASON,
SEARCH_DISABLED_REASON,
)
from app.services.domains.scholars.exceptions import ScholarServiceError
from app.services.domains.scholars.search_hints import (
from app.services.scholars.exceptions import ScholarServiceError
from app.services.scholars.search_hints import (
_merge_warnings,
_policy_blocked_author_search_result,
_trim_author_search_result,
)
from app.services.domains.scholars.uploads import (
from app.services.scholars.uploads import (
_ensure_upload_root,
_resolve_upload_path,
_safe_remove_upload,
)
from app.services.domains.scholars.validators import (
from app.services.scholars.validators import (
normalize_display_name,
normalize_profile_image_url,
validate_scholar_id,
@ -205,7 +205,7 @@ def _deserialize_parsed_author_search_page(payload: object) -> ParsedAuthorSearc
marker_counts = _payload_marker_counts(payload)
warnings = _payload_warnings(payload)
from app.services.domains.scholar.parser import ScholarSearchCandidate
from app.services.scholar.parser import ScholarSearchCandidate
normalized_candidates = _deserialize_candidates(payload)

4
app/services/scholars/search_hints.py
View file

@ -1,8 +1,8 @@
from __future__ import annotations
from app.db.models import ScholarProfile
from app.services.domains.scholar.parser import ParsedAuthorSearchPage, ParseState
from app.services.domains.scholars.constants import (
from app.services.scholar.parser import ParsedAuthorSearchPage, ParseState
from app.services.scholars.constants import (
MAX_AUTHOR_SEARCH_LIMIT,
STATE_REASON_HINTS,
)

2
app/services/scholars/uploads.py
View file

@ -2,7 +2,7 @@ from __future__ import annotations
from pathlib import Path
from app.services.domains.scholars.exceptions import ScholarServiceError
from app.services.scholars.exceptions import ScholarServiceError
def _ensure_upload_root(upload_dir: str, *, create: bool) -> Path:

4
app/services/scholars/validators.py
View file

@ -2,8 +2,8 @@ from __future__ import annotations
from urllib.parse import urlparse
from app.services.domains.scholars.constants import MAX_IMAGE_URL_LENGTH, SCHOLAR_ID_PATTERN
from app.services.domains.scholars.exceptions import ScholarServiceError
from app.services.scholars.constants import MAX_IMAGE_URL_LENGTH, SCHOLAR_ID_PATTERN
from app.services.scholars.exceptions import ScholarServiceError
def validate_scholar_id(value: str) -> str:

2
app/services/unpaywall/__init__.py
View file

@ -2,7 +2,7 @@ from __future__ import annotations
async def resolve_publication_pdf_urls(*args, **kwargs):
from app.services.domains.unpaywall.application import resolve_publication_pdf_urls as _impl
from app.services.unpaywall.application import resolve_publication_pdf_urls as _impl
return await _impl(*args, **kwargs)

10
app/services/unpaywall/application.py
View file

@ -7,17 +7,17 @@ from typing import TYPE_CHECKING
from urllib.parse import unquote
from app.logging_utils import structured_log
from app.services.domains.crossref.application import discover_doi_for_publication
from app.services.domains.doi.normalize import normalize_doi
from app.services.domains.unpaywall.pdf_discovery import (
from app.services.crossref.application import discover_doi_for_publication
from app.services.doi.normalize import normalize_doi
from app.services.unpaywall.pdf_discovery import (
looks_like_pdf_url,
resolve_pdf_from_landing_page,
)
from app.services.domains.unpaywall.rate_limit import wait_for_unpaywall_slot
from app.services.unpaywall.rate_limit import wait_for_unpaywall_slot
from app.settings import settings
if TYPE_CHECKING:
from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem
from app.services.publications.types import PublicationListItem, UnreadPublicationItem
DOI_PATTERN = re.compile(r"10\.\d{4,9}/[-._;()/:A-Z0-9]+", re.I)
DOI_PREFIX_RE = re.compile(r"\bdoi\s*[:=]\s*(10\.\d{4,9}/[-._;()/:A-Z0-9]+)", re.I)

2
app/services/unpaywall/pdf_discovery.py
View file

@ -4,7 +4,7 @@ import re
from html.parser import HTMLParser
from urllib.parse import urljoin, urlparse
from app.services.domains.unpaywall.rate_limit import wait_for_unpaywall_slot
from app.services.unpaywall.rate_limit import wait_for_unpaywall_slot
from app.settings import settings
PDF_MIME = "application/pdf"