Initial release prep
This commit is contained in:
parent
c9b9d892f4
commit
92395b2b4b
100 changed files with 405 additions and 401 deletions
|
|
@ -10,8 +10,8 @@ from app.api.deps import get_api_current_user
|
|||
from app.api.errors import ApiException
|
||||
from app.db.models import User
|
||||
from app.db.session import get_db_session
|
||||
from app.services.domains.scholars import application as scholar_service
|
||||
from app.services.domains.scholars import uploads as scholar_uploads
|
||||
from app.services.scholars import application as scholar_service
|
||||
from app.services.scholars import uploads as scholar_uploads
|
||||
from app.settings import settings
|
||||
|
||||
router = APIRouter(tags=["media"])
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ from app.auth.service import AuthService
|
|||
from app.db.models import User
|
||||
from app.db.session import get_db_session
|
||||
from app.logging_utils import structured_log
|
||||
from app.services.domains.users import application as user_service
|
||||
from app.services.users import application as user_service
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
|
|
|||
|
|
@ -22,13 +22,13 @@ from app.api.schemas import (
|
|||
from app.db.models import DataRepairJob, User
|
||||
from app.db.session import get_db_session
|
||||
from app.logging_utils import structured_log
|
||||
from app.services.domains.dbops import (
|
||||
from app.services.dbops import (
|
||||
collect_integrity_report,
|
||||
list_repair_jobs,
|
||||
run_publication_link_repair,
|
||||
run_publication_near_duplicate_repair,
|
||||
)
|
||||
from app.services.domains.publications import application as publication_service
|
||||
from app.services.publications import application as publication_service
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
|
|
|||
|
|
@ -26,7 +26,7 @@ from app.db.models import User
|
|||
from app.db.session import get_db_session
|
||||
from app.logging_utils import structured_log
|
||||
from app.security.csrf import ensure_csrf_token
|
||||
from app.services.domains.users import application as user_service
|
||||
from app.services.users import application as user_service
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
|
|
|||
|
|
@ -23,9 +23,9 @@ from app.api.schemas import (
|
|||
from app.db.models import User
|
||||
from app.db.session import get_db_session
|
||||
from app.logging_utils import structured_log
|
||||
from app.services.domains.publication_identifiers import application as identifier_service
|
||||
from app.services.domains.publications import application as publication_service
|
||||
from app.services.domains.scholars import application as scholar_service
|
||||
from app.services.publication_identifiers import application as identifier_service
|
||||
from app.services.publications import application as publication_service
|
||||
from app.services.scholars import application as scholar_service
|
||||
from app.settings import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
|
|
|||
|
|
@ -25,11 +25,11 @@ from app.api.schemas import (
|
|||
from app.db.models import RunStatus, RunTriggerType, User
|
||||
from app.db.session import get_db_session
|
||||
from app.logging_utils import structured_log
|
||||
from app.services.domains.ingestion import application as ingestion_service
|
||||
from app.services.domains.ingestion import safety as run_safety_service
|
||||
from app.services.domains.runs import application as run_service
|
||||
from app.services.domains.runs.events import event_generator
|
||||
from app.services.domains.settings import application as user_settings_service
|
||||
from app.services.ingestion import application as ingestion_service
|
||||
from app.services.ingestion import safety as run_safety_service
|
||||
from app.services.runs import application as run_service
|
||||
from app.services.runs.events import event_generator
|
||||
from app.services.settings import application as user_settings_service
|
||||
from app.settings import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
|
|
|||
|
|
@ -25,12 +25,12 @@ from app.api.schemas import (
|
|||
from app.db.models import User
|
||||
from app.db.session import get_db_session
|
||||
from app.logging_utils import structured_log
|
||||
from app.services.domains.ingestion import queue as ingestion_queue_service
|
||||
from app.services.domains.portability import application as import_export_service
|
||||
from app.services.domains.scholar import rate_limit as scholar_rate_limit
|
||||
from app.services.domains.scholar.source import ScholarSource
|
||||
from app.services.domains.scholars import application as scholar_service
|
||||
from app.services.domains.scholars import search_hints as scholar_search_hints
|
||||
from app.services.ingestion import queue as ingestion_queue_service
|
||||
from app.services.portability import application as import_export_service
|
||||
from app.services.scholar import rate_limit as scholar_rate_limit
|
||||
from app.services.scholar.source import ScholarSource
|
||||
from app.services.scholars import application as scholar_service
|
||||
from app.services.scholars import search_hints as scholar_search_hints
|
||||
from app.settings import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
|
|
|||
|
|
@ -12,8 +12,8 @@ from app.api.schemas import SettingsEnvelope, SettingsUpdateRequest
|
|||
from app.db.models import User
|
||||
from app.db.session import get_db_session
|
||||
from app.logging_utils import structured_log
|
||||
from app.services.domains.ingestion import safety as run_safety_service
|
||||
from app.services.domains.settings import application as user_settings_service
|
||||
from app.services.ingestion import safety as run_safety_service
|
||||
from app.services.settings import application as user_settings_service
|
||||
from app.settings import settings as settings_module
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
|
|
|||
|
|
@ -2,8 +2,8 @@ from __future__ import annotations
|
|||
|
||||
from fastapi import Depends
|
||||
|
||||
from app.services.domains.ingestion import application as ingestion_service
|
||||
from app.services.domains.scholar import source as scholar_source_service
|
||||
from app.services.ingestion import application as ingestion_service
|
||||
from app.services.scholar import source as scholar_source_service
|
||||
|
||||
|
||||
def get_scholar_source() -> scholar_source_service.ScholarSource:
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ from app.auth.session import clear_session_user, get_session_user, set_session_u
|
|||
from app.db.models import User
|
||||
from app.logging_utils import structured_log
|
||||
from app.security.csrf import CSRF_SESSION_KEY
|
||||
from app.services.domains.users import application as user_service
|
||||
from app.services.users import application as user_service
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ from app.http.middleware import (
|
|||
from app.logging_config import configure_logging, parse_redact_fields
|
||||
from app.logging_utils import structured_log
|
||||
from app.security.csrf import CSRFMiddleware
|
||||
from app.services.domains.ingestion.scheduler import SchedulerService
|
||||
from app.services.ingestion.scheduler import SchedulerService
|
||||
from app.settings import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
|
|
|||
|
|
@ -2,13 +2,13 @@ from __future__ import annotations
|
|||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from app.services.domains.arxiv.gateway import (
|
||||
from app.services.arxiv.gateway import (
|
||||
build_arxiv_query,
|
||||
get_arxiv_gateway,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem
|
||||
from app.services.publications.types import PublicationListItem, UnreadPublicationItem
|
||||
|
||||
|
||||
def _build_arxiv_query(title: str, author_surname: str | None) -> str | None:
|
||||
|
|
|
|||
|
|
@ -12,8 +12,8 @@ from sqlalchemy import delete, func, select
|
|||
|
||||
from app.db.models import ArxivQueryCacheEntry
|
||||
from app.db.session import get_session_factory
|
||||
from app.services.domains.arxiv.constants import ARXIV_CACHE_FINGERPRINT_VERSION
|
||||
from app.services.domains.arxiv.types import ArxivEntry, ArxivFeed, ArxivOpenSearchMeta
|
||||
from app.services.arxiv.constants import ARXIV_CACHE_FINGERPRINT_VERSION
|
||||
from app.services.arxiv.types import ArxivEntry, ArxivFeed, ArxivOpenSearchMeta
|
||||
|
||||
_INFLIGHT_LOCK = asyncio.Lock()
|
||||
_INFLIGHT_FEEDS: dict[str, asyncio.Future[ArxivFeed]] = {}
|
||||
|
|
|
|||
|
|
@ -6,21 +6,21 @@ from collections.abc import Awaitable, Callable
|
|||
import httpx
|
||||
|
||||
from app.logging_utils import structured_log
|
||||
from app.services.domains.arxiv.cache import (
|
||||
from app.services.arxiv.cache import (
|
||||
build_query_fingerprint,
|
||||
get_cached_feed,
|
||||
run_with_inflight_dedupe,
|
||||
set_cached_feed,
|
||||
)
|
||||
from app.services.domains.arxiv.constants import (
|
||||
from app.services.arxiv.constants import (
|
||||
ARXIV_SOURCE_PATH_LOOKUP_IDS,
|
||||
ARXIV_SOURCE_PATH_SEARCH,
|
||||
ARXIV_SOURCE_PATH_UNKNOWN,
|
||||
)
|
||||
from app.services.domains.arxiv.errors import ArxivClientValidationError, ArxivRateLimitError
|
||||
from app.services.domains.arxiv.parser import parse_arxiv_feed
|
||||
from app.services.domains.arxiv.rate_limit import get_arxiv_cooldown_status, run_with_global_arxiv_limit
|
||||
from app.services.domains.arxiv.types import ArxivFeed
|
||||
from app.services.arxiv.errors import ArxivClientValidationError, ArxivRateLimitError
|
||||
from app.services.arxiv.parser import parse_arxiv_feed
|
||||
from app.services.arxiv.rate_limit import get_arxiv_cooldown_status, run_with_global_arxiv_limit
|
||||
from app.services.arxiv.types import ArxivFeed
|
||||
from app.settings import settings
|
||||
|
||||
_ARXIV_API_URL = "https://export.arxiv.org/api/query"
|
||||
|
|
|
|||
|
|
@ -6,13 +6,13 @@ import unicodedata
|
|||
from typing import TYPE_CHECKING, Protocol
|
||||
|
||||
from app.logging_utils import structured_log
|
||||
from app.services.domains.arxiv.client import ArxivClient
|
||||
from app.services.domains.arxiv.errors import ArxivRateLimitError
|
||||
from app.services.domains.arxiv.types import ArxivFeed
|
||||
from app.services.arxiv.client import ArxivClient
|
||||
from app.services.arxiv.errors import ArxivRateLimitError
|
||||
from app.services.arxiv.types import ArxivFeed
|
||||
from app.settings import settings
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem
|
||||
from app.services.publications.types import PublicationListItem, UnreadPublicationItem
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
|
|
|||
|
|
@ -3,17 +3,17 @@ from __future__ import annotations
|
|||
import re
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from app.services.domains.arxiv.constants import (
|
||||
from app.services.arxiv.constants import (
|
||||
ARXIV_STRONG_IDENTIFIER_CONFIDENCE,
|
||||
ARXIV_TITLE_MIN_ALPHA_TOKENS,
|
||||
ARXIV_TITLE_MIN_TOKENS,
|
||||
ARXIV_TITLE_TOKEN_MIN_LENGTH,
|
||||
)
|
||||
from app.services.domains.doi.normalize import normalize_doi
|
||||
from app.services.domains.publication_identifiers.normalize import normalize_arxiv_id
|
||||
from app.services.doi.normalize import normalize_doi
|
||||
from app.services.publication_identifiers.normalize import normalize_arxiv_id
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem
|
||||
from app.services.publications.types import PublicationListItem, UnreadPublicationItem
|
||||
|
||||
_TITLE_TOKEN_RE = re.compile(r"[a-z0-9]+")
|
||||
|
||||
|
|
|
|||
|
|
@ -2,9 +2,9 @@ from __future__ import annotations
|
|||
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
from app.services.domains.arxiv.errors import ArxivParseError
|
||||
from app.services.domains.arxiv.types import ArxivEntry, ArxivFeed, ArxivOpenSearchMeta
|
||||
from app.services.domains.publication_identifiers.normalize import normalize_arxiv_id
|
||||
from app.services.arxiv.errors import ArxivParseError
|
||||
from app.services.arxiv.types import ArxivEntry, ArxivFeed, ArxivOpenSearchMeta
|
||||
from app.services.publication_identifiers.normalize import normalize_arxiv_id
|
||||
|
||||
_NAMESPACES = {
|
||||
"atom": "http://www.w3.org/2005/Atom",
|
||||
|
|
|
|||
|
|
@ -13,13 +13,13 @@ from sqlalchemy.ext.asyncio import AsyncSession
|
|||
from app.db.models import ArxivRuntimeState
|
||||
from app.db.session import get_session_factory
|
||||
from app.logging_utils import structured_log
|
||||
from app.services.domains.arxiv.constants import (
|
||||
from app.services.arxiv.constants import (
|
||||
ARXIV_RATE_LIMIT_LOCK_KEY,
|
||||
ARXIV_RATE_LIMIT_LOCK_NAMESPACE,
|
||||
ARXIV_RUNTIME_STATE_KEY,
|
||||
ARXIV_SOURCE_PATH_UNKNOWN,
|
||||
)
|
||||
from app.services.domains.arxiv.errors import ArxivRateLimitError
|
||||
from app.services.arxiv.errors import ArxivRateLimitError
|
||||
from app.settings import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from app.services.domains.crossref.application import discover_doi_for_publication
|
||||
from app.services.crossref.application import discover_doi_for_publication
|
||||
|
||||
__all__ = ["discover_doi_for_publication"]
|
||||
|
|
|
|||
|
|
@ -1,9 +1,9 @@
|
|||
from app.services.domains.dbops.application import run_publication_link_repair
|
||||
from app.services.domains.dbops.integrity import collect_integrity_report
|
||||
from app.services.domains.dbops.near_duplicate_repair import (
|
||||
from app.services.dbops.application import run_publication_link_repair
|
||||
from app.services.dbops.integrity import collect_integrity_report
|
||||
from app.services.dbops.near_duplicate_repair import (
|
||||
run_publication_near_duplicate_repair,
|
||||
)
|
||||
from app.services.domains.dbops.query import list_repair_jobs
|
||||
from app.services.dbops.query import list_repair_jobs
|
||||
|
||||
__all__ = [
|
||||
"collect_integrity_report",
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ from typing import Any
|
|||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.models import DataRepairJob
|
||||
from app.services.domains.publications import dedup as dedup_service
|
||||
from app.services.publications import dedup as dedup_service
|
||||
|
||||
REPAIR_STATUS_PLANNED = "planned"
|
||||
REPAIR_STATUS_RUNNING = "running"
|
||||
|
|
|
|||
|
|
@ -20,11 +20,11 @@ from app.db.models import (
|
|||
ScholarPublication,
|
||||
)
|
||||
from app.logging_utils import structured_log
|
||||
from app.services.domains.arxiv.errors import ArxivRateLimitError
|
||||
from app.services.domains.doi.normalize import first_doi_from_texts
|
||||
from app.services.domains.ingestion import queue as queue_service
|
||||
from app.services.domains.ingestion import safety as run_safety_service
|
||||
from app.services.domains.ingestion.constants import (
|
||||
from app.services.arxiv.errors import ArxivRateLimitError
|
||||
from app.services.doi.normalize import first_doi_from_texts
|
||||
from app.services.ingestion import queue as queue_service
|
||||
from app.services.ingestion import safety as run_safety_service
|
||||
from app.services.ingestion.constants import (
|
||||
FAILED_STATES,
|
||||
FAILURE_BUCKET_BLOCKED,
|
||||
FAILURE_BUCKET_INGESTION,
|
||||
|
|
@ -35,7 +35,7 @@ from app.services.domains.ingestion.constants import (
|
|||
RESUMABLE_PARTIAL_REASONS,
|
||||
RUN_LOCK_NAMESPACE,
|
||||
)
|
||||
from app.services.domains.ingestion.fingerprints import (
|
||||
from app.services.ingestion.fingerprints import (
|
||||
_build_body_excerpt,
|
||||
_dedupe_publication_candidates,
|
||||
_next_cstart_value,
|
||||
|
|
@ -45,7 +45,7 @@ from app.services.domains.ingestion.fingerprints import (
|
|||
canonical_title_for_dedup,
|
||||
normalize_title,
|
||||
)
|
||||
from app.services.domains.ingestion.types import (
|
||||
from app.services.ingestion.types import (
|
||||
PagedLoopState,
|
||||
PagedParseResult,
|
||||
RunAlertSummary,
|
||||
|
|
@ -56,17 +56,17 @@ from app.services.domains.ingestion.types import (
|
|||
RunProgress,
|
||||
ScholarProcessingOutcome,
|
||||
)
|
||||
from app.services.domains.publication_identifiers import application as identifier_service
|
||||
from app.services.domains.runs.events import run_events
|
||||
from app.services.domains.scholar.parser import (
|
||||
from app.services.publication_identifiers import application as identifier_service
|
||||
from app.services.runs.events import run_events
|
||||
from app.services.scholar.parser import (
|
||||
ParsedProfilePage,
|
||||
ParseState,
|
||||
PublicationCandidate,
|
||||
ScholarParserError,
|
||||
parse_profile_page,
|
||||
)
|
||||
from app.services.domains.scholar.source import FetchResult, ScholarSource
|
||||
from app.services.domains.settings import application as user_settings_service
|
||||
from app.services.scholar.source import FetchResult, ScholarSource
|
||||
from app.services.settings import application as user_settings_service
|
||||
from app.settings import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
|
@ -2369,12 +2369,12 @@ class ScholarIngestionService:
|
|||
Stops immediately on budget exhaustion (429 with $0 remaining).
|
||||
Sleeps 60s and continues on transient rate limits.
|
||||
"""
|
||||
from app.services.domains.openalex.client import (
|
||||
from app.services.openalex.client import (
|
||||
OpenAlexBudgetExhaustedError,
|
||||
OpenAlexClient,
|
||||
OpenAlexRateLimitError,
|
||||
)
|
||||
from app.services.domains.openalex.matching import find_best_match
|
||||
from app.services.openalex.matching import find_best_match
|
||||
|
||||
run_result = await db_session.execute(select(CrawlRun.user_id).where(CrawlRun.id == run_id))
|
||||
user_id = run_result.scalar_one()
|
||||
|
|
@ -2480,7 +2480,7 @@ class ScholarIngestionService:
|
|||
|
||||
await db_session.flush()
|
||||
|
||||
from app.services.domains.publications.dedup import sweep_identifier_duplicates
|
||||
from app.services.publications.dedup import sweep_identifier_duplicates
|
||||
|
||||
merge_count = await sweep_identifier_duplicates(db_session)
|
||||
if merge_count:
|
||||
|
|
@ -2579,8 +2579,8 @@ class ScholarIngestionService:
|
|||
if not publications:
|
||||
return publications
|
||||
|
||||
from app.services.domains.openalex.client import OpenAlexClient
|
||||
from app.services.domains.openalex.matching import find_best_match
|
||||
from app.services.openalex.client import OpenAlexClient
|
||||
from app.services.openalex.matching import find_best_match
|
||||
|
||||
client = OpenAlexClient(api_key=settings.openalex_api_key, mailto=settings.crossref_api_mailto)
|
||||
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ from __future__ import annotations
|
|||
|
||||
import re
|
||||
|
||||
from app.services.domains.scholar.parser import ParseState
|
||||
from app.services.scholar.parser import ParseState
|
||||
|
||||
TITLE_ALNUM_RE = re.compile(r"[^a-z0-9]+")
|
||||
WORD_RE = re.compile(r"[a-z0-9]+")
|
||||
|
|
|
|||
|
|
@ -7,14 +7,14 @@ import unicodedata
|
|||
from typing import Any
|
||||
from urllib.parse import urljoin
|
||||
|
||||
from app.services.domains.ingestion.constants import (
|
||||
from app.services.ingestion.constants import (
|
||||
HTML_TAG_RE,
|
||||
INITIAL_PAGE_FINGERPRINT_MAX_PUBLICATIONS,
|
||||
SPACE_RE,
|
||||
TITLE_ALNUM_RE,
|
||||
WORD_RE,
|
||||
)
|
||||
from app.services.domains.scholar.parser import ParsedProfilePage, ParseState, PublicationCandidate
|
||||
from app.services.scholar.parser import ParsedProfilePage, ParseState, PublicationCandidate
|
||||
|
||||
# Scholar-specific noise patterns stripped before canonical comparison.
|
||||
# Applied in order; each targets a different Scholar metadata injection style.
|
||||
|
|
|
|||
|
|
@ -18,14 +18,14 @@ from app.db.models import (
|
|||
)
|
||||
from app.db.session import get_session_factory
|
||||
from app.logging_utils import structured_log
|
||||
from app.services.domains.ingestion import queue as queue_service
|
||||
from app.services.domains.ingestion.application import (
|
||||
from app.services.ingestion import queue as queue_service
|
||||
from app.services.ingestion.application import (
|
||||
RunAlreadyInProgressError,
|
||||
RunBlockedBySafetyPolicyError,
|
||||
ScholarIngestionService,
|
||||
)
|
||||
from app.services.domains.scholar.source import LiveScholarSource
|
||||
from app.services.domains.settings import application as user_settings_service
|
||||
from app.services.scholar.source import LiveScholarSource
|
||||
from app.services.settings import application as user_settings_service
|
||||
from app.settings import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
|
@ -589,7 +589,7 @@ class SchedulerService:
|
|||
await self._finalize_queue_job_after_run(job, run_summary)
|
||||
|
||||
async def _drain_pdf_queue(self) -> None:
|
||||
from app.services.domains.publications.pdf_queue import drain_ready_jobs
|
||||
from app.services.publications.pdf_queue import drain_ready_jobs
|
||||
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as session:
|
||||
|
|
|
|||
|
|
@ -4,8 +4,8 @@ from dataclasses import dataclass, field
|
|||
from typing import Any
|
||||
|
||||
from app.db.models import RunStatus
|
||||
from app.services.domains.scholar.parser import ParsedProfilePage, PublicationCandidate
|
||||
from app.services.domains.scholar.source import FetchResult
|
||||
from app.services.scholar.parser import ParsedProfilePage, PublicationCandidate
|
||||
from app.services.scholar.source import FetchResult
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ from tenacity import (
|
|||
wait_exponential,
|
||||
)
|
||||
|
||||
from app.services.domains.openalex.types import OpenAlexWork
|
||||
from app.services.openalex.types import OpenAlexWork
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ import re
|
|||
|
||||
from rapidfuzz import fuzz
|
||||
|
||||
from app.services.domains.openalex.types import OpenAlexWork
|
||||
from app.services.openalex.types import OpenAlexWork
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,21 +1,21 @@
|
|||
from app.services.domains.portability.application import (
|
||||
from app.services.portability.application import (
|
||||
EXPORT_SCHEMA_VERSION as EXPORT_SCHEMA_VERSION,
|
||||
)
|
||||
from app.services.domains.portability.application import (
|
||||
from app.services.portability.application import (
|
||||
MAX_IMPORT_PUBLICATIONS as MAX_IMPORT_PUBLICATIONS,
|
||||
)
|
||||
from app.services.domains.portability.application import (
|
||||
from app.services.portability.application import (
|
||||
MAX_IMPORT_SCHOLARS as MAX_IMPORT_SCHOLARS,
|
||||
)
|
||||
from app.services.domains.portability.application import (
|
||||
from app.services.portability.application import (
|
||||
ImportedPublicationInput as ImportedPublicationInput,
|
||||
)
|
||||
from app.services.domains.portability.application import (
|
||||
from app.services.portability.application import (
|
||||
ImportExportError as ImportExportError,
|
||||
)
|
||||
from app.services.domains.portability.application import (
|
||||
from app.services.portability.application import (
|
||||
export_user_data as export_user_data,
|
||||
)
|
||||
from app.services.domains.portability.application import (
|
||||
from app.services.portability.application import (
|
||||
import_user_data as import_user_data,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -5,20 +5,20 @@ from typing import Any
|
|||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.models import Publication
|
||||
from app.services.domains.portability.constants import (
|
||||
from app.services.portability.constants import (
|
||||
EXPORT_SCHEMA_VERSION,
|
||||
MAX_IMPORT_PUBLICATIONS,
|
||||
MAX_IMPORT_SCHOLARS,
|
||||
)
|
||||
from app.services.domains.portability.exporting import export_user_data
|
||||
from app.services.domains.portability.normalize import _validate_import_sizes
|
||||
from app.services.domains.portability.publication_import import (
|
||||
from app.services.portability.exporting import export_user_data
|
||||
from app.services.portability.normalize import _validate_import_sizes
|
||||
from app.services.portability.publication_import import (
|
||||
_build_imported_publication_input,
|
||||
_initialize_import_counters,
|
||||
_upsert_imported_publication,
|
||||
)
|
||||
from app.services.domains.portability.scholar_import import _upsert_imported_scholars
|
||||
from app.services.domains.portability.types import ImportedPublicationInput, ImportExportError
|
||||
from app.services.portability.scholar_import import _upsert_imported_scholars
|
||||
from app.services.portability.types import ImportedPublicationInput, ImportExportError
|
||||
|
||||
|
||||
async def import_user_data(
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ from sqlalchemy import select
|
|||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.models import Publication, ScholarProfile, ScholarPublication
|
||||
from app.services.domains.portability.constants import EXPORT_SCHEMA_VERSION
|
||||
from app.services.portability.constants import EXPORT_SCHEMA_VERSION
|
||||
|
||||
|
||||
def _exported_at_iso() -> str:
|
||||
|
|
|
|||
|
|
@ -3,14 +3,14 @@ from __future__ import annotations
|
|||
import hashlib
|
||||
from typing import Any
|
||||
|
||||
from app.services.domains.ingestion.application import normalize_title
|
||||
from app.services.domains.portability.constants import (
|
||||
from app.services.ingestion.application import normalize_title
|
||||
from app.services.portability.constants import (
|
||||
MAX_IMPORT_PUBLICATIONS,
|
||||
MAX_IMPORT_SCHOLARS,
|
||||
SHA256_RE,
|
||||
WORD_RE,
|
||||
)
|
||||
from app.services.domains.portability.types import ImportExportError
|
||||
from app.services.portability.types import ImportExportError
|
||||
|
||||
|
||||
def _normalize_optional_text(value: Any) -> str | None:
|
||||
|
|
|
|||
|
|
@ -6,15 +6,15 @@ from sqlalchemy import select
|
|||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.models import Publication, ScholarProfile, ScholarPublication
|
||||
from app.services.domains.doi.normalize import normalize_doi
|
||||
from app.services.domains.ingestion.application import build_publication_url, normalize_title
|
||||
from app.services.domains.portability.normalize import (
|
||||
from app.services.doi.normalize import normalize_doi
|
||||
from app.services.ingestion.application import build_publication_url, normalize_title
|
||||
from app.services.portability.normalize import (
|
||||
_normalize_citation_count,
|
||||
_normalize_optional_text,
|
||||
_normalize_optional_year,
|
||||
_resolve_fingerprint,
|
||||
)
|
||||
from app.services.domains.portability.types import ImportedPublicationInput
|
||||
from app.services.portability.types import ImportedPublicationInput
|
||||
|
||||
|
||||
async def _find_publication_by_cluster(
|
||||
|
|
|
|||
|
|
@ -6,8 +6,8 @@ from sqlalchemy import select
|
|||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.models import ScholarProfile
|
||||
from app.services.domains.portability.normalize import _normalize_optional_text
|
||||
from app.services.domains.scholars import application as scholar_service
|
||||
from app.services.portability.normalize import _normalize_optional_text
|
||||
from app.services.scholars import application as scholar_service
|
||||
|
||||
|
||||
async def _load_user_scholar_map(
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
from app.services.domains.publication_identifiers.application import (
|
||||
from app.services.publication_identifiers.application import (
|
||||
DisplayIdentifier,
|
||||
derive_display_identifier_from_values,
|
||||
display_identifier_for_publication_id,
|
||||
|
|
|
|||
|
|
@ -7,22 +7,22 @@ from sqlalchemy import select
|
|||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.models import Publication, PublicationIdentifier
|
||||
from app.services.domains.arxiv.guards import arxiv_skip_reason_for_item
|
||||
from app.services.domains.doi.normalize import normalize_doi
|
||||
from app.services.domains.publication_identifiers.normalize import (
|
||||
from app.services.arxiv.guards import arxiv_skip_reason_for_item
|
||||
from app.services.doi.normalize import normalize_doi
|
||||
from app.services.publication_identifiers.normalize import (
|
||||
normalize_arxiv_id,
|
||||
normalize_pmcid,
|
||||
normalize_pmid,
|
||||
)
|
||||
from app.services.domains.publication_identifiers.types import (
|
||||
from app.services.publication_identifiers.types import (
|
||||
DisplayIdentifier,
|
||||
IdentifierCandidate,
|
||||
IdentifierKind,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from app.services.domains.publications.pdf_queue import PdfQueueListItem
|
||||
from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem
|
||||
from app.services.publications.pdf_queue import PdfQueueListItem
|
||||
from app.services.publications.types import PublicationListItem, UnreadPublicationItem
|
||||
|
||||
CONFIDENCE_HIGH = 0.98
|
||||
CONFIDENCE_MEDIUM = 0.9
|
||||
|
|
@ -163,7 +163,7 @@ def _identifier_lookup_item(
|
|||
publication: Publication,
|
||||
scholar_label: str,
|
||||
) -> UnreadPublicationItem:
|
||||
from app.services.domains.publications.types import UnreadPublicationItem
|
||||
from app.services.publications.types import UnreadPublicationItem
|
||||
|
||||
return UnreadPublicationItem(
|
||||
publication_id=int(publication.id),
|
||||
|
|
@ -184,7 +184,7 @@ async def _discover_crossref_doi(
|
|||
publication_id: int,
|
||||
item: UnreadPublicationItem,
|
||||
) -> bool:
|
||||
from app.services.domains.crossref import application as crossref_service
|
||||
from app.services.crossref import application as crossref_service
|
||||
|
||||
discovered_doi = await crossref_service.discover_doi_for_publication(item=item)
|
||||
normalized_doi = normalize_doi(discovered_doi)
|
||||
|
|
@ -212,7 +212,7 @@ async def _discover_arxiv_identifier(
|
|||
publication_id: int,
|
||||
item: UnreadPublicationItem,
|
||||
) -> None:
|
||||
from app.services.domains.arxiv import application as arxiv_service
|
||||
from app.services.arxiv import application as arxiv_service
|
||||
|
||||
discovered_arxiv = await arxiv_service.discover_arxiv_id_for_publication(item=item)
|
||||
normalized_arxiv = normalize_arxiv_id(discovered_arxiv)
|
||||
|
|
|
|||
|
|
@ -3,8 +3,8 @@ from __future__ import annotations
|
|||
import re
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from app.services.domains.doi.normalize import normalize_doi
|
||||
from app.services.domains.publication_identifiers.types import IdentifierKind
|
||||
from app.services.doi.normalize import normalize_doi
|
||||
from app.services.publication_identifiers.types import IdentifierKind
|
||||
|
||||
ARXIV_ABS_RE = re.compile(r"\barxiv:\s*([a-z-]+/\d{7}|\d{4}\.\d{4,5})(v\d+)?\b", re.I)
|
||||
ARXIV_RAW_RE = re.compile(r"^([a-z-]+/\d{7}|\d{4}\.\d{4,5})(v\d+)?$", re.I)
|
||||
|
|
|
|||
|
|
@ -1,84 +1,84 @@
|
|||
from app.services.domains.publications.application import (
|
||||
from app.services.publications.application import (
|
||||
MODE_ALL as MODE_ALL,
|
||||
)
|
||||
from app.services.domains.publications.application import (
|
||||
from app.services.publications.application import (
|
||||
MODE_LATEST as MODE_LATEST,
|
||||
)
|
||||
from app.services.domains.publications.application import (
|
||||
from app.services.publications.application import (
|
||||
MODE_NEW as MODE_NEW,
|
||||
)
|
||||
from app.services.domains.publications.application import (
|
||||
from app.services.publications.application import (
|
||||
MODE_UNREAD as MODE_UNREAD,
|
||||
)
|
||||
from app.services.domains.publications.application import (
|
||||
from app.services.publications.application import (
|
||||
PublicationListItem as PublicationListItem,
|
||||
)
|
||||
from app.services.domains.publications.application import (
|
||||
from app.services.publications.application import (
|
||||
UnreadPublicationItem as UnreadPublicationItem,
|
||||
)
|
||||
from app.services.domains.publications.application import (
|
||||
from app.services.publications.application import (
|
||||
count_favorite_for_user as count_favorite_for_user,
|
||||
)
|
||||
from app.services.domains.publications.application import (
|
||||
from app.services.publications.application import (
|
||||
count_for_user as count_for_user,
|
||||
)
|
||||
from app.services.domains.publications.application import (
|
||||
from app.services.publications.application import (
|
||||
count_latest_for_user as count_latest_for_user,
|
||||
)
|
||||
from app.services.domains.publications.application import (
|
||||
from app.services.publications.application import (
|
||||
count_pdf_queue_items as count_pdf_queue_items,
|
||||
)
|
||||
from app.services.domains.publications.application import (
|
||||
from app.services.publications.application import (
|
||||
count_unread_for_user as count_unread_for_user,
|
||||
)
|
||||
from app.services.domains.publications.application import (
|
||||
from app.services.publications.application import (
|
||||
enqueue_all_missing_pdf_jobs as enqueue_all_missing_pdf_jobs,
|
||||
)
|
||||
from app.services.domains.publications.application import (
|
||||
from app.services.publications.application import (
|
||||
enqueue_retry_pdf_job_for_publication_id as enqueue_retry_pdf_job_for_publication_id,
|
||||
)
|
||||
from app.services.domains.publications.application import (
|
||||
from app.services.publications.application import (
|
||||
get_latest_run_id_for_user as get_latest_run_id_for_user,
|
||||
)
|
||||
from app.services.domains.publications.application import (
|
||||
from app.services.publications.application import (
|
||||
get_publication_item_for_user as get_publication_item_for_user,
|
||||
)
|
||||
from app.services.domains.publications.application import (
|
||||
from app.services.publications.application import (
|
||||
hydrate_pdf_enrichment_state as hydrate_pdf_enrichment_state,
|
||||
)
|
||||
from app.services.domains.publications.application import (
|
||||
from app.services.publications.application import (
|
||||
list_for_user as list_for_user,
|
||||
)
|
||||
from app.services.domains.publications.application import (
|
||||
from app.services.publications.application import (
|
||||
list_pdf_queue_items as list_pdf_queue_items,
|
||||
)
|
||||
from app.services.domains.publications.application import (
|
||||
from app.services.publications.application import (
|
||||
list_pdf_queue_page as list_pdf_queue_page,
|
||||
)
|
||||
from app.services.domains.publications.application import (
|
||||
from app.services.publications.application import (
|
||||
list_unread_for_user as list_unread_for_user,
|
||||
)
|
||||
from app.services.domains.publications.application import (
|
||||
from app.services.publications.application import (
|
||||
mark_all_unread_as_read_for_user as mark_all_unread_as_read_for_user,
|
||||
)
|
||||
from app.services.domains.publications.application import (
|
||||
from app.services.publications.application import (
|
||||
mark_selected_as_read_for_user as mark_selected_as_read_for_user,
|
||||
)
|
||||
from app.services.domains.publications.application import (
|
||||
from app.services.publications.application import (
|
||||
publications_query as publications_query,
|
||||
)
|
||||
from app.services.domains.publications.application import (
|
||||
from app.services.publications.application import (
|
||||
resolve_publication_view_mode as resolve_publication_view_mode,
|
||||
)
|
||||
from app.services.domains.publications.application import (
|
||||
from app.services.publications.application import (
|
||||
retry_pdf_for_user as retry_pdf_for_user,
|
||||
)
|
||||
from app.services.domains.publications.application import (
|
||||
from app.services.publications.application import (
|
||||
schedule_missing_pdf_enrichment_for_user as schedule_missing_pdf_enrichment_for_user,
|
||||
)
|
||||
from app.services.domains.publications.application import (
|
||||
from app.services.publications.application import (
|
||||
schedule_retry_pdf_enrichment_for_row as schedule_retry_pdf_enrichment_for_row,
|
||||
)
|
||||
from app.services.domains.publications.application import (
|
||||
from app.services.publications.application import (
|
||||
set_publication_favorite_for_user as set_publication_favorite_for_user,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,46 +1,46 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from app.services.domains.publications.counts import (
|
||||
from app.services.publications.counts import (
|
||||
count_favorite_for_user,
|
||||
count_for_user,
|
||||
count_latest_for_user,
|
||||
count_unread_for_user,
|
||||
)
|
||||
from app.services.domains.publications.enrichment import (
|
||||
from app.services.publications.enrichment import (
|
||||
hydrate_pdf_enrichment_state,
|
||||
schedule_missing_pdf_enrichment_for_user,
|
||||
schedule_retry_pdf_enrichment_for_row,
|
||||
)
|
||||
from app.services.domains.publications.listing import (
|
||||
from app.services.publications.listing import (
|
||||
list_for_user,
|
||||
list_unread_for_user,
|
||||
retry_pdf_for_user,
|
||||
)
|
||||
from app.services.domains.publications.modes import (
|
||||
from app.services.publications.modes import (
|
||||
MODE_ALL,
|
||||
MODE_LATEST,
|
||||
MODE_NEW,
|
||||
MODE_UNREAD,
|
||||
resolve_publication_view_mode,
|
||||
)
|
||||
from app.services.domains.publications.pdf_queue import (
|
||||
from app.services.publications.pdf_queue import (
|
||||
count_pdf_queue_items,
|
||||
enqueue_all_missing_pdf_jobs,
|
||||
enqueue_retry_pdf_job_for_publication_id,
|
||||
list_pdf_queue_items,
|
||||
list_pdf_queue_page,
|
||||
)
|
||||
from app.services.domains.publications.queries import (
|
||||
from app.services.publications.queries import (
|
||||
get_latest_run_id_for_user,
|
||||
get_publication_item_for_user,
|
||||
publications_query,
|
||||
)
|
||||
from app.services.domains.publications.read_state import (
|
||||
from app.services.publications.read_state import (
|
||||
mark_all_unread_as_read_for_user,
|
||||
mark_selected_as_read_for_user,
|
||||
set_publication_favorite_for_user,
|
||||
)
|
||||
from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem
|
||||
from app.services.publications.types import PublicationListItem, UnreadPublicationItem
|
||||
|
||||
__all__ = [
|
||||
"MODE_ALL",
|
||||
|
|
|
|||
|
|
@ -6,13 +6,13 @@ from sqlalchemy import distinct, func, select
|
|||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.models import Publication, ScholarProfile, ScholarPublication
|
||||
from app.services.domains.publications.modes import (
|
||||
from app.services.publications.modes import (
|
||||
MODE_ALL,
|
||||
MODE_LATEST,
|
||||
MODE_UNREAD,
|
||||
resolve_publication_view_mode,
|
||||
)
|
||||
from app.services.domains.publications.queries import get_latest_run_id_for_user
|
||||
from app.services.publications.queries import get_latest_run_id_for_user
|
||||
|
||||
|
||||
async def count_for_user(
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ from sqlalchemy.orm import aliased
|
|||
|
||||
from app.db.models import Publication, PublicationIdentifier, ScholarPublication
|
||||
from app.logging_utils import structured_log
|
||||
from app.services.domains.ingestion.fingerprints import (
|
||||
from app.services.ingestion.fingerprints import (
|
||||
canonical_title_text_for_dedup,
|
||||
canonical_title_tokens_for_dedup,
|
||||
normalize_title,
|
||||
|
|
|
|||
|
|
@ -5,12 +5,12 @@ import logging
|
|||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.logging_utils import structured_log
|
||||
from app.services.domains.publications.pdf_queue import (
|
||||
from app.services.publications.pdf_queue import (
|
||||
enqueue_missing_pdf_jobs,
|
||||
enqueue_retry_pdf_job,
|
||||
overlay_pdf_job_state,
|
||||
)
|
||||
from app.services.domains.publications.types import PublicationListItem
|
||||
from app.services.publications.types import PublicationListItem
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
|
|
|||
|
|
@ -4,20 +4,20 @@ from datetime import datetime
|
|||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.services.domains.publication_identifiers import application as identifier_service
|
||||
from app.services.domains.publications.modes import (
|
||||
from app.services.publication_identifiers import application as identifier_service
|
||||
from app.services.publications.modes import (
|
||||
MODE_ALL,
|
||||
MODE_UNREAD,
|
||||
resolve_publication_view_mode,
|
||||
)
|
||||
from app.services.domains.publications.queries import (
|
||||
from app.services.publications.queries import (
|
||||
get_latest_run_id_for_user,
|
||||
get_publication_item_for_user,
|
||||
publication_list_item_from_row,
|
||||
publications_query,
|
||||
unread_item_from_row,
|
||||
)
|
||||
from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem
|
||||
from app.services.publications.types import PublicationListItem, UnreadPublicationItem
|
||||
|
||||
|
||||
async def list_for_user(
|
||||
|
|
|
|||
|
|
@ -18,13 +18,13 @@ from app.db.models import (
|
|||
)
|
||||
from app.db.session import get_session_factory
|
||||
from app.logging_utils import structured_log
|
||||
from app.services.domains.publication_identifiers import application as identifier_service
|
||||
from app.services.domains.publication_identifiers.types import DisplayIdentifier
|
||||
from app.services.domains.publications.pdf_resolution_pipeline import (
|
||||
from app.services.publication_identifiers import application as identifier_service
|
||||
from app.services.publication_identifiers.types import DisplayIdentifier
|
||||
from app.services.publications.pdf_resolution_pipeline import (
|
||||
resolve_publication_pdf_outcome_for_row,
|
||||
)
|
||||
from app.services.domains.publications.types import PublicationListItem
|
||||
from app.services.domains.unpaywall.application import (
|
||||
from app.services.publications.types import PublicationListItem
|
||||
from app.services.unpaywall.application import (
|
||||
FAILURE_RESOLUTION_EXCEPTION,
|
||||
OaResolutionOutcome,
|
||||
)
|
||||
|
|
@ -450,7 +450,7 @@ async def _resolve_publication_row(
|
|||
openalex_api_key: str | None = None,
|
||||
allow_arxiv_lookup: bool = True,
|
||||
) -> bool:
|
||||
from app.services.domains.openalex.client import OpenAlexBudgetExhaustedError
|
||||
from app.services.openalex.client import OpenAlexBudgetExhaustedError
|
||||
|
||||
await _mark_attempt_started(publication_id=row.publication_id, user_id=user_id)
|
||||
try:
|
||||
|
|
@ -493,8 +493,8 @@ async def _run_resolution_task(
|
|||
request_email: str | None,
|
||||
rows: list[PublicationListItem],
|
||||
) -> None:
|
||||
from app.services.domains.openalex.client import OpenAlexBudgetExhaustedError
|
||||
from app.services.domains.settings import application as user_settings_service
|
||||
from app.services.openalex.client import OpenAlexBudgetExhaustedError
|
||||
from app.services.settings import application as user_settings_service
|
||||
|
||||
# Resolve the best available API key: per-user setting → env var fallback.
|
||||
openalex_api_key: str | None = None
|
||||
|
|
|
|||
|
|
@ -5,11 +5,11 @@ from dataclasses import dataclass
|
|||
from typing import Any
|
||||
|
||||
from app.logging_utils import structured_log
|
||||
from app.services.domains.arxiv.errors import ArxivRateLimitError
|
||||
from app.services.domains.arxiv.guards import arxiv_skip_reason_for_item
|
||||
from app.services.domains.openalex.client import OpenAlexBudgetExhaustedError
|
||||
from app.services.domains.publications.types import PublicationListItem
|
||||
from app.services.domains.unpaywall.application import OaResolutionOutcome, resolve_publication_oa_outcomes
|
||||
from app.services.arxiv.errors import ArxivRateLimitError
|
||||
from app.services.arxiv.guards import arxiv_skip_reason_for_item
|
||||
from app.services.openalex.client import OpenAlexBudgetExhaustedError
|
||||
from app.services.publications.types import PublicationListItem
|
||||
from app.services.unpaywall.application import OaResolutionOutcome, resolve_publication_oa_outcomes
|
||||
from app.settings import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
|
@ -59,8 +59,8 @@ async def _openalex_outcome(
|
|||
request_email: str | None,
|
||||
openalex_api_key: str | None = None,
|
||||
) -> OaResolutionOutcome | None:
|
||||
from app.services.domains.openalex.client import OpenAlexClient
|
||||
from app.services.domains.openalex.matching import find_best_match
|
||||
from app.services.openalex.client import OpenAlexClient
|
||||
from app.services.openalex.matching import find_best_match
|
||||
|
||||
if not row.title:
|
||||
return None
|
||||
|
|
@ -105,7 +105,7 @@ async def _arxiv_outcome(
|
|||
request_email: str | None,
|
||||
allow_lookup: bool = True,
|
||||
) -> OaResolutionOutcome | None:
|
||||
from app.services.domains.arxiv.application import discover_arxiv_id_for_publication
|
||||
from app.services.arxiv.application import discover_arxiv_id_for_publication
|
||||
|
||||
if not allow_lookup:
|
||||
structured_log(
|
||||
|
|
|
|||
|
|
@ -13,8 +13,8 @@ from app.db.models import (
|
|||
ScholarProfile,
|
||||
ScholarPublication,
|
||||
)
|
||||
from app.services.domains.publications.modes import MODE_LATEST, MODE_UNREAD
|
||||
from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem
|
||||
from app.services.publications.modes import MODE_LATEST, MODE_UNREAD
|
||||
from app.services.publications.types import PublicationListItem, UnreadPublicationItem
|
||||
|
||||
|
||||
def _normalized_citation_count(value: object) -> int:
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ from __future__ import annotations
|
|||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
|
||||
from app.services.domains.publication_identifiers.types import DisplayIdentifier
|
||||
from app.services.publication_identifiers.types import DisplayIdentifier
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
|
|
|
|||
|
|
@ -1,51 +1,51 @@
|
|||
from app.services.domains.runs.application import (
|
||||
from app.services.runs.application import (
|
||||
QUEUE_STATUS_DROPPED as QUEUE_STATUS_DROPPED,
|
||||
)
|
||||
from app.services.domains.runs.application import (
|
||||
from app.services.runs.application import (
|
||||
QUEUE_STATUS_QUEUED as QUEUE_STATUS_QUEUED,
|
||||
)
|
||||
from app.services.domains.runs.application import (
|
||||
from app.services.runs.application import (
|
||||
QUEUE_STATUS_RETRYING as QUEUE_STATUS_RETRYING,
|
||||
)
|
||||
from app.services.domains.runs.application import (
|
||||
from app.services.runs.application import (
|
||||
QueueClearResult as QueueClearResult,
|
||||
)
|
||||
from app.services.domains.runs.application import (
|
||||
from app.services.runs.application import (
|
||||
QueueListItem as QueueListItem,
|
||||
)
|
||||
from app.services.domains.runs.application import (
|
||||
from app.services.runs.application import (
|
||||
QueueTransitionError as QueueTransitionError,
|
||||
)
|
||||
from app.services.domains.runs.application import (
|
||||
from app.services.runs.application import (
|
||||
clear_queue_item_for_user as clear_queue_item_for_user,
|
||||
)
|
||||
from app.services.domains.runs.application import (
|
||||
from app.services.runs.application import (
|
||||
drop_queue_item_for_user as drop_queue_item_for_user,
|
||||
)
|
||||
from app.services.domains.runs.application import (
|
||||
from app.services.runs.application import (
|
||||
extract_run_summary as extract_run_summary,
|
||||
)
|
||||
from app.services.domains.runs.application import (
|
||||
from app.services.runs.application import (
|
||||
get_manual_run_by_idempotency_key as get_manual_run_by_idempotency_key,
|
||||
)
|
||||
from app.services.domains.runs.application import (
|
||||
from app.services.runs.application import (
|
||||
get_queue_item_for_user as get_queue_item_for_user,
|
||||
)
|
||||
from app.services.domains.runs.application import (
|
||||
from app.services.runs.application import (
|
||||
get_run_for_user as get_run_for_user,
|
||||
)
|
||||
from app.services.domains.runs.application import (
|
||||
from app.services.runs.application import (
|
||||
list_queue_items_for_user as list_queue_items_for_user,
|
||||
)
|
||||
from app.services.domains.runs.application import (
|
||||
from app.services.runs.application import (
|
||||
list_recent_runs_for_user as list_recent_runs_for_user,
|
||||
)
|
||||
from app.services.domains.runs.application import (
|
||||
from app.services.runs.application import (
|
||||
list_runs_for_user as list_runs_for_user,
|
||||
)
|
||||
from app.services.domains.runs.application import (
|
||||
from app.services.runs.application import (
|
||||
queue_status_counts_for_user as queue_status_counts_for_user,
|
||||
)
|
||||
from app.services.domains.runs.application import (
|
||||
from app.services.runs.application import (
|
||||
retry_queue_item_for_user as retry_queue_item_for_user,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from app.services.domains.runs.queue_service import (
|
||||
from app.services.runs.queue_service import (
|
||||
clear_queue_item_for_user,
|
||||
drop_queue_item_for_user,
|
||||
get_queue_item_for_user,
|
||||
|
|
@ -8,14 +8,14 @@ from app.services.domains.runs.queue_service import (
|
|||
queue_status_counts_for_user,
|
||||
retry_queue_item_for_user,
|
||||
)
|
||||
from app.services.domains.runs.runs_service import (
|
||||
from app.services.runs.runs_service import (
|
||||
get_manual_run_by_idempotency_key,
|
||||
get_run_for_user,
|
||||
list_recent_runs_for_user,
|
||||
list_runs_for_user,
|
||||
)
|
||||
from app.services.domains.runs.summary import extract_run_summary
|
||||
from app.services.domains.runs.types import (
|
||||
from app.services.runs.summary import extract_run_summary
|
||||
from app.services.runs.types import (
|
||||
QUEUE_STATUS_DROPPED,
|
||||
QUEUE_STATUS_QUEUED,
|
||||
QUEUE_STATUS_RETRYING,
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ from __future__ import annotations
|
|||
from sqlalchemy import and_, select
|
||||
|
||||
from app.db.models import IngestionQueueItem, ScholarProfile
|
||||
from app.services.domains.runs.types import QueueListItem
|
||||
from app.services.runs.types import QueueListItem
|
||||
|
||||
|
||||
def queue_item_columns() -> tuple:
|
||||
|
|
|
|||
|
|
@ -4,9 +4,9 @@ from sqlalchemy import case, func, select
|
|||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.models import IngestionQueueItem
|
||||
from app.services.domains.ingestion import queue as queue_mutations
|
||||
from app.services.domains.runs.queue_queries import queue_item_select, queue_list_item_from_row
|
||||
from app.services.domains.runs.types import (
|
||||
from app.services.ingestion import queue as queue_mutations
|
||||
from app.services.runs.queue_queries import queue_item_select, queue_list_item_from_row
|
||||
from app.services.runs.types import (
|
||||
QUEUE_STATUS_DROPPED,
|
||||
QUEUE_STATUS_QUEUED,
|
||||
QUEUE_STATUS_RETRYING,
|
||||
|
|
|
|||
|
|
@ -5,9 +5,9 @@ from html.parser import HTMLParser
|
|||
from typing import Any
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
|
||||
from app.services.domains.scholar.parser_constants import AUTHOR_SEARCH_MARKER_KEYS
|
||||
from app.services.domains.scholar.parser_types import ScholarSearchCandidate
|
||||
from app.services.domains.scholar.parser_utils import (
|
||||
from app.services.scholar.parser_constants import AUTHOR_SEARCH_MARKER_KEYS
|
||||
from app.services.scholar.parser_types import ScholarSearchCandidate
|
||||
from app.services.scholar.parser_utils import (
|
||||
attr_class,
|
||||
attr_href,
|
||||
attr_src,
|
||||
|
|
|
|||
|
|
@ -1,30 +1,30 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from app.services.domains.scholar.author_rows import (
|
||||
from app.services.scholar.author_rows import (
|
||||
ScholarAuthorSearchParser,
|
||||
count_author_search_markers,
|
||||
)
|
||||
from app.services.domains.scholar.parser_constants import SCRIPT_STYLE_RE
|
||||
from app.services.domains.scholar.parser_types import (
|
||||
from app.services.scholar.parser_constants import SCRIPT_STYLE_RE
|
||||
from app.services.scholar.parser_types import (
|
||||
ParsedAuthorSearchPage,
|
||||
ParsedProfilePage,
|
||||
PublicationCandidate,
|
||||
ScholarDomInvariantError,
|
||||
ScholarMalformedDataError,
|
||||
)
|
||||
from app.services.domains.scholar.parser_types import (
|
||||
from app.services.scholar.parser_types import (
|
||||
ParseState as ParseState,
|
||||
)
|
||||
from app.services.domains.scholar.parser_types import (
|
||||
from app.services.scholar.parser_types import (
|
||||
ScholarParserError as ScholarParserError,
|
||||
)
|
||||
from app.services.domains.scholar.parser_types import (
|
||||
from app.services.scholar.parser_types import (
|
||||
ScholarSearchCandidate as ScholarSearchCandidate,
|
||||
)
|
||||
from app.services.domains.scholar.parser_utils import (
|
||||
from app.services.scholar.parser_utils import (
|
||||
strip_tags,
|
||||
)
|
||||
from app.services.domains.scholar.profile_rows import (
|
||||
from app.services.scholar.profile_rows import (
|
||||
count_markers,
|
||||
extract_articles_range,
|
||||
extract_profile_image_url,
|
||||
|
|
@ -33,8 +33,8 @@ from app.services.domains.scholar.profile_rows import (
|
|||
has_show_more_button,
|
||||
parse_publications,
|
||||
)
|
||||
from app.services.domains.scholar.source import FetchResult
|
||||
from app.services.domains.scholar.state_detection import (
|
||||
from app.services.scholar.source import FetchResult
|
||||
from app.services.scholar.state_detection import (
|
||||
detect_author_search_state,
|
||||
detect_state,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ from __future__ import annotations
|
|||
|
||||
from html import unescape
|
||||
|
||||
from app.services.domains.scholar.parser_constants import TAG_RE
|
||||
from app.services.scholar.parser_constants import TAG_RE
|
||||
|
||||
|
||||
def normalize_space(value: str) -> str:
|
||||
|
|
|
|||
|
|
@ -5,12 +5,12 @@ from html.parser import HTMLParser
|
|||
from typing import Any
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
|
||||
from app.services.domains.scholar.parser_constants import (
|
||||
from app.services.scholar.parser_constants import (
|
||||
MARKER_KEYS,
|
||||
SHOW_MORE_BUTTON_RE,
|
||||
)
|
||||
from app.services.domains.scholar.parser_types import PublicationCandidate
|
||||
from app.services.domains.scholar.parser_utils import (
|
||||
from app.services.scholar.parser_types import PublicationCandidate
|
||||
from app.services.scholar.parser_utils import (
|
||||
attr_class,
|
||||
attr_href,
|
||||
build_absolute_scholar_url,
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ from urllib.parse import urlencode
|
|||
from urllib.request import Request, urlopen
|
||||
|
||||
from app.logging_utils import structured_log
|
||||
from app.services.domains.scholar import rate_limit as scholar_rate_limit
|
||||
from app.services.scholar import rate_limit as scholar_rate_limit
|
||||
from app.settings import settings
|
||||
|
||||
SCHOLAR_PROFILE_URL = "https://scholar.google.com/citations"
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from app.services.domains.scholar.parser_constants import (
|
||||
from app.services.scholar.parser_constants import (
|
||||
BLOCKED_KEYWORDS,
|
||||
NETWORK_DNS_ERROR_KEYWORDS,
|
||||
NETWORK_TIMEOUT_KEYWORDS,
|
||||
|
|
@ -8,8 +8,8 @@ from app.services.domains.scholar.parser_constants import (
|
|||
NO_AUTHOR_RESULTS_KEYWORDS,
|
||||
NO_RESULTS_KEYWORDS,
|
||||
)
|
||||
from app.services.domains.scholar.parser_types import ParseState, PublicationCandidate, ScholarSearchCandidate
|
||||
from app.services.domains.scholar.source import FetchResult
|
||||
from app.services.scholar.parser_types import ParseState, PublicationCandidate, ScholarSearchCandidate
|
||||
from app.services.scholar.source import FetchResult
|
||||
|
||||
|
||||
def classify_network_error_reason(fetch_error: str | None) -> str:
|
||||
|
|
|
|||
|
|
@ -15,15 +15,15 @@ from sqlalchemy.ext.asyncio import AsyncSession
|
|||
|
||||
from app.db.models import AuthorSearchCacheEntry, AuthorSearchRuntimeState, ScholarProfile
|
||||
from app.logging_utils import structured_log
|
||||
from app.services.domains.scholar.parser import (
|
||||
from app.services.scholar.parser import (
|
||||
ParsedAuthorSearchPage,
|
||||
ParseState,
|
||||
ScholarParserError,
|
||||
parse_author_search_page,
|
||||
parse_profile_page,
|
||||
)
|
||||
from app.services.domains.scholar.source import ScholarSource
|
||||
from app.services.domains.scholars.constants import (
|
||||
from app.services.scholar.source import ScholarSource
|
||||
from app.services.scholars.constants import (
|
||||
ALLOWED_IMAGE_UPLOAD_CONTENT_TYPES,
|
||||
AUTHOR_SEARCH_LOCK_KEY,
|
||||
AUTHOR_SEARCH_LOCK_NAMESPACE,
|
||||
|
|
@ -41,18 +41,18 @@ from app.services.domains.scholars.constants import (
|
|||
SEARCH_COOLDOWN_REASON,
|
||||
SEARCH_DISABLED_REASON,
|
||||
)
|
||||
from app.services.domains.scholars.exceptions import ScholarServiceError
|
||||
from app.services.domains.scholars.search_hints import (
|
||||
from app.services.scholars.exceptions import ScholarServiceError
|
||||
from app.services.scholars.search_hints import (
|
||||
_merge_warnings,
|
||||
_policy_blocked_author_search_result,
|
||||
_trim_author_search_result,
|
||||
)
|
||||
from app.services.domains.scholars.uploads import (
|
||||
from app.services.scholars.uploads import (
|
||||
_ensure_upload_root,
|
||||
_resolve_upload_path,
|
||||
_safe_remove_upload,
|
||||
)
|
||||
from app.services.domains.scholars.validators import (
|
||||
from app.services.scholars.validators import (
|
||||
normalize_display_name,
|
||||
normalize_profile_image_url,
|
||||
validate_scholar_id,
|
||||
|
|
@ -205,7 +205,7 @@ def _deserialize_parsed_author_search_page(payload: object) -> ParsedAuthorSearc
|
|||
|
||||
marker_counts = _payload_marker_counts(payload)
|
||||
warnings = _payload_warnings(payload)
|
||||
from app.services.domains.scholar.parser import ScholarSearchCandidate
|
||||
from app.services.scholar.parser import ScholarSearchCandidate
|
||||
|
||||
normalized_candidates = _deserialize_candidates(payload)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from app.db.models import ScholarProfile
|
||||
from app.services.domains.scholar.parser import ParsedAuthorSearchPage, ParseState
|
||||
from app.services.domains.scholars.constants import (
|
||||
from app.services.scholar.parser import ParsedAuthorSearchPage, ParseState
|
||||
from app.services.scholars.constants import (
|
||||
MAX_AUTHOR_SEARCH_LIMIT,
|
||||
STATE_REASON_HINTS,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ from __future__ import annotations
|
|||
|
||||
from pathlib import Path
|
||||
|
||||
from app.services.domains.scholars.exceptions import ScholarServiceError
|
||||
from app.services.scholars.exceptions import ScholarServiceError
|
||||
|
||||
|
||||
def _ensure_upload_root(upload_dir: str, *, create: bool) -> Path:
|
||||
|
|
|
|||
|
|
@ -2,8 +2,8 @@ from __future__ import annotations
|
|||
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from app.services.domains.scholars.constants import MAX_IMAGE_URL_LENGTH, SCHOLAR_ID_PATTERN
|
||||
from app.services.domains.scholars.exceptions import ScholarServiceError
|
||||
from app.services.scholars.constants import MAX_IMAGE_URL_LENGTH, SCHOLAR_ID_PATTERN
|
||||
from app.services.scholars.exceptions import ScholarServiceError
|
||||
|
||||
|
||||
def validate_scholar_id(value: str) -> str:
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ from __future__ import annotations
|
|||
|
||||
|
||||
async def resolve_publication_pdf_urls(*args, **kwargs):
|
||||
from app.services.domains.unpaywall.application import resolve_publication_pdf_urls as _impl
|
||||
from app.services.unpaywall.application import resolve_publication_pdf_urls as _impl
|
||||
|
||||
return await _impl(*args, **kwargs)
|
||||
|
||||
|
|
|
|||
|
|
@ -7,17 +7,17 @@ from typing import TYPE_CHECKING
|
|||
from urllib.parse import unquote
|
||||
|
||||
from app.logging_utils import structured_log
|
||||
from app.services.domains.crossref.application import discover_doi_for_publication
|
||||
from app.services.domains.doi.normalize import normalize_doi
|
||||
from app.services.domains.unpaywall.pdf_discovery import (
|
||||
from app.services.crossref.application import discover_doi_for_publication
|
||||
from app.services.doi.normalize import normalize_doi
|
||||
from app.services.unpaywall.pdf_discovery import (
|
||||
looks_like_pdf_url,
|
||||
resolve_pdf_from_landing_page,
|
||||
)
|
||||
from app.services.domains.unpaywall.rate_limit import wait_for_unpaywall_slot
|
||||
from app.services.unpaywall.rate_limit import wait_for_unpaywall_slot
|
||||
from app.settings import settings
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem
|
||||
from app.services.publications.types import PublicationListItem, UnreadPublicationItem
|
||||
|
||||
DOI_PATTERN = re.compile(r"10\.\d{4,9}/[-._;()/:A-Z0-9]+", re.I)
|
||||
DOI_PREFIX_RE = re.compile(r"\bdoi\s*[:=]\s*(10\.\d{4,9}/[-._;()/:A-Z0-9]+)", re.I)
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ import re
|
|||
from html.parser import HTMLParser
|
||||
from urllib.parse import urljoin, urlparse
|
||||
|
||||
from app.services.domains.unpaywall.rate_limit import wait_for_unpaywall_slot
|
||||
from app.services.unpaywall.rate_limit import wait_for_unpaywall_slot
|
||||
from app.settings import settings
|
||||
|
||||
PDF_MIME = "application/pdf"
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue