Feat/decomposition #19
24 changed files with 219 additions and 57 deletions
|
|
@ -11,6 +11,7 @@ from app.api.errors import ApiException
|
||||||
from app.db.models import User
|
from app.db.models import User
|
||||||
from app.db.session import get_db_session
|
from app.db.session import get_db_session
|
||||||
from app.services.domains.scholars import application as scholar_service
|
from app.services.domains.scholars import application as scholar_service
|
||||||
|
from app.services.domains.scholars import uploads as scholar_uploads
|
||||||
from app.settings import settings
|
from app.settings import settings
|
||||||
|
|
||||||
router = APIRouter(tags=["media"])
|
router = APIRouter(tags=["media"])
|
||||||
|
|
@ -41,7 +42,7 @@ async def get_uploaded_scholar_image(
|
||||||
)
|
)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
image_path = scholar_service.resolve_upload_file_path(
|
image_path = scholar_uploads.resolve_upload_file_path(
|
||||||
upload_dir=settings.scholar_image_upload_dir,
|
upload_dir=settings.scholar_image_upload_dir,
|
||||||
relative_path=profile.profile_image_upload_path,
|
relative_path=profile.profile_image_upload_path,
|
||||||
)
|
)
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,7 @@
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
|
from typing import NoReturn
|
||||||
|
|
||||||
from fastapi import APIRouter, Depends, Request
|
from fastapi import APIRouter, Depends, Request
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
@ -61,7 +62,7 @@ def _serialize_user_payload(user: User) -> dict[str, object]:
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
def _raise_invalid_credentials(*, normalized_email: str) -> None:
|
def _raise_invalid_credentials(*, normalized_email: str) -> NoReturn:
|
||||||
structured_log(logger, "info", "api.auth.login_failed", email=normalized_email)
|
structured_log(logger, "info", "api.auth.login_failed", email=normalized_email)
|
||||||
raise ApiException(
|
raise ApiException(
|
||||||
status_code=401,
|
status_code=401,
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,6 @@
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
import logging
|
import logging
|
||||||
import re
|
import re
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
@ -32,6 +33,7 @@ from app.services.domains.settings import application as user_settings_service
|
||||||
from app.settings import settings
|
from app.settings import settings
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
_background_tasks: set[asyncio.Task[Any]] = set()
|
||||||
|
|
||||||
router = APIRouter(prefix="/runs", tags=["api-runs"])
|
router = APIRouter(prefix="/runs", tags=["api-runs"])
|
||||||
ACTIVE_RUN_STATUSES = {RunStatus.RUNNING, RunStatus.RESOLVING}
|
ACTIVE_RUN_STATUSES = {RunStatus.RUNNING, RunStatus.RESOLVING}
|
||||||
|
|
@ -656,11 +658,9 @@ async def run_manual(
|
||||||
await db_session.commit()
|
await db_session.commit()
|
||||||
|
|
||||||
# Kick off background execution
|
# Kick off background execution
|
||||||
import asyncio
|
|
||||||
|
|
||||||
from app.db.session import get_session_factory
|
from app.db.session import get_session_factory
|
||||||
|
|
||||||
asyncio.create_task(
|
task = asyncio.create_task(
|
||||||
ingest_service.execute_run(
|
ingest_service.execute_run(
|
||||||
session_factory=get_session_factory(),
|
session_factory=get_session_factory(),
|
||||||
run_id=run.id,
|
run_id=run.id,
|
||||||
|
|
@ -680,6 +680,8 @@ async def run_manual(
|
||||||
idempotency_key=idempotency_key,
|
idempotency_key=idempotency_key,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
_background_tasks.add(task)
|
||||||
|
task.add_done_callback(_background_tasks.discard)
|
||||||
|
|
||||||
return success_payload(
|
return success_payload(
|
||||||
request,
|
request,
|
||||||
|
|
|
||||||
|
|
@ -2,6 +2,7 @@ from __future__ import annotations
|
||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
import logging
|
import logging
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
from fastapi import APIRouter, Depends, File, Query, Request, UploadFile
|
from fastapi import APIRouter, Depends, File, Query, Request, UploadFile
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
@ -29,6 +30,7 @@ from app.services.domains.portability import application as import_export_servic
|
||||||
from app.services.domains.scholar import rate_limit as scholar_rate_limit
|
from app.services.domains.scholar import rate_limit as scholar_rate_limit
|
||||||
from app.services.domains.scholar.source import ScholarSource
|
from app.services.domains.scholar.source import ScholarSource
|
||||||
from app.services.domains.scholars import application as scholar_service
|
from app.services.domains.scholars import application as scholar_service
|
||||||
|
from app.services.domains.scholars import search_hints as scholar_search_hints
|
||||||
from app.settings import settings
|
from app.settings import settings
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
@ -110,7 +112,7 @@ def _serialize_scholar(profile) -> dict[str, object]:
|
||||||
if profile.profile_image_upload_path:
|
if profile.profile_image_upload_path:
|
||||||
uploaded_image_url = _uploaded_image_media_path(int(profile.id))
|
uploaded_image_url = _uploaded_image_media_path(int(profile.id))
|
||||||
|
|
||||||
profile_image_url, profile_image_source = scholar_service.resolve_profile_image(
|
profile_image_url, profile_image_source = scholar_search_hints.resolve_profile_image(
|
||||||
profile,
|
profile,
|
||||||
uploaded_image_url=uploaded_image_url,
|
uploaded_image_url=uploaded_image_url,
|
||||||
)
|
)
|
||||||
|
|
@ -180,7 +182,7 @@ async def _hydrate_scholar_metadata_if_needed(
|
||||||
return profile
|
return profile
|
||||||
|
|
||||||
|
|
||||||
def _search_kwargs() -> dict[str, object]:
|
def _search_kwargs() -> dict[str, Any]:
|
||||||
return {
|
return {
|
||||||
"network_error_retries": settings.ingestion_network_error_retries,
|
"network_error_retries": settings.ingestion_network_error_retries,
|
||||||
"retry_backoff_seconds": settings.ingestion_retry_backoff_seconds,
|
"retry_backoff_seconds": settings.ingestion_retry_backoff_seconds,
|
||||||
|
|
@ -202,7 +204,7 @@ def _search_response_data(query: str, parsed) -> dict[str, object]:
|
||||||
"query": query.strip(),
|
"query": query.strip(),
|
||||||
"state": parsed.state.value,
|
"state": parsed.state.value,
|
||||||
"state_reason": parsed.state_reason,
|
"state_reason": parsed.state_reason,
|
||||||
"action_hint": scholar_service.scrape_state_hint(
|
"action_hint": scholar_search_hints.scrape_state_hint(
|
||||||
state=parsed.state,
|
state=parsed.state,
|
||||||
state_reason=parsed.state_reason,
|
state_reason=parsed.state_reason,
|
||||||
),
|
),
|
||||||
|
|
|
||||||
|
|
@ -54,7 +54,7 @@ class CSRFMiddleware(BaseHTTPMiddleware):
|
||||||
if request_token is None and self._is_form_payload(request):
|
if request_token is None and self._is_form_payload(request):
|
||||||
body = await request.body()
|
body = await request.body()
|
||||||
request_token = self._token_from_form_body(request, body)
|
request_token = self._token_from_form_body(request, body)
|
||||||
request._receive = self._build_receive(body) # type: ignore[attr-defined]
|
request._receive = self._build_receive(body)
|
||||||
|
|
||||||
if not request_token or not compare_digest(str(session_token), str(request_token)):
|
if not request_token or not compare_digest(str(session_token), str(request_token)):
|
||||||
structured_log(
|
structured_log(
|
||||||
|
|
|
||||||
|
|
@ -68,8 +68,7 @@ async def _run_serialized_fetch(
|
||||||
source_path: str,
|
source_path: str,
|
||||||
) -> tuple[httpx.Response, bool]:
|
) -> tuple[httpx.Response, bool]:
|
||||||
session_factory = get_session_factory()
|
session_factory = get_session_factory()
|
||||||
async with session_factory() as db_session:
|
async with session_factory() as db_session, db_session.begin():
|
||||||
async with db_session.begin():
|
|
||||||
await _acquire_arxiv_lock(db_session)
|
await _acquire_arxiv_lock(db_session)
|
||||||
runtime_state = await _load_runtime_state_for_update(db_session)
|
runtime_state = await _load_runtime_state_for_update(db_session)
|
||||||
wait_seconds = await _wait_for_allowed_slot_or_raise(
|
wait_seconds = await _wait_for_allowed_slot_or_raise(
|
||||||
|
|
|
||||||
|
|
@ -109,6 +109,9 @@ def _classify_failure_bucket(*, state: str, state_reason: str) -> str:
|
||||||
return FAILURE_BUCKET_OTHER
|
return FAILURE_BUCKET_OTHER
|
||||||
|
|
||||||
|
|
||||||
|
_background_tasks: set[asyncio.Task[Any]] = set()
|
||||||
|
|
||||||
|
|
||||||
class ScholarIngestionService:
|
class ScholarIngestionService:
|
||||||
def __init__(self, *, source: ScholarSource) -> None:
|
def __init__(self, *, source: ScholarSource) -> None:
|
||||||
self._source = source
|
self._source = source
|
||||||
|
|
@ -287,9 +290,8 @@ class ScholarIngestionService:
|
||||||
paged_parse_result: PagedParseResult,
|
paged_parse_result: PagedParseResult,
|
||||||
) -> None:
|
) -> None:
|
||||||
parsed_page = paged_parse_result.parsed_page
|
parsed_page = paged_parse_result.parsed_page
|
||||||
if parsed_page.state in {ParseState.OK, ParseState.NO_RESULTS}:
|
if parsed_page.state in {ParseState.OK, ParseState.NO_RESULTS} and any(code.startswith("layout_") for code in parsed_page.warnings):
|
||||||
if any(code.startswith("layout_") for code in parsed_page.warnings):
|
raise RuntimeError(f"Layout warning marked as terminal for scholar_id={scholar_id}.")
|
||||||
raise RuntimeError(f"Layout warning marked as terminal for scholar_id={scholar_id}.")
|
|
||||||
for publication in paged_parse_result.publications:
|
for publication in paged_parse_result.publications:
|
||||||
if not publication.title.strip():
|
if not publication.title.strip():
|
||||||
raise RuntimeError(f"Malformed publication title for scholar_id={scholar_id}.")
|
raise RuntimeError(f"Malformed publication title for scholar_id={scholar_id}.")
|
||||||
|
|
@ -1490,7 +1492,7 @@ class ScholarIngestionService:
|
||||||
|
|
||||||
# Fire-and-forget enrichment in a separate background task
|
# Fire-and-forget enrichment in a separate background task
|
||||||
if intended_final_status not in (RunStatus.CANCELED,):
|
if intended_final_status not in (RunStatus.CANCELED,):
|
||||||
asyncio.create_task(
|
task = asyncio.create_task(
|
||||||
self._background_enrich(
|
self._background_enrich(
|
||||||
session_factory,
|
session_factory,
|
||||||
run_id=run.id,
|
run_id=run.id,
|
||||||
|
|
@ -1498,6 +1500,8 @@ class ScholarIngestionService:
|
||||||
openalex_api_key=getattr(user_settings, "openalex_api_key", None),
|
openalex_api_key=getattr(user_settings, "openalex_api_key", None),
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
_background_tasks.add(task)
|
||||||
|
task.add_done_callback(_background_tasks.discard)
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
await db_session.rollback()
|
await db_session.rollback()
|
||||||
logger.exception("ingestion.background_run_failed", extra={"run_id": run_id, "user_id": user_id})
|
logger.exception("ingestion.background_run_failed", extra={"run_id": run_id, "user_id": user_id})
|
||||||
|
|
@ -2807,7 +2811,7 @@ class ScholarIngestionService:
|
||||||
publication.venue_text = candidate.venue_text
|
publication.venue_text = candidate.venue_text
|
||||||
if candidate.title_url:
|
if candidate.title_url:
|
||||||
publication.pub_url = build_publication_url(candidate.title_url)
|
publication.pub_url = build_publication_url(candidate.title_url)
|
||||||
local_doi = first_doi_from_texts(candidate.title_url, candidate.venue_text, candidate.title)
|
first_doi_from_texts(candidate.title_url, candidate.venue_text, candidate.title)
|
||||||
|
|
||||||
async def _resolve_publication(
|
async def _resolve_publication(
|
||||||
self,
|
self,
|
||||||
|
|
|
||||||
|
|
@ -367,10 +367,7 @@ def _publication_identity(pub: PublicationCandidate) -> str:
|
||||||
|
|
||||||
|
|
||||||
def _is_fuzzy_dup(tokens: set[str], seen: list[set[str]]) -> bool:
|
def _is_fuzzy_dup(tokens: set[str], seen: list[set[str]]) -> bool:
|
||||||
for existing in seen:
|
return any(_jaccard(tokens, existing) >= _CANONICAL_DEDUP_THRESHOLD for existing in seen)
|
||||||
if _jaccard(tokens, existing) >= _CANONICAL_DEDUP_THRESHOLD:
|
|
||||||
return True
|
|
||||||
return False
|
|
||||||
|
|
||||||
|
|
||||||
def _build_body_excerpt(body: str, *, max_chars: int = 220) -> str | None:
|
def _build_body_excerpt(body: str, *, max_chars: int = 220) -> str | None:
|
||||||
|
|
|
||||||
|
|
@ -330,9 +330,7 @@ class SchedulerService:
|
||||||
await session.commit()
|
await session.commit()
|
||||||
if queue_item is None:
|
if queue_item is None:
|
||||||
return False
|
return False
|
||||||
if queue_item.status == QueueItemStatus.DROPPED.value:
|
return queue_item.status != QueueItemStatus.DROPPED.value
|
||||||
return False
|
|
||||||
return True
|
|
||||||
|
|
||||||
async def _queue_job_has_available_scholar(
|
async def _queue_job_has_available_scholar(
|
||||||
self,
|
self,
|
||||||
|
|
|
||||||
|
|
@ -93,8 +93,7 @@ def find_best_match(
|
||||||
|
|
||||||
for original_score, cand in top_scored_candidates:
|
for original_score, cand in top_scored_candidates:
|
||||||
tb_score = 0
|
tb_score = 0
|
||||||
if target_year is not None and cand.publication_year is not None:
|
if target_year is not None and cand.publication_year is not None and abs(target_year - cand.publication_year) <= 1:
|
||||||
if abs(target_year - cand.publication_year) <= 1:
|
|
||||||
tb_score += 1
|
tb_score += 1
|
||||||
|
|
||||||
candidate_author_names = [a.display_name for a in cand.authors if a.display_name]
|
candidate_author_names = [a.display_name for a in cand.authors if a.display_name]
|
||||||
|
|
|
||||||
|
|
@ -1 +1,21 @@
|
||||||
from app.services.domains.portability.application import *
|
from app.services.domains.portability.application import (
|
||||||
|
EXPORT_SCHEMA_VERSION as EXPORT_SCHEMA_VERSION,
|
||||||
|
)
|
||||||
|
from app.services.domains.portability.application import (
|
||||||
|
MAX_IMPORT_PUBLICATIONS as MAX_IMPORT_PUBLICATIONS,
|
||||||
|
)
|
||||||
|
from app.services.domains.portability.application import (
|
||||||
|
MAX_IMPORT_SCHOLARS as MAX_IMPORT_SCHOLARS,
|
||||||
|
)
|
||||||
|
from app.services.domains.portability.application import (
|
||||||
|
ImportedPublicationInput as ImportedPublicationInput,
|
||||||
|
)
|
||||||
|
from app.services.domains.portability.application import (
|
||||||
|
ImportExportError as ImportExportError,
|
||||||
|
)
|
||||||
|
from app.services.domains.portability.application import (
|
||||||
|
export_user_data as export_user_data,
|
||||||
|
)
|
||||||
|
from app.services.domains.portability.application import (
|
||||||
|
import_user_data as import_user_data,
|
||||||
|
)
|
||||||
|
|
|
||||||
|
|
@ -188,7 +188,7 @@ async def _discover_crossref_doi(
|
||||||
|
|
||||||
discovered_doi = await crossref_service.discover_doi_for_publication(item=item)
|
discovered_doi = await crossref_service.discover_doi_for_publication(item=item)
|
||||||
normalized_doi = normalize_doi(discovered_doi)
|
normalized_doi = normalize_doi(discovered_doi)
|
||||||
if normalized_doi is None:
|
if discovered_doi is None or normalized_doi is None:
|
||||||
return False
|
return False
|
||||||
candidate = _candidate(
|
candidate = _candidate(
|
||||||
IdentifierKind.DOI,
|
IdentifierKind.DOI,
|
||||||
|
|
@ -216,7 +216,7 @@ async def _discover_arxiv_identifier(
|
||||||
|
|
||||||
discovered_arxiv = await arxiv_service.discover_arxiv_id_for_publication(item=item)
|
discovered_arxiv = await arxiv_service.discover_arxiv_id_for_publication(item=item)
|
||||||
normalized_arxiv = normalize_arxiv_id(discovered_arxiv)
|
normalized_arxiv = normalize_arxiv_id(discovered_arxiv)
|
||||||
if normalized_arxiv is None:
|
if discovered_arxiv is None or normalized_arxiv is None:
|
||||||
return
|
return
|
||||||
candidate = _candidate(
|
candidate = _candidate(
|
||||||
IdentifierKind.ARXIV,
|
IdentifierKind.ARXIV,
|
||||||
|
|
|
||||||
|
|
@ -1 +1,84 @@
|
||||||
from app.services.domains.publications.application import *
|
from app.services.domains.publications.application import (
|
||||||
|
MODE_ALL as MODE_ALL,
|
||||||
|
)
|
||||||
|
from app.services.domains.publications.application import (
|
||||||
|
MODE_LATEST as MODE_LATEST,
|
||||||
|
)
|
||||||
|
from app.services.domains.publications.application import (
|
||||||
|
MODE_NEW as MODE_NEW,
|
||||||
|
)
|
||||||
|
from app.services.domains.publications.application import (
|
||||||
|
MODE_UNREAD as MODE_UNREAD,
|
||||||
|
)
|
||||||
|
from app.services.domains.publications.application import (
|
||||||
|
PublicationListItem as PublicationListItem,
|
||||||
|
)
|
||||||
|
from app.services.domains.publications.application import (
|
||||||
|
UnreadPublicationItem as UnreadPublicationItem,
|
||||||
|
)
|
||||||
|
from app.services.domains.publications.application import (
|
||||||
|
count_favorite_for_user as count_favorite_for_user,
|
||||||
|
)
|
||||||
|
from app.services.domains.publications.application import (
|
||||||
|
count_for_user as count_for_user,
|
||||||
|
)
|
||||||
|
from app.services.domains.publications.application import (
|
||||||
|
count_latest_for_user as count_latest_for_user,
|
||||||
|
)
|
||||||
|
from app.services.domains.publications.application import (
|
||||||
|
count_pdf_queue_items as count_pdf_queue_items,
|
||||||
|
)
|
||||||
|
from app.services.domains.publications.application import (
|
||||||
|
count_unread_for_user as count_unread_for_user,
|
||||||
|
)
|
||||||
|
from app.services.domains.publications.application import (
|
||||||
|
enqueue_all_missing_pdf_jobs as enqueue_all_missing_pdf_jobs,
|
||||||
|
)
|
||||||
|
from app.services.domains.publications.application import (
|
||||||
|
enqueue_retry_pdf_job_for_publication_id as enqueue_retry_pdf_job_for_publication_id,
|
||||||
|
)
|
||||||
|
from app.services.domains.publications.application import (
|
||||||
|
get_latest_run_id_for_user as get_latest_run_id_for_user,
|
||||||
|
)
|
||||||
|
from app.services.domains.publications.application import (
|
||||||
|
get_publication_item_for_user as get_publication_item_for_user,
|
||||||
|
)
|
||||||
|
from app.services.domains.publications.application import (
|
||||||
|
hydrate_pdf_enrichment_state as hydrate_pdf_enrichment_state,
|
||||||
|
)
|
||||||
|
from app.services.domains.publications.application import (
|
||||||
|
list_for_user as list_for_user,
|
||||||
|
)
|
||||||
|
from app.services.domains.publications.application import (
|
||||||
|
list_pdf_queue_items as list_pdf_queue_items,
|
||||||
|
)
|
||||||
|
from app.services.domains.publications.application import (
|
||||||
|
list_pdf_queue_page as list_pdf_queue_page,
|
||||||
|
)
|
||||||
|
from app.services.domains.publications.application import (
|
||||||
|
list_unread_for_user as list_unread_for_user,
|
||||||
|
)
|
||||||
|
from app.services.domains.publications.application import (
|
||||||
|
mark_all_unread_as_read_for_user as mark_all_unread_as_read_for_user,
|
||||||
|
)
|
||||||
|
from app.services.domains.publications.application import (
|
||||||
|
mark_selected_as_read_for_user as mark_selected_as_read_for_user,
|
||||||
|
)
|
||||||
|
from app.services.domains.publications.application import (
|
||||||
|
publications_query as publications_query,
|
||||||
|
)
|
||||||
|
from app.services.domains.publications.application import (
|
||||||
|
resolve_publication_view_mode as resolve_publication_view_mode,
|
||||||
|
)
|
||||||
|
from app.services.domains.publications.application import (
|
||||||
|
retry_pdf_for_user as retry_pdf_for_user,
|
||||||
|
)
|
||||||
|
from app.services.domains.publications.application import (
|
||||||
|
schedule_missing_pdf_enrichment_for_user as schedule_missing_pdf_enrichment_for_user,
|
||||||
|
)
|
||||||
|
from app.services.domains.publications.application import (
|
||||||
|
schedule_retry_pdf_enrichment_for_row as schedule_retry_pdf_enrichment_for_row,
|
||||||
|
)
|
||||||
|
from app.services.domains.publications.application import (
|
||||||
|
set_publication_favorite_for_user as set_publication_favorite_for_user,
|
||||||
|
)
|
||||||
|
|
|
||||||
|
|
@ -190,12 +190,10 @@ def _can_enqueue_job(
|
||||||
return False
|
return False
|
||||||
if int(job.attempt_count) >= _auto_retry_max_attempts():
|
if int(job.attempt_count) >= _auto_retry_max_attempts():
|
||||||
return False
|
return False
|
||||||
if _cooldown_active(
|
return not _cooldown_active(
|
||||||
last_attempt_at=job.last_attempt_at,
|
last_attempt_at=job.last_attempt_at,
|
||||||
attempt_count=int(job.attempt_count),
|
attempt_count=int(job.attempt_count),
|
||||||
):
|
)
|
||||||
return False
|
|
||||||
return True
|
|
||||||
|
|
||||||
|
|
||||||
def _event_row(
|
def _event_row(
|
||||||
|
|
|
||||||
|
|
@ -5,7 +5,7 @@ from dataclasses import dataclass
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from app.logging_utils import structured_log
|
from app.logging_utils import structured_log
|
||||||
from app.services.domains.arxiv.application import ArxivRateLimitError
|
from app.services.domains.arxiv.errors import ArxivRateLimitError
|
||||||
from app.services.domains.arxiv.guards import arxiv_skip_reason_for_item
|
from app.services.domains.arxiv.guards import arxiv_skip_reason_for_item
|
||||||
from app.services.domains.openalex.client import OpenAlexBudgetExhaustedError
|
from app.services.domains.openalex.client import OpenAlexBudgetExhaustedError
|
||||||
from app.services.domains.publications.types import PublicationListItem
|
from app.services.domains.publications.types import PublicationListItem
|
||||||
|
|
|
||||||
|
|
@ -19,7 +19,7 @@ from app.services.domains.publications.types import PublicationListItem, UnreadP
|
||||||
|
|
||||||
def _normalized_citation_count(value: object) -> int:
|
def _normalized_citation_count(value: object) -> int:
|
||||||
try:
|
try:
|
||||||
return int(value or 0)
|
return int(value or 0) # type: ignore[call-overload] # intentionally accepts any object
|
||||||
except (TypeError, ValueError):
|
except (TypeError, ValueError):
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1 +1,51 @@
|
||||||
from app.services.domains.runs.application import *
|
from app.services.domains.runs.application import (
|
||||||
|
QUEUE_STATUS_DROPPED as QUEUE_STATUS_DROPPED,
|
||||||
|
)
|
||||||
|
from app.services.domains.runs.application import (
|
||||||
|
QUEUE_STATUS_QUEUED as QUEUE_STATUS_QUEUED,
|
||||||
|
)
|
||||||
|
from app.services.domains.runs.application import (
|
||||||
|
QUEUE_STATUS_RETRYING as QUEUE_STATUS_RETRYING,
|
||||||
|
)
|
||||||
|
from app.services.domains.runs.application import (
|
||||||
|
QueueClearResult as QueueClearResult,
|
||||||
|
)
|
||||||
|
from app.services.domains.runs.application import (
|
||||||
|
QueueListItem as QueueListItem,
|
||||||
|
)
|
||||||
|
from app.services.domains.runs.application import (
|
||||||
|
QueueTransitionError as QueueTransitionError,
|
||||||
|
)
|
||||||
|
from app.services.domains.runs.application import (
|
||||||
|
clear_queue_item_for_user as clear_queue_item_for_user,
|
||||||
|
)
|
||||||
|
from app.services.domains.runs.application import (
|
||||||
|
drop_queue_item_for_user as drop_queue_item_for_user,
|
||||||
|
)
|
||||||
|
from app.services.domains.runs.application import (
|
||||||
|
extract_run_summary as extract_run_summary,
|
||||||
|
)
|
||||||
|
from app.services.domains.runs.application import (
|
||||||
|
get_manual_run_by_idempotency_key as get_manual_run_by_idempotency_key,
|
||||||
|
)
|
||||||
|
from app.services.domains.runs.application import (
|
||||||
|
get_queue_item_for_user as get_queue_item_for_user,
|
||||||
|
)
|
||||||
|
from app.services.domains.runs.application import (
|
||||||
|
get_run_for_user as get_run_for_user,
|
||||||
|
)
|
||||||
|
from app.services.domains.runs.application import (
|
||||||
|
list_queue_items_for_user as list_queue_items_for_user,
|
||||||
|
)
|
||||||
|
from app.services.domains.runs.application import (
|
||||||
|
list_recent_runs_for_user as list_recent_runs_for_user,
|
||||||
|
)
|
||||||
|
from app.services.domains.runs.application import (
|
||||||
|
list_runs_for_user as list_runs_for_user,
|
||||||
|
)
|
||||||
|
from app.services.domains.runs.application import (
|
||||||
|
queue_status_counts_for_user as queue_status_counts_for_user,
|
||||||
|
)
|
||||||
|
from app.services.domains.runs.application import (
|
||||||
|
retry_queue_item_for_user as retry_queue_item_for_user,
|
||||||
|
)
|
||||||
|
|
|
||||||
|
|
@ -1,11 +1,11 @@
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from typing import Any
|
from typing import Any, cast
|
||||||
|
|
||||||
|
|
||||||
def _safe_int(value: object, default: int = 0) -> int:
|
def _safe_int(value: object, default: int = 0) -> int:
|
||||||
try:
|
try:
|
||||||
return int(value)
|
return int(cast(Any, value))
|
||||||
except (TypeError, ValueError):
|
except (TypeError, ValueError):
|
||||||
return default
|
return default
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -12,6 +12,15 @@ from app.services.domains.scholar.parser_types import (
|
||||||
ScholarDomInvariantError,
|
ScholarDomInvariantError,
|
||||||
ScholarMalformedDataError,
|
ScholarMalformedDataError,
|
||||||
)
|
)
|
||||||
|
from app.services.domains.scholar.parser_types import (
|
||||||
|
ParseState as ParseState,
|
||||||
|
)
|
||||||
|
from app.services.domains.scholar.parser_types import (
|
||||||
|
ScholarParserError as ScholarParserError,
|
||||||
|
)
|
||||||
|
from app.services.domains.scholar.parser_types import (
|
||||||
|
ScholarSearchCandidate as ScholarSearchCandidate,
|
||||||
|
)
|
||||||
from app.services.domains.scholar.parser_utils import (
|
from app.services.domains.scholar.parser_utils import (
|
||||||
strip_tags,
|
strip_tags,
|
||||||
)
|
)
|
||||||
|
|
|
||||||
|
|
@ -2,6 +2,7 @@ from __future__ import annotations
|
||||||
|
|
||||||
import re
|
import re
|
||||||
from html.parser import HTMLParser
|
from html.parser import HTMLParser
|
||||||
|
from typing import Any
|
||||||
from urllib.parse import parse_qs, urlparse
|
from urllib.parse import parse_qs, urlparse
|
||||||
|
|
||||||
from app.services.domains.scholar.parser_constants import (
|
from app.services.domains.scholar.parser_constants import (
|
||||||
|
|
@ -30,7 +31,7 @@ class ScholarRowParser(HTMLParser):
|
||||||
self._title_depth = 0
|
self._title_depth = 0
|
||||||
self._citation_depth = 0
|
self._citation_depth = 0
|
||||||
self._year_depth = 0
|
self._year_depth = 0
|
||||||
self._gray_stack: list[dict[str, object]] = []
|
self._gray_stack: list[dict[str, Any]] = []
|
||||||
|
|
||||||
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
||||||
if self._title_depth > 0:
|
if self._title_depth > 0:
|
||||||
|
|
@ -252,9 +253,7 @@ def has_show_more_button(html: str) -> bool:
|
||||||
return False
|
return False
|
||||||
if 'aria-disabled="true"' in button_tag or "aria-disabled='true'" in button_tag:
|
if 'aria-disabled="true"' in button_tag or "aria-disabled='true'" in button_tag:
|
||||||
return False
|
return False
|
||||||
if "gs_dis" in button_tag:
|
return "gs_dis" not in button_tag
|
||||||
return False
|
|
||||||
return True
|
|
||||||
|
|
||||||
|
|
||||||
def has_operation_error_banner(html: str) -> bool:
|
def has_operation_error_banner(html: str) -> bool:
|
||||||
|
|
|
||||||
|
|
@ -6,6 +6,7 @@ import os
|
||||||
import random
|
import random
|
||||||
from dataclasses import replace
|
from dataclasses import replace
|
||||||
from datetime import UTC, datetime, timedelta
|
from datetime import UTC, datetime, timedelta
|
||||||
|
from typing import Any
|
||||||
from uuid import uuid4
|
from uuid import uuid4
|
||||||
|
|
||||||
from sqlalchemy import delete, func, select, text
|
from sqlalchemy import delete, func, select, text
|
||||||
|
|
@ -182,11 +183,11 @@ def _deserialize_candidate_payload(value: object) -> dict[str, object] | None:
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
def _deserialize_candidates(payload: dict[str, object]) -> list[dict[str, object]]:
|
def _deserialize_candidates(payload: dict[str, object]) -> list[dict[str, Any]]:
|
||||||
candidates_payload = payload.get("candidates")
|
candidates_payload = payload.get("candidates")
|
||||||
if not isinstance(candidates_payload, list):
|
if not isinstance(candidates_payload, list):
|
||||||
return []
|
return []
|
||||||
normalized: list[dict[str, object]] = []
|
normalized: list[dict[str, Any]] = []
|
||||||
for value in candidates_payload:
|
for value in candidates_payload:
|
||||||
candidate = _deserialize_candidate_payload(value)
|
candidate = _deserialize_candidate_payload(value)
|
||||||
if candidate is not None:
|
if candidate is not None:
|
||||||
|
|
@ -790,7 +791,7 @@ async def _perform_live_author_search(
|
||||||
cache_ttl_seconds: int,
|
cache_ttl_seconds: int,
|
||||||
cache_max_entries: int,
|
cache_max_entries: int,
|
||||||
) -> tuple[ParsedAuthorSearchPage, bool]:
|
) -> tuple[ParsedAuthorSearchPage, bool]:
|
||||||
runtime_state_updated = await _wait_for_author_search_throttle(
|
await _wait_for_author_search_throttle(
|
||||||
runtime_state=runtime_state,
|
runtime_state=runtime_state,
|
||||||
normalized_query=normalized_query,
|
normalized_query=normalized_query,
|
||||||
now_utc=datetime.now(UTC),
|
now_utc=datetime.now(UTC),
|
||||||
|
|
|
||||||
|
|
@ -4,6 +4,7 @@ from sqlalchemy import select
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
from app.db.models import UserSetting
|
from app.db.models import UserSetting
|
||||||
|
from app.settings import settings as app_settings
|
||||||
|
|
||||||
|
|
||||||
class UserSettingsServiceError(ValueError):
|
class UserSettingsServiceError(ValueError):
|
||||||
|
|
@ -103,9 +104,6 @@ def parse_nav_visible_pages(value: object) -> list[str]:
|
||||||
return deduped
|
return deduped
|
||||||
|
|
||||||
|
|
||||||
from app.settings import settings as app_settings
|
|
||||||
|
|
||||||
|
|
||||||
async def get_or_create_settings(
|
async def get_or_create_settings(
|
||||||
db_session: AsyncSession,
|
db_session: AsyncSession,
|
||||||
*,
|
*,
|
||||||
|
|
|
||||||
|
|
@ -65,8 +65,9 @@ def _extract_explicit_doi(text: str | None) -> str | None:
|
||||||
|
|
||||||
def _publication_doi(item: PublicationListItem | UnreadPublicationItem) -> str | None:
|
def _publication_doi(item: PublicationListItem | UnreadPublicationItem) -> str | None:
|
||||||
stored = None
|
stored = None
|
||||||
if getattr(item, "display_identifier", None) and item.display_identifier.kind == "doi":
|
di = getattr(item, "display_identifier", None)
|
||||||
stored = normalize_doi(item.display_identifier.value)
|
if di is not None and di.kind == "doi":
|
||||||
|
stored = normalize_doi(di.value)
|
||||||
|
|
||||||
explicit_doi = _extract_explicit_doi(item.pub_url) or _extract_explicit_doi(item.venue_text)
|
explicit_doi = _extract_explicit_doi(item.pub_url) or _extract_explicit_doi(item.venue_text)
|
||||||
if explicit_doi:
|
if explicit_doi:
|
||||||
|
|
|
||||||
|
|
@ -22,7 +22,7 @@ def test_parse_run_interval_minutes_accepts_valid_value() -> None:
|
||||||
def test_parse_run_interval_minutes_rejects_below_minimum() -> None:
|
def test_parse_run_interval_minutes_rejects_below_minimum() -> None:
|
||||||
with pytest.raises(
|
with pytest.raises(
|
||||||
UserSettingsServiceError,
|
UserSettingsServiceError,
|
||||||
match="Check interval must be at least 15 minutes.",
|
match=r"Check interval must be at least 15 minutes.",
|
||||||
):
|
):
|
||||||
parse_run_interval_minutes("14")
|
parse_run_interval_minutes("14")
|
||||||
|
|
||||||
|
|
@ -34,7 +34,7 @@ def test_parse_request_delay_seconds_accepts_valid_value() -> None:
|
||||||
def test_parse_request_delay_seconds_rejects_below_minimum() -> None:
|
def test_parse_request_delay_seconds_rejects_below_minimum() -> None:
|
||||||
with pytest.raises(
|
with pytest.raises(
|
||||||
UserSettingsServiceError,
|
UserSettingsServiceError,
|
||||||
match="Request delay must be at least 2 seconds.",
|
match=r"Request delay must be at least 2 seconds.",
|
||||||
):
|
):
|
||||||
parse_request_delay_seconds("1")
|
parse_request_delay_seconds("1")
|
||||||
|
|
||||||
|
|
@ -42,7 +42,7 @@ def test_parse_request_delay_seconds_rejects_below_minimum() -> None:
|
||||||
def test_parse_run_interval_minutes_rejects_below_configured_minimum() -> None:
|
def test_parse_run_interval_minutes_rejects_below_configured_minimum() -> None:
|
||||||
with pytest.raises(
|
with pytest.raises(
|
||||||
UserSettingsServiceError,
|
UserSettingsServiceError,
|
||||||
match="Check interval must be at least 30 minutes.",
|
match=r"Check interval must be at least 30 minutes.",
|
||||||
):
|
):
|
||||||
parse_run_interval_minutes("29", minimum=30)
|
parse_run_interval_minutes("29", minimum=30)
|
||||||
|
|
||||||
|
|
@ -50,7 +50,7 @@ def test_parse_run_interval_minutes_rejects_below_configured_minimum() -> None:
|
||||||
def test_parse_request_delay_seconds_rejects_below_configured_minimum() -> None:
|
def test_parse_request_delay_seconds_rejects_below_configured_minimum() -> None:
|
||||||
with pytest.raises(
|
with pytest.raises(
|
||||||
UserSettingsServiceError,
|
UserSettingsServiceError,
|
||||||
match="Request delay must be at least 8 seconds.",
|
match=r"Request delay must be at least 8 seconds.",
|
||||||
):
|
):
|
||||||
parse_request_delay_seconds("7", minimum=8)
|
parse_request_delay_seconds("7", minimum=8)
|
||||||
|
|
||||||
|
|
@ -82,7 +82,7 @@ def test_parse_nav_visible_pages_accepts_valid_pages() -> None:
|
||||||
def test_parse_nav_visible_pages_rejects_missing_required_pages() -> None:
|
def test_parse_nav_visible_pages_rejects_missing_required_pages() -> None:
|
||||||
with pytest.raises(
|
with pytest.raises(
|
||||||
UserSettingsServiceError,
|
UserSettingsServiceError,
|
||||||
match="Dashboard, Scholars, and Settings must remain visible.",
|
match=r"Dashboard, Scholars, and Settings must remain visible.",
|
||||||
):
|
):
|
||||||
parse_nav_visible_pages(["dashboard", "publications"])
|
parse_nav_visible_pages(["dashboard", "publications"])
|
||||||
|
|
||||||
|
|
@ -90,6 +90,6 @@ def test_parse_nav_visible_pages_rejects_missing_required_pages() -> None:
|
||||||
def test_parse_nav_visible_pages_rejects_unknown_page() -> None:
|
def test_parse_nav_visible_pages_rejects_unknown_page() -> None:
|
||||||
with pytest.raises(
|
with pytest.raises(
|
||||||
UserSettingsServiceError,
|
UserSettingsServiceError,
|
||||||
match="Unsupported navigation page id: reports",
|
match=r"Unsupported navigation page id: reports",
|
||||||
):
|
):
|
||||||
parse_nav_visible_pages(DEFAULT_NAV_VISIBLE_PAGES + ["reports"])
|
parse_nav_visible_pages([*DEFAULT_NAV_VISIBLE_PAGES, "reports"])
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue