Feat/decomposition #19
9 changed files with 37 additions and 41 deletions
|
|
@ -69,30 +69,30 @@ async def _run_serialized_fetch(
|
|||
) -> tuple[httpx.Response, bool]:
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as db_session, db_session.begin():
|
||||
await _acquire_arxiv_lock(db_session)
|
||||
runtime_state = await _load_runtime_state_for_update(db_session)
|
||||
wait_seconds = await _wait_for_allowed_slot_or_raise(
|
||||
runtime_state,
|
||||
source_path=source_path,
|
||||
)
|
||||
response = await fetch()
|
||||
hit_rate_limit = _record_post_response_state(
|
||||
runtime_state,
|
||||
response_status=int(response.status_code),
|
||||
source_path=source_path,
|
||||
)
|
||||
structured_log(
|
||||
logger,
|
||||
"info",
|
||||
"arxiv.request_completed",
|
||||
status_code=int(response.status_code),
|
||||
wait_seconds=wait_seconds,
|
||||
cooldown_remaining_seconds=_cooldown_remaining_seconds(
|
||||
runtime_state.cooldown_until, now_utc=datetime.now(UTC)
|
||||
),
|
||||
source_path=source_path,
|
||||
)
|
||||
return response, hit_rate_limit
|
||||
await _acquire_arxiv_lock(db_session)
|
||||
runtime_state = await _load_runtime_state_for_update(db_session)
|
||||
wait_seconds = await _wait_for_allowed_slot_or_raise(
|
||||
runtime_state,
|
||||
source_path=source_path,
|
||||
)
|
||||
response = await fetch()
|
||||
hit_rate_limit = _record_post_response_state(
|
||||
runtime_state,
|
||||
response_status=int(response.status_code),
|
||||
source_path=source_path,
|
||||
)
|
||||
structured_log(
|
||||
logger,
|
||||
"info",
|
||||
"arxiv.request_completed",
|
||||
status_code=int(response.status_code),
|
||||
wait_seconds=wait_seconds,
|
||||
cooldown_remaining_seconds=_cooldown_remaining_seconds(
|
||||
runtime_state.cooldown_until, now_utc=datetime.now(UTC)
|
||||
),
|
||||
source_path=source_path,
|
||||
)
|
||||
return response, hit_rate_limit
|
||||
|
||||
|
||||
async def _acquire_arxiv_lock(db_session: AsyncSession) -> None:
|
||||
|
|
|
|||
|
|
@ -22,8 +22,8 @@ from app.services.ingestion.constants import RUN_LOCK_NAMESPACE
|
|||
from app.services.ingestion.enrichment import EnrichmentRunner
|
||||
from app.services.ingestion.pagination import PaginationEngine
|
||||
from app.services.ingestion.run_completion import (
|
||||
int_or_default,
|
||||
complete_run_for_user,
|
||||
int_or_default,
|
||||
run_execution_summary,
|
||||
)
|
||||
from app.services.ingestion.scholar_processing import run_scholar_iteration
|
||||
|
|
|
|||
|
|
@ -286,9 +286,7 @@ class PaginationEngine:
|
|||
) -> None:
|
||||
deduped = _dedupe_publication_candidates(list(publications), seen_canonical=seen_canonical)
|
||||
if deduped:
|
||||
discovered_count = await upsert_publications_fn(
|
||||
db_session, run=run, scholar=scholar, publications=deduped
|
||||
)
|
||||
discovered_count = await upsert_publications_fn(db_session, run=run, scholar=scholar, publications=deduped)
|
||||
state.discovered_publication_count += discovered_count
|
||||
|
||||
async def _paginate_loop(
|
||||
|
|
|
|||
|
|
@ -184,9 +184,7 @@ class SchedulerService:
|
|||
run_interval_minutes=int(run_interval_minutes),
|
||||
request_delay_seconds=effective_request_delay_seconds(
|
||||
request_delay_seconds,
|
||||
floor=user_settings_service.resolve_request_delay_minimum(
|
||||
settings.ingestion_min_request_delay_seconds
|
||||
),
|
||||
floor=user_settings_service.resolve_request_delay_minimum(settings.ingestion_min_request_delay_seconds),
|
||||
),
|
||||
cooldown_until=cooldown_until,
|
||||
cooldown_reason=(str(cooldown_reason).strip() if cooldown_reason else None),
|
||||
|
|
|
|||
|
|
@ -93,8 +93,12 @@ def find_best_match(
|
|||
|
||||
for original_score, cand in top_scored_candidates:
|
||||
tb_score = 0
|
||||
if target_year is not None and cand.publication_year is not None and abs(target_year - cand.publication_year) <= 1:
|
||||
tb_score += 1
|
||||
if (
|
||||
target_year is not None
|
||||
and cand.publication_year is not None
|
||||
and abs(target_year - cand.publication_year) <= 1
|
||||
):
|
||||
tb_score += 1
|
||||
|
||||
candidate_author_names = [a.display_name for a in cand.authors if a.display_name]
|
||||
if _author_overlap_score(target_authors, candidate_author_names):
|
||||
|
|
|
|||
|
|
@ -9,7 +9,6 @@ from app.logging_utils import structured_log
|
|||
from app.services.publication_identifiers import application as identifier_service
|
||||
from app.services.publications.pdf_queue_common import (
|
||||
PDF_STATUS_FAILED,
|
||||
PDF_STATUS_QUEUED,
|
||||
PDF_STATUS_RESOLVED,
|
||||
PDF_STATUS_RUNNING,
|
||||
event_row,
|
||||
|
|
|
|||
|
|
@ -3,7 +3,8 @@ from __future__ import annotations
|
|||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import Select, case, false as sa_false, func, select
|
||||
from sqlalchemy import Select, case, func, select
|
||||
from sqlalchemy import false as sa_false
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.models import (
|
||||
|
|
|
|||
|
|
@ -74,9 +74,7 @@ class LiveScholarSource:
|
|||
self._min_interval_seconds = max(configured_interval, 0.0)
|
||||
self._user_agents = user_agents or DEFAULT_USER_AGENTS
|
||||
self._rotate_user_agents = (
|
||||
bool(settings.scholar_http_rotate_user_agent)
|
||||
if rotate_user_agents is None
|
||||
else bool(rotate_user_agents)
|
||||
bool(settings.scholar_http_rotate_user_agent) if rotate_user_agents is None else bool(rotate_user_agents)
|
||||
)
|
||||
configured_user_agent = settings.scholar_http_user_agent.strip()
|
||||
self._configured_user_agent = configured_user_agent or None
|
||||
|
|
|
|||
|
|
@ -84,9 +84,7 @@ async def test_deferred_enrichment_sweeps_previous_runs(db_session: AsyncSession
|
|||
# We patch the client at its source, and also mock arXiv to avoid real HTTP calls
|
||||
with (
|
||||
patch("app.services.openalex.client.OpenAlexClient") as MockClient,
|
||||
patch(
|
||||
"app.services.arxiv.application.discover_arxiv_id_for_publication", new=AsyncMock(return_value=None)
|
||||
),
|
||||
patch("app.services.arxiv.application.discover_arxiv_id_for_publication", new=AsyncMock(return_value=None)),
|
||||
):
|
||||
mock_instance = MockClient.return_value
|
||||
mock_instance.get_works_by_filter = AsyncMock(return_value=[mock_work])
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue