justinzeus/scholarr
1
0
Fork 0
Code Issues 1 Pull requests 18 Projects Releases Packages Wiki Activity Actions

refactor: fix naming, remove duplicate code from decomposition

Browse source 
- Rename _int_or_default → int_or_default (public API, was using private convention)
- Extract shared pdf_queue utilities (utcnow, event_row, queued_job, status constants) to pdf_queue_common.py to avoid circular imports
- Consolidate _effective_request_delay_seconds into queue_runner.py with explicit floor parameter
- Update tests to match new function locations and names

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Justin Visser 2026-02-27 16:07:24 +01:00
parent 74249fa24c
commit 3fd9132176
9 changed files with 104 additions and 128 deletions
Download patch file Download diff file Expand all files Collapse all files

8
app/services/ingestion/application.py
View file

@ -22,7 +22,7 @@ from app.services.ingestion.constants import RUN_LOCK_NAMESPACE
from app.services.ingestion.enrichment import EnrichmentRunner
from app.services.ingestion.pagination import PaginationEngine
from app.services.ingestion.run_completion import (
_int_or_default,
int_or_default,
complete_run_for_user,
run_execution_summary,
)
@ -135,7 +135,7 @@ class ScholarIngestionService:
policy_minimum = user_settings_service.resolve_request_delay_minimum(
settings.ingestion_min_request_delay_seconds
)
return max(policy_minimum, _int_or_default(value, policy_minimum))
return max(policy_minimum, int_or_default(value, policy_minimum))
async def _load_user_settings_for_run(
self,
@ -333,13 +333,13 @@ class ScholarIngestionService:
alert_retry_scheduled_threshold: int = 3,
) -> tuple[CrawlRun, list[ScholarProfile], dict[int, int]]:
effective_delay = self._effective_request_delay_seconds(request_delay_seconds)
if effective_delay != _int_or_default(request_delay_seconds, effective_delay):
if effective_delay != int_or_default(request_delay_seconds, effective_delay):
structured_log(
logger,
"warning",
"ingestion.delay_coerced",
user_id=user_id,
requested_request_delay_seconds=_int_or_default(request_delay_seconds, 0),
requested_request_delay_seconds=int_or_default(request_delay_seconds, 0),
effective_request_delay_seconds=effective_delay,
policy_minimum_request_delay_seconds=user_settings_service.resolve_request_delay_minimum(
settings.ingestion_min_request_delay_seconds

4
app/services/ingestion/queue_runner.py
View file

@ -20,7 +20,7 @@ from app.settings import settings
logger = logging.getLogger(__name__)
def _effective_request_delay_seconds(value: int | None, *, floor: int) -> int:
def effective_request_delay_seconds(value: int | None, *, floor: int) -> int:
try:
parsed = int(value) if value is not None else floor
except (TypeError, ValueError):
@ -238,7 +238,7 @@ class QueueJobRunner:
select(UserSetting.request_delay_seconds).where(UserSetting.user_id == user_id)
)
delay = result.scalar_one_or_none()
return _effective_request_delay_seconds(delay, floor=floor)
return effective_request_delay_seconds(delay, floor=floor)
async def _run_ingestion_for_queue_job(
self,

8
app/services/ingestion/run_completion.py
View file

@ -35,7 +35,7 @@ from app.settings import settings
logger = logging.getLogger(__name__)
def _int_or_default(value: Any, default: int = 0) -> int:
def int_or_default(value: Any, default: int = 0) -> int:
try:
return int(value)
except (TypeError, ValueError):
@ -68,7 +68,7 @@ def summarize_failures(
scholars_with_retries_count = 0
retry_exhausted_count = 0
for entry in scholar_results:
retries_for_entry = max(0, _int_or_default(entry.get("attempt_count"), 0) - 1)
retries_for_entry = max(0, int_or_default(entry.get("attempt_count"), 0) - 1)
if retries_for_entry > 0:
retries_scheduled_count += retries_for_entry
scholars_with_retries_count += 1
@ -347,7 +347,7 @@ def find_scholar_result_index(
scholar_profile_id: int,
) -> int | None:
for index, result_entry in enumerate(scholar_results):
current_scholar_id = _int_or_default(result_entry.get("scholar_profile_id"), 0)
current_scholar_id = int_or_default(result_entry.get("scholar_profile_id"), 0)
if current_scholar_id == scholar_profile_id:
return index
return None
@ -372,7 +372,7 @@ def apply_outcome_to_progress(
progress: RunProgress,
outcome: ScholarProcessingOutcome,
) -> None:
scholar_profile_id = _int_or_default(outcome.result_entry.get("scholar_profile_id"), 0)
scholar_profile_id = int_or_default(outcome.result_entry.get("scholar_profile_id"), 0)
if scholar_profile_id <= 0:
raise RuntimeError("Scholar outcome missing valid scholar_profile_id.")
prior_index = find_scholar_result_index(

22
app/services/ingestion/scheduler.py
View file

@ -21,7 +21,7 @@ from app.services.ingestion.application import (
RunBlockedBySafetyPolicyError,
ScholarIngestionService,
)
from app.services.ingestion.queue_runner import QueueJobRunner
from app.services.ingestion.queue_runner import QueueJobRunner, effective_request_delay_seconds
from app.services.scholar.source import LiveScholarSource
from app.services.settings import application as user_settings_service
from app.settings import settings
@ -29,19 +29,6 @@ from app.settings import settings
logger = logging.getLogger(__name__)
def _request_delay_floor_seconds() -> int:
return user_settings_service.resolve_request_delay_minimum(settings.ingestion_min_request_delay_seconds)
def _effective_request_delay_seconds(value: int | None) -> int:
floor = _request_delay_floor_seconds()
try:
parsed = int(value) if value is not None else floor
except (TypeError, ValueError):
parsed = floor
return max(floor, parsed)
@dataclass(frozen=True)
class _AutoRunCandidate:
user_id: int
@ -195,7 +182,12 @@ class SchedulerService:
return _AutoRunCandidate(
user_id=int(user_id),
run_interval_minutes=int(run_interval_minutes),
request_delay_seconds=_effective_request_delay_seconds(request_delay_seconds),
request_delay_seconds=effective_request_delay_seconds(
request_delay_seconds,
floor=user_settings_service.resolve_request_delay_minimum(
settings.ingestion_min_request_delay_seconds
),
),
cooldown_until=cooldown_until,
cooldown_reason=(str(cooldown_reason).strip() if cooldown_reason else None),
)