justinzeus/scholarr
1
0
Fork 0
Code Issues 1 Pull requests 18 Projects Releases Packages Wiki Activity Actions

temp commit

Browse source
This commit is contained in:
Justin Visser 2026-02-26 12:54:19 +01:00
parent 8760f27b51
commit 0e9e49df16
193 changed files with 23228 additions and 935 deletions
Download patch file Download diff file Expand all files Collapse all files

62
app/api/routers/admin_dbops.py
View file

@ -57,7 +57,6 @@ def _serialize_pdf_queue_item(item) -> dict[str, object]:
return {
"publication_id": item.publication_id,
"title": item.title,
"doi": item.doi,
"display_identifier": _serialize_display_identifier(item.display_identifier),
"pdf_url": item.pdf_url,
"status": item.status,
@ -348,3 +347,64 @@ async def trigger_publication_link_repair(
},
)
return success_payload(request, data=result)
DROP_PUBLICATIONS_CONFIRMATION = "DROP ALL PUBLICATIONS"
@router.post("/drop-all-publications")
async def drop_all_publications(
request: Request,
db_session: AsyncSession = Depends(get_db_session),
admin_user: User = Depends(get_api_admin_user),
):
body = await request.json()
confirmation_text = (body.get("confirmation_text") or "").strip()
if confirmation_text != DROP_PUBLICATIONS_CONFIRMATION:
raise ApiException(
status_code=400,
code="confirmation_required",
message=f"Type '{DROP_PUBLICATIONS_CONFIRMATION}' to confirm this destructive action.",
)
from sqlalchemy import delete, func, select, update
from app.db.models import (
Publication,
PublicationIdentifier,
PublicationPdfJob,
PublicationPdfJobEvent,
ScholarProfile,
ScholarPublication,
)
count_result = await db_session.execute(select(func.count()).select_from(Publication))
total_publications = count_result.scalar_one()
await db_session.execute(delete(ScholarPublication))
await db_session.execute(delete(PublicationIdentifier))
await db_session.execute(delete(PublicationPdfJobEvent))
await db_session.execute(delete(PublicationPdfJob))
await db_session.execute(delete(Publication))
await db_session.execute(
update(ScholarProfile).values(baseline_completed=False)
)
await db_session.commit()
logger.warning(
"api.admin.db.all_publications_dropped",
extra={
"event": "api.admin.db.all_publications_dropped",
"admin_user_id": int(admin_user.id),
"admin_email": admin_user.email,
"deleted_count": int(total_publications),
},
)
return success_payload(
request,
data={
"deleted_count": int(total_publications),
"message": f"Dropped {total_publications} publication(s) and all related data. "
"Scholar baselines have been reset; the next run will re-discover all publications.",
},
)

53
app/api/routers/publications.py
View file

@ -1,5 +1,6 @@
from __future__ import annotations
from datetime import datetime, timezone
import logging
from typing import Literal
@ -62,7 +63,6 @@ def _serialize_publication_item(item) -> dict[str, object]:
"citation_count": item.citation_count,
"venue_text": item.venue_text,
"pub_url": item.pub_url,
"doi": item.doi,
"display_identifier": _serialize_display_identifier(item.display_identifier),
"pdf_url": item.pdf_url,
"pdf_status": item.pdf_status,
@ -94,23 +94,27 @@ async def _publication_counts(
user_id: int,
selected_scholar_id: int | None,
favorite_only: bool,
snapshot_before: datetime | None,
) -> tuple[int, int, int, int]:
unread_count = await publication_service.count_unread_for_user(
db_session,
user_id=user_id,
scholar_profile_id=selected_scholar_id,
favorite_only=favorite_only,
snapshot_before=snapshot_before,
)
favorites_count = await publication_service.count_favorite_for_user(
db_session,
user_id=user_id,
scholar_profile_id=selected_scholar_id,
snapshot_before=snapshot_before,
)
latest_count = await publication_service.count_latest_for_user(
db_session,
user_id=user_id,
scholar_profile_id=selected_scholar_id,
favorite_only=favorite_only,
snapshot_before=snapshot_before,
)
total_count = await publication_service.count_for_user(
db_session,
@ -118,6 +122,7 @@ async def _publication_counts(
mode=publication_service.MODE_ALL,
scholar_profile_id=selected_scholar_id,
favorite_only=favorite_only,
snapshot_before=snapshot_before,
)
return unread_count, favorites_count, latest_count, total_count
@ -129,8 +134,12 @@ async def _list_publications_for_request(
mode: Literal["all", "unread", "latest", "new"] | None,
favorite_only: bool,
scholar_profile_id: int | None,
search: str | None,
sort_by: str,
sort_dir: str,
limit: int,
offset: int,
snapshot_before: datetime | None,
) -> tuple[str, int | None, list]:
resolved_mode = publication_service.resolve_publication_view_mode(mode)
selected_scholar_id = scholar_profile_id
@ -145,8 +154,12 @@ async def _list_publications_for_request(
mode=resolved_mode,
scholar_profile_id=selected_scholar_id,
favorite_only=favorite_only,
search=search,
sort_by=sort_by,
sort_dir=sort_dir,
limit=limit,
offset=offset,
snapshot_before=snapshot_before,
)
await publication_service.schedule_missing_pdf_enrichment_for_user(
db_session,
@ -162,6 +175,27 @@ async def _list_publications_for_request(
return resolved_mode, selected_scholar_id, hydrated
def _resolve_publications_snapshot(
*,
snapshot: str | None,
) -> tuple[datetime, str]:
if snapshot is None:
now_utc = datetime.now(timezone.utc)
return now_utc, now_utc.isoformat()
try:
parsed = datetime.fromisoformat(snapshot)
except ValueError as exc:
raise ApiException(
status_code=400,
code="invalid_snapshot",
message="Invalid publications snapshot cursor.",
) from exc
if parsed.tzinfo is None:
parsed = parsed.replace(tzinfo=timezone.utc)
normalized = parsed.astimezone(timezone.utc)
return normalized, normalized.isoformat()
def _resolve_publications_paging(
*,
page: int,
@ -188,6 +222,7 @@ def _publications_list_data(
page: int,
page_size: int,
offset: int,
snapshot: str,
) -> dict[str, object]:
return {
"mode": mode,
@ -200,6 +235,7 @@ def _publications_list_data(
"total_count": total_count,
"page": int(page),
"page_size": int(page_size),
"snapshot": snapshot,
"has_prev": int(offset) > 0,
"has_next": int(offset) + int(page_size) < int(total_count),
"publications": [_serialize_publication_item(item) for item in publications],
@ -322,7 +358,6 @@ def _log_retry_pdf_result(
"queued": queued,
"resolved_pdf": resolved_pdf,
"pdf_status": pdf_status,
"has_doi": bool(doi),
},
)
@ -336,10 +371,14 @@ async def list_publications(
mode: Literal["all", "unread", "latest", "new"] | None = Query(default=None),
favorite_only: bool = Query(default=False),
scholar_profile_id: int | None = Query(default=None, ge=1),
search: str | None = Query(default=None, min_length=1, max_length=200),
sort_by: Literal["first_seen", "title", "year", "citations", "scholar"] = Query(default="first_seen"),
sort_dir: Literal["asc", "desc"] = Query(default="desc"),
page: int = Query(default=1, ge=1),
page_size: int = Query(default=100, ge=1, le=500),
limit: int | None = Query(default=None, ge=1, le=1000),
offset: int | None = Query(default=None, ge=0),
snapshot: str | None = Query(default=None, min_length=1, max_length=64),
db_session: AsyncSession = Depends(get_db_session),
current_user: User = Depends(get_api_current_user),
):
@ -349,20 +388,27 @@ async def list_publications(
limit=limit,
offset=offset,
)
snapshot_before, snapshot_cursor = _resolve_publications_snapshot(snapshot=snapshot)
normalized_search = (search or "").strip() or None
resolved_mode, selected_scholar_id, publications = await _list_publications_for_request(
db_session,
current_user=current_user,
mode=mode,
favorite_only=favorite_only,
scholar_profile_id=scholar_profile_id,
search=normalized_search,
sort_by=sort_by,
sort_dir=sort_dir,
limit=resolved_limit,
offset=resolved_offset,
snapshot_before=snapshot_before,
)
unread_count, favorites_count, latest_count, total_count = await _publication_counts(
db_session,
user_id=current_user.id,
selected_scholar_id=selected_scholar_id,
favorite_only=favorite_only,
snapshot_before=snapshot_before,
)
data = _publications_list_data(
mode=resolved_mode,
@ -376,6 +422,7 @@ async def list_publications(
page=resolved_page,
page_size=resolved_limit,
offset=resolved_offset,
snapshot=snapshot_cursor,
)
return success_payload(request, data=data)
@ -480,7 +527,7 @@ async def retry_publication_pdf(
queued=queued,
resolved_pdf=resolved_pdf,
pdf_status=current.pdf_status,
doi=current.doi,
doi=None,
)
return success_payload(
request,

194
app/api/routers/runs.py
View file

@ -11,6 +11,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
from app.api.deps import get_api_current_user
from app.api.errors import ApiException
from app.api.responses import success_payload
from fastapi.responses import StreamingResponse
from app.api.schemas import (
ManualRunEnvelope,
QueueClearEnvelope,
@ -24,6 +25,7 @@ from app.db.session import get_db_session
from app.services.domains.ingestion import application as ingestion_service
from app.services.domains.ingestion import safety as run_safety_service
from app.services.domains.runs import application as run_service
from app.services.domains.runs.events import event_generator
from app.services.domains.settings import application as user_settings_service
from app.settings import settings
from app.api.runtime_deps import get_ingestion_service
@ -31,6 +33,7 @@ from app.api.runtime_deps import get_ingestion_service
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/runs", tags=["api-runs"])
ACTIVE_RUN_STATUSES = {RunStatus.RUNNING, RunStatus.RESOLVING}
IDEMPOTENCY_HEADER = "Idempotency-Key"
IDEMPOTENCY_MAX_LENGTH = 128
@ -338,7 +341,7 @@ async def _reused_manual_run_payload(
)
if previous_run is None:
return None
if previous_run.status == RunStatus.RUNNING:
if previous_run.status in (RunStatus.RUNNING, RunStatus.RESOLVING):
raise ApiException(
status_code=409,
code="run_in_progress",
@ -410,7 +413,7 @@ async def _recover_integrity_error(
extra={"event": "api.runs.manual_integrity_error", "user_id": user_id},
)
raise ApiException(status_code=500, code="manual_run_failed", message="Manual run failed.") from original_exc
if existing_run.status == RunStatus.RUNNING:
if existing_run.status in (RunStatus.RUNNING, RunStatus.RESOLVING):
raise ApiException(
status_code=409,
code="run_in_progress",
@ -586,6 +589,61 @@ async def get_run(
)
@router.post(
"/{run_id}/cancel",
response_model=RunDetailEnvelope,
)
async def cancel_run(
run_id: int,
request: Request,
db_session: AsyncSession = Depends(get_db_session),
current_user: User = Depends(get_api_current_user),
):
run = await run_service.get_run_for_user(
db_session,
user_id=current_user.id,
run_id=run_id,
)
if run is None:
raise ApiException(
status_code=404,
code="run_not_found",
message="Run not found.",
)
if run.status in ACTIVE_RUN_STATUSES:
run.status = RunStatus.CANCELED
await db_session.commit()
await db_session.refresh(run)
else:
raise ApiException(
status_code=409,
code="run_not_cancelable",
message="Run is already terminal and cannot be canceled.",
details={"run_id": int(run.id), "status": run.status.value},
)
error_log = run.error_log if isinstance(run.error_log, dict) else {}
scholar_results = error_log.get("scholar_results")
if not isinstance(scholar_results, list):
scholar_results = []
safety_state = await _load_safety_state(
db_session,
user_id=current_user.id,
)
return success_payload(
request,
data={
"run": _serialize_run(run),
"summary": run_service.extract_run_summary(error_log),
"scholar_results": [_normalize_scholar_result(item) for item in scholar_results],
"safety_state": safety_state,
},
)
@router.post(
"/manual",
response_model=ManualRunEnvelope,
@ -596,10 +654,7 @@ async def run_manual(
current_user: User = Depends(get_api_current_user),
ingest_service: ingestion_service.ScholarIngestionService = Depends(get_ingestion_service),
):
safety_state = await _load_safety_state(
db_session,
user_id=current_user.id,
)
safety_state = await _load_safety_state(db_session, user_id=current_user.id)
if not settings.ingestion_manual_run_allowed:
_raise_manual_runs_disabled(user_id=current_user.id, safety_state=safety_state)
@ -614,28 +669,90 @@ async def run_manual(
if reused_payload is not None:
return reused_payload
run_summary = await _execute_manual_run(
db_session,
request=request,
ingest_service=ingest_service,
user_id=current_user.id,
idempotency_key=idempotency_key,
)
if isinstance(run_summary, dict):
return run_summary
current_safety_state = await _load_safety_state(
db_session,
user_id=current_user.id,
)
return success_payload(
request,
data=_manual_run_success_payload(
run_summary=run_summary,
try:
user_settings = await user_settings_service.get_or_create_settings(db_session, user_id=current_user.id)
# Initialize run (creates the record and performs safety checks)
run, scholars, start_cstart_map = await ingest_service.initialize_run(
db_session,
user_id=current_user.id,
trigger_type=RunTriggerType.MANUAL,
request_delay_seconds=user_settings.request_delay_seconds,
network_error_retries=settings.ingestion_network_error_retries,
retry_backoff_seconds=settings.ingestion_retry_backoff_seconds,
max_pages_per_scholar=settings.ingestion_max_pages_per_scholar,
page_size=settings.ingestion_page_size,
idempotency_key=idempotency_key,
safety_state=current_safety_state,
),
)
alert_blocked_failure_threshold=settings.ingestion_alert_blocked_failure_threshold,
alert_network_failure_threshold=settings.ingestion_alert_network_failure_threshold,
alert_retry_scheduled_threshold=settings.ingestion_alert_retry_scheduled_threshold,
)
await db_session.commit()
# Kick off background execution
from app.db.session import get_session_factory
import asyncio
asyncio.create_task(
ingest_service.execute_run(
session_factory=get_session_factory(),
run_id=run.id,
user_id=current_user.id,
scholars=scholars,
start_cstart_map=start_cstart_map,
request_delay_seconds=user_settings.request_delay_seconds,
network_error_retries=settings.ingestion_network_error_retries,
retry_backoff_seconds=settings.ingestion_retry_backoff_seconds,
max_pages_per_scholar=settings.ingestion_max_pages_per_scholar,
page_size=settings.ingestion_page_size,
auto_queue_continuations=settings.ingestion_continuation_queue_enabled,
queue_delay_seconds=settings.ingestion_continuation_base_delay_seconds,
alert_blocked_failure_threshold=settings.ingestion_alert_blocked_failure_threshold,
alert_network_failure_threshold=settings.ingestion_alert_network_failure_threshold,
alert_retry_scheduled_threshold=settings.ingestion_alert_retry_scheduled_threshold,
idempotency_key=idempotency_key,
)
)
return success_payload(
request,
data={
"run_id": int(run.id),
"status": run.status.value,
"scholar_count": int(run.scholar_count or 0),
"succeeded_count": 0,
"failed_count": 0,
"partial_count": 0,
"new_publication_count": 0,
"reused_existing_run": False,
"idempotency_key": idempotency_key,
"safety_state": await _load_safety_state(db_session, user_id=current_user.id),
}
)
except ingestion_service.RunBlockedBySafetyPolicyError as exc:
await db_session.rollback()
_raise_manual_blocked_safety(exc=exc, user_id=current_user.id)
except ingestion_service.RunAlreadyInProgressError as exc:
await db_session.rollback()
raise ApiException(
status_code=409,
code="run_in_progress",
message="A run is already in progress for this account.",
) from exc
except IntegrityError as exc:
await db_session.rollback()
return await _recover_integrity_error(
db_session,
request=request,
user_id=current_user.id,
idempotency_key=idempotency_key,
original_exc=exc,
)
except Exception as exc:
await db_session.rollback()
_raise_manual_failed(exc=exc, user_id=current_user.id)
@router.get(
@ -769,3 +886,26 @@ async def clear_queue_item(
"message": "Queue item cleared.",
},
)
@router.get("/{run_id}/stream")
async def stream_run_events(
run_id: int,
db_session: AsyncSession = Depends(get_db_session),
current_user: User = Depends(get_api_current_user),
):
run = await run_service.get_run_for_user(
db_session,
user_id=current_user.id,
run_id=run_id,
)
if run is None:
raise ApiException(
status_code=404,
code="run_not_found",
message="Run not found.",
)
return StreamingResponse(
event_generator(run_id),
media_type="text/event-stream"
)

7
app/api/routers/settings.py
View file

@ -38,6 +38,9 @@ def _serialize_settings(user_settings) -> dict[str, object]:
"cooldown_network_seconds": max(60, int(settings_module.ingestion_safety_cooldown_network_seconds)),
},
"safety_state": run_safety_service.get_safety_state_payload(user_settings),
"openalex_api_key": user_settings.openalex_api_key,
"crossref_api_token": user_settings.crossref_api_token,
"crossref_api_mailto": user_settings.crossref_api_mailto,
}
@ -107,6 +110,7 @@ def _log_settings_update(*, user_id: int, updated) -> None:
"run_interval_minutes": updated.run_interval_minutes,
"request_delay_seconds": updated.request_delay_seconds,
"nav_visible_pages": updated.nav_visible_pages,
"openalex_api_key": "SET" if updated.openalex_api_key else "UNSET",
},
)
@ -169,6 +173,9 @@ async def update_settings(
run_interval_minutes=parsed_interval,
request_delay_seconds=parsed_delay,
nav_visible_pages=parsed_nav_visible_pages,
openalex_api_key=payload.openalex_api_key,
crossref_api_token=payload.crossref_api_token,
crossref_api_mailto=payload.crossref_api_mailto,
)
await _clear_expired_cooldown_with_log(
db_session,

12
app/api/schemas.py
View file

@ -205,7 +205,6 @@ class PublicationExportItemData(BaseModel):
author_text: str | None = None
venue_text: str | None = None
pub_url: str | None = None
doi: str | None = None
pdf_url: str | None = None
is_read: bool = False
@ -594,7 +593,6 @@ class DisplayIdentifierData(BaseModel):
class AdminPdfQueueItemData(BaseModel):
publication_id: int
title: str
doi: str | None
display_identifier: DisplayIdentifierData | None = None
pdf_url: str | None
status: str
@ -725,6 +723,10 @@ class SettingsData(BaseModel):
nav_visible_pages: list[str]
policy: SettingsPolicyData
safety_state: ScrapeSafetyStateData
openalex_api_key: str | None = None
crossref_api_token: str | None = None
crossref_api_mailto: str | None = None
model_config = ConfigDict(extra="forbid")
@ -742,6 +744,10 @@ class SettingsUpdateRequest(BaseModel):
request_delay_seconds: int
nav_visible_pages: list[str] | None = None
openalex_api_key: str | None = None
crossref_api_token: str | None = None
crossref_api_mailto: str | None = None
model_config = ConfigDict(extra="forbid")
@ -754,7 +760,6 @@ class PublicationItemData(BaseModel):
citation_count: int
venue_text: str | None
pub_url: str | None
doi: str | None
display_identifier: DisplayIdentifierData | None = None
pdf_url: str | None
pdf_status: str = "untracked"
@ -780,6 +785,7 @@ class PublicationsListData(BaseModel):
total_count: int
page: int
page_size: int
snapshot: str
has_next: bool = False
has_prev: bool = False
publications: list[PublicationItemData]