justinzeus/scholarr
1
0
Fork 0
Code Issues 1 Pull requests 18 Projects Releases Packages Wiki Activity Actions

Intermediate commit

Browse source
This commit is contained in:
Justin Visser 2026-02-26 16:09:57 +01:00
parent 0e9e49df16
commit 3d4cfeff1a
65 changed files with 5507 additions and 333 deletions
Download patch file Download diff file Expand all files Collapse all files

46
app/api/routers/admin_dbops.py
View file

@ -14,6 +14,8 @@ from app.api.schemas import (
AdminPdfQueueRequeueEnvelope,
AdminPdfQueueEnvelope,
AdminDbRepairJobsEnvelope,
AdminRepairPublicationNearDuplicatesEnvelope,
AdminRepairPublicationNearDuplicatesRequest,
AdminRepairPublicationLinksEnvelope,
AdminRepairPublicationLinksRequest,
)
@ -22,6 +24,7 @@ from app.db.session import get_db_session
from app.services.domains.dbops import (
collect_integrity_report,
list_repair_jobs,
run_publication_near_duplicate_repair,
run_publication_link_repair,
)
from app.services.domains.publications import application as publication_service
@ -48,7 +51,7 @@ def _serialize_repair_job(job: DataRepairJob) -> dict[str, object]:
}
def _requested_by_value(*, payload: AdminRepairPublicationLinksRequest, admin_user: User) -> str:
def _requested_by_value(*, payload, admin_user: User) -> str:
from_payload = (payload.requested_by or "").strip()
return from_payload or admin_user.email
@ -349,6 +352,47 @@ async def trigger_publication_link_repair(
return success_payload(request, data=result)
@router.post(
"/repairs/publication-near-duplicates",
response_model=AdminRepairPublicationNearDuplicatesEnvelope,
)
async def trigger_publication_near_duplicate_repair(
payload: AdminRepairPublicationNearDuplicatesRequest,
request: Request,
db_session: AsyncSession = Depends(get_db_session),
admin_user: User = Depends(get_api_admin_user),
):
try:
result = await run_publication_near_duplicate_repair(
db_session,
dry_run=bool(payload.dry_run),
similarity_threshold=float(payload.similarity_threshold),
min_shared_tokens=int(payload.min_shared_tokens),
max_year_delta=int(payload.max_year_delta),
max_clusters=int(payload.max_clusters),
selected_cluster_keys=list(payload.selected_cluster_keys),
requested_by=_requested_by_value(payload=payload, admin_user=admin_user),
)
except ValueError as exc:
raise ApiException(
status_code=400,
code="invalid_near_duplicate_repair_request",
message=str(exc),
) from exc
logger.info(
"api.admin.db.publication_near_duplicate_repair_triggered",
extra={
"event": "api.admin.db.publication_near_duplicate_repair_triggered",
"admin_user_id": int(admin_user.id),
"dry_run": bool(payload.dry_run),
"selected_cluster_count": len(payload.selected_cluster_keys),
"job_id": int(result["job_id"]),
"status": result["status"],
},
)
return success_payload(request, data=result)
DROP_PUBLICATIONS_CONFIRMATION = "DROP ALL PUBLICATIONS"

5
app/api/routers/publications.py
View file

@ -94,6 +94,7 @@ async def _publication_counts(
user_id: int,
selected_scholar_id: int | None,
favorite_only: bool,
search: str | None,
snapshot_before: datetime | None,
) -> tuple[int, int, int, int]:
unread_count = await publication_service.count_unread_for_user(
@ -122,6 +123,7 @@ async def _publication_counts(
mode=publication_service.MODE_ALL,
scholar_profile_id=selected_scholar_id,
favorite_only=favorite_only,
search=search,
snapshot_before=snapshot_before,
)
return unread_count, favorites_count, latest_count, total_count
@ -372,7 +374,7 @@ async def list_publications(
favorite_only: bool = Query(default=False),
scholar_profile_id: int | None = Query(default=None, ge=1),
search: str | None = Query(default=None, min_length=1, max_length=200),
sort_by: Literal["first_seen", "title", "year", "citations", "scholar"] = Query(default="first_seen"),
sort_by: Literal["first_seen", "title", "year", "citations", "scholar", "pdf_status"] = Query(default="first_seen"),
sort_dir: Literal["asc", "desc"] = Query(default="desc"),
page: int = Query(default=1, ge=1),
page_size: int = Query(default=100, ge=1, le=500),
@ -408,6 +410,7 @@ async def list_publications(
user_id=current_user.id,
selected_scholar_id=selected_scholar_id,
favorite_only=favorite_only,
search=normalized_search,
snapshot_before=snapshot_before,
)
data = _publications_list_data(

124
app/api/routers/scholars.py
View file

@ -23,7 +23,9 @@ from app.api.schemas import (
)
from app.db.models import User
from app.db.session import get_db_session
from app.services.domains.ingestion import queue as ingestion_queue_service
from app.services.domains.portability import application as import_export_service
from app.services.domains.scholar import rate_limit as scholar_rate_limit
from app.services.domains.scholars import application as scholar_service
from app.services.domains.scholar.source import ScholarSource
from app.settings import settings
@ -31,6 +33,73 @@ from app.settings import settings
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/scholars", tags=["api-scholars"])
CREATE_METADATA_HYDRATION_TIMEOUT_SECONDS = 5.0
INITIAL_SCHOLAR_SCRAPE_QUEUE_DELAY_SECONDS = 0
INITIAL_SCHOLAR_SCRAPE_QUEUE_REASON = "scholar_added_initial_scrape"
def _needs_metadata_hydration(profile) -> bool:
if not profile.profile_image_url:
return True
return not (profile.display_name or "").strip()
def _is_create_hydration_rate_limited() -> tuple[bool, float]:
remaining_seconds = scholar_rate_limit.remaining_scholar_slot_seconds(
min_interval_seconds=float(settings.ingestion_min_request_delay_seconds),
)
return remaining_seconds > 0, remaining_seconds
def _auto_enqueue_new_scholar_enabled() -> bool:
if not settings.scheduler_enabled:
return False
if not settings.ingestion_automation_allowed:
return False
return bool(settings.ingestion_continuation_queue_enabled)
async def _enqueue_initial_scrape_job_for_scholar(
db_session: AsyncSession,
*,
profile,
user_id: int,
) -> bool:
if not _auto_enqueue_new_scholar_enabled():
return False
try:
await ingestion_queue_service.upsert_job(
db_session,
user_id=user_id,
scholar_profile_id=int(profile.id),
resume_cstart=0,
reason=INITIAL_SCHOLAR_SCRAPE_QUEUE_REASON,
run_id=None,
delay_seconds=INITIAL_SCHOLAR_SCRAPE_QUEUE_DELAY_SECONDS,
)
await db_session.commit()
except Exception:
await db_session.rollback()
logger.warning(
"api.scholars.initial_scrape_enqueue_failed",
extra={
"event": "api.scholars.initial_scrape_enqueue_failed",
"user_id": user_id,
"scholar_profile_id": profile.id,
},
)
return False
logger.info(
"api.scholars.initial_scrape_enqueued",
extra={
"event": "api.scholars.initial_scrape_enqueued",
"user_id": user_id,
"scholar_profile_id": profile.id,
"reason": INITIAL_SCHOLAR_SCRAPE_QUEUE_REASON,
},
)
return True
def _uploaded_image_media_path(scholar_profile_id: int) -> str:
@ -69,16 +138,42 @@ async def _hydrate_scholar_metadata_if_needed(
source: ScholarSource,
user_id: int,
):
if not _needs_metadata_hydration(profile):
return profile
should_skip, remaining_seconds = _is_create_hydration_rate_limited()
if should_skip:
logger.info(
"api.scholars.create_metadata_hydration_skipped",
extra={
"event": "api.scholars.create_metadata_hydration_skipped",
"reason": "scholar_request_throttle_active",
"user_id": user_id,
"scholar_profile_id": profile.id,
"retry_after_seconds": round(remaining_seconds, 3),
},
)
return profile
try:
if not profile.profile_image_url or not (profile.display_name or "").strip():
return await asyncio.wait_for(
scholar_service.hydrate_profile_metadata(
db_session,
profile=profile,
source=source,
),
timeout=5.0,
)
return await asyncio.wait_for(
scholar_service.hydrate_profile_metadata(
db_session,
profile=profile,
source=source,
),
timeout=CREATE_METADATA_HYDRATION_TIMEOUT_SECONDS,
)
except TimeoutError:
logger.info(
"api.scholars.create_metadata_hydration_skipped",
extra={
"event": "api.scholars.create_metadata_hydration_skipped",
"reason": "create_timeout",
"user_id": user_id,
"scholar_profile_id": profile.id,
},
)
except Exception:
logger.warning(
"api.scholars.create_metadata_hydration_failed",
@ -280,12 +375,18 @@ async def create_scholar(
"scholar_profile_id": created.id,
},
)
created = await _hydrate_scholar_metadata_if_needed(
did_queue_initial_scrape = await _enqueue_initial_scrape_job_for_scholar(
db_session,
profile=created,
source=source,
user_id=current_user.id,
)
if not did_queue_initial_scrape:
created = await _hydrate_scholar_metadata_if_needed(
db_session,
profile=created,
source=source,
user_id=current_user.id,
)
return success_payload(
request,
@ -554,4 +655,3 @@ async def clear_scholar_image_customization(
request,
data=_serialize_scholar(updated),
)

64
app/api/schemas.py
View file

@ -703,6 +703,70 @@ class AdminRepairPublicationLinksEnvelope(BaseModel):
model_config = ConfigDict(extra="forbid")
class AdminNearDuplicateClusterMemberData(BaseModel):
publication_id: int
title: str
year: int | None
citation_count: int
model_config = ConfigDict(extra="forbid")
class AdminNearDuplicateClusterData(BaseModel):
cluster_key: str
winner_publication_id: int
member_count: int
similarity_score: float = Field(ge=0.0, le=1.0)
members: list[AdminNearDuplicateClusterMemberData] = Field(default_factory=list)
model_config = ConfigDict(extra="forbid")
class AdminRepairPublicationNearDuplicatesRequest(BaseModel):
dry_run: bool = True
similarity_threshold: float = Field(default=0.78, ge=0.5, le=1.0)
min_shared_tokens: int = Field(default=3, ge=1, le=8)
max_year_delta: int = Field(default=1, ge=0, le=5)
max_clusters: int = Field(default=25, ge=1, le=200)
selected_cluster_keys: list[str] = Field(default_factory=list, max_length=200)
requested_by: str | None = None
confirmation_text: str | None = None
model_config = ConfigDict(extra="forbid")
@model_validator(mode="after")
def validate_apply_mode(self) -> "AdminRepairPublicationNearDuplicatesRequest":
if self.dry_run:
return self
if not self.selected_cluster_keys:
raise ValueError("selected_cluster_keys is required when dry_run=false.")
expected = "MERGE SELECTED DUPLICATES"
provided = (self.confirmation_text or "").strip()
if provided != expected:
raise ValueError(
"confirmation_text must equal 'MERGE SELECTED DUPLICATES' "
"when applying near-duplicate merges."
)
return self
class AdminRepairPublicationNearDuplicatesResultData(BaseModel):
job_id: int
status: str
scope: dict[str, Any] = Field(default_factory=dict)
summary: dict[str, Any] = Field(default_factory=dict)
clusters: list[AdminNearDuplicateClusterData] = Field(default_factory=list)
model_config = ConfigDict(extra="forbid")
class AdminRepairPublicationNearDuplicatesEnvelope(BaseModel):
data: AdminRepairPublicationNearDuplicatesResultData
meta: ApiMeta
model_config = ConfigDict(extra="forbid")
class SettingsPolicyData(BaseModel):
min_run_interval_minutes: int
min_request_delay_seconds: int