chore: remove stale build artifacts, fix gitignore, use npm ci in CI
- Remove 114 tracked build/lib/ files (stale setuptools artifact that broke the repo-hygiene CI guard) - Add build/ to .gitignore - Remove AGENTS.MD from .gitignore (track agents.md instead) - Delete ad-hoc scripts: diag_pubs.py, test_enrich.py - Delete orphaned frontend/docs/website/package-lock.json - Replace npm install with npm ci in ci.yml and docs-pages.yml for deterministic lockfile installs Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
3d4cfeff1a
commit
7736cab141
120 changed files with 4 additions and 18380 deletions
4
.github/workflows/ci.yml
vendored
4
.github/workflows/ci.yml
vendored
|
|
@ -91,7 +91,7 @@ jobs:
|
|||
|
||||
- name: Install frontend dependencies
|
||||
working-directory: frontend
|
||||
run: npm install
|
||||
run: npm ci
|
||||
|
||||
- name: Frontend theme token policy
|
||||
working-directory: frontend
|
||||
|
|
@ -124,7 +124,7 @@ jobs:
|
|||
node-version: "20"
|
||||
|
||||
- name: Install docs dependencies
|
||||
run: npm install --prefix docs/website
|
||||
run: npm ci --prefix docs/website
|
||||
|
||||
- name: Docs build
|
||||
run: npm --prefix docs/website run build
|
||||
|
|
|
|||
2
.github/workflows/docs-pages.yml
vendored
2
.github/workflows/docs-pages.yml
vendored
|
|
@ -32,7 +32,7 @@ jobs:
|
|||
node-version: "20"
|
||||
|
||||
- name: Install docs dependencies
|
||||
run: npm install --prefix docs/website
|
||||
run: npm ci --prefix docs/website
|
||||
|
||||
- name: Build docs site
|
||||
run: npm --prefix docs/website run build
|
||||
|
|
|
|||
2
.gitignore
vendored
2
.gitignore
vendored
|
|
@ -12,9 +12,9 @@ htmlcov/
|
|||
*.log
|
||||
.DS_Store
|
||||
planning/
|
||||
build/
|
||||
frontend/node_modules/
|
||||
frontend/dist/
|
||||
frontend/.vite/
|
||||
docs/website/node_modules/
|
||||
docs/website/build/
|
||||
AGENTS.MD
|
||||
|
|
|
|||
|
|
@ -1 +0,0 @@
|
|||
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
|
|
@ -1,35 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from fastapi import Depends, Request
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.api.errors import ApiException
|
||||
from app.auth import runtime as auth_runtime
|
||||
from app.db.models import User
|
||||
from app.db.session import get_db_session
|
||||
|
||||
|
||||
async def get_api_current_user(
|
||||
request: Request,
|
||||
db_session: AsyncSession = Depends(get_db_session),
|
||||
) -> User:
|
||||
current_user = await auth_runtime.get_authenticated_user(request, db_session)
|
||||
if current_user is None:
|
||||
raise ApiException(
|
||||
status_code=401,
|
||||
code="auth_required",
|
||||
message="Authentication required.",
|
||||
)
|
||||
return current_user
|
||||
|
||||
|
||||
async def get_api_admin_user(
|
||||
current_user: User = Depends(get_api_current_user),
|
||||
) -> User:
|
||||
if not current_user.is_admin:
|
||||
raise ApiException(
|
||||
status_code=403,
|
||||
code="forbidden",
|
||||
message="Admin access required.",
|
||||
)
|
||||
return current_user
|
||||
|
|
@ -1,85 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from fastapi import FastAPI, HTTPException, Request
|
||||
from fastapi.exception_handlers import (
|
||||
http_exception_handler as fastapi_http_exception_handler,
|
||||
)
|
||||
from fastapi.exception_handlers import (
|
||||
request_validation_exception_handler as fastapi_validation_exception_handler,
|
||||
)
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
|
||||
from app.api.responses import error_response
|
||||
|
||||
|
||||
def _is_api_path(path: str) -> bool:
|
||||
return path.startswith("/api/")
|
||||
|
||||
|
||||
def _status_code_to_error_code(status_code: int) -> str:
|
||||
mapping = {
|
||||
400: "bad_request",
|
||||
401: "auth_required",
|
||||
403: "forbidden",
|
||||
404: "not_found",
|
||||
409: "conflict",
|
||||
422: "validation_error",
|
||||
429: "rate_limited",
|
||||
500: "internal_error",
|
||||
}
|
||||
return mapping.get(status_code, "error")
|
||||
|
||||
|
||||
class ApiException(Exception):
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
status_code: int,
|
||||
code: str,
|
||||
message: str,
|
||||
details: Any | None = None,
|
||||
) -> None:
|
||||
super().__init__(message)
|
||||
self.status_code = status_code
|
||||
self.code = code
|
||||
self.message = message
|
||||
self.details = details
|
||||
|
||||
|
||||
def register_api_exception_handlers(app: FastAPI) -> None:
|
||||
@app.exception_handler(ApiException)
|
||||
async def _handle_api_exception(request: Request, exc: ApiException):
|
||||
return error_response(
|
||||
request,
|
||||
status_code=exc.status_code,
|
||||
code=exc.code,
|
||||
message=exc.message,
|
||||
details=exc.details,
|
||||
)
|
||||
|
||||
@app.exception_handler(HTTPException)
|
||||
async def _handle_http_exception(request: Request, exc: HTTPException):
|
||||
if not _is_api_path(request.url.path):
|
||||
return await fastapi_http_exception_handler(request, exc)
|
||||
return error_response(
|
||||
request,
|
||||
status_code=exc.status_code,
|
||||
code=_status_code_to_error_code(exc.status_code),
|
||||
message=str(exc.detail) if exc.detail is not None else "Request failed.",
|
||||
details=exc.detail if isinstance(exc.detail, (dict, list)) else None,
|
||||
)
|
||||
|
||||
@app.exception_handler(RequestValidationError)
|
||||
async def _handle_validation_exception(request: Request, exc: RequestValidationError):
|
||||
if not _is_api_path(request.url.path):
|
||||
return await fastapi_validation_exception_handler(request, exc)
|
||||
return error_response(
|
||||
request,
|
||||
status_code=422,
|
||||
code="validation_error",
|
||||
message="Request validation failed.",
|
||||
details=exc.errors(),
|
||||
)
|
||||
|
||||
|
|
@ -1,63 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import mimetypes
|
||||
|
||||
from fastapi import APIRouter, Depends
|
||||
from fastapi.responses import FileResponse
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.api.deps import get_api_current_user
|
||||
from app.api.errors import ApiException
|
||||
from app.db.models import User
|
||||
from app.db.session import get_db_session
|
||||
from app.services.domains.scholars import application as scholar_service
|
||||
from app.settings import settings
|
||||
|
||||
router = APIRouter(tags=["media"])
|
||||
|
||||
|
||||
@router.get("/scholar-images/{scholar_profile_id}/upload")
|
||||
async def get_uploaded_scholar_image(
|
||||
scholar_profile_id: int,
|
||||
db_session: AsyncSession = Depends(get_db_session),
|
||||
current_user: User = Depends(get_api_current_user),
|
||||
):
|
||||
profile = await scholar_service.get_user_scholar_by_id(
|
||||
db_session,
|
||||
user_id=current_user.id,
|
||||
scholar_profile_id=scholar_profile_id,
|
||||
)
|
||||
if profile is None:
|
||||
raise ApiException(
|
||||
status_code=404,
|
||||
code="scholar_not_found",
|
||||
message="Scholar not found.",
|
||||
)
|
||||
if not profile.profile_image_upload_path:
|
||||
raise ApiException(
|
||||
status_code=404,
|
||||
code="scholar_image_not_found",
|
||||
message="Scholar image not found.",
|
||||
)
|
||||
|
||||
try:
|
||||
image_path = scholar_service.resolve_upload_file_path(
|
||||
upload_dir=settings.scholar_image_upload_dir,
|
||||
relative_path=profile.profile_image_upload_path,
|
||||
)
|
||||
except scholar_service.ScholarServiceError as exc:
|
||||
raise ApiException(
|
||||
status_code=404,
|
||||
code="scholar_image_not_found",
|
||||
message="Scholar image not found.",
|
||||
) from exc
|
||||
|
||||
if not image_path.exists() or not image_path.is_file():
|
||||
raise ApiException(
|
||||
status_code=404,
|
||||
code="scholar_image_not_found",
|
||||
message="Scholar image not found.",
|
||||
)
|
||||
|
||||
media_type = mimetypes.guess_type(str(image_path))[0] or "application/octet-stream"
|
||||
return FileResponse(path=image_path, media_type=media_type)
|
||||
|
|
@ -1,64 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from fastapi.encoders import jsonable_encoder
|
||||
from starlette.requests import Request
|
||||
from starlette.responses import JSONResponse
|
||||
|
||||
|
||||
def _request_id(request: Request) -> str | None:
|
||||
request_state = getattr(request, "state", None)
|
||||
if request_state is None:
|
||||
return None
|
||||
return getattr(request_state, "request_id", None)
|
||||
|
||||
|
||||
def success_payload(
|
||||
request: Request,
|
||||
*,
|
||||
data: Any,
|
||||
) -> dict[str, Any]:
|
||||
return {
|
||||
"data": data,
|
||||
"meta": {
|
||||
"request_id": _request_id(request),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def success_response(
|
||||
request: Request,
|
||||
*,
|
||||
data: Any,
|
||||
status_code: int = 200,
|
||||
) -> JSONResponse:
|
||||
payload = success_payload(request, data=data)
|
||||
return JSONResponse(
|
||||
status_code=status_code,
|
||||
content=jsonable_encoder(payload),
|
||||
)
|
||||
|
||||
|
||||
def error_response(
|
||||
request: Request,
|
||||
*,
|
||||
status_code: int,
|
||||
code: str,
|
||||
message: str,
|
||||
details: Any | None = None,
|
||||
) -> JSONResponse:
|
||||
payload = {
|
||||
"error": {
|
||||
"code": code,
|
||||
"message": message,
|
||||
"details": details,
|
||||
},
|
||||
"meta": {
|
||||
"request_id": _request_id(request),
|
||||
},
|
||||
}
|
||||
return JSONResponse(
|
||||
status_code=status_code,
|
||||
content=jsonable_encoder(payload),
|
||||
)
|
||||
|
|
@ -1,14 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from fastapi import APIRouter
|
||||
|
||||
from app.api.routers import admin, admin_dbops, auth, publications, runs, scholars, settings
|
||||
|
||||
router = APIRouter(prefix="/api/v1")
|
||||
router.include_router(auth.router)
|
||||
router.include_router(admin.router)
|
||||
router.include_router(admin_dbops.router)
|
||||
router.include_router(scholars.router)
|
||||
router.include_router(settings.router)
|
||||
router.include_router(runs.router)
|
||||
router.include_router(publications.router)
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
|
|
@ -1,202 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from fastapi import APIRouter, Depends, Request
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.api.deps import get_api_admin_user
|
||||
from app.api.errors import ApiException
|
||||
from app.api.responses import success_payload
|
||||
from app.api.schemas import (
|
||||
AdminResetPasswordRequest,
|
||||
AdminUserActiveUpdateRequest,
|
||||
AdminUserCreateRequest,
|
||||
AdminUserEnvelope,
|
||||
AdminUsersListEnvelope,
|
||||
MessageEnvelope,
|
||||
)
|
||||
from app.auth.deps import get_auth_service
|
||||
from app.auth.service import AuthService
|
||||
from app.db.models import User
|
||||
from app.db.session import get_db_session
|
||||
from app.services.domains.users import application as user_service
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/admin/users", tags=["api-admin-users"])
|
||||
|
||||
|
||||
def _serialize_user(user: User) -> dict[str, object]:
|
||||
return {
|
||||
"id": int(user.id),
|
||||
"email": user.email,
|
||||
"is_active": bool(user.is_active),
|
||||
"is_admin": bool(user.is_admin),
|
||||
"created_at": user.created_at,
|
||||
"updated_at": user.updated_at,
|
||||
}
|
||||
|
||||
|
||||
@router.get(
|
||||
"",
|
||||
response_model=AdminUsersListEnvelope,
|
||||
)
|
||||
async def list_users(
|
||||
request: Request,
|
||||
db_session: AsyncSession = Depends(get_db_session),
|
||||
admin_user: User = Depends(get_api_admin_user),
|
||||
):
|
||||
users = await user_service.list_users(db_session)
|
||||
logger.info(
|
||||
"api.admin.users_listed",
|
||||
extra={
|
||||
"event": "api.admin.users_listed",
|
||||
"admin_user_id": int(admin_user.id),
|
||||
"user_count": len(users),
|
||||
},
|
||||
)
|
||||
return success_payload(
|
||||
request,
|
||||
data={
|
||||
"users": [_serialize_user(user) for user in users],
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@router.post(
|
||||
"",
|
||||
response_model=AdminUserEnvelope,
|
||||
status_code=201,
|
||||
)
|
||||
async def create_user(
|
||||
payload: AdminUserCreateRequest,
|
||||
request: Request,
|
||||
db_session: AsyncSession = Depends(get_db_session),
|
||||
auth_service: AuthService = Depends(get_auth_service),
|
||||
admin_user: User = Depends(get_api_admin_user),
|
||||
):
|
||||
try:
|
||||
validated_email = user_service.validate_email(payload.email)
|
||||
validated_password = user_service.validate_password(payload.password)
|
||||
created_user = await user_service.create_user(
|
||||
db_session,
|
||||
email=validated_email,
|
||||
password_hash=auth_service.hash_password(validated_password),
|
||||
is_admin=bool(payload.is_admin),
|
||||
)
|
||||
except user_service.UserServiceError as exc:
|
||||
raise ApiException(
|
||||
status_code=400,
|
||||
code="invalid_user_input",
|
||||
message=str(exc),
|
||||
) from exc
|
||||
|
||||
logger.info(
|
||||
"api.admin.user_created",
|
||||
extra={
|
||||
"event": "api.admin.user_created",
|
||||
"admin_user_id": int(admin_user.id),
|
||||
"target_user_id": int(created_user.id),
|
||||
"target_is_admin": bool(created_user.is_admin),
|
||||
},
|
||||
)
|
||||
return success_payload(
|
||||
request,
|
||||
data=_serialize_user(created_user),
|
||||
)
|
||||
|
||||
|
||||
@router.patch(
|
||||
"/{user_id}/active",
|
||||
response_model=AdminUserEnvelope,
|
||||
)
|
||||
async def set_user_active(
|
||||
user_id: int,
|
||||
payload: AdminUserActiveUpdateRequest,
|
||||
request: Request,
|
||||
db_session: AsyncSession = Depends(get_db_session),
|
||||
admin_user: User = Depends(get_api_admin_user),
|
||||
):
|
||||
target_user = await user_service.get_user_by_id(db_session, user_id)
|
||||
if target_user is None:
|
||||
raise ApiException(
|
||||
status_code=404,
|
||||
code="user_not_found",
|
||||
message="User not found.",
|
||||
)
|
||||
if (
|
||||
int(target_user.id) == int(admin_user.id)
|
||||
and bool(target_user.is_active)
|
||||
and not bool(payload.is_active)
|
||||
):
|
||||
raise ApiException(
|
||||
status_code=400,
|
||||
code="cannot_deactivate_self",
|
||||
message="You cannot deactivate your own account.",
|
||||
)
|
||||
updated_user = await user_service.set_user_active(
|
||||
db_session,
|
||||
user=target_user,
|
||||
is_active=bool(payload.is_active),
|
||||
)
|
||||
logger.info(
|
||||
"api.admin.user_active_updated",
|
||||
extra={
|
||||
"event": "api.admin.user_active_updated",
|
||||
"admin_user_id": int(admin_user.id),
|
||||
"target_user_id": int(updated_user.id),
|
||||
"is_active": bool(updated_user.is_active),
|
||||
},
|
||||
)
|
||||
return success_payload(
|
||||
request,
|
||||
data=_serialize_user(updated_user),
|
||||
)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/{user_id}/reset-password",
|
||||
response_model=MessageEnvelope,
|
||||
)
|
||||
async def reset_user_password(
|
||||
user_id: int,
|
||||
payload: AdminResetPasswordRequest,
|
||||
request: Request,
|
||||
db_session: AsyncSession = Depends(get_db_session),
|
||||
auth_service: AuthService = Depends(get_auth_service),
|
||||
admin_user: User = Depends(get_api_admin_user),
|
||||
):
|
||||
target_user = await user_service.get_user_by_id(db_session, user_id)
|
||||
if target_user is None:
|
||||
raise ApiException(
|
||||
status_code=404,
|
||||
code="user_not_found",
|
||||
message="User not found.",
|
||||
)
|
||||
try:
|
||||
validated_password = user_service.validate_password(payload.new_password)
|
||||
except user_service.UserServiceError as exc:
|
||||
raise ApiException(
|
||||
status_code=400,
|
||||
code="invalid_password",
|
||||
message=str(exc),
|
||||
) from exc
|
||||
|
||||
await user_service.set_user_password_hash(
|
||||
db_session,
|
||||
user=target_user,
|
||||
password_hash=auth_service.hash_password(validated_password),
|
||||
)
|
||||
logger.info(
|
||||
"api.admin.user_password_reset",
|
||||
extra={
|
||||
"event": "api.admin.user_password_reset",
|
||||
"admin_user_id": int(admin_user.id),
|
||||
"target_user_id": int(target_user.id),
|
||||
},
|
||||
)
|
||||
return success_payload(
|
||||
request,
|
||||
data={"message": f"Password reset: {target_user.email}"},
|
||||
)
|
||||
|
|
@ -1,349 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from fastapi import APIRouter, Depends, Path, Query, Request
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.api.deps import get_api_admin_user
|
||||
from app.api.errors import ApiException
|
||||
from app.api.responses import success_payload
|
||||
from app.api.schemas import (
|
||||
AdminDbIntegrityEnvelope,
|
||||
AdminPdfQueueBulkEnqueueEnvelope,
|
||||
AdminPdfQueueRequeueEnvelope,
|
||||
AdminPdfQueueEnvelope,
|
||||
AdminDbRepairJobsEnvelope,
|
||||
AdminRepairPublicationLinksEnvelope,
|
||||
AdminRepairPublicationLinksRequest,
|
||||
)
|
||||
from app.db.models import DataRepairJob, User
|
||||
from app.db.session import get_db_session
|
||||
from app.services.domains.dbops import (
|
||||
collect_integrity_report,
|
||||
list_repair_jobs,
|
||||
run_publication_link_repair,
|
||||
)
|
||||
from app.services.domains.publications import application as publication_service
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/admin/db", tags=["api-admin-dbops"])
|
||||
|
||||
|
||||
def _serialize_repair_job(job: DataRepairJob) -> dict[str, object]:
|
||||
return {
|
||||
"id": int(job.id),
|
||||
"job_name": job.job_name,
|
||||
"requested_by": job.requested_by,
|
||||
"dry_run": bool(job.dry_run),
|
||||
"status": job.status,
|
||||
"scope": dict(job.scope or {}),
|
||||
"summary": dict(job.summary or {}),
|
||||
"error_text": job.error_text,
|
||||
"started_at": job.started_at,
|
||||
"finished_at": job.finished_at,
|
||||
"created_at": job.created_at,
|
||||
"updated_at": job.updated_at,
|
||||
}
|
||||
|
||||
|
||||
def _requested_by_value(*, payload: AdminRepairPublicationLinksRequest, admin_user: User) -> str:
|
||||
from_payload = (payload.requested_by or "").strip()
|
||||
return from_payload or admin_user.email
|
||||
|
||||
|
||||
def _serialize_pdf_queue_item(item) -> dict[str, object]:
|
||||
return {
|
||||
"publication_id": item.publication_id,
|
||||
"title": item.title,
|
||||
"display_identifier": _serialize_display_identifier(item.display_identifier),
|
||||
"pdf_url": item.pdf_url,
|
||||
"status": item.status,
|
||||
"attempt_count": item.attempt_count,
|
||||
"last_failure_reason": item.last_failure_reason,
|
||||
"last_failure_detail": item.last_failure_detail,
|
||||
"last_source": item.last_source,
|
||||
"requested_by_user_id": item.requested_by_user_id,
|
||||
"requested_by_email": item.requested_by_email,
|
||||
"queued_at": item.queued_at,
|
||||
"last_attempt_at": item.last_attempt_at,
|
||||
"resolved_at": item.resolved_at,
|
||||
"updated_at": item.updated_at,
|
||||
}
|
||||
|
||||
|
||||
def _serialize_display_identifier(value) -> dict[str, object] | None:
|
||||
if value is None:
|
||||
return None
|
||||
return {
|
||||
"kind": value.kind,
|
||||
"value": value.value,
|
||||
"label": value.label,
|
||||
"url": value.url,
|
||||
"confidence_score": float(value.confidence_score),
|
||||
}
|
||||
|
||||
|
||||
def _requeue_response_state(*, queued: bool) -> tuple[str, str]:
|
||||
if queued:
|
||||
return "queued", "PDF lookup queued."
|
||||
return "blocked", "PDF lookup is already queued or currently running."
|
||||
|
||||
|
||||
def _bulk_enqueue_message(*, queued_count: int, requested_count: int) -> str:
|
||||
if requested_count == 0:
|
||||
return "No missing-PDF publications were found."
|
||||
if queued_count == 0:
|
||||
return "No publications were queued; all candidates were already in-flight."
|
||||
return f"Queued {queued_count} publication(s) for PDF lookup."
|
||||
|
||||
|
||||
def _resolve_pdf_queue_paging(
|
||||
*,
|
||||
page: int,
|
||||
page_size: int,
|
||||
limit: int | None,
|
||||
offset: int | None,
|
||||
) -> tuple[int, int, int]:
|
||||
resolved_limit = int(limit) if limit is not None else int(page_size)
|
||||
resolved_offset = int(offset) if offset is not None else max((int(page) - 1) * resolved_limit, 0)
|
||||
resolved_page = max((resolved_offset // max(resolved_limit, 1)) + 1, 1)
|
||||
return resolved_page, max(resolved_limit, 1), max(resolved_offset, 0)
|
||||
|
||||
|
||||
def _pdf_queue_page_data(*, total_count: int, page: int, page_size: int, offset: int) -> dict[str, object]:
|
||||
return {
|
||||
"total_count": int(total_count),
|
||||
"page": int(page),
|
||||
"page_size": int(page_size),
|
||||
"has_prev": int(offset) > 0,
|
||||
"has_next": int(offset) + int(page_size) < int(total_count),
|
||||
}
|
||||
|
||||
|
||||
@router.get(
|
||||
"/integrity",
|
||||
response_model=AdminDbIntegrityEnvelope,
|
||||
)
|
||||
async def get_integrity_report(
|
||||
request: Request,
|
||||
db_session: AsyncSession = Depends(get_db_session),
|
||||
admin_user: User = Depends(get_api_admin_user),
|
||||
):
|
||||
report = await collect_integrity_report(db_session)
|
||||
logger.info(
|
||||
"api.admin.db.integrity_checked",
|
||||
extra={
|
||||
"event": "api.admin.db.integrity_checked",
|
||||
"admin_user_id": int(admin_user.id),
|
||||
"status": report.get("status"),
|
||||
"failure_count": len(report.get("failures", [])),
|
||||
"warning_count": len(report.get("warnings", [])),
|
||||
},
|
||||
)
|
||||
return success_payload(request, data=report)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/repair-jobs",
|
||||
response_model=AdminDbRepairJobsEnvelope,
|
||||
)
|
||||
async def get_repair_jobs(
|
||||
request: Request,
|
||||
limit: int = Query(default=50, ge=1, le=200),
|
||||
db_session: AsyncSession = Depends(get_db_session),
|
||||
admin_user: User = Depends(get_api_admin_user),
|
||||
):
|
||||
jobs = await list_repair_jobs(db_session, limit=limit)
|
||||
logger.info(
|
||||
"api.admin.db.repair_jobs_listed",
|
||||
extra={
|
||||
"event": "api.admin.db.repair_jobs_listed",
|
||||
"admin_user_id": int(admin_user.id),
|
||||
"limit": int(limit),
|
||||
"job_count": len(jobs),
|
||||
},
|
||||
)
|
||||
return success_payload(
|
||||
request,
|
||||
data={"jobs": [_serialize_repair_job(job) for job in jobs]},
|
||||
)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/pdf-queue",
|
||||
response_model=AdminPdfQueueEnvelope,
|
||||
)
|
||||
async def get_pdf_queue(
|
||||
request: Request,
|
||||
page: int = Query(default=1, ge=1),
|
||||
page_size: int = Query(default=100, ge=1, le=500),
|
||||
limit: int | None = Query(default=None, ge=1, le=500),
|
||||
offset: int | None = Query(default=None, ge=0),
|
||||
status: str | None = Query(default=None),
|
||||
db_session: AsyncSession = Depends(get_db_session),
|
||||
admin_user: User = Depends(get_api_admin_user),
|
||||
):
|
||||
normalized_status = (status or "").strip().lower() or None
|
||||
resolved_page, resolved_limit, resolved_offset = _resolve_pdf_queue_paging(
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
limit=limit,
|
||||
offset=offset,
|
||||
)
|
||||
queue_page = await publication_service.list_pdf_queue_page(
|
||||
db_session,
|
||||
limit=resolved_limit,
|
||||
offset=resolved_offset,
|
||||
status=normalized_status,
|
||||
)
|
||||
logger.info(
|
||||
"api.admin.db.pdf_queue_listed",
|
||||
extra={
|
||||
"event": "api.admin.db.pdf_queue_listed",
|
||||
"admin_user_id": int(admin_user.id),
|
||||
"page": int(resolved_page),
|
||||
"page_size": int(resolved_limit),
|
||||
"offset": int(resolved_offset),
|
||||
"status": normalized_status,
|
||||
"item_count": len(queue_page.items),
|
||||
"total_count": int(queue_page.total_count),
|
||||
},
|
||||
)
|
||||
return success_payload(
|
||||
request,
|
||||
data={
|
||||
"items": [_serialize_pdf_queue_item(item) for item in queue_page.items],
|
||||
**_pdf_queue_page_data(
|
||||
total_count=queue_page.total_count,
|
||||
page=resolved_page,
|
||||
page_size=resolved_limit,
|
||||
offset=resolved_offset,
|
||||
),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/pdf-queue/{publication_id}/requeue",
|
||||
response_model=AdminPdfQueueRequeueEnvelope,
|
||||
)
|
||||
async def requeue_pdf_lookup(
|
||||
request: Request,
|
||||
publication_id: int = Path(ge=1),
|
||||
db_session: AsyncSession = Depends(get_db_session),
|
||||
admin_user: User = Depends(get_api_admin_user),
|
||||
):
|
||||
result = await publication_service.enqueue_retry_pdf_job_for_publication_id(
|
||||
db_session,
|
||||
user_id=int(admin_user.id),
|
||||
request_email=admin_user.email,
|
||||
publication_id=publication_id,
|
||||
)
|
||||
if not result.publication_exists:
|
||||
raise ApiException(
|
||||
status_code=404,
|
||||
code="publication_not_found",
|
||||
message="Publication not found.",
|
||||
)
|
||||
status, message = _requeue_response_state(queued=result.queued)
|
||||
logger.info(
|
||||
"api.admin.db.pdf_queue_requeue_requested",
|
||||
extra={
|
||||
"event": "api.admin.db.pdf_queue_requeue_requested",
|
||||
"admin_user_id": int(admin_user.id),
|
||||
"publication_id": int(publication_id),
|
||||
"queued": bool(result.queued),
|
||||
},
|
||||
)
|
||||
return success_payload(
|
||||
request,
|
||||
data={
|
||||
"publication_id": int(publication_id),
|
||||
"queued": bool(result.queued),
|
||||
"status": status,
|
||||
"message": message,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/pdf-queue/requeue-all",
|
||||
response_model=AdminPdfQueueBulkEnqueueEnvelope,
|
||||
)
|
||||
async def requeue_all_missing_pdfs(
|
||||
request: Request,
|
||||
limit: int = Query(default=1000, ge=1, le=5000),
|
||||
db_session: AsyncSession = Depends(get_db_session),
|
||||
admin_user: User = Depends(get_api_admin_user),
|
||||
):
|
||||
result = await publication_service.enqueue_all_missing_pdf_jobs(
|
||||
db_session,
|
||||
user_id=int(admin_user.id),
|
||||
request_email=admin_user.email,
|
||||
limit=limit,
|
||||
)
|
||||
logger.info(
|
||||
"api.admin.db.pdf_queue_requeue_all",
|
||||
extra={
|
||||
"event": "api.admin.db.pdf_queue_requeue_all",
|
||||
"admin_user_id": int(admin_user.id),
|
||||
"limit": int(limit),
|
||||
"requested_count": int(result.requested_count),
|
||||
"queued_count": int(result.queued_count),
|
||||
},
|
||||
)
|
||||
return success_payload(
|
||||
request,
|
||||
data={
|
||||
"requested_count": int(result.requested_count),
|
||||
"queued_count": int(result.queued_count),
|
||||
"message": _bulk_enqueue_message(
|
||||
queued_count=int(result.queued_count),
|
||||
requested_count=int(result.requested_count),
|
||||
),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/repairs/publication-links",
|
||||
response_model=AdminRepairPublicationLinksEnvelope,
|
||||
)
|
||||
async def trigger_publication_link_repair(
|
||||
payload: AdminRepairPublicationLinksRequest,
|
||||
request: Request,
|
||||
db_session: AsyncSession = Depends(get_db_session),
|
||||
admin_user: User = Depends(get_api_admin_user),
|
||||
):
|
||||
try:
|
||||
result = await run_publication_link_repair(
|
||||
db_session,
|
||||
scope_mode=payload.scope_mode,
|
||||
user_id=payload.user_id,
|
||||
scholar_profile_ids=payload.scholar_profile_ids,
|
||||
dry_run=bool(payload.dry_run),
|
||||
gc_orphan_publications=bool(payload.gc_orphan_publications),
|
||||
requested_by=_requested_by_value(payload=payload, admin_user=admin_user),
|
||||
)
|
||||
except ValueError as exc:
|
||||
raise ApiException(
|
||||
status_code=400,
|
||||
code="invalid_repair_scope",
|
||||
message=str(exc),
|
||||
) from exc
|
||||
logger.info(
|
||||
"api.admin.db.publication_link_repair_triggered",
|
||||
extra={
|
||||
"event": "api.admin.db.publication_link_repair_triggered",
|
||||
"admin_user_id": int(admin_user.id),
|
||||
"scope_mode": payload.scope_mode,
|
||||
"target_user_id": int(payload.user_id) if payload.user_id is not None else None,
|
||||
"dry_run": bool(payload.dry_run),
|
||||
"gc_orphan_publications": bool(payload.gc_orphan_publications),
|
||||
"job_id": int(result["job_id"]),
|
||||
"status": result["status"],
|
||||
},
|
||||
)
|
||||
return success_payload(request, data=result)
|
||||
|
|
@ -1,237 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from fastapi import APIRouter, Depends, Request
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.api.errors import ApiException
|
||||
from app.api.deps import get_api_current_user
|
||||
from app.api.responses import success_payload
|
||||
from app.api.schemas import (
|
||||
AuthMeEnvelope,
|
||||
ChangePasswordRequest,
|
||||
CsrfBootstrapEnvelope,
|
||||
LoginEnvelope,
|
||||
LoginRequest,
|
||||
MessageEnvelope,
|
||||
)
|
||||
from app.auth.deps import get_auth_service, get_login_rate_limiter
|
||||
from app.auth.rate_limit import SlidingWindowRateLimiter
|
||||
from app.auth import runtime as auth_runtime
|
||||
from app.auth.service import AuthService
|
||||
from app.auth.session import set_session_user
|
||||
from app.db.models import User
|
||||
from app.db.session import get_db_session
|
||||
from app.security.csrf import ensure_csrf_token
|
||||
from app.services.domains.users import application as user_service
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/auth", tags=["api-auth"])
|
||||
|
||||
|
||||
def _login_limiter_key_and_email(request: Request, payload: LoginRequest) -> tuple[str, str]:
|
||||
return auth_runtime.login_rate_limit_key(request, payload.email), payload.email.strip().lower()
|
||||
|
||||
|
||||
def _raise_rate_limited(normalized_email: str, retry_after_seconds: int) -> None:
|
||||
logger.warning(
|
||||
"api.auth.login_rate_limited",
|
||||
extra={
|
||||
"event": "api.auth.login_rate_limited",
|
||||
"email": normalized_email,
|
||||
"retry_after_seconds": retry_after_seconds,
|
||||
},
|
||||
)
|
||||
raise ApiException(
|
||||
status_code=429,
|
||||
code="rate_limited",
|
||||
message="Too many login attempts. Please try again later.",
|
||||
details={"retry_after_seconds": retry_after_seconds},
|
||||
)
|
||||
|
||||
|
||||
def _serialize_user_payload(user: User) -> dict[str, object]:
|
||||
return {
|
||||
"id": int(user.id),
|
||||
"email": user.email,
|
||||
"is_admin": bool(user.is_admin),
|
||||
"is_active": bool(user.is_active),
|
||||
}
|
||||
|
||||
|
||||
def _raise_invalid_credentials(*, normalized_email: str) -> None:
|
||||
logger.info(
|
||||
"api.auth.login_failed",
|
||||
extra={"event": "api.auth.login_failed", "email": normalized_email},
|
||||
)
|
||||
raise ApiException(
|
||||
status_code=401,
|
||||
code="invalid_credentials",
|
||||
message="Invalid email or password.",
|
||||
)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/login",
|
||||
response_model=LoginEnvelope,
|
||||
)
|
||||
async def login(
|
||||
payload: LoginRequest,
|
||||
request: Request,
|
||||
db_session: AsyncSession = Depends(get_db_session),
|
||||
auth_service: AuthService = Depends(get_auth_service),
|
||||
rate_limiter: SlidingWindowRateLimiter = Depends(get_login_rate_limiter),
|
||||
):
|
||||
limiter_key, normalized_email = _login_limiter_key_and_email(request, payload)
|
||||
decision = rate_limiter.check(limiter_key)
|
||||
if not decision.allowed:
|
||||
_raise_rate_limited(normalized_email, int(decision.retry_after_seconds))
|
||||
|
||||
user = await auth_service.authenticate_user(
|
||||
db_session,
|
||||
email=payload.email,
|
||||
password=payload.password,
|
||||
)
|
||||
if user is None:
|
||||
rate_limiter.record_failure(limiter_key)
|
||||
_raise_invalid_credentials(normalized_email=normalized_email)
|
||||
|
||||
rate_limiter.reset(limiter_key)
|
||||
set_session_user(
|
||||
request,
|
||||
user_id=user.id,
|
||||
email=user.email,
|
||||
is_admin=user.is_admin,
|
||||
)
|
||||
logger.info(
|
||||
"api.auth.login_succeeded",
|
||||
extra={
|
||||
"event": "api.auth.login_succeeded",
|
||||
"user_id": user.id,
|
||||
"is_admin": user.is_admin,
|
||||
},
|
||||
)
|
||||
return success_payload(
|
||||
request,
|
||||
data={
|
||||
"authenticated": True,
|
||||
"csrf_token": ensure_csrf_token(request),
|
||||
"user": _serialize_user_payload(user),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/me",
|
||||
response_model=AuthMeEnvelope,
|
||||
)
|
||||
async def get_current_session(
|
||||
request: Request,
|
||||
current_user: User = Depends(get_api_current_user),
|
||||
):
|
||||
return success_payload(
|
||||
request,
|
||||
data={
|
||||
"authenticated": True,
|
||||
"csrf_token": ensure_csrf_token(request),
|
||||
"user": _serialize_user_payload(current_user),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/csrf",
|
||||
response_model=CsrfBootstrapEnvelope,
|
||||
)
|
||||
async def get_csrf_bootstrap(
|
||||
request: Request,
|
||||
db_session: AsyncSession = Depends(get_db_session),
|
||||
):
|
||||
current_user = await auth_runtime.get_authenticated_user(request, db_session)
|
||||
return success_payload(
|
||||
request,
|
||||
data={
|
||||
"csrf_token": ensure_csrf_token(request),
|
||||
"authenticated": current_user is not None,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/change-password",
|
||||
response_model=MessageEnvelope,
|
||||
)
|
||||
async def change_password(
|
||||
payload: ChangePasswordRequest,
|
||||
request: Request,
|
||||
current_user: User = Depends(get_api_current_user),
|
||||
db_session: AsyncSession = Depends(get_db_session),
|
||||
auth_service: AuthService = Depends(get_auth_service),
|
||||
):
|
||||
if not auth_service.verify_password(
|
||||
password_hash=current_user.password_hash,
|
||||
password=payload.current_password,
|
||||
):
|
||||
raise ApiException(
|
||||
status_code=400,
|
||||
code="invalid_current_password",
|
||||
message="Current password is incorrect.",
|
||||
)
|
||||
if payload.new_password != payload.confirm_password:
|
||||
raise ApiException(
|
||||
status_code=400,
|
||||
code="password_confirmation_mismatch",
|
||||
message="New password and confirmation do not match.",
|
||||
)
|
||||
try:
|
||||
validated_password = user_service.validate_password(payload.new_password)
|
||||
except user_service.UserServiceError as exc:
|
||||
raise ApiException(
|
||||
status_code=400,
|
||||
code="invalid_password",
|
||||
message=str(exc),
|
||||
) from exc
|
||||
|
||||
await user_service.set_user_password_hash(
|
||||
db_session,
|
||||
user=current_user,
|
||||
password_hash=auth_service.hash_password(validated_password),
|
||||
)
|
||||
logger.info(
|
||||
"api.auth.password_changed",
|
||||
extra={
|
||||
"event": "api.auth.password_changed",
|
||||
"user_id": int(current_user.id),
|
||||
},
|
||||
)
|
||||
return success_payload(
|
||||
request,
|
||||
data={"message": "Password updated successfully."},
|
||||
)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/logout",
|
||||
response_model=MessageEnvelope,
|
||||
)
|
||||
async def logout(
|
||||
request: Request,
|
||||
db_session: AsyncSession = Depends(get_db_session),
|
||||
):
|
||||
current_user = await auth_runtime.get_authenticated_user(request, db_session)
|
||||
auth_runtime.invalidate_session(request)
|
||||
logger.info(
|
||||
"api.auth.logout",
|
||||
extra={
|
||||
"event": "api.auth.logout",
|
||||
"user_id": int(current_user.id) if current_user is not None else None,
|
||||
},
|
||||
)
|
||||
return success_payload(
|
||||
request,
|
||||
data={
|
||||
"message": "Logged out.",
|
||||
},
|
||||
)
|
||||
|
|
@ -1,528 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Literal
|
||||
|
||||
from fastapi import APIRouter, Depends, Path, Query, Request
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.api.deps import get_api_current_user
|
||||
from app.api.errors import ApiException
|
||||
from app.api.responses import success_payload
|
||||
from app.api.schemas import (
|
||||
MarkAllReadEnvelope,
|
||||
MarkSelectedReadEnvelope,
|
||||
MarkSelectedReadRequest,
|
||||
PublicationsListEnvelope,
|
||||
RetryPublicationPdfEnvelope,
|
||||
RetryPublicationPdfRequest,
|
||||
TogglePublicationFavoriteEnvelope,
|
||||
TogglePublicationFavoriteRequest,
|
||||
)
|
||||
from app.db.models import User
|
||||
from app.db.session import get_db_session
|
||||
from app.services.domains.publication_identifiers import application as identifier_service
|
||||
from app.services.domains.publications import application as publication_service
|
||||
from app.services.domains.scholars import application as scholar_service
|
||||
from app.settings import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/publications", tags=["api-publications"])
|
||||
|
||||
|
||||
async def _require_selected_profile(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
selected_scholar_id: int | None,
|
||||
) -> None:
|
||||
if selected_scholar_id is None:
|
||||
return
|
||||
selected_profile = await scholar_service.get_user_scholar_by_id(
|
||||
db_session,
|
||||
user_id=user_id,
|
||||
scholar_profile_id=selected_scholar_id,
|
||||
)
|
||||
if selected_profile is None:
|
||||
raise ApiException(
|
||||
status_code=404,
|
||||
code="scholar_not_found",
|
||||
message="Scholar filter not found.",
|
||||
)
|
||||
|
||||
|
||||
def _serialize_publication_item(item) -> dict[str, object]:
|
||||
return {
|
||||
"publication_id": item.publication_id,
|
||||
"scholar_profile_id": item.scholar_profile_id,
|
||||
"scholar_label": item.scholar_label,
|
||||
"title": item.title,
|
||||
"year": item.year,
|
||||
"citation_count": item.citation_count,
|
||||
"venue_text": item.venue_text,
|
||||
"pub_url": item.pub_url,
|
||||
"display_identifier": _serialize_display_identifier(item.display_identifier),
|
||||
"pdf_url": item.pdf_url,
|
||||
"pdf_status": item.pdf_status,
|
||||
"pdf_attempt_count": item.pdf_attempt_count,
|
||||
"pdf_failure_reason": item.pdf_failure_reason,
|
||||
"pdf_failure_detail": item.pdf_failure_detail,
|
||||
"is_read": item.is_read,
|
||||
"is_favorite": item.is_favorite,
|
||||
"first_seen_at": item.first_seen_at,
|
||||
"is_new_in_latest_run": item.is_new_in_latest_run,
|
||||
}
|
||||
|
||||
|
||||
def _serialize_display_identifier(value) -> dict[str, object] | None:
|
||||
if value is None:
|
||||
return None
|
||||
return {
|
||||
"kind": value.kind,
|
||||
"value": value.value,
|
||||
"label": value.label,
|
||||
"url": value.url,
|
||||
"confidence_score": float(value.confidence_score),
|
||||
}
|
||||
|
||||
|
||||
async def _publication_counts(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
selected_scholar_id: int | None,
|
||||
favorite_only: bool,
|
||||
) -> tuple[int, int, int, int]:
|
||||
unread_count = await publication_service.count_unread_for_user(
|
||||
db_session,
|
||||
user_id=user_id,
|
||||
scholar_profile_id=selected_scholar_id,
|
||||
favorite_only=favorite_only,
|
||||
)
|
||||
favorites_count = await publication_service.count_favorite_for_user(
|
||||
db_session,
|
||||
user_id=user_id,
|
||||
scholar_profile_id=selected_scholar_id,
|
||||
)
|
||||
latest_count = await publication_service.count_latest_for_user(
|
||||
db_session,
|
||||
user_id=user_id,
|
||||
scholar_profile_id=selected_scholar_id,
|
||||
favorite_only=favorite_only,
|
||||
)
|
||||
total_count = await publication_service.count_for_user(
|
||||
db_session,
|
||||
user_id=user_id,
|
||||
mode=publication_service.MODE_ALL,
|
||||
scholar_profile_id=selected_scholar_id,
|
||||
favorite_only=favorite_only,
|
||||
)
|
||||
return unread_count, favorites_count, latest_count, total_count
|
||||
|
||||
|
||||
async def _list_publications_for_request(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
current_user: User,
|
||||
mode: Literal["all", "unread", "latest", "new"] | None,
|
||||
favorite_only: bool,
|
||||
scholar_profile_id: int | None,
|
||||
limit: int,
|
||||
offset: int,
|
||||
) -> tuple[str, int | None, list]:
|
||||
resolved_mode = publication_service.resolve_publication_view_mode(mode)
|
||||
selected_scholar_id = scholar_profile_id
|
||||
await _require_selected_profile(
|
||||
db_session,
|
||||
user_id=current_user.id,
|
||||
selected_scholar_id=selected_scholar_id,
|
||||
)
|
||||
publications = await publication_service.list_for_user(
|
||||
db_session,
|
||||
user_id=current_user.id,
|
||||
mode=resolved_mode,
|
||||
scholar_profile_id=selected_scholar_id,
|
||||
favorite_only=favorite_only,
|
||||
limit=limit,
|
||||
offset=offset,
|
||||
)
|
||||
await publication_service.schedule_missing_pdf_enrichment_for_user(
|
||||
db_session,
|
||||
user_id=current_user.id,
|
||||
request_email=current_user.email,
|
||||
items=publications,
|
||||
max_items=settings.unpaywall_max_items_per_request,
|
||||
)
|
||||
hydrated = await publication_service.hydrate_pdf_enrichment_state(
|
||||
db_session,
|
||||
items=publications,
|
||||
)
|
||||
return resolved_mode, selected_scholar_id, hydrated
|
||||
|
||||
|
||||
def _resolve_publications_paging(
|
||||
*,
|
||||
page: int,
|
||||
page_size: int,
|
||||
limit: int | None,
|
||||
offset: int | None,
|
||||
) -> tuple[int, int, int]:
|
||||
resolved_limit = int(limit) if limit is not None else int(page_size)
|
||||
resolved_offset = int(offset) if offset is not None else max((int(page) - 1) * resolved_limit, 0)
|
||||
resolved_page = max((resolved_offset // max(resolved_limit, 1)) + 1, 1)
|
||||
return resolved_page, max(resolved_limit, 1), max(resolved_offset, 0)
|
||||
|
||||
|
||||
def _publications_list_data(
|
||||
*,
|
||||
mode: str,
|
||||
favorite_only: bool,
|
||||
selected_scholar_id: int | None,
|
||||
unread_count: int,
|
||||
favorites_count: int,
|
||||
latest_count: int,
|
||||
total_count: int,
|
||||
publications: list,
|
||||
page: int,
|
||||
page_size: int,
|
||||
offset: int,
|
||||
) -> dict[str, object]:
|
||||
return {
|
||||
"mode": mode,
|
||||
"favorite_only": favorite_only,
|
||||
"selected_scholar_profile_id": selected_scholar_id,
|
||||
"unread_count": unread_count,
|
||||
"favorites_count": favorites_count,
|
||||
"latest_count": latest_count,
|
||||
"new_count": latest_count,
|
||||
"total_count": total_count,
|
||||
"page": int(page),
|
||||
"page_size": int(page_size),
|
||||
"has_prev": int(offset) > 0,
|
||||
"has_next": int(offset) + int(page_size) < int(total_count),
|
||||
"publications": [_serialize_publication_item(item) for item in publications],
|
||||
}
|
||||
|
||||
|
||||
def _retry_pdf_message(*, queued: bool, resolved_pdf: bool, pdf_status: str) -> str:
|
||||
if resolved_pdf:
|
||||
return "Open-access PDF link already resolved."
|
||||
if queued:
|
||||
return "PDF lookup queued."
|
||||
if pdf_status in {"queued", "running"}:
|
||||
return "PDF lookup is already queued."
|
||||
return "No open-access PDF link found."
|
||||
|
||||
|
||||
def _favorite_message(*, is_favorite: bool) -> str:
|
||||
if is_favorite:
|
||||
return "Publication marked as favorite."
|
||||
return "Publication removed from favorites."
|
||||
|
||||
|
||||
async def _retry_publication_state(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
current_user: User,
|
||||
scholar_profile_id: int,
|
||||
publication_id: int,
|
||||
):
|
||||
publication = await publication_service.retry_pdf_for_user(
|
||||
db_session,
|
||||
user_id=current_user.id,
|
||||
scholar_profile_id=scholar_profile_id,
|
||||
publication_id=publication_id,
|
||||
)
|
||||
if publication is None:
|
||||
raise ApiException(
|
||||
status_code=404,
|
||||
code="publication_not_found",
|
||||
message="Publication not found.",
|
||||
)
|
||||
queued = False
|
||||
if not publication.pdf_url:
|
||||
queued = await publication_service.schedule_retry_pdf_enrichment_for_row(
|
||||
db_session,
|
||||
user_id=current_user.id,
|
||||
request_email=current_user.email,
|
||||
item=publication,
|
||||
)
|
||||
hydrated = await publication_service.hydrate_pdf_enrichment_state(
|
||||
db_session,
|
||||
items=[publication],
|
||||
)
|
||||
current = hydrated[0] if hydrated else publication
|
||||
return current, queued
|
||||
|
||||
|
||||
async def _favorite_publication_state(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
current_user: User,
|
||||
scholar_profile_id: int,
|
||||
publication_id: int,
|
||||
is_favorite: bool,
|
||||
):
|
||||
updated_count = await publication_service.set_publication_favorite_for_user(
|
||||
db_session,
|
||||
user_id=current_user.id,
|
||||
scholar_profile_id=scholar_profile_id,
|
||||
publication_id=publication_id,
|
||||
is_favorite=is_favorite,
|
||||
)
|
||||
if updated_count <= 0:
|
||||
raise ApiException(
|
||||
status_code=404,
|
||||
code="publication_not_found",
|
||||
message="Publication not found.",
|
||||
)
|
||||
publication = await publication_service.get_publication_item_for_user(
|
||||
db_session,
|
||||
user_id=current_user.id,
|
||||
scholar_profile_id=scholar_profile_id,
|
||||
publication_id=publication_id,
|
||||
)
|
||||
if publication is None:
|
||||
raise ApiException(
|
||||
status_code=404,
|
||||
code="publication_not_found",
|
||||
message="Publication not found.",
|
||||
)
|
||||
hydrated = await publication_service.hydrate_pdf_enrichment_state(
|
||||
db_session,
|
||||
items=[publication],
|
||||
)
|
||||
current = hydrated[0] if hydrated else publication
|
||||
identifiers = await identifier_service.overlay_publication_items_with_display_identifiers(
|
||||
db_session,
|
||||
items=[current],
|
||||
)
|
||||
return identifiers[0] if identifiers else current
|
||||
|
||||
|
||||
def _log_retry_pdf_result(
|
||||
*,
|
||||
current_user: User,
|
||||
scholar_profile_id: int,
|
||||
publication_id: int,
|
||||
queued: bool,
|
||||
resolved_pdf: bool,
|
||||
pdf_status: str,
|
||||
doi: str | None,
|
||||
) -> None:
|
||||
logger.info(
|
||||
"api.publications.retry_pdf",
|
||||
extra={
|
||||
"event": "api.publications.retry_pdf",
|
||||
"user_id": current_user.id,
|
||||
"scholar_profile_id": scholar_profile_id,
|
||||
"publication_id": publication_id,
|
||||
"queued": queued,
|
||||
"resolved_pdf": resolved_pdf,
|
||||
"pdf_status": pdf_status,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@router.get(
|
||||
"",
|
||||
response_model=PublicationsListEnvelope,
|
||||
)
|
||||
async def list_publications(
|
||||
request: Request,
|
||||
mode: Literal["all", "unread", "latest", "new"] | None = Query(default=None),
|
||||
favorite_only: bool = Query(default=False),
|
||||
scholar_profile_id: int | None = Query(default=None, ge=1),
|
||||
page: int = Query(default=1, ge=1),
|
||||
page_size: int = Query(default=100, ge=1, le=500),
|
||||
limit: int | None = Query(default=None, ge=1, le=1000),
|
||||
offset: int | None = Query(default=None, ge=0),
|
||||
db_session: AsyncSession = Depends(get_db_session),
|
||||
current_user: User = Depends(get_api_current_user),
|
||||
):
|
||||
resolved_page, resolved_limit, resolved_offset = _resolve_publications_paging(
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
limit=limit,
|
||||
offset=offset,
|
||||
)
|
||||
resolved_mode, selected_scholar_id, publications = await _list_publications_for_request(
|
||||
db_session,
|
||||
current_user=current_user,
|
||||
mode=mode,
|
||||
favorite_only=favorite_only,
|
||||
scholar_profile_id=scholar_profile_id,
|
||||
limit=resolved_limit,
|
||||
offset=resolved_offset,
|
||||
)
|
||||
unread_count, favorites_count, latest_count, total_count = await _publication_counts(
|
||||
db_session,
|
||||
user_id=current_user.id,
|
||||
selected_scholar_id=selected_scholar_id,
|
||||
favorite_only=favorite_only,
|
||||
)
|
||||
data = _publications_list_data(
|
||||
mode=resolved_mode,
|
||||
favorite_only=favorite_only,
|
||||
selected_scholar_id=selected_scholar_id,
|
||||
unread_count=unread_count,
|
||||
favorites_count=favorites_count,
|
||||
latest_count=latest_count,
|
||||
total_count=total_count,
|
||||
publications=publications,
|
||||
page=resolved_page,
|
||||
page_size=resolved_limit,
|
||||
offset=resolved_offset,
|
||||
)
|
||||
return success_payload(request, data=data)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/mark-all-read",
|
||||
response_model=MarkAllReadEnvelope,
|
||||
)
|
||||
async def mark_all_publications_read(
|
||||
request: Request,
|
||||
db_session: AsyncSession = Depends(get_db_session),
|
||||
current_user: User = Depends(get_api_current_user),
|
||||
):
|
||||
updated_count = await publication_service.mark_all_unread_as_read_for_user(
|
||||
db_session,
|
||||
user_id=current_user.id,
|
||||
)
|
||||
logger.info(
|
||||
"api.publications.mark_all_read",
|
||||
extra={
|
||||
"event": "api.publications.mark_all_read",
|
||||
"user_id": current_user.id,
|
||||
"updated_count": updated_count,
|
||||
},
|
||||
)
|
||||
return success_payload(
|
||||
request,
|
||||
data={
|
||||
"message": "Marked all unread publications as read.",
|
||||
"updated_count": updated_count,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/mark-read",
|
||||
response_model=MarkSelectedReadEnvelope,
|
||||
)
|
||||
async def mark_selected_publications_read(
|
||||
payload: MarkSelectedReadRequest,
|
||||
request: Request,
|
||||
db_session: AsyncSession = Depends(get_db_session),
|
||||
current_user: User = Depends(get_api_current_user),
|
||||
):
|
||||
selection_pairs = sorted(
|
||||
{
|
||||
(int(item.scholar_profile_id), int(item.publication_id))
|
||||
for item in payload.selections
|
||||
}
|
||||
)
|
||||
updated_count = await publication_service.mark_selected_as_read_for_user(
|
||||
db_session,
|
||||
user_id=current_user.id,
|
||||
selections=selection_pairs,
|
||||
)
|
||||
logger.info(
|
||||
"api.publications.mark_selected_read",
|
||||
extra={
|
||||
"event": "api.publications.mark_selected_read",
|
||||
"user_id": current_user.id,
|
||||
"requested_count": len(selection_pairs),
|
||||
"updated_count": updated_count,
|
||||
},
|
||||
)
|
||||
return success_payload(
|
||||
request,
|
||||
data={
|
||||
"message": "Marked selected publications as read.",
|
||||
"requested_count": len(selection_pairs),
|
||||
"updated_count": updated_count,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/{publication_id}/retry-pdf",
|
||||
response_model=RetryPublicationPdfEnvelope,
|
||||
)
|
||||
async def retry_publication_pdf(
|
||||
payload: RetryPublicationPdfRequest,
|
||||
request: Request,
|
||||
publication_id: int = Path(ge=1),
|
||||
db_session: AsyncSession = Depends(get_db_session),
|
||||
current_user: User = Depends(get_api_current_user),
|
||||
):
|
||||
current, queued = await _retry_publication_state(
|
||||
db_session,
|
||||
current_user=current_user,
|
||||
scholar_profile_id=payload.scholar_profile_id,
|
||||
publication_id=publication_id,
|
||||
)
|
||||
resolved_pdf = bool(current.pdf_url)
|
||||
message = _retry_pdf_message(
|
||||
queued=queued,
|
||||
resolved_pdf=resolved_pdf,
|
||||
pdf_status=current.pdf_status,
|
||||
)
|
||||
_log_retry_pdf_result(
|
||||
current_user=current_user,
|
||||
scholar_profile_id=payload.scholar_profile_id,
|
||||
publication_id=publication_id,
|
||||
queued=queued,
|
||||
resolved_pdf=resolved_pdf,
|
||||
pdf_status=current.pdf_status,
|
||||
doi=None,
|
||||
)
|
||||
return success_payload(
|
||||
request,
|
||||
data={
|
||||
"message": message,
|
||||
"queued": queued,
|
||||
"resolved_pdf": resolved_pdf,
|
||||
"publication": _serialize_publication_item(current),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/{publication_id}/favorite",
|
||||
response_model=TogglePublicationFavoriteEnvelope,
|
||||
)
|
||||
async def toggle_publication_favorite(
|
||||
payload: TogglePublicationFavoriteRequest,
|
||||
request: Request,
|
||||
publication_id: int = Path(ge=1),
|
||||
db_session: AsyncSession = Depends(get_db_session),
|
||||
current_user: User = Depends(get_api_current_user),
|
||||
):
|
||||
current = await _favorite_publication_state(
|
||||
db_session,
|
||||
current_user=current_user,
|
||||
scholar_profile_id=payload.scholar_profile_id,
|
||||
publication_id=publication_id,
|
||||
is_favorite=payload.is_favorite,
|
||||
)
|
||||
logger.info(
|
||||
"api.publications.favorite",
|
||||
extra={
|
||||
"event": "api.publications.favorite",
|
||||
"user_id": current_user.id,
|
||||
"scholar_profile_id": payload.scholar_profile_id,
|
||||
"publication_id": publication_id,
|
||||
"is_favorite": bool(payload.is_favorite),
|
||||
},
|
||||
)
|
||||
return success_payload(
|
||||
request,
|
||||
data={
|
||||
"message": _favorite_message(is_favorite=bool(payload.is_favorite)),
|
||||
"publication": _serialize_publication_item(current),
|
||||
},
|
||||
)
|
||||
|
|
@ -1,845 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, Depends, Query, Request
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.api.deps import get_api_current_user
|
||||
from app.api.errors import ApiException
|
||||
from app.api.responses import success_payload
|
||||
from fastapi.responses import StreamingResponse
|
||||
from app.api.schemas import (
|
||||
ManualRunEnvelope,
|
||||
QueueClearEnvelope,
|
||||
QueueItemEnvelope,
|
||||
QueueListEnvelope,
|
||||
RunDetailEnvelope,
|
||||
RunsListEnvelope,
|
||||
)
|
||||
from app.db.models import RunStatus, RunTriggerType, User
|
||||
from app.db.session import get_db_session
|
||||
from app.services.domains.ingestion import application as ingestion_service
|
||||
from app.services.domains.ingestion import safety as run_safety_service
|
||||
from app.services.domains.runs import application as run_service
|
||||
from app.services.domains.runs.events import event_generator
|
||||
from app.services.domains.settings import application as user_settings_service
|
||||
from app.settings import settings
|
||||
from app.api.runtime_deps import get_ingestion_service
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/runs", tags=["api-runs"])
|
||||
|
||||
IDEMPOTENCY_HEADER = "Idempotency-Key"
|
||||
IDEMPOTENCY_MAX_LENGTH = 128
|
||||
IDEMPOTENCY_PATTERN = re.compile(r"^[A-Za-z0-9._:-]{8,128}$")
|
||||
|
||||
|
||||
def _int_value(value: Any, default: int = 0) -> int:
|
||||
try:
|
||||
return int(value)
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
|
||||
|
||||
def _str_value(value: Any) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
text = str(value).strip()
|
||||
return text if text else None
|
||||
|
||||
|
||||
def _bool_value(value: Any, default: bool = False) -> bool:
|
||||
if isinstance(value, bool):
|
||||
return value
|
||||
if isinstance(value, str):
|
||||
lowered = value.strip().lower()
|
||||
if lowered in {"1", "true", "yes", "on"}:
|
||||
return True
|
||||
if lowered in {"0", "false", "no", "off"}:
|
||||
return False
|
||||
return default
|
||||
|
||||
|
||||
def _normalize_idempotency_key(raw_value: str | None) -> str | None:
|
||||
if raw_value is None:
|
||||
return None
|
||||
candidate = raw_value.strip()
|
||||
if not candidate:
|
||||
return None
|
||||
if len(candidate) > IDEMPOTENCY_MAX_LENGTH or not IDEMPOTENCY_PATTERN.match(candidate):
|
||||
raise ApiException(
|
||||
status_code=400,
|
||||
code="invalid_idempotency_key",
|
||||
message=(
|
||||
"Invalid Idempotency-Key. Use 8-128 characters from: "
|
||||
"A-Z a-z 0-9 . _ : -"
|
||||
),
|
||||
)
|
||||
return candidate
|
||||
|
||||
|
||||
def _serialize_run(run) -> dict[str, Any]:
|
||||
summary = run_service.extract_run_summary(run.error_log)
|
||||
return {
|
||||
"id": int(run.id),
|
||||
"trigger_type": run.trigger_type.value,
|
||||
"status": run.status.value,
|
||||
"start_dt": run.start_dt,
|
||||
"end_dt": run.end_dt,
|
||||
"scholar_count": int(run.scholar_count or 0),
|
||||
"new_publication_count": int(run.new_pub_count or 0),
|
||||
"failed_count": int(summary["failed_count"]),
|
||||
"partial_count": int(summary["partial_count"]),
|
||||
}
|
||||
|
||||
|
||||
def _serialize_queue_item(item) -> dict[str, Any]:
|
||||
return {
|
||||
"id": int(item.id),
|
||||
"scholar_profile_id": int(item.scholar_profile_id),
|
||||
"scholar_label": item.scholar_label,
|
||||
"status": item.status,
|
||||
"reason": item.reason,
|
||||
"dropped_reason": item.dropped_reason,
|
||||
"attempt_count": int(item.attempt_count),
|
||||
"resume_cstart": int(item.resume_cstart),
|
||||
"next_attempt_dt": item.next_attempt_dt,
|
||||
"updated_at": item.updated_at,
|
||||
"last_error": item.last_error,
|
||||
"last_run_id": item.last_run_id,
|
||||
}
|
||||
|
||||
|
||||
def _normalize_attempt_log(value: Any) -> list[dict[str, Any]]:
|
||||
if not isinstance(value, list):
|
||||
return []
|
||||
normalized: list[dict[str, Any]] = []
|
||||
for item in value:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
normalized.append(
|
||||
{
|
||||
"attempt": _int_value(item.get("attempt"), 0),
|
||||
"cstart": _int_value(item.get("cstart"), 0),
|
||||
"state": _str_value(item.get("state")),
|
||||
"state_reason": _str_value(item.get("state_reason")),
|
||||
"status_code": (
|
||||
_int_value(item.get("status_code"))
|
||||
if item.get("status_code") is not None
|
||||
else None
|
||||
),
|
||||
"fetch_error": _str_value(item.get("fetch_error")),
|
||||
}
|
||||
)
|
||||
return normalized
|
||||
|
||||
|
||||
def _normalize_page_logs(value: Any) -> list[dict[str, Any]]:
|
||||
if not isinstance(value, list):
|
||||
return []
|
||||
normalized: list[dict[str, Any]] = []
|
||||
for item in value:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
warning_codes = item.get("warning_codes")
|
||||
normalized.append(
|
||||
{
|
||||
"page": _int_value(item.get("page"), 0),
|
||||
"cstart": _int_value(item.get("cstart"), 0),
|
||||
"state": _str_value(item.get("state")) or "unknown",
|
||||
"state_reason": _str_value(item.get("state_reason")),
|
||||
"status_code": (
|
||||
_int_value(item.get("status_code"))
|
||||
if item.get("status_code") is not None
|
||||
else None
|
||||
),
|
||||
"publication_count": _int_value(item.get("publication_count"), 0),
|
||||
"attempt_count": _int_value(item.get("attempt_count"), 0),
|
||||
"has_show_more_button": _bool_value(item.get("has_show_more_button"), False),
|
||||
"articles_range": _str_value(item.get("articles_range")),
|
||||
"warning_codes": [
|
||||
str(code)
|
||||
for code in (warning_codes if isinstance(warning_codes, list) else [])
|
||||
],
|
||||
}
|
||||
)
|
||||
return normalized
|
||||
|
||||
|
||||
def _normalize_debug(value: Any) -> dict[str, Any] | None:
|
||||
if not isinstance(value, dict):
|
||||
return None
|
||||
marker_counts = value.get("marker_counts_nonzero")
|
||||
warning_codes = value.get("warning_codes")
|
||||
return {
|
||||
"status_code": (
|
||||
_int_value(value.get("status_code"))
|
||||
if value.get("status_code") is not None
|
||||
else None
|
||||
),
|
||||
"final_url": _str_value(value.get("final_url")),
|
||||
"fetch_error": _str_value(value.get("fetch_error")),
|
||||
"body_sha256": _str_value(value.get("body_sha256")),
|
||||
"body_length": (
|
||||
_int_value(value.get("body_length"))
|
||||
if value.get("body_length") is not None
|
||||
else None
|
||||
),
|
||||
"has_show_more_button": (
|
||||
_bool_value(value.get("has_show_more_button"), False)
|
||||
if value.get("has_show_more_button") is not None
|
||||
else None
|
||||
),
|
||||
"articles_range": _str_value(value.get("articles_range")),
|
||||
"state_reason": _str_value(value.get("state_reason")),
|
||||
"warning_codes": [
|
||||
str(code)
|
||||
for code in (warning_codes if isinstance(warning_codes, list) else [])
|
||||
],
|
||||
"marker_counts_nonzero": {
|
||||
str(key): _int_value(count, 0)
|
||||
for key, count in (marker_counts.items() if isinstance(marker_counts, dict) else [])
|
||||
},
|
||||
"page_logs": _normalize_page_logs(value.get("page_logs")),
|
||||
"attempt_log": _normalize_attempt_log(value.get("attempt_log")),
|
||||
}
|
||||
|
||||
|
||||
def _normalize_scholar_result(value: Any) -> dict[str, Any]:
|
||||
if not isinstance(value, dict):
|
||||
return {
|
||||
"scholar_profile_id": 0,
|
||||
"scholar_id": "unknown",
|
||||
"state": "unknown",
|
||||
"state_reason": None,
|
||||
"outcome": "failed",
|
||||
"attempt_count": 0,
|
||||
"publication_count": 0,
|
||||
"start_cstart": 0,
|
||||
"continuation_cstart": None,
|
||||
"continuation_enqueued": False,
|
||||
"continuation_cleared": False,
|
||||
"warnings": [],
|
||||
"error": None,
|
||||
"debug": None,
|
||||
}
|
||||
warnings = value.get("warnings")
|
||||
return {
|
||||
"scholar_profile_id": _int_value(value.get("scholar_profile_id"), 0),
|
||||
"scholar_id": _str_value(value.get("scholar_id")) or "unknown",
|
||||
"state": _str_value(value.get("state")) or "unknown",
|
||||
"state_reason": _str_value(value.get("state_reason")),
|
||||
"outcome": _str_value(value.get("outcome")) or "failed",
|
||||
"attempt_count": _int_value(value.get("attempt_count"), 0),
|
||||
"publication_count": _int_value(value.get("publication_count"), 0),
|
||||
"start_cstart": _int_value(value.get("start_cstart"), 0),
|
||||
"continuation_cstart": (
|
||||
_int_value(value.get("continuation_cstart"))
|
||||
if value.get("continuation_cstart") is not None
|
||||
else None
|
||||
),
|
||||
"continuation_enqueued": _bool_value(value.get("continuation_enqueued"), False),
|
||||
"continuation_cleared": _bool_value(value.get("continuation_cleared"), False),
|
||||
"warnings": [str(item) for item in (warnings if isinstance(warnings, list) else [])],
|
||||
"error": _str_value(value.get("error")),
|
||||
"debug": _normalize_debug(value.get("debug")),
|
||||
}
|
||||
|
||||
|
||||
def _manual_run_payload_from_run(
|
||||
run,
|
||||
*,
|
||||
idempotency_key: str | None,
|
||||
reused_existing_run: bool,
|
||||
safety_state: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
summary = run_service.extract_run_summary(run.error_log)
|
||||
return {
|
||||
"run_id": int(run.id),
|
||||
"status": run.status.value,
|
||||
"scholar_count": int(run.scholar_count or 0),
|
||||
"succeeded_count": int(summary["succeeded_count"]),
|
||||
"failed_count": int(summary["failed_count"]),
|
||||
"partial_count": int(summary["partial_count"]),
|
||||
"new_publication_count": int(run.new_pub_count or 0),
|
||||
"reused_existing_run": reused_existing_run,
|
||||
"idempotency_key": idempotency_key,
|
||||
"safety_state": safety_state,
|
||||
}
|
||||
|
||||
|
||||
async def _load_safety_state(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
) -> dict[str, Any]:
|
||||
user_settings = await user_settings_service.get_or_create_settings(
|
||||
db_session,
|
||||
user_id=user_id,
|
||||
)
|
||||
previous_safety_state = run_safety_service.get_safety_event_context(user_settings)
|
||||
if run_safety_service.clear_expired_cooldown(user_settings):
|
||||
await db_session.commit()
|
||||
await db_session.refresh(user_settings)
|
||||
logger.info(
|
||||
"api.runs.safety_cooldown_cleared",
|
||||
extra={
|
||||
"event": "api.runs.safety_cooldown_cleared",
|
||||
"user_id": user_id,
|
||||
"reason": previous_safety_state.get("cooldown_reason"),
|
||||
"cooldown_until": previous_safety_state.get("cooldown_until"),
|
||||
"metric_name": "api_runs_safety_cooldown_cleared_total",
|
||||
"metric_value": 1,
|
||||
},
|
||||
)
|
||||
return run_safety_service.get_safety_state_payload(user_settings)
|
||||
|
||||
|
||||
def _raise_manual_runs_disabled(*, user_id: int, safety_state: dict[str, Any]) -> None:
|
||||
logger.warning(
|
||||
"api.runs.manual_blocked_policy",
|
||||
extra={
|
||||
"event": "api.runs.manual_blocked_policy",
|
||||
"user_id": user_id,
|
||||
"policy": {"manual_run_allowed": False},
|
||||
"safety_state": safety_state,
|
||||
"metric_name": "api_runs_manual_blocked_policy_total",
|
||||
"metric_value": 1,
|
||||
},
|
||||
)
|
||||
raise ApiException(
|
||||
status_code=403,
|
||||
code="manual_runs_disabled",
|
||||
message="Manual checks are disabled by server policy.",
|
||||
details={
|
||||
"policy": {"manual_run_allowed": False},
|
||||
"safety_state": safety_state,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
async def _reused_manual_run_payload(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
request: Request,
|
||||
user_id: int,
|
||||
idempotency_key: str | None,
|
||||
safety_state: dict[str, Any],
|
||||
) -> dict[str, Any] | None:
|
||||
if idempotency_key is None:
|
||||
return None
|
||||
previous_run = await run_service.get_manual_run_by_idempotency_key(
|
||||
db_session,
|
||||
user_id=user_id,
|
||||
idempotency_key=idempotency_key,
|
||||
)
|
||||
if previous_run is None:
|
||||
return None
|
||||
if previous_run.status == RunStatus.RUNNING:
|
||||
raise ApiException(
|
||||
status_code=409,
|
||||
code="run_in_progress",
|
||||
message="A run with this idempotency key is still in progress.",
|
||||
details={"run_id": int(previous_run.id), "idempotency_key": idempotency_key},
|
||||
)
|
||||
return success_payload(
|
||||
request,
|
||||
data=_manual_run_payload_from_run(
|
||||
previous_run,
|
||||
idempotency_key=idempotency_key,
|
||||
reused_existing_run=True,
|
||||
safety_state=safety_state,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
async def _run_ingestion_for_manual(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
ingest_service: ingestion_service.ScholarIngestionService,
|
||||
user_id: int,
|
||||
idempotency_key: str | None,
|
||||
):
|
||||
user_settings = await user_settings_service.get_or_create_settings(
|
||||
db_session,
|
||||
user_id=user_id,
|
||||
)
|
||||
return await ingest_service.run_for_user(
|
||||
db_session,
|
||||
user_id=user_id,
|
||||
trigger_type=RunTriggerType.MANUAL,
|
||||
request_delay_seconds=user_settings.request_delay_seconds,
|
||||
network_error_retries=settings.ingestion_network_error_retries,
|
||||
retry_backoff_seconds=settings.ingestion_retry_backoff_seconds,
|
||||
max_pages_per_scholar=settings.ingestion_max_pages_per_scholar,
|
||||
page_size=settings.ingestion_page_size,
|
||||
auto_queue_continuations=settings.ingestion_continuation_queue_enabled,
|
||||
queue_delay_seconds=settings.ingestion_continuation_base_delay_seconds,
|
||||
idempotency_key=idempotency_key,
|
||||
alert_blocked_failure_threshold=settings.ingestion_alert_blocked_failure_threshold,
|
||||
alert_network_failure_threshold=settings.ingestion_alert_network_failure_threshold,
|
||||
alert_retry_scheduled_threshold=settings.ingestion_alert_retry_scheduled_threshold,
|
||||
)
|
||||
|
||||
|
||||
async def _recover_integrity_error(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
request: Request,
|
||||
user_id: int,
|
||||
idempotency_key: str | None,
|
||||
original_exc: IntegrityError,
|
||||
) -> dict[str, Any]:
|
||||
if idempotency_key is None:
|
||||
logger.exception(
|
||||
"api.runs.manual_integrity_error",
|
||||
extra={"event": "api.runs.manual_integrity_error", "user_id": user_id},
|
||||
)
|
||||
raise ApiException(status_code=500, code="manual_run_failed", message="Manual run failed.") from original_exc
|
||||
existing_run = await run_service.get_manual_run_by_idempotency_key(
|
||||
db_session,
|
||||
user_id=user_id,
|
||||
idempotency_key=idempotency_key,
|
||||
)
|
||||
if existing_run is None:
|
||||
logger.exception(
|
||||
"api.runs.manual_integrity_error",
|
||||
extra={"event": "api.runs.manual_integrity_error", "user_id": user_id},
|
||||
)
|
||||
raise ApiException(status_code=500, code="manual_run_failed", message="Manual run failed.") from original_exc
|
||||
if existing_run.status == RunStatus.RUNNING:
|
||||
raise ApiException(
|
||||
status_code=409,
|
||||
code="run_in_progress",
|
||||
message="A run with this idempotency key is still in progress.",
|
||||
details={"run_id": int(existing_run.id), "idempotency_key": idempotency_key},
|
||||
) from original_exc
|
||||
return success_payload(
|
||||
request,
|
||||
data=_manual_run_payload_from_run(
|
||||
existing_run,
|
||||
idempotency_key=idempotency_key,
|
||||
reused_existing_run=True,
|
||||
safety_state=await _load_safety_state(db_session, user_id=user_id),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
async def _execute_manual_run(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
request: Request,
|
||||
ingest_service: ingestion_service.ScholarIngestionService,
|
||||
user_id: int,
|
||||
idempotency_key: str | None,
|
||||
):
|
||||
try:
|
||||
return await _run_ingestion_for_manual(
|
||||
db_session,
|
||||
ingest_service=ingest_service,
|
||||
user_id=user_id,
|
||||
idempotency_key=idempotency_key,
|
||||
)
|
||||
except ingestion_service.RunAlreadyInProgressError as exc:
|
||||
await db_session.rollback()
|
||||
raise ApiException(
|
||||
status_code=409,
|
||||
code="run_in_progress",
|
||||
message="A run is already in progress for this account.",
|
||||
) from exc
|
||||
except ingestion_service.RunBlockedBySafetyPolicyError as exc:
|
||||
await db_session.rollback()
|
||||
_raise_manual_blocked_safety(exc=exc, user_id=user_id)
|
||||
except IntegrityError as exc:
|
||||
await db_session.rollback()
|
||||
return await _recover_integrity_error(
|
||||
db_session,
|
||||
request=request,
|
||||
user_id=user_id,
|
||||
idempotency_key=idempotency_key,
|
||||
original_exc=exc,
|
||||
)
|
||||
except Exception as exc:
|
||||
await db_session.rollback()
|
||||
_raise_manual_failed(exc=exc, user_id=user_id)
|
||||
|
||||
|
||||
def _raise_manual_blocked_safety(*, exc, user_id: int) -> None:
|
||||
logger.info(
|
||||
"api.runs.manual_blocked_safety",
|
||||
extra={
|
||||
"event": "api.runs.manual_blocked_safety",
|
||||
"user_id": user_id,
|
||||
"reason": exc.safety_state.get("cooldown_reason"),
|
||||
"cooldown_until": exc.safety_state.get("cooldown_until"),
|
||||
"cooldown_remaining_seconds": exc.safety_state.get("cooldown_remaining_seconds"),
|
||||
"metric_name": "api_runs_manual_blocked_safety_total",
|
||||
"metric_value": 1,
|
||||
},
|
||||
)
|
||||
raise ApiException(
|
||||
status_code=429,
|
||||
code=exc.code,
|
||||
message=exc.message,
|
||||
details={"safety_state": exc.safety_state},
|
||||
) from exc
|
||||
|
||||
|
||||
def _raise_manual_failed(*, exc: Exception, user_id: int) -> None:
|
||||
logger.exception(
|
||||
"api.runs.manual_failed",
|
||||
extra={"event": "api.runs.manual_failed", "user_id": user_id},
|
||||
)
|
||||
raise ApiException(status_code=500, code="manual_run_failed", message="Manual run failed.") from exc
|
||||
|
||||
|
||||
def _manual_run_success_payload(
|
||||
*,
|
||||
run_summary,
|
||||
idempotency_key: str | None,
|
||||
safety_state: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
return {
|
||||
"run_id": run_summary.crawl_run_id,
|
||||
"status": run_summary.status.value,
|
||||
"scholar_count": run_summary.scholar_count,
|
||||
"succeeded_count": run_summary.succeeded_count,
|
||||
"failed_count": run_summary.failed_count,
|
||||
"partial_count": run_summary.partial_count,
|
||||
"new_publication_count": run_summary.new_publication_count,
|
||||
"reused_existing_run": False,
|
||||
"idempotency_key": idempotency_key,
|
||||
"safety_state": safety_state,
|
||||
}
|
||||
|
||||
|
||||
@router.get(
|
||||
"",
|
||||
response_model=RunsListEnvelope,
|
||||
)
|
||||
async def list_runs(
|
||||
request: Request,
|
||||
failed_only: bool = Query(default=False),
|
||||
limit: int = Query(default=100, ge=1, le=500),
|
||||
db_session: AsyncSession = Depends(get_db_session),
|
||||
current_user: User = Depends(get_api_current_user),
|
||||
):
|
||||
runs = await run_service.list_runs_for_user(
|
||||
db_session,
|
||||
user_id=current_user.id,
|
||||
limit=limit,
|
||||
failed_only=failed_only,
|
||||
)
|
||||
safety_state = await _load_safety_state(
|
||||
db_session,
|
||||
user_id=current_user.id,
|
||||
)
|
||||
return success_payload(
|
||||
request,
|
||||
data={
|
||||
"runs": [_serialize_run(run) for run in runs],
|
||||
"safety_state": safety_state,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/{run_id}",
|
||||
response_model=RunDetailEnvelope,
|
||||
)
|
||||
async def get_run(
|
||||
run_id: int,
|
||||
request: Request,
|
||||
db_session: AsyncSession = Depends(get_db_session),
|
||||
current_user: User = Depends(get_api_current_user),
|
||||
):
|
||||
run = await run_service.get_run_for_user(
|
||||
db_session,
|
||||
user_id=current_user.id,
|
||||
run_id=run_id,
|
||||
)
|
||||
if run is None:
|
||||
raise ApiException(
|
||||
status_code=404,
|
||||
code="run_not_found",
|
||||
message="Run not found.",
|
||||
)
|
||||
error_log = run.error_log if isinstance(run.error_log, dict) else {}
|
||||
scholar_results = error_log.get("scholar_results")
|
||||
if not isinstance(scholar_results, list):
|
||||
scholar_results = []
|
||||
safety_state = await _load_safety_state(
|
||||
db_session,
|
||||
user_id=current_user.id,
|
||||
)
|
||||
return success_payload(
|
||||
request,
|
||||
data={
|
||||
"run": _serialize_run(run),
|
||||
"summary": run_service.extract_run_summary(error_log),
|
||||
"scholar_results": [_normalize_scholar_result(item) for item in scholar_results],
|
||||
"safety_state": safety_state,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/{run_id}/cancel",
|
||||
response_model=RunDetailEnvelope,
|
||||
)
|
||||
async def cancel_run(
|
||||
run_id: int,
|
||||
request: Request,
|
||||
db_session: AsyncSession = Depends(get_db_session),
|
||||
current_user: User = Depends(get_api_current_user),
|
||||
):
|
||||
run = await run_service.get_run_for_user(
|
||||
db_session,
|
||||
user_id=current_user.id,
|
||||
run_id=run_id,
|
||||
)
|
||||
if run is None:
|
||||
raise ApiException(
|
||||
status_code=404,
|
||||
code="run_not_found",
|
||||
message="Run not found.",
|
||||
)
|
||||
|
||||
if run.status == RunStatus.RUNNING:
|
||||
run.status = RunStatus.CANCELED
|
||||
await db_session.commit()
|
||||
await db_session.refresh(run)
|
||||
|
||||
error_log = run.error_log if isinstance(run.error_log, dict) else {}
|
||||
scholar_results = error_log.get("scholar_results")
|
||||
if not isinstance(scholar_results, list):
|
||||
scholar_results = []
|
||||
|
||||
safety_state = await _load_safety_state(
|
||||
db_session,
|
||||
user_id=current_user.id,
|
||||
)
|
||||
|
||||
return success_payload(
|
||||
request,
|
||||
data={
|
||||
"run": _serialize_run(run),
|
||||
"summary": run_service.extract_run_summary(error_log),
|
||||
"scholar_results": [_normalize_scholar_result(item) for item in scholar_results],
|
||||
"safety_state": safety_state,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/manual",
|
||||
response_model=ManualRunEnvelope,
|
||||
)
|
||||
async def run_manual(
|
||||
request: Request,
|
||||
db_session: AsyncSession = Depends(get_db_session),
|
||||
current_user: User = Depends(get_api_current_user),
|
||||
ingest_service: ingestion_service.ScholarIngestionService = Depends(get_ingestion_service),
|
||||
):
|
||||
safety_state = await _load_safety_state(
|
||||
db_session,
|
||||
user_id=current_user.id,
|
||||
)
|
||||
if not settings.ingestion_manual_run_allowed:
|
||||
_raise_manual_runs_disabled(user_id=current_user.id, safety_state=safety_state)
|
||||
|
||||
idempotency_key = _normalize_idempotency_key(request.headers.get(IDEMPOTENCY_HEADER))
|
||||
reused_payload = await _reused_manual_run_payload(
|
||||
db_session,
|
||||
request=request,
|
||||
user_id=current_user.id,
|
||||
idempotency_key=idempotency_key,
|
||||
safety_state=safety_state,
|
||||
)
|
||||
if reused_payload is not None:
|
||||
return reused_payload
|
||||
|
||||
run_summary = await _execute_manual_run(
|
||||
db_session,
|
||||
request=request,
|
||||
ingest_service=ingest_service,
|
||||
user_id=current_user.id,
|
||||
idempotency_key=idempotency_key,
|
||||
)
|
||||
if isinstance(run_summary, dict):
|
||||
return run_summary
|
||||
|
||||
current_safety_state = await _load_safety_state(
|
||||
db_session,
|
||||
user_id=current_user.id,
|
||||
)
|
||||
return success_payload(
|
||||
request,
|
||||
data=_manual_run_success_payload(
|
||||
run_summary=run_summary,
|
||||
idempotency_key=idempotency_key,
|
||||
safety_state=current_safety_state,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
|
||||
@router.get(
|
||||
"/queue/items",
|
||||
response_model=QueueListEnvelope,
|
||||
)
|
||||
async def list_queue_items(
|
||||
request: Request,
|
||||
limit: int = Query(default=200, ge=1, le=500),
|
||||
db_session: AsyncSession = Depends(get_db_session),
|
||||
current_user: User = Depends(get_api_current_user),
|
||||
):
|
||||
items = await run_service.list_queue_items_for_user(
|
||||
db_session,
|
||||
user_id=current_user.id,
|
||||
limit=limit,
|
||||
)
|
||||
return success_payload(
|
||||
request,
|
||||
data={
|
||||
"queue_items": [_serialize_queue_item(item) for item in items],
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/queue/{queue_item_id}/retry",
|
||||
response_model=QueueItemEnvelope,
|
||||
)
|
||||
async def retry_queue_item(
|
||||
queue_item_id: int,
|
||||
request: Request,
|
||||
db_session: AsyncSession = Depends(get_db_session),
|
||||
current_user: User = Depends(get_api_current_user),
|
||||
):
|
||||
try:
|
||||
queue_item = await run_service.retry_queue_item_for_user(
|
||||
db_session,
|
||||
user_id=current_user.id,
|
||||
queue_item_id=queue_item_id,
|
||||
)
|
||||
except run_service.QueueTransitionError as exc:
|
||||
raise ApiException(
|
||||
status_code=409,
|
||||
code=exc.code,
|
||||
message=exc.message,
|
||||
details={"current_status": exc.current_status},
|
||||
) from exc
|
||||
if queue_item is None:
|
||||
raise ApiException(
|
||||
status_code=404,
|
||||
code="queue_item_not_found",
|
||||
message="Queue item not found.",
|
||||
)
|
||||
return success_payload(
|
||||
request,
|
||||
data=_serialize_queue_item(queue_item),
|
||||
)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/queue/{queue_item_id}/drop",
|
||||
response_model=QueueItemEnvelope,
|
||||
)
|
||||
async def drop_queue_item(
|
||||
queue_item_id: int,
|
||||
request: Request,
|
||||
db_session: AsyncSession = Depends(get_db_session),
|
||||
current_user: User = Depends(get_api_current_user),
|
||||
):
|
||||
try:
|
||||
dropped = await run_service.drop_queue_item_for_user(
|
||||
db_session,
|
||||
user_id=current_user.id,
|
||||
queue_item_id=queue_item_id,
|
||||
)
|
||||
except run_service.QueueTransitionError as exc:
|
||||
raise ApiException(
|
||||
status_code=409,
|
||||
code=exc.code,
|
||||
message=exc.message,
|
||||
details={"current_status": exc.current_status},
|
||||
) from exc
|
||||
if dropped is None:
|
||||
raise ApiException(
|
||||
status_code=404,
|
||||
code="queue_item_not_found",
|
||||
message="Queue item not found.",
|
||||
)
|
||||
return success_payload(
|
||||
request,
|
||||
data=_serialize_queue_item(dropped),
|
||||
)
|
||||
|
||||
|
||||
@router.delete(
|
||||
"/queue/{queue_item_id}",
|
||||
response_model=QueueClearEnvelope,
|
||||
)
|
||||
async def clear_queue_item(
|
||||
queue_item_id: int,
|
||||
request: Request,
|
||||
db_session: AsyncSession = Depends(get_db_session),
|
||||
current_user: User = Depends(get_api_current_user),
|
||||
):
|
||||
try:
|
||||
deleted = await run_service.clear_queue_item_for_user(
|
||||
db_session,
|
||||
user_id=current_user.id,
|
||||
queue_item_id=queue_item_id,
|
||||
)
|
||||
except run_service.QueueTransitionError as exc:
|
||||
raise ApiException(
|
||||
status_code=409,
|
||||
code=exc.code,
|
||||
message=exc.message,
|
||||
details={"current_status": exc.current_status},
|
||||
) from exc
|
||||
if deleted is None:
|
||||
raise ApiException(
|
||||
status_code=404,
|
||||
code="queue_item_not_found",
|
||||
message="Queue item not found.",
|
||||
)
|
||||
return success_payload(
|
||||
request,
|
||||
data={
|
||||
"queue_item_id": deleted.queue_item_id,
|
||||
"previous_status": deleted.previous_status,
|
||||
"status": "cleared",
|
||||
"message": "Queue item cleared.",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@router.get("/{run_id}/stream")
|
||||
async def stream_run_events(
|
||||
run_id: int,
|
||||
db_session: AsyncSession = Depends(get_db_session),
|
||||
current_user: User = Depends(get_api_current_user),
|
||||
):
|
||||
run = await run_service.get_run_for_user(
|
||||
db_session,
|
||||
user_id=current_user.id,
|
||||
run_id=run_id,
|
||||
)
|
||||
if run is None:
|
||||
raise ApiException(
|
||||
status_code=404,
|
||||
code="run_not_found",
|
||||
message="Run not found.",
|
||||
)
|
||||
return StreamingResponse(
|
||||
event_generator(run_id),
|
||||
media_type="text/event-stream"
|
||||
)
|
||||
|
|
@ -1,557 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
from fastapi import APIRouter, Depends, File, Query, Request, UploadFile
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.api.deps import get_api_current_user
|
||||
from app.api.errors import ApiException
|
||||
from app.api.responses import success_payload
|
||||
from app.api.runtime_deps import get_scholar_source
|
||||
from app.api.schemas import (
|
||||
DataExportEnvelope,
|
||||
DataImportEnvelope,
|
||||
DataImportRequest,
|
||||
MessageEnvelope,
|
||||
ScholarCreateRequest,
|
||||
ScholarEnvelope,
|
||||
ScholarImageUrlUpdateRequest,
|
||||
ScholarSearchEnvelope,
|
||||
ScholarsListEnvelope,
|
||||
)
|
||||
from app.db.models import User
|
||||
from app.db.session import get_db_session
|
||||
from app.services.domains.portability import application as import_export_service
|
||||
from app.services.domains.scholars import application as scholar_service
|
||||
from app.services.domains.scholar.source import ScholarSource
|
||||
from app.settings import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/scholars", tags=["api-scholars"])
|
||||
|
||||
|
||||
def _uploaded_image_media_path(scholar_profile_id: int) -> str:
|
||||
return f"/scholar-images/{scholar_profile_id}/upload"
|
||||
|
||||
|
||||
def _serialize_scholar(profile) -> dict[str, object]:
|
||||
uploaded_image_url = None
|
||||
if profile.profile_image_upload_path:
|
||||
uploaded_image_url = _uploaded_image_media_path(int(profile.id))
|
||||
|
||||
profile_image_url, profile_image_source = scholar_service.resolve_profile_image(
|
||||
profile,
|
||||
uploaded_image_url=uploaded_image_url,
|
||||
)
|
||||
|
||||
return {
|
||||
"id": int(profile.id),
|
||||
"scholar_id": profile.scholar_id,
|
||||
"display_name": profile.display_name,
|
||||
"profile_image_url": profile_image_url,
|
||||
"profile_image_source": profile_image_source,
|
||||
"is_enabled": bool(profile.is_enabled),
|
||||
"baseline_completed": bool(profile.baseline_completed),
|
||||
"last_run_dt": profile.last_run_dt,
|
||||
"last_run_status": (
|
||||
profile.last_run_status.value if profile.last_run_status is not None else None
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
async def _hydrate_scholar_metadata_if_needed(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
profile,
|
||||
source: ScholarSource,
|
||||
user_id: int,
|
||||
):
|
||||
try:
|
||||
if not profile.profile_image_url or not (profile.display_name or "").strip():
|
||||
return await asyncio.wait_for(
|
||||
scholar_service.hydrate_profile_metadata(
|
||||
db_session,
|
||||
profile=profile,
|
||||
source=source,
|
||||
),
|
||||
timeout=5.0,
|
||||
)
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"api.scholars.create_metadata_hydration_failed",
|
||||
extra={
|
||||
"event": "api.scholars.create_metadata_hydration_failed",
|
||||
"user_id": user_id,
|
||||
"scholar_profile_id": profile.id,
|
||||
},
|
||||
)
|
||||
return profile
|
||||
|
||||
|
||||
def _search_kwargs() -> dict[str, object]:
|
||||
return {
|
||||
"network_error_retries": settings.ingestion_network_error_retries,
|
||||
"retry_backoff_seconds": settings.ingestion_retry_backoff_seconds,
|
||||
"search_enabled": settings.scholar_name_search_enabled,
|
||||
"cache_ttl_seconds": settings.scholar_name_search_cache_ttl_seconds,
|
||||
"blocked_cache_ttl_seconds": settings.scholar_name_search_blocked_cache_ttl_seconds,
|
||||
"cache_max_entries": settings.scholar_name_search_cache_max_entries,
|
||||
"min_interval_seconds": settings.scholar_name_search_min_interval_seconds,
|
||||
"interval_jitter_seconds": settings.scholar_name_search_interval_jitter_seconds,
|
||||
"cooldown_block_threshold": settings.scholar_name_search_cooldown_block_threshold,
|
||||
"cooldown_seconds": settings.scholar_name_search_cooldown_seconds,
|
||||
"retry_alert_threshold": settings.scholar_name_search_alert_retry_count_threshold,
|
||||
"cooldown_rejection_alert_threshold": (
|
||||
settings.scholar_name_search_alert_cooldown_rejections_threshold
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def _search_response_data(query: str, parsed) -> dict[str, object]:
|
||||
return {
|
||||
"query": query.strip(),
|
||||
"state": parsed.state.value,
|
||||
"state_reason": parsed.state_reason,
|
||||
"action_hint": scholar_service.scrape_state_hint(
|
||||
state=parsed.state,
|
||||
state_reason=parsed.state_reason,
|
||||
),
|
||||
"candidates": [
|
||||
{
|
||||
"scholar_id": item.scholar_id,
|
||||
"display_name": item.display_name,
|
||||
"affiliation": item.affiliation,
|
||||
"email_domain": item.email_domain,
|
||||
"cited_by_count": item.cited_by_count,
|
||||
"interests": item.interests,
|
||||
"profile_url": item.profile_url,
|
||||
"profile_image_url": item.profile_image_url,
|
||||
}
|
||||
for item in parsed.candidates
|
||||
],
|
||||
"warnings": parsed.warnings,
|
||||
}
|
||||
|
||||
|
||||
async def _read_uploaded_image(image: UploadFile) -> bytes:
|
||||
try:
|
||||
return await image.read()
|
||||
finally:
|
||||
await image.close()
|
||||
|
||||
|
||||
async def _require_user_profile(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
scholar_profile_id: int,
|
||||
):
|
||||
profile = await scholar_service.get_user_scholar_by_id(
|
||||
db_session,
|
||||
user_id=user_id,
|
||||
scholar_profile_id=scholar_profile_id,
|
||||
)
|
||||
if profile is None:
|
||||
raise ApiException(
|
||||
status_code=404,
|
||||
code="scholar_not_found",
|
||||
message="Scholar not found.",
|
||||
)
|
||||
return profile
|
||||
|
||||
|
||||
@router.get(
|
||||
"",
|
||||
response_model=ScholarsListEnvelope,
|
||||
)
|
||||
async def list_scholars(
|
||||
request: Request,
|
||||
db_session: AsyncSession = Depends(get_db_session),
|
||||
current_user: User = Depends(get_api_current_user),
|
||||
):
|
||||
scholars = await scholar_service.list_scholars_for_user(
|
||||
db_session,
|
||||
user_id=current_user.id,
|
||||
)
|
||||
return success_payload(
|
||||
request,
|
||||
data={
|
||||
"scholars": [_serialize_scholar(profile) for profile in scholars],
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/export",
|
||||
response_model=DataExportEnvelope,
|
||||
)
|
||||
async def export_scholars_and_publications(
|
||||
request: Request,
|
||||
db_session: AsyncSession = Depends(get_db_session),
|
||||
current_user: User = Depends(get_api_current_user),
|
||||
):
|
||||
data = await import_export_service.export_user_data(
|
||||
db_session,
|
||||
user_id=current_user.id,
|
||||
)
|
||||
return success_payload(request, data=data)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/import",
|
||||
response_model=DataImportEnvelope,
|
||||
)
|
||||
async def import_scholars_and_publications(
|
||||
payload: DataImportRequest,
|
||||
request: Request,
|
||||
db_session: AsyncSession = Depends(get_db_session),
|
||||
current_user: User = Depends(get_api_current_user),
|
||||
):
|
||||
if (
|
||||
payload.schema_version is not None
|
||||
and int(payload.schema_version) != import_export_service.EXPORT_SCHEMA_VERSION
|
||||
):
|
||||
raise ApiException(
|
||||
status_code=400,
|
||||
code="invalid_import_schema_version",
|
||||
message=(
|
||||
"Import schema version is not supported. "
|
||||
f"Expected {import_export_service.EXPORT_SCHEMA_VERSION}."
|
||||
),
|
||||
)
|
||||
try:
|
||||
result = await import_export_service.import_user_data(
|
||||
db_session,
|
||||
user_id=current_user.id,
|
||||
scholars=[item.model_dump() for item in payload.scholars],
|
||||
publications=[item.model_dump() for item in payload.publications],
|
||||
)
|
||||
except import_export_service.ImportExportError as exc:
|
||||
raise ApiException(
|
||||
status_code=400,
|
||||
code="invalid_import_payload",
|
||||
message=str(exc),
|
||||
) from exc
|
||||
logger.info(
|
||||
"api.scholars.imported",
|
||||
extra={
|
||||
"event": "api.scholars.imported",
|
||||
"user_id": current_user.id,
|
||||
**result,
|
||||
},
|
||||
)
|
||||
return success_payload(request, data=result)
|
||||
|
||||
|
||||
@router.post(
|
||||
"",
|
||||
response_model=ScholarEnvelope,
|
||||
status_code=201,
|
||||
)
|
||||
async def create_scholar(
|
||||
payload: ScholarCreateRequest,
|
||||
request: Request,
|
||||
db_session: AsyncSession = Depends(get_db_session),
|
||||
source: ScholarSource = Depends(get_scholar_source),
|
||||
current_user: User = Depends(get_api_current_user),
|
||||
):
|
||||
try:
|
||||
created = await scholar_service.create_scholar_for_user(
|
||||
db_session,
|
||||
user_id=current_user.id,
|
||||
scholar_id=payload.scholar_id,
|
||||
display_name="",
|
||||
profile_image_url=payload.profile_image_url,
|
||||
)
|
||||
except scholar_service.ScholarServiceError as exc:
|
||||
raise ApiException(
|
||||
status_code=400,
|
||||
code="invalid_scholar",
|
||||
message=str(exc),
|
||||
) from exc
|
||||
logger.info(
|
||||
"api.scholars.created",
|
||||
extra={
|
||||
"event": "api.scholars.created",
|
||||
"user_id": current_user.id,
|
||||
"scholar_profile_id": created.id,
|
||||
},
|
||||
)
|
||||
created = await _hydrate_scholar_metadata_if_needed(
|
||||
db_session,
|
||||
profile=created,
|
||||
source=source,
|
||||
user_id=current_user.id,
|
||||
)
|
||||
|
||||
return success_payload(
|
||||
request,
|
||||
data=_serialize_scholar(created),
|
||||
)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/search",
|
||||
response_model=ScholarSearchEnvelope,
|
||||
)
|
||||
async def search_scholars(
|
||||
request: Request,
|
||||
query: str = Query(..., min_length=2, max_length=120),
|
||||
limit: int = Query(10, ge=1, le=25),
|
||||
db_session: AsyncSession = Depends(get_db_session),
|
||||
source: ScholarSource = Depends(get_scholar_source),
|
||||
current_user: User = Depends(get_api_current_user),
|
||||
):
|
||||
try:
|
||||
parsed = await scholar_service.search_author_candidates(
|
||||
source=source,
|
||||
db_session=db_session,
|
||||
query=query,
|
||||
limit=limit,
|
||||
**_search_kwargs(),
|
||||
)
|
||||
except scholar_service.ScholarServiceError as exc:
|
||||
raise ApiException(
|
||||
status_code=400,
|
||||
code="invalid_scholar_search",
|
||||
message=str(exc),
|
||||
) from exc
|
||||
|
||||
logger.info(
|
||||
"api.scholars.search.completed",
|
||||
extra={
|
||||
"event": "api.scholars.search.completed",
|
||||
"user_id": current_user.id,
|
||||
"query": query.strip(),
|
||||
"candidate_count": len(parsed.candidates),
|
||||
"state": parsed.state.value,
|
||||
},
|
||||
)
|
||||
return success_payload(
|
||||
request,
|
||||
data=_search_response_data(query, parsed),
|
||||
)
|
||||
|
||||
|
||||
@router.patch(
|
||||
"/{scholar_profile_id}/toggle",
|
||||
response_model=ScholarEnvelope,
|
||||
)
|
||||
async def toggle_scholar(
|
||||
scholar_profile_id: int,
|
||||
request: Request,
|
||||
db_session: AsyncSession = Depends(get_db_session),
|
||||
current_user: User = Depends(get_api_current_user),
|
||||
):
|
||||
profile = await scholar_service.get_user_scholar_by_id(
|
||||
db_session,
|
||||
user_id=current_user.id,
|
||||
scholar_profile_id=scholar_profile_id,
|
||||
)
|
||||
if profile is None:
|
||||
raise ApiException(
|
||||
status_code=404,
|
||||
code="scholar_not_found",
|
||||
message="Scholar not found.",
|
||||
)
|
||||
updated = await scholar_service.toggle_scholar_enabled(db_session, profile=profile)
|
||||
logger.info(
|
||||
"api.scholars.toggled",
|
||||
extra={
|
||||
"event": "api.scholars.toggled",
|
||||
"user_id": current_user.id,
|
||||
"scholar_profile_id": updated.id,
|
||||
"is_enabled": updated.is_enabled,
|
||||
},
|
||||
)
|
||||
return success_payload(
|
||||
request,
|
||||
data=_serialize_scholar(updated),
|
||||
)
|
||||
|
||||
|
||||
@router.delete(
|
||||
"/{scholar_profile_id}",
|
||||
response_model=MessageEnvelope,
|
||||
)
|
||||
async def delete_scholar(
|
||||
scholar_profile_id: int,
|
||||
request: Request,
|
||||
db_session: AsyncSession = Depends(get_db_session),
|
||||
current_user: User = Depends(get_api_current_user),
|
||||
):
|
||||
profile = await scholar_service.get_user_scholar_by_id(
|
||||
db_session,
|
||||
user_id=current_user.id,
|
||||
scholar_profile_id=scholar_profile_id,
|
||||
)
|
||||
if profile is None:
|
||||
raise ApiException(
|
||||
status_code=404,
|
||||
code="scholar_not_found",
|
||||
message="Scholar not found.",
|
||||
)
|
||||
await scholar_service.delete_scholar(
|
||||
db_session,
|
||||
profile=profile,
|
||||
upload_dir=settings.scholar_image_upload_dir,
|
||||
)
|
||||
logger.info(
|
||||
"api.scholars.deleted",
|
||||
extra={
|
||||
"event": "api.scholars.deleted",
|
||||
"user_id": current_user.id,
|
||||
"scholar_profile_id": scholar_profile_id,
|
||||
},
|
||||
)
|
||||
return success_payload(
|
||||
request,
|
||||
data={"message": "Scholar deleted."},
|
||||
)
|
||||
|
||||
|
||||
@router.put(
|
||||
"/{scholar_profile_id}/image/url",
|
||||
response_model=ScholarEnvelope,
|
||||
)
|
||||
async def update_scholar_image_url(
|
||||
scholar_profile_id: int,
|
||||
payload: ScholarImageUrlUpdateRequest,
|
||||
request: Request,
|
||||
db_session: AsyncSession = Depends(get_db_session),
|
||||
current_user: User = Depends(get_api_current_user),
|
||||
):
|
||||
profile = await scholar_service.get_user_scholar_by_id(
|
||||
db_session,
|
||||
user_id=current_user.id,
|
||||
scholar_profile_id=scholar_profile_id,
|
||||
)
|
||||
if profile is None:
|
||||
raise ApiException(
|
||||
status_code=404,
|
||||
code="scholar_not_found",
|
||||
message="Scholar not found.",
|
||||
)
|
||||
|
||||
try:
|
||||
updated = await scholar_service.set_profile_image_override_url(
|
||||
db_session,
|
||||
profile=profile,
|
||||
image_url=payload.image_url,
|
||||
upload_dir=settings.scholar_image_upload_dir,
|
||||
)
|
||||
except scholar_service.ScholarServiceError as exc:
|
||||
raise ApiException(
|
||||
status_code=400,
|
||||
code="invalid_scholar_image",
|
||||
message=str(exc),
|
||||
) from exc
|
||||
|
||||
logger.info(
|
||||
"api.scholars.image_url_updated",
|
||||
extra={
|
||||
"event": "api.scholars.image_url_updated",
|
||||
"user_id": current_user.id,
|
||||
"scholar_profile_id": updated.id,
|
||||
},
|
||||
)
|
||||
return success_payload(
|
||||
request,
|
||||
data=_serialize_scholar(updated),
|
||||
)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/{scholar_profile_id}/image/upload",
|
||||
response_model=ScholarEnvelope,
|
||||
)
|
||||
async def upload_scholar_image(
|
||||
scholar_profile_id: int,
|
||||
request: Request,
|
||||
image: UploadFile = File(...),
|
||||
db_session: AsyncSession = Depends(get_db_session),
|
||||
current_user: User = Depends(get_api_current_user),
|
||||
):
|
||||
profile = await _require_user_profile(
|
||||
db_session,
|
||||
user_id=current_user.id,
|
||||
scholar_profile_id=scholar_profile_id,
|
||||
)
|
||||
|
||||
image_bytes = await _read_uploaded_image(image)
|
||||
try:
|
||||
updated = await scholar_service.set_profile_image_upload(
|
||||
db_session,
|
||||
profile=profile,
|
||||
content_type=image.content_type,
|
||||
image_bytes=image_bytes,
|
||||
upload_dir=settings.scholar_image_upload_dir,
|
||||
max_upload_bytes=settings.scholar_image_upload_max_bytes,
|
||||
)
|
||||
except scholar_service.ScholarServiceError as exc:
|
||||
raise ApiException(
|
||||
status_code=400,
|
||||
code="invalid_scholar_image",
|
||||
message=str(exc),
|
||||
) from exc
|
||||
|
||||
image_size = len(image_bytes)
|
||||
logger.info(
|
||||
"api.scholars.image_uploaded",
|
||||
extra={
|
||||
"event": "api.scholars.image_uploaded",
|
||||
"user_id": current_user.id,
|
||||
"scholar_profile_id": updated.id,
|
||||
"content_type": image.content_type,
|
||||
"size_bytes": image_size,
|
||||
},
|
||||
)
|
||||
return success_payload(
|
||||
request,
|
||||
data=_serialize_scholar(updated),
|
||||
)
|
||||
|
||||
|
||||
@router.delete(
|
||||
"/{scholar_profile_id}/image",
|
||||
response_model=ScholarEnvelope,
|
||||
)
|
||||
async def clear_scholar_image_customization(
|
||||
scholar_profile_id: int,
|
||||
request: Request,
|
||||
db_session: AsyncSession = Depends(get_db_session),
|
||||
current_user: User = Depends(get_api_current_user),
|
||||
):
|
||||
profile = await scholar_service.get_user_scholar_by_id(
|
||||
db_session,
|
||||
user_id=current_user.id,
|
||||
scholar_profile_id=scholar_profile_id,
|
||||
)
|
||||
if profile is None:
|
||||
raise ApiException(
|
||||
status_code=404,
|
||||
code="scholar_not_found",
|
||||
message="Scholar not found.",
|
||||
)
|
||||
|
||||
updated = await scholar_service.clear_profile_image_customization(
|
||||
db_session,
|
||||
profile=profile,
|
||||
upload_dir=settings.scholar_image_upload_dir,
|
||||
)
|
||||
logger.info(
|
||||
"api.scholars.image_customization_cleared",
|
||||
extra={
|
||||
"event": "api.scholars.image_customization_cleared",
|
||||
"user_id": current_user.id,
|
||||
"scholar_profile_id": updated.id,
|
||||
},
|
||||
)
|
||||
return success_payload(
|
||||
request,
|
||||
data=_serialize_scholar(updated),
|
||||
)
|
||||
|
||||
|
|
@ -1,189 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from fastapi import APIRouter, Depends, Request
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.api.deps import get_api_current_user
|
||||
from app.api.errors import ApiException
|
||||
from app.api.responses import success_payload
|
||||
from app.api.schemas import SettingsEnvelope, SettingsUpdateRequest
|
||||
from app.db.models import User
|
||||
from app.db.session import get_db_session
|
||||
from app.services.domains.ingestion import safety as run_safety_service
|
||||
from app.services.domains.settings import application as user_settings_service
|
||||
from app.settings import settings as settings_module
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/settings", tags=["api-settings"])
|
||||
|
||||
|
||||
def _serialize_settings(user_settings) -> dict[str, object]:
|
||||
min_run_interval_minutes, min_request_delay_seconds = _minimum_policy()
|
||||
return {
|
||||
"auto_run_enabled": bool(user_settings.auto_run_enabled) and settings_module.ingestion_automation_allowed,
|
||||
"run_interval_minutes": int(user_settings.run_interval_minutes),
|
||||
"request_delay_seconds": int(user_settings.request_delay_seconds),
|
||||
"nav_visible_pages": list(user_settings.nav_visible_pages or []),
|
||||
"policy": {
|
||||
"min_run_interval_minutes": min_run_interval_minutes,
|
||||
"min_request_delay_seconds": min_request_delay_seconds,
|
||||
"automation_allowed": bool(settings_module.ingestion_automation_allowed),
|
||||
"manual_run_allowed": bool(settings_module.ingestion_manual_run_allowed),
|
||||
"blocked_failure_threshold": max(1, int(settings_module.ingestion_alert_blocked_failure_threshold)),
|
||||
"network_failure_threshold": max(1, int(settings_module.ingestion_alert_network_failure_threshold)),
|
||||
"cooldown_blocked_seconds": max(60, int(settings_module.ingestion_safety_cooldown_blocked_seconds)),
|
||||
"cooldown_network_seconds": max(60, int(settings_module.ingestion_safety_cooldown_network_seconds)),
|
||||
},
|
||||
"safety_state": run_safety_service.get_safety_state_payload(user_settings),
|
||||
"openalex_api_key": user_settings.openalex_api_key,
|
||||
"crossref_api_token": user_settings.crossref_api_token,
|
||||
"crossref_api_mailto": user_settings.crossref_api_mailto,
|
||||
}
|
||||
|
||||
|
||||
def _minimum_policy() -> tuple[int, int]:
|
||||
min_run_interval_minutes = user_settings_service.resolve_run_interval_minimum(
|
||||
settings_module.ingestion_min_run_interval_minutes
|
||||
)
|
||||
min_request_delay_seconds = user_settings_service.resolve_request_delay_minimum(
|
||||
settings_module.ingestion_min_request_delay_seconds
|
||||
)
|
||||
return min_run_interval_minutes, min_request_delay_seconds
|
||||
|
||||
|
||||
def _parse_settings_payload(payload: SettingsUpdateRequest, user_settings) -> tuple[int, int, list[str]]:
|
||||
min_run_interval_minutes, min_request_delay_seconds = _minimum_policy()
|
||||
parsed_interval = user_settings_service.parse_run_interval_minutes(
|
||||
str(payload.run_interval_minutes),
|
||||
minimum=min_run_interval_minutes,
|
||||
)
|
||||
parsed_delay = user_settings_service.parse_request_delay_seconds(
|
||||
str(payload.request_delay_seconds),
|
||||
minimum=min_request_delay_seconds,
|
||||
)
|
||||
parsed_nav_visible_pages = user_settings_service.parse_nav_visible_pages(
|
||||
payload.nav_visible_pages
|
||||
if payload.nav_visible_pages is not None
|
||||
else list(user_settings.nav_visible_pages or user_settings_service.DEFAULT_NAV_VISIBLE_PAGES)
|
||||
)
|
||||
return parsed_interval, parsed_delay, parsed_nav_visible_pages
|
||||
|
||||
|
||||
async def _clear_expired_cooldown_with_log(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
user_settings,
|
||||
) -> None:
|
||||
previous_safety_state = run_safety_service.get_safety_event_context(user_settings)
|
||||
if not run_safety_service.clear_expired_cooldown(user_settings):
|
||||
return
|
||||
await db_session.commit()
|
||||
await db_session.refresh(user_settings)
|
||||
logger.info(
|
||||
"api.settings.safety_cooldown_cleared",
|
||||
extra={
|
||||
"event": "api.settings.safety_cooldown_cleared",
|
||||
"user_id": user_id,
|
||||
"reason": previous_safety_state.get("cooldown_reason"),
|
||||
"cooldown_until": previous_safety_state.get("cooldown_until"),
|
||||
"metric_name": "api_settings_safety_cooldown_cleared_total",
|
||||
"metric_value": 1,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _effective_auto_run_enabled(payload: SettingsUpdateRequest) -> bool:
|
||||
return bool(payload.auto_run_enabled) and settings_module.ingestion_automation_allowed
|
||||
|
||||
|
||||
def _log_settings_update(*, user_id: int, updated) -> None:
|
||||
logger.info(
|
||||
"api.settings.updated",
|
||||
extra={
|
||||
"event": "api.settings.updated",
|
||||
"user_id": user_id,
|
||||
"auto_run_enabled": updated.auto_run_enabled,
|
||||
"run_interval_minutes": updated.run_interval_minutes,
|
||||
"request_delay_seconds": updated.request_delay_seconds,
|
||||
"nav_visible_pages": updated.nav_visible_pages,
|
||||
"openalex_api_key": "SET" if updated.openalex_api_key else "UNSET",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@router.get(
|
||||
"",
|
||||
response_model=SettingsEnvelope,
|
||||
)
|
||||
async def get_settings(
|
||||
request: Request,
|
||||
db_session: AsyncSession = Depends(get_db_session),
|
||||
current_user: User = Depends(get_api_current_user),
|
||||
):
|
||||
user_settings = await user_settings_service.get_or_create_settings(
|
||||
db_session,
|
||||
user_id=current_user.id,
|
||||
)
|
||||
await _clear_expired_cooldown_with_log(
|
||||
db_session,
|
||||
user_id=current_user.id,
|
||||
user_settings=user_settings,
|
||||
)
|
||||
return success_payload(
|
||||
request,
|
||||
data=_serialize_settings(user_settings),
|
||||
)
|
||||
|
||||
|
||||
@router.put(
|
||||
"",
|
||||
response_model=SettingsEnvelope,
|
||||
)
|
||||
async def update_settings(
|
||||
payload: SettingsUpdateRequest,
|
||||
request: Request,
|
||||
db_session: AsyncSession = Depends(get_db_session),
|
||||
current_user: User = Depends(get_api_current_user),
|
||||
):
|
||||
user_settings = await user_settings_service.get_or_create_settings(
|
||||
db_session,
|
||||
user_id=current_user.id,
|
||||
)
|
||||
|
||||
try:
|
||||
parsed_interval, parsed_delay, parsed_nav_visible_pages = _parse_settings_payload(
|
||||
payload,
|
||||
user_settings,
|
||||
)
|
||||
except user_settings_service.UserSettingsServiceError as exc:
|
||||
raise ApiException(
|
||||
status_code=400,
|
||||
code="invalid_settings",
|
||||
message=str(exc),
|
||||
) from exc
|
||||
|
||||
updated = await user_settings_service.update_settings(
|
||||
db_session,
|
||||
settings=user_settings,
|
||||
auto_run_enabled=_effective_auto_run_enabled(payload),
|
||||
run_interval_minutes=parsed_interval,
|
||||
request_delay_seconds=parsed_delay,
|
||||
nav_visible_pages=parsed_nav_visible_pages,
|
||||
openalex_api_key=payload.openalex_api_key,
|
||||
crossref_api_token=payload.crossref_api_token,
|
||||
crossref_api_mailto=payload.crossref_api_mailto,
|
||||
)
|
||||
await _clear_expired_cooldown_with_log(
|
||||
db_session,
|
||||
user_id=current_user.id,
|
||||
user_settings=updated,
|
||||
)
|
||||
_log_settings_update(user_id=current_user.id, updated=updated)
|
||||
return success_payload(
|
||||
request,
|
||||
data=_serialize_settings(updated),
|
||||
)
|
||||
|
|
@ -1,16 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from fastapi import Depends
|
||||
|
||||
from app.services.domains.ingestion import application as ingestion_service
|
||||
from app.services.domains.scholar import source as scholar_source_service
|
||||
|
||||
|
||||
def get_scholar_source() -> scholar_source_service.ScholarSource:
|
||||
return scholar_source_service.LiveScholarSource()
|
||||
|
||||
|
||||
def get_ingestion_service(
|
||||
source: scholar_source_service.ScholarSource = Depends(get_scholar_source),
|
||||
) -> ingestion_service.ScholarIngestionService:
|
||||
return ingestion_service.ScholarIngestionService(source=source)
|
||||
|
|
@ -1,884 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Any, Literal
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, model_validator
|
||||
|
||||
|
||||
class ApiMeta(BaseModel):
|
||||
request_id: str | None = None
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class ApiErrorData(BaseModel):
|
||||
code: str
|
||||
message: str
|
||||
details: Any | None = None
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class ApiErrorEnvelope(BaseModel):
|
||||
error: ApiErrorData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class MessageData(BaseModel):
|
||||
message: str
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class MessageEnvelope(BaseModel):
|
||||
data: MessageData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class SessionUserData(BaseModel):
|
||||
id: int
|
||||
email: str
|
||||
is_admin: bool
|
||||
is_active: bool
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class AuthMeData(BaseModel):
|
||||
authenticated: bool
|
||||
csrf_token: str
|
||||
user: SessionUserData
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class AuthMeEnvelope(BaseModel):
|
||||
data: AuthMeData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class CsrfBootstrapData(BaseModel):
|
||||
csrf_token: str
|
||||
authenticated: bool
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class CsrfBootstrapEnvelope(BaseModel):
|
||||
data: CsrfBootstrapData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class LoginRequest(BaseModel):
|
||||
email: str
|
||||
password: str
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class LoginData(BaseModel):
|
||||
authenticated: bool
|
||||
csrf_token: str
|
||||
user: SessionUserData
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class LoginEnvelope(BaseModel):
|
||||
data: LoginData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class ChangePasswordRequest(BaseModel):
|
||||
current_password: str
|
||||
new_password: str
|
||||
confirm_password: str
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class ScholarItemData(BaseModel):
|
||||
id: int
|
||||
scholar_id: str
|
||||
display_name: str | None
|
||||
profile_image_url: str | None
|
||||
profile_image_source: str
|
||||
is_enabled: bool
|
||||
baseline_completed: bool
|
||||
last_run_dt: datetime | None
|
||||
last_run_status: str | None
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class ScholarsListData(BaseModel):
|
||||
scholars: list[ScholarItemData]
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class ScholarsListEnvelope(BaseModel):
|
||||
data: ScholarsListData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class ScholarCreateRequest(BaseModel):
|
||||
scholar_id: str
|
||||
profile_image_url: str | None = None
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class ScholarEnvelope(BaseModel):
|
||||
data: ScholarItemData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class ScholarSearchCandidateData(BaseModel):
|
||||
scholar_id: str
|
||||
display_name: str
|
||||
affiliation: str | None
|
||||
email_domain: str | None
|
||||
cited_by_count: int | None
|
||||
interests: list[str] = Field(default_factory=list)
|
||||
profile_url: str
|
||||
profile_image_url: str | None
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class ScholarSearchData(BaseModel):
|
||||
query: str
|
||||
state: str
|
||||
state_reason: str
|
||||
action_hint: str | None = None
|
||||
candidates: list[ScholarSearchCandidateData]
|
||||
warnings: list[str] = Field(default_factory=list)
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class ScholarSearchEnvelope(BaseModel):
|
||||
data: ScholarSearchData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class ScholarImageUrlUpdateRequest(BaseModel):
|
||||
image_url: str
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class ScholarExportItemData(BaseModel):
|
||||
scholar_id: str
|
||||
display_name: str | None = None
|
||||
is_enabled: bool = True
|
||||
profile_image_override_url: str | None = None
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class PublicationExportItemData(BaseModel):
|
||||
scholar_id: str
|
||||
cluster_id: str | None = None
|
||||
fingerprint_sha256: str | None = None
|
||||
title: str
|
||||
year: int | None = None
|
||||
citation_count: int = 0
|
||||
author_text: str | None = None
|
||||
venue_text: str | None = None
|
||||
pub_url: str | None = None
|
||||
pdf_url: str | None = None
|
||||
is_read: bool = False
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class DataExportData(BaseModel):
|
||||
schema_version: int
|
||||
exported_at: str
|
||||
scholars: list[ScholarExportItemData]
|
||||
publications: list[PublicationExportItemData]
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class DataExportEnvelope(BaseModel):
|
||||
data: DataExportData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class DataImportRequest(BaseModel):
|
||||
schema_version: int | None = None
|
||||
scholars: list[ScholarExportItemData] = Field(default_factory=list)
|
||||
publications: list[PublicationExportItemData] = Field(default_factory=list)
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class DataImportResultData(BaseModel):
|
||||
scholars_created: int
|
||||
scholars_updated: int
|
||||
publications_created: int
|
||||
publications_updated: int
|
||||
links_created: int
|
||||
links_updated: int
|
||||
skipped_records: int
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class DataImportEnvelope(BaseModel):
|
||||
data: DataImportResultData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class RunListItemData(BaseModel):
|
||||
id: int
|
||||
trigger_type: str
|
||||
status: str
|
||||
start_dt: datetime
|
||||
end_dt: datetime | None
|
||||
scholar_count: int
|
||||
new_publication_count: int
|
||||
failed_count: int
|
||||
partial_count: int
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class RunsListData(BaseModel):
|
||||
runs: list[RunListItemData]
|
||||
safety_state: ScrapeSafetyStateData
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class RunsListEnvelope(BaseModel):
|
||||
data: RunsListData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class RunSummaryData(BaseModel):
|
||||
succeeded_count: int
|
||||
failed_count: int
|
||||
partial_count: int
|
||||
failed_state_counts: dict[str, int] = Field(default_factory=dict)
|
||||
failed_reason_counts: dict[str, int] = Field(default_factory=dict)
|
||||
scrape_failure_counts: dict[str, int] = Field(default_factory=dict)
|
||||
retry_counts: dict[str, int] = Field(default_factory=dict)
|
||||
alert_thresholds: dict[str, int] = Field(default_factory=dict)
|
||||
alert_flags: dict[str, bool] = Field(default_factory=dict)
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class ScrapeSafetyCountersData(BaseModel):
|
||||
consecutive_blocked_runs: int = 0
|
||||
consecutive_network_runs: int = 0
|
||||
cooldown_entry_count: int = 0
|
||||
blocked_start_count: int = 0
|
||||
last_blocked_failure_count: int = 0
|
||||
last_network_failure_count: int = 0
|
||||
last_evaluated_run_id: int | None = None
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class ScrapeSafetyStateData(BaseModel):
|
||||
cooldown_active: bool
|
||||
cooldown_reason: str | None = None
|
||||
cooldown_reason_label: str | None = None
|
||||
cooldown_until: datetime | None = None
|
||||
cooldown_remaining_seconds: int = 0
|
||||
recommended_action: str | None = None
|
||||
counters: ScrapeSafetyCountersData = Field(default_factory=ScrapeSafetyCountersData)
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class RunAttemptLogData(BaseModel):
|
||||
attempt: int
|
||||
cstart: int
|
||||
state: str | None = None
|
||||
state_reason: str | None = None
|
||||
status_code: int | None = None
|
||||
fetch_error: str | None = None
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class RunPageLogData(BaseModel):
|
||||
page: int
|
||||
cstart: int
|
||||
state: str
|
||||
state_reason: str | None = None
|
||||
status_code: int | None = None
|
||||
publication_count: int = 0
|
||||
attempt_count: int = 0
|
||||
has_show_more_button: bool = False
|
||||
articles_range: str | None = None
|
||||
warning_codes: list[str] = Field(default_factory=list)
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class RunDebugData(BaseModel):
|
||||
status_code: int | None = None
|
||||
final_url: str | None = None
|
||||
fetch_error: str | None = None
|
||||
body_sha256: str | None = None
|
||||
body_length: int | None = None
|
||||
has_show_more_button: bool | None = None
|
||||
articles_range: str | None = None
|
||||
state_reason: str | None = None
|
||||
warning_codes: list[str] = Field(default_factory=list)
|
||||
marker_counts_nonzero: dict[str, int] = Field(default_factory=dict)
|
||||
page_logs: list[RunPageLogData] = Field(default_factory=list)
|
||||
attempt_log: list[RunAttemptLogData] = Field(default_factory=list)
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class RunScholarResultData(BaseModel):
|
||||
scholar_profile_id: int
|
||||
scholar_id: str
|
||||
state: str
|
||||
state_reason: str | None = None
|
||||
outcome: str
|
||||
attempt_count: int = 0
|
||||
publication_count: int = 0
|
||||
start_cstart: int = 0
|
||||
continuation_cstart: int | None = None
|
||||
continuation_enqueued: bool = False
|
||||
continuation_cleared: bool = False
|
||||
warnings: list[str] = Field(default_factory=list)
|
||||
error: str | None = None
|
||||
debug: RunDebugData | None = None
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class RunDetailData(BaseModel):
|
||||
run: RunListItemData
|
||||
summary: RunSummaryData
|
||||
scholar_results: list[RunScholarResultData]
|
||||
safety_state: ScrapeSafetyStateData
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class RunDetailEnvelope(BaseModel):
|
||||
data: RunDetailData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class ManualRunData(BaseModel):
|
||||
run_id: int
|
||||
status: str
|
||||
scholar_count: int
|
||||
succeeded_count: int
|
||||
failed_count: int
|
||||
partial_count: int
|
||||
new_publication_count: int
|
||||
reused_existing_run: bool
|
||||
idempotency_key: str | None = None
|
||||
safety_state: ScrapeSafetyStateData
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class ManualRunEnvelope(BaseModel):
|
||||
data: ManualRunData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class QueueItemData(BaseModel):
|
||||
id: int
|
||||
scholar_profile_id: int
|
||||
scholar_label: str
|
||||
status: str
|
||||
reason: str
|
||||
dropped_reason: str | None
|
||||
attempt_count: int
|
||||
resume_cstart: int
|
||||
next_attempt_dt: datetime | None
|
||||
updated_at: datetime
|
||||
last_error: str | None
|
||||
last_run_id: int | None
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class QueueListData(BaseModel):
|
||||
queue_items: list[QueueItemData]
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class QueueListEnvelope(BaseModel):
|
||||
data: QueueListData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class QueueItemEnvelope(BaseModel):
|
||||
data: QueueItemData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class QueueClearData(BaseModel):
|
||||
queue_item_id: int
|
||||
previous_status: str
|
||||
status: str
|
||||
message: str
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class QueueClearEnvelope(BaseModel):
|
||||
data: QueueClearData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class AdminUserData(BaseModel):
|
||||
id: int
|
||||
email: str
|
||||
is_active: bool
|
||||
is_admin: bool
|
||||
created_at: datetime
|
||||
updated_at: datetime
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class AdminUsersListData(BaseModel):
|
||||
users: list[AdminUserData]
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class AdminUsersListEnvelope(BaseModel):
|
||||
data: AdminUsersListData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class AdminUserCreateRequest(BaseModel):
|
||||
email: str
|
||||
password: str
|
||||
is_admin: bool = False
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class AdminUserActiveUpdateRequest(BaseModel):
|
||||
is_active: bool
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class AdminUserEnvelope(BaseModel):
|
||||
data: AdminUserData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class AdminResetPasswordRequest(BaseModel):
|
||||
new_password: str
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class AdminDbIntegrityCheckData(BaseModel):
|
||||
name: str
|
||||
count: int
|
||||
severity: str
|
||||
message: str
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class AdminDbIntegrityData(BaseModel):
|
||||
status: str
|
||||
checked_at: datetime
|
||||
failures: list[str] = Field(default_factory=list)
|
||||
warnings: list[str] = Field(default_factory=list)
|
||||
checks: list[AdminDbIntegrityCheckData] = Field(default_factory=list)
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class AdminDbIntegrityEnvelope(BaseModel):
|
||||
data: AdminDbIntegrityData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class AdminDbRepairJobData(BaseModel):
|
||||
id: int
|
||||
job_name: str
|
||||
requested_by: str | None
|
||||
dry_run: bool
|
||||
status: str
|
||||
scope: dict[str, Any] = Field(default_factory=dict)
|
||||
summary: dict[str, Any] = Field(default_factory=dict)
|
||||
error_text: str | None
|
||||
started_at: datetime | None
|
||||
finished_at: datetime | None
|
||||
created_at: datetime
|
||||
updated_at: datetime
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class AdminDbRepairJobsData(BaseModel):
|
||||
jobs: list[AdminDbRepairJobData] = Field(default_factory=list)
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class AdminDbRepairJobsEnvelope(BaseModel):
|
||||
data: AdminDbRepairJobsData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class DisplayIdentifierData(BaseModel):
|
||||
kind: str
|
||||
value: str
|
||||
label: str
|
||||
url: str | None
|
||||
confidence_score: float = Field(ge=0.0, le=1.0)
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class AdminPdfQueueItemData(BaseModel):
|
||||
publication_id: int
|
||||
title: str
|
||||
display_identifier: DisplayIdentifierData | None = None
|
||||
pdf_url: str | None
|
||||
status: str
|
||||
attempt_count: int
|
||||
last_failure_reason: str | None
|
||||
last_failure_detail: str | None
|
||||
last_source: str | None
|
||||
requested_by_user_id: int | None
|
||||
requested_by_email: str | None
|
||||
queued_at: datetime | None
|
||||
last_attempt_at: datetime | None
|
||||
resolved_at: datetime | None
|
||||
updated_at: datetime
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class AdminPdfQueueData(BaseModel):
|
||||
items: list[AdminPdfQueueItemData] = Field(default_factory=list)
|
||||
total_count: int = 0
|
||||
page: int = 1
|
||||
page_size: int = 100
|
||||
has_next: bool = False
|
||||
has_prev: bool = False
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class AdminPdfQueueEnvelope(BaseModel):
|
||||
data: AdminPdfQueueData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class AdminPdfQueueRequeueData(BaseModel):
|
||||
publication_id: int
|
||||
queued: bool
|
||||
status: str
|
||||
message: str
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class AdminPdfQueueRequeueEnvelope(BaseModel):
|
||||
data: AdminPdfQueueRequeueData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class AdminPdfQueueBulkEnqueueData(BaseModel):
|
||||
requested_count: int
|
||||
queued_count: int
|
||||
message: str
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class AdminPdfQueueBulkEnqueueEnvelope(BaseModel):
|
||||
data: AdminPdfQueueBulkEnqueueData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class AdminRepairPublicationLinksRequest(BaseModel):
|
||||
scope_mode: Literal["single_user", "all_users"] = "single_user"
|
||||
user_id: int | None = Field(default=None, ge=1)
|
||||
scholar_profile_ids: list[int] = Field(default_factory=list, max_length=200)
|
||||
dry_run: bool = True
|
||||
gc_orphan_publications: bool = False
|
||||
requested_by: str | None = None
|
||||
confirmation_text: str | None = None
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_scope(self) -> "AdminRepairPublicationLinksRequest":
|
||||
if self.scope_mode == "single_user" and self.user_id is None:
|
||||
raise ValueError("user_id is required when scope_mode=single_user.")
|
||||
if self.scope_mode == "all_users" and self.user_id is not None:
|
||||
raise ValueError("user_id must be omitted when scope_mode=all_users.")
|
||||
if not self.dry_run and self.scope_mode == "all_users":
|
||||
expected = "REPAIR ALL USERS"
|
||||
provided = (self.confirmation_text or "").strip()
|
||||
if provided != expected:
|
||||
raise ValueError(
|
||||
"confirmation_text must equal 'REPAIR ALL USERS' "
|
||||
"when applying a repair to all users."
|
||||
)
|
||||
return self
|
||||
|
||||
|
||||
class AdminRepairPublicationLinksResultData(BaseModel):
|
||||
job_id: int
|
||||
status: str
|
||||
scope: dict[str, Any] = Field(default_factory=dict)
|
||||
summary: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class AdminRepairPublicationLinksEnvelope(BaseModel):
|
||||
data: AdminRepairPublicationLinksResultData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class SettingsPolicyData(BaseModel):
|
||||
min_run_interval_minutes: int
|
||||
min_request_delay_seconds: int
|
||||
automation_allowed: bool
|
||||
manual_run_allowed: bool
|
||||
blocked_failure_threshold: int
|
||||
network_failure_threshold: int
|
||||
cooldown_blocked_seconds: int
|
||||
cooldown_network_seconds: int
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class SettingsData(BaseModel):
|
||||
auto_run_enabled: bool
|
||||
run_interval_minutes: int
|
||||
request_delay_seconds: int
|
||||
nav_visible_pages: list[str]
|
||||
policy: SettingsPolicyData
|
||||
safety_state: ScrapeSafetyStateData
|
||||
|
||||
openalex_api_key: str | None = None
|
||||
crossref_api_token: str | None = None
|
||||
crossref_api_mailto: str | None = None
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class SettingsEnvelope(BaseModel):
|
||||
data: SettingsData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class SettingsUpdateRequest(BaseModel):
|
||||
auto_run_enabled: bool
|
||||
run_interval_minutes: int
|
||||
request_delay_seconds: int
|
||||
nav_visible_pages: list[str] | None = None
|
||||
|
||||
openalex_api_key: str | None = None
|
||||
crossref_api_token: str | None = None
|
||||
crossref_api_mailto: str | None = None
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class PublicationItemData(BaseModel):
|
||||
publication_id: int
|
||||
scholar_profile_id: int
|
||||
scholar_label: str
|
||||
title: str
|
||||
year: int | None
|
||||
citation_count: int
|
||||
venue_text: str | None
|
||||
pub_url: str | None
|
||||
display_identifier: DisplayIdentifierData | None = None
|
||||
pdf_url: str | None
|
||||
pdf_status: str = "untracked"
|
||||
pdf_attempt_count: int = 0
|
||||
pdf_failure_reason: str | None = None
|
||||
pdf_failure_detail: str | None = None
|
||||
is_read: bool
|
||||
is_favorite: bool = False
|
||||
first_seen_at: datetime
|
||||
is_new_in_latest_run: bool
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class PublicationsListData(BaseModel):
|
||||
mode: str
|
||||
favorite_only: bool = False
|
||||
selected_scholar_profile_id: int | None
|
||||
unread_count: int
|
||||
favorites_count: int
|
||||
latest_count: int
|
||||
new_count: int
|
||||
total_count: int
|
||||
page: int
|
||||
page_size: int
|
||||
has_next: bool = False
|
||||
has_prev: bool = False
|
||||
publications: list[PublicationItemData]
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class PublicationsListEnvelope(BaseModel):
|
||||
data: PublicationsListData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class MarkAllReadData(BaseModel):
|
||||
message: str
|
||||
updated_count: int
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class MarkAllReadEnvelope(BaseModel):
|
||||
data: MarkAllReadData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class PublicationSelectionItem(BaseModel):
|
||||
scholar_profile_id: int
|
||||
publication_id: int
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class MarkSelectedReadRequest(BaseModel):
|
||||
selections: list[PublicationSelectionItem] = Field(min_length=1, max_length=500)
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class MarkSelectedReadData(BaseModel):
|
||||
message: str
|
||||
requested_count: int
|
||||
updated_count: int
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class MarkSelectedReadEnvelope(BaseModel):
|
||||
data: MarkSelectedReadData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class RetryPublicationPdfRequest(BaseModel):
|
||||
scholar_profile_id: int = Field(ge=1)
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class RetryPublicationPdfData(BaseModel):
|
||||
message: str
|
||||
queued: bool
|
||||
resolved_pdf: bool
|
||||
publication: PublicationItemData
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class RetryPublicationPdfEnvelope(BaseModel):
|
||||
data: RetryPublicationPdfData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class TogglePublicationFavoriteRequest(BaseModel):
|
||||
scholar_profile_id: int = Field(ge=1)
|
||||
is_favorite: bool
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class TogglePublicationFavoriteData(BaseModel):
|
||||
message: str
|
||||
publication: PublicationItemData
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class TogglePublicationFavoriteEnvelope(BaseModel):
|
||||
data: TogglePublicationFavoriteData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
"""Authentication package for scholarr."""
|
||||
|
||||
|
|
@ -1,27 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from functools import lru_cache
|
||||
|
||||
from app.auth.rate_limit import SlidingWindowRateLimiter
|
||||
from app.auth.security import PasswordService
|
||||
from app.auth.service import AuthService
|
||||
from app.settings import settings
|
||||
|
||||
|
||||
@lru_cache
|
||||
def get_password_service() -> PasswordService:
|
||||
return PasswordService()
|
||||
|
||||
|
||||
@lru_cache
|
||||
def get_auth_service() -> AuthService:
|
||||
return AuthService(password_service=get_password_service())
|
||||
|
||||
|
||||
@lru_cache
|
||||
def get_login_rate_limiter() -> SlidingWindowRateLimiter:
|
||||
return SlidingWindowRateLimiter(
|
||||
max_attempts=settings.login_rate_limit_attempts,
|
||||
window_seconds=settings.login_rate_limit_window_seconds,
|
||||
)
|
||||
|
||||
|
|
@ -1,69 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from collections import deque
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from math import ceil
|
||||
from threading import Lock
|
||||
from time import monotonic
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RateLimitDecision:
|
||||
allowed: bool
|
||||
retry_after_seconds: int = 0
|
||||
|
||||
|
||||
class SlidingWindowRateLimiter:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
max_attempts: int,
|
||||
window_seconds: int,
|
||||
now: Callable[[], float] = monotonic,
|
||||
) -> None:
|
||||
self._max_attempts = max_attempts
|
||||
self._window_seconds = window_seconds
|
||||
self._now = now
|
||||
self._attempts: dict[str, deque[float]] = {}
|
||||
self._lock = Lock()
|
||||
|
||||
def check(self, key: str) -> RateLimitDecision:
|
||||
with self._lock:
|
||||
now_value = self._now()
|
||||
attempts = self._attempts.get(key)
|
||||
if attempts is None:
|
||||
return RateLimitDecision(allowed=True)
|
||||
self._trim_expired(attempts, now_value)
|
||||
if not attempts:
|
||||
self._attempts.pop(key, None)
|
||||
return RateLimitDecision(allowed=True)
|
||||
if len(attempts) >= self._max_attempts:
|
||||
retry_after = self._window_seconds - (now_value - attempts[0])
|
||||
return RateLimitDecision(
|
||||
allowed=False,
|
||||
retry_after_seconds=max(1, ceil(retry_after)),
|
||||
)
|
||||
return RateLimitDecision(allowed=True)
|
||||
|
||||
def record_failure(self, key: str) -> None:
|
||||
with self._lock:
|
||||
now_value = self._now()
|
||||
attempts = self._attempts.get(key)
|
||||
if attempts is None:
|
||||
attempts = deque()
|
||||
self._attempts[key] = attempts
|
||||
self._trim_expired(attempts, now_value)
|
||||
attempts.append(now_value)
|
||||
|
||||
def reset(self, key: str) -> None:
|
||||
with self._lock:
|
||||
self._attempts.pop(key, None)
|
||||
|
||||
def clear_all(self) -> None:
|
||||
with self._lock:
|
||||
self._attempts.clear()
|
||||
|
||||
def _trim_expired(self, attempts: deque[float], now_value: float) -> None:
|
||||
while attempts and now_value - attempts[0] >= self._window_seconds:
|
||||
attempts.popleft()
|
||||
|
|
@ -1,55 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from fastapi import Request
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.auth.session import clear_session_user, get_session_user, set_session_user
|
||||
from app.db.models import User
|
||||
from app.security.csrf import CSRF_SESSION_KEY
|
||||
from app.services.domains.users import application as user_service
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def invalidate_session(request: Request) -> None:
|
||||
clear_session_user(request)
|
||||
request.session.pop(CSRF_SESSION_KEY, None)
|
||||
|
||||
|
||||
async def get_authenticated_user(
|
||||
request: Request,
|
||||
db_session: AsyncSession,
|
||||
) -> User | None:
|
||||
session_user = get_session_user(request)
|
||||
if session_user is None:
|
||||
return None
|
||||
|
||||
user = await user_service.get_user_by_id(db_session, session_user.id)
|
||||
if user is None or not user.is_active:
|
||||
logger.info(
|
||||
"auth.session_invalidated",
|
||||
extra={
|
||||
"event": "auth.session_invalidated",
|
||||
"session_user_id": session_user.id,
|
||||
},
|
||||
)
|
||||
invalidate_session(request)
|
||||
return None
|
||||
|
||||
if user.email != session_user.email or user.is_admin != session_user.is_admin:
|
||||
set_session_user(
|
||||
request,
|
||||
user_id=user.id,
|
||||
email=user.email,
|
||||
is_admin=user.is_admin,
|
||||
)
|
||||
|
||||
return user
|
||||
|
||||
|
||||
def login_rate_limit_key(request: Request, email: str) -> str:
|
||||
client_host = request.client.host if request.client is not None else "unknown"
|
||||
normalized_email = email.strip().lower()
|
||||
return f"{client_host}:{normalized_email or '<empty>'}"
|
||||
|
|
@ -1,19 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from argon2 import PasswordHasher
|
||||
from argon2.exceptions import InvalidHashError, VerificationError
|
||||
|
||||
|
||||
class PasswordService:
|
||||
def __init__(self, hasher: PasswordHasher | None = None) -> None:
|
||||
self._hasher = hasher or PasswordHasher()
|
||||
|
||||
def hash_password(self, password: str) -> str:
|
||||
return self._hasher.hash(password)
|
||||
|
||||
def verify_password(self, password_hash: str, password: str) -> bool:
|
||||
try:
|
||||
return bool(self._hasher.verify(password_hash, password))
|
||||
except (InvalidHashError, VerificationError):
|
||||
return False
|
||||
|
||||
|
|
@ -1,38 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.auth.security import PasswordService
|
||||
from app.db.models import User
|
||||
|
||||
|
||||
class AuthService:
|
||||
def __init__(self, password_service: PasswordService) -> None:
|
||||
self._password_service = password_service
|
||||
|
||||
async def authenticate_user(
|
||||
self,
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
email: str,
|
||||
password: str,
|
||||
) -> User | None:
|
||||
normalized_email = email.strip().lower()
|
||||
if not normalized_email or not password:
|
||||
return None
|
||||
result = await db_session.execute(
|
||||
select(User).where(User.email == normalized_email)
|
||||
)
|
||||
user = result.scalar_one_or_none()
|
||||
if user is None or not user.is_active:
|
||||
return None
|
||||
if not self._password_service.verify_password(user.password_hash, password):
|
||||
return None
|
||||
return user
|
||||
|
||||
def hash_password(self, password: str) -> str:
|
||||
return self._password_service.hash_password(password)
|
||||
|
||||
def verify_password(self, *, password_hash: str, password: str) -> bool:
|
||||
return self._password_service.verify_password(password_hash, password)
|
||||
|
|
@ -1,50 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
from starlette.requests import Request
|
||||
|
||||
|
||||
SESSION_USER_ID_KEY = "auth_user_id"
|
||||
SESSION_USER_EMAIL_KEY = "auth_user_email"
|
||||
SESSION_USER_IS_ADMIN_KEY = "auth_user_is_admin"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SessionUser:
|
||||
id: int
|
||||
email: str
|
||||
is_admin: bool
|
||||
|
||||
|
||||
def get_session_user(request: Request) -> SessionUser | None:
|
||||
user_id = request.session.get(SESSION_USER_ID_KEY)
|
||||
email = request.session.get(SESSION_USER_EMAIL_KEY)
|
||||
is_admin = request.session.get(SESSION_USER_IS_ADMIN_KEY)
|
||||
if user_id is None or email is None or is_admin is None:
|
||||
return None
|
||||
try:
|
||||
parsed_user_id = int(user_id)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
if not isinstance(email, str):
|
||||
return None
|
||||
return SessionUser(id=parsed_user_id, email=email, is_admin=bool(is_admin))
|
||||
|
||||
|
||||
def set_session_user(
|
||||
request: Request,
|
||||
*,
|
||||
user_id: int,
|
||||
email: str,
|
||||
is_admin: bool,
|
||||
) -> None:
|
||||
request.session[SESSION_USER_ID_KEY] = int(user_id)
|
||||
request.session[SESSION_USER_EMAIL_KEY] = email
|
||||
request.session[SESSION_USER_IS_ADMIN_KEY] = bool(is_admin)
|
||||
|
||||
|
||||
def clear_session_user(request: Request) -> None:
|
||||
request.session.pop(SESSION_USER_ID_KEY, None)
|
||||
request.session.pop(SESSION_USER_EMAIL_KEY, None)
|
||||
request.session.pop(SESSION_USER_IS_ADMIN_KEY, None)
|
||||
|
|
@ -1 +0,0 @@
|
|||
|
||||
|
|
@ -1,27 +0,0 @@
|
|||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy import MetaData, event
|
||||
from sqlalchemy.orm import DeclarativeBase
|
||||
|
||||
|
||||
NAMING_CONVENTION = {
|
||||
"ix": "ix_%(column_0_label)s",
|
||||
"uq": "uq_%(table_name)s_%(column_0_name)s",
|
||||
"ck": "ck_%(table_name)s_%(constraint_name)s",
|
||||
"fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s",
|
||||
"pk": "pk_%(table_name)s",
|
||||
}
|
||||
|
||||
|
||||
class Base(DeclarativeBase):
|
||||
metadata = MetaData(naming_convention=NAMING_CONVENTION)
|
||||
|
||||
|
||||
@event.listens_for(Base, "before_update", propagate=True)
|
||||
def _set_updated_at_before_update(_mapper, _connection, target) -> None:
|
||||
# Keep audit timestamps current for ORM-managed updates.
|
||||
if hasattr(target, "updated_at"):
|
||||
target.updated_at = datetime.now(timezone.utc)
|
||||
|
||||
|
||||
metadata = Base.metadata
|
||||
|
|
@ -1,546 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from enum import StrEnum
|
||||
|
||||
from sqlalchemy import (
|
||||
Boolean,
|
||||
CheckConstraint,
|
||||
DateTime,
|
||||
Enum,
|
||||
Float,
|
||||
ForeignKey,
|
||||
Index,
|
||||
Integer,
|
||||
String,
|
||||
Text,
|
||||
UniqueConstraint,
|
||||
func,
|
||||
text,
|
||||
)
|
||||
from sqlalchemy.dialects.postgresql import JSONB
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from app.db.base import Base
|
||||
|
||||
|
||||
class RunTriggerType(StrEnum):
|
||||
MANUAL = "manual"
|
||||
SCHEDULED = "scheduled"
|
||||
|
||||
|
||||
class RunStatus(StrEnum):
|
||||
RUNNING = "running"
|
||||
SUCCESS = "success"
|
||||
PARTIAL_FAILURE = "partial_failure"
|
||||
FAILED = "failed"
|
||||
CANCELED = "canceled"
|
||||
|
||||
|
||||
class QueueItemStatus(StrEnum):
|
||||
QUEUED = "queued"
|
||||
RETRYING = "retrying"
|
||||
DROPPED = "dropped"
|
||||
|
||||
|
||||
RUN_STATUS_DB_ENUM = Enum(
|
||||
RunStatus,
|
||||
name="run_status",
|
||||
values_callable=lambda members: [member.value for member in members],
|
||||
)
|
||||
RUN_TRIGGER_TYPE_DB_ENUM = Enum(
|
||||
RunTriggerType,
|
||||
name="run_trigger_type",
|
||||
values_callable=lambda members: [member.value for member in members],
|
||||
)
|
||||
|
||||
|
||||
class User(Base):
|
||||
__tablename__ = "users"
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
email: Mapped[str] = mapped_column(String(255), nullable=False, unique=True)
|
||||
password_hash: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
is_active: Mapped[bool] = mapped_column(
|
||||
Boolean, nullable=False, server_default=text("true")
|
||||
)
|
||||
is_admin: Mapped[bool] = mapped_column(
|
||||
Boolean, nullable=False, server_default=text("false")
|
||||
)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
|
||||
|
||||
class UserSetting(Base):
|
||||
__tablename__ = "user_settings"
|
||||
__table_args__ = (
|
||||
CheckConstraint(
|
||||
"run_interval_minutes >= 15",
|
||||
name="run_interval_minutes_min_15",
|
||||
),
|
||||
CheckConstraint(
|
||||
"request_delay_seconds >= 2",
|
||||
name="request_delay_seconds_min_2",
|
||||
),
|
||||
)
|
||||
|
||||
user_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("users.id", ondelete="CASCADE"), primary_key=True
|
||||
)
|
||||
auto_run_enabled: Mapped[bool] = mapped_column(
|
||||
Boolean, nullable=False, server_default=text("false")
|
||||
)
|
||||
run_interval_minutes: Mapped[int] = mapped_column(
|
||||
Integer, nullable=False, server_default=text("1440")
|
||||
)
|
||||
request_delay_seconds: Mapped[int] = mapped_column(
|
||||
Integer, nullable=False, server_default=text("10")
|
||||
)
|
||||
nav_visible_pages: Mapped[list[str]] = mapped_column(
|
||||
JSONB,
|
||||
nullable=False,
|
||||
server_default=text(
|
||||
'\'["dashboard","scholars","publications","settings","style-guide","runs","users"]\'::jsonb'
|
||||
),
|
||||
)
|
||||
scrape_safety_state: Mapped[dict] = mapped_column(
|
||||
JSONB,
|
||||
nullable=False,
|
||||
server_default=text("'{}'::jsonb"),
|
||||
)
|
||||
scrape_cooldown_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
scrape_cooldown_reason: Mapped[str | None] = mapped_column(String(64))
|
||||
|
||||
openalex_api_key: Mapped[str | None] = mapped_column(String(255))
|
||||
crossref_api_token: Mapped[str | None] = mapped_column(String(255))
|
||||
crossref_api_mailto: Mapped[str | None] = mapped_column(String(255))
|
||||
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
|
||||
|
||||
class ScholarProfile(Base):
|
||||
__tablename__ = "scholar_profiles"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("user_id", "scholar_id", name="uq_scholar_profiles_user_scholar"),
|
||||
Index("ix_scholar_profiles_user_enabled", "user_id", "is_enabled"),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
user_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("users.id", ondelete="CASCADE"), nullable=False
|
||||
)
|
||||
scholar_id: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
display_name: Mapped[str | None] = mapped_column(String(255))
|
||||
profile_image_url: Mapped[str | None] = mapped_column(Text)
|
||||
profile_image_override_url: Mapped[str | None] = mapped_column(Text)
|
||||
profile_image_upload_path: Mapped[str | None] = mapped_column(Text)
|
||||
last_initial_page_fingerprint_sha256: Mapped[str | None] = mapped_column(String(64))
|
||||
last_initial_page_checked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
is_enabled: Mapped[bool] = mapped_column(
|
||||
Boolean, nullable=False, server_default=text("true")
|
||||
)
|
||||
baseline_completed: Mapped[bool] = mapped_column(
|
||||
Boolean, nullable=False, server_default=text("false")
|
||||
)
|
||||
last_run_dt: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
last_run_status: Mapped[RunStatus | None] = mapped_column(
|
||||
RUN_STATUS_DB_ENUM,
|
||||
)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
|
||||
|
||||
class CrawlRun(Base):
|
||||
__tablename__ = "crawl_runs"
|
||||
__table_args__ = (
|
||||
Index("ix_crawl_runs_user_start", "user_id", "start_dt"),
|
||||
Index(
|
||||
"uq_crawl_runs_user_manual_idempotency_key",
|
||||
"user_id",
|
||||
"idempotency_key",
|
||||
unique=True,
|
||||
postgresql_where=text(
|
||||
"idempotency_key IS NOT NULL AND trigger_type = 'manual'::run_trigger_type"
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
user_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("users.id", ondelete="CASCADE"), nullable=False
|
||||
)
|
||||
trigger_type: Mapped[RunTriggerType] = mapped_column(
|
||||
RUN_TRIGGER_TYPE_DB_ENUM, nullable=False
|
||||
)
|
||||
status: Mapped[RunStatus] = mapped_column(
|
||||
RUN_STATUS_DB_ENUM, nullable=False
|
||||
)
|
||||
start_dt: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
end_dt: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
scholar_count: Mapped[int] = mapped_column(
|
||||
Integer, nullable=False, server_default=text("0")
|
||||
)
|
||||
new_pub_count: Mapped[int] = mapped_column(
|
||||
Integer, nullable=False, server_default=text("0")
|
||||
)
|
||||
idempotency_key: Mapped[str | None] = mapped_column(String(128))
|
||||
error_log: Mapped[dict] = mapped_column(
|
||||
JSONB, nullable=False, server_default=text("'{}'::jsonb")
|
||||
)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
|
||||
|
||||
class Publication(Base):
|
||||
__tablename__ = "publications"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("fingerprint_sha256", name="uq_publications_fingerprint"),
|
||||
Index(
|
||||
"uq_publications_cluster_id_not_null",
|
||||
"cluster_id",
|
||||
unique=True,
|
||||
postgresql_where=text("cluster_id IS NOT NULL"),
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
cluster_id: Mapped[str | None] = mapped_column(String(64))
|
||||
fingerprint_sha256: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
title_raw: Mapped[str] = mapped_column(Text, nullable=False)
|
||||
title_normalized: Mapped[str] = mapped_column(Text, nullable=False)
|
||||
year: Mapped[int | None] = mapped_column(Integer)
|
||||
citation_count: Mapped[int] = mapped_column(
|
||||
Integer, nullable=False, server_default=text("0")
|
||||
)
|
||||
author_text: Mapped[str | None] = mapped_column(Text)
|
||||
venue_text: Mapped[str | None] = mapped_column(Text)
|
||||
pub_url: Mapped[str | None] = mapped_column(Text)
|
||||
pdf_url: Mapped[str | None] = mapped_column(Text)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
|
||||
|
||||
class PublicationIdentifier(Base):
|
||||
__tablename__ = "publication_identifiers"
|
||||
__table_args__ = (
|
||||
UniqueConstraint(
|
||||
"publication_id",
|
||||
"kind",
|
||||
"value_normalized",
|
||||
name="uq_publication_identifiers_publication_kind_value",
|
||||
),
|
||||
CheckConstraint(
|
||||
"confidence_score >= 0 AND confidence_score <= 1",
|
||||
name="publication_identifiers_confidence_score_range",
|
||||
),
|
||||
Index(
|
||||
"ix_publication_identifiers_kind_value",
|
||||
"kind",
|
||||
"value_normalized",
|
||||
),
|
||||
Index(
|
||||
"ix_publication_identifiers_publication_id",
|
||||
"publication_id",
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
publication_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("publications.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
)
|
||||
kind: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
value_raw: Mapped[str] = mapped_column(Text, nullable=False)
|
||||
value_normalized: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
source: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
confidence_score: Mapped[float] = mapped_column(
|
||||
Float,
|
||||
nullable=False,
|
||||
server_default=text("0"),
|
||||
)
|
||||
evidence_url: Mapped[str | None] = mapped_column(Text)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
|
||||
|
||||
class PublicationPdfJob(Base):
|
||||
__tablename__ = "publication_pdf_jobs"
|
||||
__table_args__ = (
|
||||
CheckConstraint(
|
||||
"status IN ('queued', 'running', 'resolved', 'failed')",
|
||||
name="publication_pdf_jobs_status_valid",
|
||||
),
|
||||
Index("ix_publication_pdf_jobs_status", "status"),
|
||||
Index("ix_publication_pdf_jobs_updated_at", "updated_at"),
|
||||
Index("ix_publication_pdf_jobs_queued_at", "queued_at"),
|
||||
)
|
||||
|
||||
publication_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("publications.id", ondelete="CASCADE"),
|
||||
primary_key=True,
|
||||
)
|
||||
status: Mapped[str] = mapped_column(
|
||||
String(16),
|
||||
nullable=False,
|
||||
server_default=text("'queued'"),
|
||||
)
|
||||
attempt_count: Mapped[int] = mapped_column(
|
||||
Integer,
|
||||
nullable=False,
|
||||
server_default=text("0"),
|
||||
)
|
||||
queued_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
last_attempt_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
resolved_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
last_failure_reason: Mapped[str | None] = mapped_column(String(64))
|
||||
last_failure_detail: Mapped[str | None] = mapped_column(Text)
|
||||
last_source: Mapped[str | None] = mapped_column(String(32))
|
||||
last_requested_by_user_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("users.id", ondelete="SET NULL"),
|
||||
)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
|
||||
|
||||
class PublicationPdfJobEvent(Base):
|
||||
__tablename__ = "publication_pdf_job_events"
|
||||
__table_args__ = (
|
||||
Index("ix_publication_pdf_job_events_publication_created", "publication_id", "created_at"),
|
||||
Index("ix_publication_pdf_job_events_created_at", "created_at"),
|
||||
Index("ix_publication_pdf_job_events_event_type", "event_type"),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
publication_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("publications.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
)
|
||||
user_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("users.id", ondelete="SET NULL"),
|
||||
)
|
||||
event_type: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
status: Mapped[str | None] = mapped_column(String(16))
|
||||
source: Mapped[str | None] = mapped_column(String(32))
|
||||
failure_reason: Mapped[str | None] = mapped_column(String(64))
|
||||
message: Mapped[str | None] = mapped_column(Text)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
|
||||
|
||||
class ScholarPublication(Base):
|
||||
__tablename__ = "scholar_publications"
|
||||
__table_args__ = (
|
||||
Index("ix_scholar_publications_is_read", "is_read"),
|
||||
Index("ix_scholar_publications_is_favorite", "is_favorite"),
|
||||
)
|
||||
|
||||
scholar_profile_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("scholar_profiles.id", ondelete="CASCADE"),
|
||||
primary_key=True,
|
||||
)
|
||||
publication_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("publications.id", ondelete="CASCADE"),
|
||||
primary_key=True,
|
||||
)
|
||||
is_read: Mapped[bool] = mapped_column(
|
||||
Boolean, nullable=False, server_default=text("false")
|
||||
)
|
||||
is_favorite: Mapped[bool] = mapped_column(
|
||||
Boolean, nullable=False, server_default=text("false")
|
||||
)
|
||||
first_seen_run_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("crawl_runs.id", ondelete="SET NULL")
|
||||
)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
|
||||
|
||||
class IngestionQueueItem(Base):
|
||||
__tablename__ = "ingestion_queue_items"
|
||||
__table_args__ = (
|
||||
UniqueConstraint(
|
||||
"user_id",
|
||||
"scholar_profile_id",
|
||||
name="uq_ingestion_queue_user_scholar",
|
||||
),
|
||||
CheckConstraint(
|
||||
"status IN ('queued', 'retrying', 'dropped')",
|
||||
name="ingestion_queue_status_valid",
|
||||
),
|
||||
Index("ix_ingestion_queue_next_attempt", "next_attempt_dt"),
|
||||
Index("ix_ingestion_queue_status_next_attempt", "status", "next_attempt_dt"),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
user_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("users.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
)
|
||||
scholar_profile_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("scholar_profiles.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
)
|
||||
resume_cstart: Mapped[int] = mapped_column(
|
||||
Integer,
|
||||
nullable=False,
|
||||
server_default=text("0"),
|
||||
)
|
||||
reason: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
status: Mapped[str] = mapped_column(
|
||||
String(16),
|
||||
nullable=False,
|
||||
server_default=text("'queued'"),
|
||||
)
|
||||
attempt_count: Mapped[int] = mapped_column(
|
||||
Integer,
|
||||
nullable=False,
|
||||
server_default=text("0"),
|
||||
)
|
||||
next_attempt_dt: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True),
|
||||
nullable=False,
|
||||
server_default=func.now(),
|
||||
)
|
||||
last_run_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("crawl_runs.id", ondelete="SET NULL"),
|
||||
)
|
||||
last_error: Mapped[str | None] = mapped_column(Text)
|
||||
dropped_reason: Mapped[str | None] = mapped_column(String(128))
|
||||
dropped_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
|
||||
|
||||
class DataRepairJob(Base):
|
||||
__tablename__ = "data_repair_jobs"
|
||||
__table_args__ = (
|
||||
CheckConstraint(
|
||||
"status IN ('planned', 'running', 'completed', 'failed')",
|
||||
name="data_repair_jobs_status_valid",
|
||||
),
|
||||
Index("ix_data_repair_jobs_created_at", "created_at"),
|
||||
Index("ix_data_repair_jobs_job_name_created_at", "job_name", "created_at"),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
job_name: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
requested_by: Mapped[str | None] = mapped_column(String(255))
|
||||
scope: Mapped[dict] = mapped_column(
|
||||
JSONB,
|
||||
nullable=False,
|
||||
server_default=text("'{}'::jsonb"),
|
||||
)
|
||||
dry_run: Mapped[bool] = mapped_column(
|
||||
Boolean,
|
||||
nullable=False,
|
||||
server_default=text("true"),
|
||||
)
|
||||
status: Mapped[str] = mapped_column(
|
||||
String(16),
|
||||
nullable=False,
|
||||
server_default=text("'planned'"),
|
||||
)
|
||||
summary: Mapped[dict] = mapped_column(
|
||||
JSONB,
|
||||
nullable=False,
|
||||
server_default=text("'{}'::jsonb"),
|
||||
)
|
||||
error_text: Mapped[str | None] = mapped_column(Text)
|
||||
started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
|
||||
|
||||
class AuthorSearchRuntimeState(Base):
|
||||
__tablename__ = "author_search_runtime_state"
|
||||
|
||||
state_key: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
last_live_request_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
cooldown_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
consecutive_blocked_count: Mapped[int] = mapped_column(
|
||||
Integer,
|
||||
nullable=False,
|
||||
server_default=text("0"),
|
||||
)
|
||||
cooldown_rejection_count: Mapped[int] = mapped_column(
|
||||
Integer,
|
||||
nullable=False,
|
||||
server_default=text("0"),
|
||||
)
|
||||
cooldown_alert_emitted: Mapped[bool] = mapped_column(
|
||||
Boolean,
|
||||
nullable=False,
|
||||
server_default=text("false"),
|
||||
)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
|
||||
|
||||
class AuthorSearchCacheEntry(Base):
|
||||
__tablename__ = "author_search_cache_entries"
|
||||
__table_args__ = (
|
||||
Index("ix_author_search_cache_expires_at", "expires_at"),
|
||||
Index("ix_author_search_cache_cached_at", "cached_at"),
|
||||
)
|
||||
|
||||
query_key: Mapped[str] = mapped_column(String(256), primary_key=True)
|
||||
payload: Mapped[dict] = mapped_column(
|
||||
JSONB,
|
||||
nullable=False,
|
||||
)
|
||||
expires_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True),
|
||||
nullable=False,
|
||||
)
|
||||
cached_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
|
|
@ -1,99 +0,0 @@
|
|||
from collections.abc import AsyncIterator
|
||||
import logging
|
||||
import os
|
||||
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.ext.asyncio import (
|
||||
AsyncEngine,
|
||||
AsyncSession,
|
||||
async_sessionmaker,
|
||||
create_async_engine,
|
||||
)
|
||||
from sqlalchemy.pool import NullPool
|
||||
|
||||
from app.settings import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_engine: AsyncEngine | None = None
|
||||
_session_factory: async_sessionmaker[AsyncSession] | None = None
|
||||
|
||||
|
||||
def _normalized_pool_mode(raw_mode: str) -> str:
|
||||
mode = (raw_mode or "").strip().lower()
|
||||
if mode == "auto":
|
||||
if os.getenv("PYTEST_CURRENT_TEST"):
|
||||
return "null"
|
||||
app_env = (os.getenv("APP_ENV") or "").strip().lower()
|
||||
if app_env in {"test", "development", "dev", "local"}:
|
||||
return "null"
|
||||
return "queue"
|
||||
if mode in {"null", "queue"}:
|
||||
return mode
|
||||
logger.warning(
|
||||
"db.invalid_pool_mode_fallback",
|
||||
extra={
|
||||
"event": "db.invalid_pool_mode_fallback",
|
||||
"database_pool_mode": raw_mode,
|
||||
"fallback_mode": "queue",
|
||||
},
|
||||
)
|
||||
return "queue"
|
||||
|
||||
|
||||
def get_engine() -> AsyncEngine:
|
||||
global _engine
|
||||
if _engine is None:
|
||||
pool_mode = _normalized_pool_mode(settings.database_pool_mode)
|
||||
engine_kwargs: dict[str, object] = {
|
||||
"pool_pre_ping": True,
|
||||
}
|
||||
if pool_mode == "null":
|
||||
engine_kwargs["poolclass"] = NullPool
|
||||
else:
|
||||
engine_kwargs["pool_size"] = max(1, int(settings.database_pool_size))
|
||||
engine_kwargs["max_overflow"] = max(0, int(settings.database_pool_max_overflow))
|
||||
engine_kwargs["pool_timeout"] = max(1, int(settings.database_pool_timeout_seconds))
|
||||
|
||||
_engine = create_async_engine(settings.database_url, **engine_kwargs)
|
||||
logger.info(
|
||||
"db.engine_initialized",
|
||||
extra={
|
||||
"event": "db.engine_initialized",
|
||||
"pool_mode": pool_mode,
|
||||
},
|
||||
)
|
||||
return _engine
|
||||
|
||||
|
||||
def get_session_factory() -> async_sessionmaker[AsyncSession]:
|
||||
global _session_factory
|
||||
if _session_factory is None:
|
||||
_session_factory = async_sessionmaker(get_engine(), expire_on_commit=False)
|
||||
return _session_factory
|
||||
|
||||
|
||||
async def get_db_session() -> AsyncIterator[AsyncSession]:
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as session:
|
||||
yield session
|
||||
|
||||
|
||||
async def check_database() -> bool:
|
||||
engine = get_engine()
|
||||
try:
|
||||
async with engine.connect() as conn:
|
||||
result = await conn.execute(text("SELECT 1"))
|
||||
return result.scalar_one() == 1
|
||||
except Exception:
|
||||
logger.exception("db.healthcheck_failed", extra={"event": "db.healthcheck_failed"})
|
||||
return False
|
||||
|
||||
|
||||
async def close_engine() -> None:
|
||||
global _engine, _session_factory
|
||||
if _engine is not None:
|
||||
await _engine.dispose()
|
||||
logger.info("db.engine_disposed", extra={"event": "db.engine_disposed"})
|
||||
_engine = None
|
||||
_session_factory = None
|
||||
|
|
@ -1 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
|
@ -1,205 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from secrets import token_urlsafe
|
||||
import logging
|
||||
import time
|
||||
|
||||
from starlette.middleware.base import BaseHTTPMiddleware
|
||||
from starlette.requests import Request
|
||||
from starlette.responses import Response
|
||||
|
||||
from app.logging_context import set_request_id
|
||||
|
||||
REQUEST_ID_HEADER = "X-Request-ID"
|
||||
DEFAULT_PERMISSIONS_POLICY = (
|
||||
"accelerometer=(), autoplay=(), camera=(), display-capture=(), "
|
||||
"geolocation=(), gyroscope=(), microphone=(), payment=(), usb=()"
|
||||
)
|
||||
DEFAULT_CSP_POLICY = (
|
||||
"default-src 'self'; "
|
||||
"base-uri 'self'; "
|
||||
"form-action 'self'; "
|
||||
"frame-ancestors 'none'; "
|
||||
"img-src 'self' data: https:; "
|
||||
"script-src 'self'; "
|
||||
"style-src 'self' 'unsafe-inline'; "
|
||||
"font-src 'self' data:; "
|
||||
"connect-src 'self'; "
|
||||
"object-src 'none'"
|
||||
)
|
||||
DEFAULT_CSP_DOCS_POLICY = (
|
||||
"default-src 'self'; "
|
||||
"img-src 'self' data: https:; "
|
||||
"script-src 'self' 'unsafe-inline'; "
|
||||
"style-src 'self' 'unsafe-inline'; "
|
||||
"font-src 'self' data:; "
|
||||
"connect-src 'self' https:; "
|
||||
"object-src 'none'; "
|
||||
"frame-ancestors 'none'"
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class RequestLoggingMiddleware(BaseHTTPMiddleware):
|
||||
def __init__(
|
||||
self,
|
||||
app,
|
||||
*,
|
||||
log_requests: bool = True,
|
||||
skip_paths: tuple[str, ...] = (),
|
||||
) -> None:
|
||||
super().__init__(app)
|
||||
self._log_requests = log_requests
|
||||
self._skip_paths = tuple(path for path in skip_paths if path)
|
||||
|
||||
async def dispatch(self, request: Request, call_next) -> Response:
|
||||
request_id = request.headers.get(REQUEST_ID_HEADER) or token_urlsafe(12)
|
||||
request.state.request_id = request_id
|
||||
set_request_id(request_id)
|
||||
|
||||
start = time.perf_counter()
|
||||
should_log = self._log_requests and not self._is_skipped_path(request.url.path)
|
||||
if should_log:
|
||||
logger.debug(
|
||||
"request.started",
|
||||
extra={
|
||||
"event": "request.started",
|
||||
"method": request.method,
|
||||
"path": request.url.path,
|
||||
},
|
||||
)
|
||||
|
||||
try:
|
||||
response = await call_next(request)
|
||||
except Exception:
|
||||
duration_ms = int((time.perf_counter() - start) * 1000)
|
||||
logger.exception(
|
||||
"request.failed",
|
||||
extra={
|
||||
"event": "request.failed",
|
||||
"method": request.method,
|
||||
"path": request.url.path,
|
||||
"duration_ms": duration_ms,
|
||||
},
|
||||
)
|
||||
raise
|
||||
else:
|
||||
duration_ms = int((time.perf_counter() - start) * 1000)
|
||||
response.headers[REQUEST_ID_HEADER] = request_id
|
||||
if should_log:
|
||||
logger.debug(
|
||||
"request.completed",
|
||||
extra={
|
||||
"event": "request.completed",
|
||||
"method": request.method,
|
||||
"path": request.url.path,
|
||||
"status_code": response.status_code,
|
||||
"duration_ms": duration_ms,
|
||||
},
|
||||
)
|
||||
return response
|
||||
finally:
|
||||
set_request_id(None)
|
||||
|
||||
def _is_skipped_path(self, path: str) -> bool:
|
||||
return any(path.startswith(prefix) for prefix in self._skip_paths)
|
||||
|
||||
|
||||
class SecurityHeadersMiddleware(BaseHTTPMiddleware):
|
||||
def __init__(
|
||||
self,
|
||||
app,
|
||||
*,
|
||||
enabled: bool = True,
|
||||
x_content_type_options: str = "nosniff",
|
||||
x_frame_options: str = "DENY",
|
||||
referrer_policy: str = "strict-origin-when-cross-origin",
|
||||
permissions_policy: str = DEFAULT_PERMISSIONS_POLICY,
|
||||
cross_origin_opener_policy: str = "same-origin",
|
||||
cross_origin_resource_policy: str = "same-origin",
|
||||
content_security_policy_enabled: bool = True,
|
||||
content_security_policy: str = DEFAULT_CSP_POLICY,
|
||||
content_security_policy_docs: str = DEFAULT_CSP_DOCS_POLICY,
|
||||
content_security_policy_report_only: bool = False,
|
||||
strict_transport_security_enabled: bool = False,
|
||||
strict_transport_security_max_age: int = 31_536_000,
|
||||
strict_transport_security_include_subdomains: bool = True,
|
||||
strict_transport_security_preload: bool = False,
|
||||
) -> None:
|
||||
super().__init__(app)
|
||||
self._enabled = enabled
|
||||
self._x_content_type_options = x_content_type_options.strip()
|
||||
self._x_frame_options = x_frame_options.strip()
|
||||
self._referrer_policy = referrer_policy.strip()
|
||||
self._permissions_policy = permissions_policy.strip()
|
||||
self._cross_origin_opener_policy = cross_origin_opener_policy.strip()
|
||||
self._cross_origin_resource_policy = cross_origin_resource_policy.strip()
|
||||
self._csp_enabled = content_security_policy_enabled
|
||||
self._csp_policy = content_security_policy.strip()
|
||||
self._csp_docs_policy = content_security_policy_docs.strip()
|
||||
self._csp_report_only = content_security_policy_report_only
|
||||
self._hsts_enabled = strict_transport_security_enabled
|
||||
self._hsts_max_age = max(0, strict_transport_security_max_age)
|
||||
self._hsts_include_subdomains = strict_transport_security_include_subdomains
|
||||
self._hsts_preload = strict_transport_security_preload
|
||||
|
||||
async def dispatch(self, request: Request, call_next) -> Response:
|
||||
response = await call_next(request)
|
||||
if not self._enabled:
|
||||
return response
|
||||
|
||||
if self._x_content_type_options:
|
||||
response.headers.setdefault("X-Content-Type-Options", self._x_content_type_options)
|
||||
if self._x_frame_options:
|
||||
response.headers.setdefault("X-Frame-Options", self._x_frame_options)
|
||||
if self._referrer_policy:
|
||||
response.headers.setdefault("Referrer-Policy", self._referrer_policy)
|
||||
if self._permissions_policy:
|
||||
response.headers.setdefault("Permissions-Policy", self._permissions_policy)
|
||||
if self._cross_origin_opener_policy:
|
||||
response.headers.setdefault(
|
||||
"Cross-Origin-Opener-Policy",
|
||||
self._cross_origin_opener_policy,
|
||||
)
|
||||
if self._cross_origin_resource_policy:
|
||||
response.headers.setdefault(
|
||||
"Cross-Origin-Resource-Policy",
|
||||
self._cross_origin_resource_policy,
|
||||
)
|
||||
|
||||
csp_policy = self._csp_policy_for_path(request.url.path)
|
||||
if self._csp_enabled and csp_policy:
|
||||
csp_header = (
|
||||
"Content-Security-Policy-Report-Only"
|
||||
if self._csp_report_only
|
||||
else "Content-Security-Policy"
|
||||
)
|
||||
response.headers.setdefault(csp_header, csp_policy)
|
||||
|
||||
hsts = self._strict_transport_security_value()
|
||||
if hsts:
|
||||
response.headers.setdefault("Strict-Transport-Security", hsts)
|
||||
|
||||
return response
|
||||
|
||||
def _csp_policy_for_path(self, path: str) -> str:
|
||||
if path.startswith("/docs") or path.startswith("/redoc"):
|
||||
return self._csp_docs_policy or self._csp_policy
|
||||
return self._csp_policy
|
||||
|
||||
def _strict_transport_security_value(self) -> str:
|
||||
if not self._hsts_enabled:
|
||||
return ""
|
||||
|
||||
directives = [f"max-age={self._hsts_max_age}"]
|
||||
if self._hsts_include_subdomains:
|
||||
directives.append("includeSubDomains")
|
||||
if self._hsts_preload:
|
||||
directives.append("preload")
|
||||
return "; ".join(directives)
|
||||
|
||||
|
||||
def parse_skip_paths(raw_value: str) -> tuple[str, ...]:
|
||||
parts = [part.strip() for part in raw_value.split(",")]
|
||||
return tuple(part for part in parts if part)
|
||||
|
|
@ -1,183 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
import json
|
||||
import logging
|
||||
import sys
|
||||
from typing import Any
|
||||
|
||||
from app.logging_context import get_request_id
|
||||
|
||||
DEFAULT_REDACT_FIELDS = {
|
||||
"authorization",
|
||||
"cookie",
|
||||
"csrf_token",
|
||||
"new_password",
|
||||
"password",
|
||||
"password_hash",
|
||||
"session",
|
||||
"session_secret_key",
|
||||
}
|
||||
|
||||
_BASE_RECORD = logging.makeLogRecord({})
|
||||
_STANDARD_RECORD_FIELDS = set(_BASE_RECORD.__dict__.keys()) | {"message", "asctime"}
|
||||
_NOISY_RECORD_FIELDS = {"color_message"}
|
||||
|
||||
|
||||
def parse_redact_fields(raw: str | None) -> set[str]:
|
||||
fields = {field.strip().lower() for field in (raw or "").split(",") if field.strip()}
|
||||
return DEFAULT_REDACT_FIELDS | fields
|
||||
|
||||
|
||||
def configure_logging(
|
||||
*,
|
||||
level: str,
|
||||
log_format: str,
|
||||
redact_fields: set[str],
|
||||
include_uvicorn_access: bool,
|
||||
) -> None:
|
||||
normalized_level = _normalize_level(level)
|
||||
root_logger = logging.getLogger()
|
||||
root_logger.handlers.clear()
|
||||
root_logger.setLevel(normalized_level)
|
||||
|
||||
handler = logging.StreamHandler(stream=sys.stdout)
|
||||
handler.setLevel(normalized_level)
|
||||
handler.addFilter(RequestContextFilter())
|
||||
|
||||
normalized_format = log_format.strip().lower()
|
||||
if normalized_format == "json":
|
||||
handler.setFormatter(JsonLogFormatter(redact_fields=redact_fields))
|
||||
else:
|
||||
handler.setFormatter(ConsoleLogFormatter(redact_fields=redact_fields))
|
||||
|
||||
root_logger.addHandler(handler)
|
||||
|
||||
# Route server/framework logs through our single root handler.
|
||||
for logger_name in ("uvicorn", "uvicorn.error"):
|
||||
framework_logger = logging.getLogger(logger_name)
|
||||
framework_logger.handlers.clear()
|
||||
framework_logger.propagate = True
|
||||
framework_logger.setLevel(normalized_level)
|
||||
|
||||
access_logger = logging.getLogger("uvicorn.access")
|
||||
access_logger.handlers.clear()
|
||||
access_logger.propagate = True
|
||||
access_logger.setLevel(normalized_level if include_uvicorn_access else logging.WARNING)
|
||||
|
||||
|
||||
class RequestContextFilter(logging.Filter):
|
||||
def filter(self, record: logging.LogRecord) -> bool:
|
||||
if not getattr(record, "request_id", None):
|
||||
request_id = get_request_id()
|
||||
if request_id:
|
||||
record.request_id = request_id
|
||||
return True
|
||||
|
||||
|
||||
class JsonLogFormatter(logging.Formatter):
|
||||
def __init__(self, *, redact_fields: set[str]) -> None:
|
||||
super().__init__()
|
||||
self._redact_fields = {field.lower() for field in redact_fields}
|
||||
|
||||
def format(self, record: logging.LogRecord) -> str:
|
||||
payload: dict[str, Any] = {
|
||||
"timestamp": _format_timestamp(record.created),
|
||||
"level": record.levelname.lower(),
|
||||
"logger": record.name,
|
||||
"event": getattr(record, "event", record.getMessage()),
|
||||
}
|
||||
request_id = getattr(record, "request_id", None)
|
||||
if request_id:
|
||||
payload["request_id"] = request_id
|
||||
payload.update(self._redact_mapping(_extra_fields(record)))
|
||||
if record.exc_info:
|
||||
payload["exception"] = self.formatException(record.exc_info)
|
||||
return json.dumps(payload, ensure_ascii=True, default=str)
|
||||
|
||||
def _redact_mapping(self, value: dict[str, Any]) -> dict[str, Any]:
|
||||
return {key: self._redact_value(key, item) for key, item in value.items()}
|
||||
|
||||
def _redact_value(self, key: str, value: Any) -> Any:
|
||||
if key.lower() in self._redact_fields:
|
||||
return "[REDACTED]"
|
||||
if isinstance(value, dict):
|
||||
return {nested_key: self._redact_value(nested_key, nested_value) for nested_key, nested_value in value.items()}
|
||||
if isinstance(value, (list, tuple)):
|
||||
return [self._redact_value(key, item) for item in value]
|
||||
return value
|
||||
|
||||
|
||||
class ConsoleLogFormatter(logging.Formatter):
|
||||
def __init__(self, *, redact_fields: set[str]) -> None:
|
||||
super().__init__()
|
||||
self._json_formatter = JsonLogFormatter(redact_fields=redact_fields)
|
||||
|
||||
def format(self, record: logging.LogRecord) -> str:
|
||||
payload = json.loads(self._json_formatter.format(record))
|
||||
timestamp = payload.get("timestamp", "")
|
||||
level = _short_level(payload.get("level", "info"))
|
||||
logger_name = str(payload.get("logger", "app"))
|
||||
event = str(payload.get("event", ""))
|
||||
|
||||
parts = [timestamp, level, logger_name, event]
|
||||
|
||||
request_id = payload.pop("request_id", None)
|
||||
method = payload.pop("method", None)
|
||||
path = payload.pop("path", None)
|
||||
status_code = payload.pop("status_code", None)
|
||||
duration_ms = payload.pop("duration_ms", None)
|
||||
|
||||
if request_id:
|
||||
parts.append(f"rid={request_id}")
|
||||
if method and path:
|
||||
parts.append(f"{method} {path}")
|
||||
if status_code is not None:
|
||||
parts.append(str(status_code))
|
||||
if duration_ms is not None:
|
||||
parts.append(f"{duration_ms}ms")
|
||||
|
||||
for key in sorted(payload.keys()):
|
||||
if key in {"timestamp", "level", "logger", "event", "exception"}:
|
||||
continue
|
||||
parts.append(f"{key}={payload[key]}")
|
||||
|
||||
if "exception" in payload:
|
||||
parts.append(f"exception={payload['exception']}")
|
||||
|
||||
return " | ".join(str(part) for part in parts if part)
|
||||
|
||||
|
||||
def _extra_fields(record: logging.LogRecord) -> dict[str, Any]:
|
||||
extras: dict[str, Any] = {}
|
||||
for key, value in record.__dict__.items():
|
||||
if key in _STANDARD_RECORD_FIELDS or key.startswith("_"):
|
||||
continue
|
||||
if key in _NOISY_RECORD_FIELDS:
|
||||
continue
|
||||
extras[key] = value
|
||||
return extras
|
||||
|
||||
|
||||
def _normalize_level(level: str) -> int:
|
||||
normalized = level.strip().upper()
|
||||
mapping = logging.getLevelNamesMapping()
|
||||
if normalized not in mapping:
|
||||
return logging.INFO
|
||||
return mapping[normalized]
|
||||
|
||||
|
||||
def _format_timestamp(created_ts: float) -> str:
|
||||
dt = datetime.fromtimestamp(created_ts, tz=timezone.utc)
|
||||
return dt.strftime("%Y-%m-%d %H:%M:%SZ")
|
||||
|
||||
|
||||
def _short_level(level: str) -> str:
|
||||
mapping = {
|
||||
"debug": "DBG",
|
||||
"info": "INF",
|
||||
"warning": "WRN",
|
||||
"error": "ERR",
|
||||
"critical": "CRT",
|
||||
}
|
||||
return mapping.get(level.lower(), level[:3].upper())
|
||||
|
|
@ -1,15 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from contextvars import ContextVar
|
||||
|
||||
|
||||
_request_id_ctx: ContextVar[str | None] = ContextVar("request_id", default=None)
|
||||
|
||||
|
||||
def get_request_id() -> str | None:
|
||||
return _request_id_ctx.get()
|
||||
|
||||
|
||||
def set_request_id(value: str | None) -> None:
|
||||
_request_id_ctx.set(value)
|
||||
|
||||
|
|
@ -1,168 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from contextlib import asynccontextmanager
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import FastAPI, HTTPException
|
||||
from fastapi.responses import FileResponse
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
from starlette.middleware.sessions import SessionMiddleware
|
||||
|
||||
from app.api.errors import register_api_exception_handlers
|
||||
from app.api.media import router as media_router
|
||||
from app.api.router import router as api_router
|
||||
from app.api.runtime_deps import get_ingestion_service, get_scholar_source
|
||||
from app.db.session import check_database
|
||||
from app.db.session import close_engine
|
||||
from app.http.middleware import (
|
||||
RequestLoggingMiddleware,
|
||||
SecurityHeadersMiddleware,
|
||||
parse_skip_paths,
|
||||
)
|
||||
from app.logging_config import configure_logging, parse_redact_fields
|
||||
from app.security.csrf import CSRFMiddleware
|
||||
from app.services.domains.ingestion.scheduler import SchedulerService
|
||||
from app.settings import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
BUILD_MARKER = "2026-02-19.phase2.direct-pdf-import-export-dashboard-sync"
|
||||
|
||||
configure_logging(
|
||||
level=settings.log_level,
|
||||
log_format=settings.log_format,
|
||||
redact_fields=parse_redact_fields(settings.log_redact_fields),
|
||||
include_uvicorn_access=settings.log_uvicorn_access,
|
||||
)
|
||||
|
||||
scheduler_service = SchedulerService(
|
||||
enabled=settings.scheduler_enabled,
|
||||
tick_seconds=settings.scheduler_tick_seconds,
|
||||
network_error_retries=settings.ingestion_network_error_retries,
|
||||
retry_backoff_seconds=settings.ingestion_retry_backoff_seconds,
|
||||
max_pages_per_scholar=settings.ingestion_max_pages_per_scholar,
|
||||
page_size=settings.ingestion_page_size,
|
||||
continuation_queue_enabled=settings.ingestion_continuation_queue_enabled,
|
||||
continuation_base_delay_seconds=settings.ingestion_continuation_base_delay_seconds,
|
||||
continuation_max_delay_seconds=settings.ingestion_continuation_max_delay_seconds,
|
||||
continuation_max_attempts=settings.ingestion_continuation_max_attempts,
|
||||
queue_batch_size=settings.scheduler_queue_batch_size,
|
||||
)
|
||||
|
||||
|
||||
def _log_startup_build_marker() -> None:
|
||||
logger.info(
|
||||
"app.startup_build_marker",
|
||||
extra={
|
||||
"event": "app.startup_build_marker",
|
||||
"build_marker": BUILD_MARKER,
|
||||
"frontend_enabled": settings.frontend_enabled,
|
||||
"scheduler_enabled": settings.scheduler_enabled,
|
||||
"log_format": settings.log_format,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(_: FastAPI):
|
||||
_log_startup_build_marker()
|
||||
|
||||
from app.db.session import get_session_factory
|
||||
from sqlalchemy import text
|
||||
try:
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as session:
|
||||
await session.execute(text("UPDATE crawl_runs SET status = 'failed' WHERE status = 'running'"))
|
||||
await session.commit()
|
||||
logger.info("app.startup_orphaned_runs_cleaned", extra={"event": "app.startup_orphaned_runs_cleaned"})
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to clean orphaned runs at startup: {e}")
|
||||
|
||||
await scheduler_service.start()
|
||||
yield
|
||||
await scheduler_service.stop()
|
||||
await close_engine()
|
||||
|
||||
|
||||
app = FastAPI(title=settings.app_name, lifespan=lifespan)
|
||||
register_api_exception_handlers(app)
|
||||
app.add_middleware(CSRFMiddleware)
|
||||
app.add_middleware(
|
||||
SessionMiddleware,
|
||||
secret_key=settings.session_secret_key,
|
||||
same_site="lax",
|
||||
https_only=settings.session_cookie_secure,
|
||||
)
|
||||
app.add_middleware(
|
||||
RequestLoggingMiddleware,
|
||||
log_requests=settings.log_requests,
|
||||
skip_paths=parse_skip_paths(settings.log_request_skip_paths),
|
||||
)
|
||||
app.add_middleware(
|
||||
SecurityHeadersMiddleware,
|
||||
enabled=settings.security_headers_enabled,
|
||||
x_content_type_options=settings.security_x_content_type_options,
|
||||
x_frame_options=settings.security_x_frame_options,
|
||||
referrer_policy=settings.security_referrer_policy,
|
||||
permissions_policy=settings.security_permissions_policy,
|
||||
cross_origin_opener_policy=settings.security_cross_origin_opener_policy,
|
||||
cross_origin_resource_policy=settings.security_cross_origin_resource_policy,
|
||||
content_security_policy_enabled=settings.security_csp_enabled,
|
||||
content_security_policy=settings.security_csp_policy,
|
||||
content_security_policy_docs=settings.security_csp_docs_policy,
|
||||
content_security_policy_report_only=settings.security_csp_report_only,
|
||||
strict_transport_security_enabled=settings.security_strict_transport_security_enabled,
|
||||
strict_transport_security_max_age=settings.security_strict_transport_security_max_age,
|
||||
strict_transport_security_include_subdomains=(
|
||||
settings.security_strict_transport_security_include_subdomains
|
||||
),
|
||||
strict_transport_security_preload=settings.security_strict_transport_security_preload,
|
||||
)
|
||||
app.include_router(api_router)
|
||||
app.include_router(media_router)
|
||||
|
||||
|
||||
@app.get("/healthz")
|
||||
async def healthz() -> dict[str, str]:
|
||||
if await check_database():
|
||||
return {"status": "ok"}
|
||||
raise HTTPException(status_code=500, detail="database unavailable")
|
||||
|
||||
|
||||
def _configure_frontend_routes(application: FastAPI) -> None:
|
||||
if not settings.frontend_enabled:
|
||||
return
|
||||
|
||||
dist_dir = Path(settings.frontend_dist_dir)
|
||||
index_file = dist_dir / "index.html"
|
||||
|
||||
if not index_file.is_file():
|
||||
return
|
||||
|
||||
assets_dir = dist_dir / "assets"
|
||||
if assets_dir.is_dir():
|
||||
application.mount("/assets", StaticFiles(directory=assets_dir), name="frontend-assets")
|
||||
|
||||
@application.get("/", include_in_schema=False)
|
||||
async def frontend_index() -> FileResponse:
|
||||
return FileResponse(index_file)
|
||||
|
||||
@application.get("/{full_path:path}", include_in_schema=False)
|
||||
async def frontend_entry(full_path: str) -> FileResponse:
|
||||
normalized = full_path.lstrip("/")
|
||||
if normalized in {"healthz", "api"} or normalized.startswith("api/"):
|
||||
raise HTTPException(status_code=404, detail="Not Found")
|
||||
|
||||
candidate = (dist_dir / normalized).resolve()
|
||||
try:
|
||||
candidate.relative_to(dist_dir.resolve())
|
||||
except ValueError as error:
|
||||
raise HTTPException(status_code=404, detail="Not Found") from error
|
||||
|
||||
if candidate.is_file():
|
||||
return FileResponse(candidate)
|
||||
|
||||
return FileResponse(index_file)
|
||||
|
||||
|
||||
_configure_frontend_routes(app)
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
"""Security middleware and helpers for scholarr."""
|
||||
|
||||
|
|
@ -1,133 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from secrets import compare_digest, token_urlsafe
|
||||
from urllib.parse import parse_qs
|
||||
|
||||
from starlette.middleware.base import BaseHTTPMiddleware
|
||||
from starlette.requests import Request
|
||||
from starlette.responses import JSONResponse, PlainTextResponse, Response
|
||||
from starlette.types import Message
|
||||
|
||||
|
||||
CSRF_SESSION_KEY = "csrf_token"
|
||||
CSRF_FORM_FIELD = "csrf_token"
|
||||
CSRF_HEADER_NAME = "x-csrf-token"
|
||||
SAFE_METHODS = {"GET", "HEAD", "OPTIONS", "TRACE"}
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def ensure_csrf_token(request: Request) -> str:
|
||||
token = request.session.get(CSRF_SESSION_KEY)
|
||||
if token is None:
|
||||
token = token_urlsafe(32)
|
||||
request.session[CSRF_SESSION_KEY] = token
|
||||
return str(token)
|
||||
|
||||
|
||||
class CSRFMiddleware(BaseHTTPMiddleware):
|
||||
def __init__(self, app, *, exempt_paths: set[str] | None = None) -> None:
|
||||
super().__init__(app)
|
||||
self._exempt_paths = exempt_paths or set()
|
||||
|
||||
async def dispatch(self, request: Request, call_next) -> Response:
|
||||
if self._should_skip(request):
|
||||
return await call_next(request)
|
||||
|
||||
session_token = request.session.get(CSRF_SESSION_KEY)
|
||||
if not session_token:
|
||||
logger.warning(
|
||||
"csrf.missing_session_token",
|
||||
extra={
|
||||
"event": "csrf.missing_session_token",
|
||||
"method": request.method,
|
||||
"path": request.url.path,
|
||||
},
|
||||
)
|
||||
return self._csrf_error_response(
|
||||
request,
|
||||
code="csrf_missing",
|
||||
message="CSRF token missing.",
|
||||
)
|
||||
|
||||
request_token = request.headers.get(CSRF_HEADER_NAME)
|
||||
if request_token is None and self._is_form_payload(request):
|
||||
body = await request.body()
|
||||
request_token = self._token_from_form_body(request, body)
|
||||
request._receive = self._build_receive(body) # type: ignore[attr-defined]
|
||||
|
||||
if not request_token or not compare_digest(str(session_token), str(request_token)):
|
||||
logger.warning(
|
||||
"csrf.invalid_token",
|
||||
extra={
|
||||
"event": "csrf.invalid_token",
|
||||
"method": request.method,
|
||||
"path": request.url.path,
|
||||
},
|
||||
)
|
||||
return self._csrf_error_response(
|
||||
request,
|
||||
code="csrf_invalid",
|
||||
message="CSRF token invalid.",
|
||||
)
|
||||
|
||||
return await call_next(request)
|
||||
|
||||
def _should_skip(self, request: Request) -> bool:
|
||||
if request.method in SAFE_METHODS:
|
||||
return True
|
||||
path = request.url.path
|
||||
return path in self._exempt_paths
|
||||
|
||||
def _is_form_payload(self, request: Request) -> bool:
|
||||
content_type = request.headers.get("content-type", "")
|
||||
return content_type.startswith("application/x-www-form-urlencoded") or (
|
||||
content_type.startswith("multipart/form-data")
|
||||
)
|
||||
|
||||
def _token_from_form_body(self, request: Request, body: bytes) -> str | None:
|
||||
content_type = request.headers.get("content-type", "")
|
||||
if not content_type.startswith("application/x-www-form-urlencoded"):
|
||||
return None
|
||||
parsed = parse_qs(body.decode("utf-8"), keep_blank_values=True)
|
||||
values = parsed.get(CSRF_FORM_FIELD)
|
||||
if not values:
|
||||
return None
|
||||
return values[0]
|
||||
|
||||
def _build_receive(self, body: bytes):
|
||||
consumed = False
|
||||
|
||||
async def receive() -> Message:
|
||||
nonlocal consumed
|
||||
if consumed:
|
||||
return {"type": "http.request", "body": b"", "more_body": False}
|
||||
consumed = True
|
||||
return {"type": "http.request", "body": body, "more_body": False}
|
||||
|
||||
return receive
|
||||
|
||||
def _csrf_error_response(
|
||||
self,
|
||||
request: Request,
|
||||
*,
|
||||
code: str,
|
||||
message: str,
|
||||
) -> Response:
|
||||
if request.url.path.startswith("/api/"):
|
||||
request_state = getattr(request, "state", None)
|
||||
request_id = getattr(request_state, "request_id", None) if request_state else None
|
||||
return JSONResponse(
|
||||
{
|
||||
"error": {
|
||||
"code": code,
|
||||
"message": message,
|
||||
"details": None,
|
||||
},
|
||||
"meta": {
|
||||
"request_id": request_id,
|
||||
},
|
||||
},
|
||||
status_code=403,
|
||||
)
|
||||
return PlainTextResponse(message, status_code=403)
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
"""Service layer for scholarr application workflows."""
|
||||
|
||||
|
|
@ -1 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
|
@ -1,84 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from typing import TYPE_CHECKING
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
import httpx
|
||||
|
||||
from app.services.domains.publication_identifiers.normalize import normalize_arxiv_id
|
||||
from app.settings import settings
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _build_arxiv_query(title: str, author_surname: str | None) -> str | None:
|
||||
parts = []
|
||||
if title:
|
||||
# arXiv api allows strict title searching using ti:
|
||||
clean_title = title.replace('"', '').replace("'", "")
|
||||
parts.append(f'ti:"{clean_title}"')
|
||||
if author_surname:
|
||||
parts.append(f'au:"{author_surname}"')
|
||||
|
||||
if not parts:
|
||||
return None
|
||||
return " AND ".join(parts)
|
||||
|
||||
|
||||
async def discover_arxiv_id_for_publication(
|
||||
*,
|
||||
item: PublicationListItem | UnreadPublicationItem,
|
||||
request_email: str | None = None,
|
||||
timeout_seconds: float = 3.0,
|
||||
) -> str | None:
|
||||
title = (item.title or "").strip()
|
||||
if not title:
|
||||
return None
|
||||
|
||||
author_surname = None
|
||||
if item.scholar_label:
|
||||
tokens = [t for t in item.scholar_label.strip().split() if t]
|
||||
if tokens:
|
||||
author_surname = tokens[-1].lower()
|
||||
|
||||
query = _build_arxiv_query(title, author_surname)
|
||||
if not query:
|
||||
return None
|
||||
|
||||
url = "https://export.arxiv.org/api/query"
|
||||
params = {
|
||||
"search_query": query,
|
||||
"start": 0,
|
||||
"max_results": 3,
|
||||
}
|
||||
|
||||
headers = {"User-Agent": f"scholar-scraper/1.0 (mailto:{request_email or settings.crossref_api_mailto or 'unknown@example.com'})"}
|
||||
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=timeout_seconds, follow_redirects=True, headers=headers) as client:
|
||||
response = await client.get(url, params=params)
|
||||
response.raise_for_status()
|
||||
|
||||
root = ET.fromstring(response.text)
|
||||
namespace = {'atom': 'http://www.w3.org/2005/Atom'}
|
||||
|
||||
for entry in root.findall('atom:entry', namespace):
|
||||
id_elem = entry.find('atom:id', namespace)
|
||||
if id_elem is not None and id_elem.text:
|
||||
candidate = str(id_elem.text)
|
||||
if '/abs/' in candidate:
|
||||
candidate = candidate.split('/abs/')[-1]
|
||||
normalized = normalize_arxiv_id(candidate)
|
||||
if normalized:
|
||||
logger.debug("arxiv.id_discovered", extra={"event": "arxiv.id_discovered", "arxiv_id": normalized})
|
||||
return normalized
|
||||
|
||||
except Exception as exc:
|
||||
logger.debug(f"Failed to query arXiv API: {exc}")
|
||||
|
||||
return None
|
||||
|
|
@ -1,5 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from app.services.domains.crossref.application import discover_doi_for_publication
|
||||
|
||||
__all__ = ["discover_doi_for_publication"]
|
||||
|
|
@ -1,382 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import re
|
||||
import threading
|
||||
import time
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from crossref.restful import Etiquette, Works
|
||||
|
||||
from app.services.domains.doi.normalize import normalize_doi
|
||||
from app.settings import settings
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem
|
||||
|
||||
TOKEN_RE = re.compile(r"[a-z0-9]+")
|
||||
NON_ALNUM_RE = re.compile(r"[^a-z0-9\\s]+")
|
||||
STOP_WORDS = {"the", "and", "for", "with", "from", "method", "study", "analysis"}
|
||||
_RATE_LOCK = threading.Lock()
|
||||
_LAST_REQUEST_AT = 0.0
|
||||
logger = logging.getLogger(__name__)
|
||||
STRICT_TITLE_MATCH_THRESHOLD = 0.75
|
||||
RELAXED_TITLE_MATCH_THRESHOLD = 0.85
|
||||
|
||||
|
||||
def _rate_limit_wait(min_interval_seconds: float) -> None:
|
||||
global _LAST_REQUEST_AT
|
||||
interval = max(float(min_interval_seconds), 0.0)
|
||||
with _RATE_LOCK:
|
||||
elapsed = time.monotonic() - _LAST_REQUEST_AT
|
||||
remaining = interval - elapsed
|
||||
if remaining > 0:
|
||||
time.sleep(remaining)
|
||||
_LAST_REQUEST_AT = time.monotonic()
|
||||
|
||||
|
||||
def _normalized_tokens(value: str) -> list[str]:
|
||||
lowered = value.lower().replace("’", "'").replace("“", "\"").replace("”", "\"")
|
||||
lowered = NON_ALNUM_RE.sub(" ", lowered)
|
||||
return [token for token in TOKEN_RE.findall(lowered) if len(token) >= 3]
|
||||
|
||||
|
||||
def _normalized_query(value: str) -> str:
|
||||
tokens = [token for token in _normalized_tokens(value) if token not in STOP_WORDS]
|
||||
if len(tokens) < 3:
|
||||
tokens = _normalized_tokens(value)
|
||||
if len(tokens) < 3:
|
||||
return ""
|
||||
return " ".join(tokens[:12]).strip()
|
||||
|
||||
|
||||
def _query_author(value: str) -> str | None:
|
||||
tokens = [token for token in value.strip().split() if token]
|
||||
if len(tokens) < 2:
|
||||
return None
|
||||
return " ".join(tokens[:2])[:64]
|
||||
|
||||
|
||||
def _author_surname(value: str) -> str | None:
|
||||
tokens = [token for token in value.strip().split() if token]
|
||||
if not tokens:
|
||||
return None
|
||||
return NON_ALNUM_RE.sub("", tokens[-1].lower()) or None
|
||||
|
||||
|
||||
def _query_filters(year: int | None) -> list[tuple[str, str] | None]:
|
||||
if year is None:
|
||||
return [None]
|
||||
return [
|
||||
(f"{year - 1}-01-01", f"{year + 1}-12-31"),
|
||||
(f"{year}-01-01", f"{year}-12-31"),
|
||||
None,
|
||||
]
|
||||
|
||||
|
||||
def _candidate_title(item: dict) -> str:
|
||||
titles = item.get("title")
|
||||
if isinstance(titles, list) and titles:
|
||||
return str(titles[0] or "")
|
||||
return str(item.get("title") or "")
|
||||
|
||||
|
||||
def _title_match_score(source: str, candidate: str) -> float:
|
||||
source_tokens = {token for token in _normalized_tokens(source) if len(token) >= 3}
|
||||
candidate_tokens = {token for token in _normalized_tokens(candidate) if len(token) >= 3}
|
||||
if not source_tokens or not candidate_tokens:
|
||||
return 0.0
|
||||
return len(source_tokens & candidate_tokens) / float(len(source_tokens))
|
||||
|
||||
|
||||
def _candidate_year(item: dict) -> int | None:
|
||||
issued = item.get("issued")
|
||||
if not isinstance(issued, dict):
|
||||
return None
|
||||
date_parts = issued.get("date-parts")
|
||||
if not isinstance(date_parts, list) or not date_parts:
|
||||
return None
|
||||
first = date_parts[0]
|
||||
if not isinstance(first, list) or not first:
|
||||
return None
|
||||
try:
|
||||
return int(first[0])
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _candidate_author_match(item: dict, surname: str | None) -> bool:
|
||||
if not surname:
|
||||
return True
|
||||
authors = item.get("author")
|
||||
if not isinstance(authors, list):
|
||||
return False
|
||||
for author in authors:
|
||||
if not isinstance(author, dict):
|
||||
continue
|
||||
family = NON_ALNUM_RE.sub("", str(author.get("family") or "").lower())
|
||||
if family and family == surname:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _candidate_rank(*, title: str, year: int | None, item: dict) -> tuple[float, str | None]:
|
||||
doi = normalize_doi(str(item.get("DOI") or ""))
|
||||
if doi is None:
|
||||
return 0.0, None
|
||||
score = _title_match_score(title, _candidate_title(item))
|
||||
candidate_year = _candidate_year(item)
|
||||
if year is not None and candidate_year is not None:
|
||||
if abs(year - candidate_year) > 1:
|
||||
return 0.0, None
|
||||
score += 0.1
|
||||
return score, doi
|
||||
|
||||
|
||||
def _year_delta(source_year: int | None, candidate_year: int | None) -> int | None:
|
||||
if source_year is None or candidate_year is None:
|
||||
return None
|
||||
return abs(int(source_year) - int(candidate_year))
|
||||
|
||||
|
||||
def _candidate_rank_relaxed(
|
||||
*,
|
||||
title: str,
|
||||
year: int | None,
|
||||
item: dict,
|
||||
author_surname: str | None,
|
||||
) -> tuple[float, str | None]:
|
||||
doi = normalize_doi(str(item.get("DOI") or ""))
|
||||
if doi is None:
|
||||
return 0.0, None
|
||||
score = _title_match_score(title, _candidate_title(item))
|
||||
if score <= 0:
|
||||
return 0.0, None
|
||||
candidate_year = _candidate_year(item)
|
||||
delta = _year_delta(year, candidate_year)
|
||||
if delta is not None:
|
||||
if delta <= 1:
|
||||
score += 0.05
|
||||
elif delta <= 3:
|
||||
score += 0.0
|
||||
elif delta <= 5:
|
||||
score -= 0.03
|
||||
else:
|
||||
score -= 0.08
|
||||
if _candidate_author_match(item, author_surname):
|
||||
score += 0.03
|
||||
return score, doi
|
||||
|
||||
|
||||
def _best_candidate_doi_strict(
|
||||
*,
|
||||
title: str,
|
||||
year: int | None,
|
||||
items: list[dict],
|
||||
author_surname: str | None,
|
||||
) -> str | None:
|
||||
best_score = 0.0
|
||||
best_doi: str | None = None
|
||||
best_year: int | None = None
|
||||
for item in items:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
if not _candidate_author_match(item, author_surname):
|
||||
continue
|
||||
score, doi = _candidate_rank(title=title, year=year, item=item)
|
||||
candidate_year = _candidate_year(item)
|
||||
if doi is None or score < STRICT_TITLE_MATCH_THRESHOLD:
|
||||
continue
|
||||
if score > best_score:
|
||||
best_score = score
|
||||
best_doi = doi
|
||||
best_year = candidate_year
|
||||
continue
|
||||
if abs(score - best_score) > 0.02:
|
||||
continue
|
||||
if best_year is None or candidate_year is None:
|
||||
continue
|
||||
if candidate_year < best_year:
|
||||
best_doi = doi
|
||||
best_year = candidate_year
|
||||
return best_doi
|
||||
|
||||
|
||||
def _best_candidate_doi_relaxed(
|
||||
*,
|
||||
title: str,
|
||||
year: int | None,
|
||||
items: list[dict],
|
||||
author_surname: str | None,
|
||||
) -> str | None:
|
||||
best_score = 0.0
|
||||
best_doi: str | None = None
|
||||
best_author_match = False
|
||||
best_delta: int | None = None
|
||||
best_year: int | None = None
|
||||
for item in items:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
score, doi = _candidate_rank_relaxed(
|
||||
title=title,
|
||||
year=year,
|
||||
item=item,
|
||||
author_surname=author_surname,
|
||||
)
|
||||
if doi is None or score < RELAXED_TITLE_MATCH_THRESHOLD:
|
||||
continue
|
||||
candidate_year = _candidate_year(item)
|
||||
candidate_author_match = _candidate_author_match(item, author_surname)
|
||||
candidate_delta = _year_delta(year, candidate_year)
|
||||
if score > best_score:
|
||||
best_score = score
|
||||
best_doi = doi
|
||||
best_author_match = candidate_author_match
|
||||
best_delta = candidate_delta
|
||||
best_year = candidate_year
|
||||
continue
|
||||
if abs(score - best_score) > 0.02:
|
||||
continue
|
||||
if candidate_author_match and not best_author_match:
|
||||
best_doi = doi
|
||||
best_author_match = True
|
||||
best_delta = candidate_delta
|
||||
best_year = candidate_year
|
||||
continue
|
||||
if best_delta is None and candidate_delta is not None:
|
||||
best_doi = doi
|
||||
best_author_match = candidate_author_match
|
||||
best_delta = candidate_delta
|
||||
best_year = candidate_year
|
||||
continue
|
||||
if best_delta is not None and candidate_delta is not None and candidate_delta < best_delta:
|
||||
best_doi = doi
|
||||
best_author_match = candidate_author_match
|
||||
best_delta = candidate_delta
|
||||
best_year = candidate_year
|
||||
continue
|
||||
if best_year is None or candidate_year is None:
|
||||
continue
|
||||
if candidate_year < best_year:
|
||||
best_doi = doi
|
||||
best_author_match = candidate_author_match
|
||||
best_delta = candidate_delta
|
||||
best_year = candidate_year
|
||||
return best_doi
|
||||
|
||||
|
||||
def _best_candidate_doi(
|
||||
*,
|
||||
title: str,
|
||||
year: int | None,
|
||||
items: list[dict],
|
||||
author_surname: str | None,
|
||||
) -> str | None:
|
||||
strict_match = _best_candidate_doi_strict(
|
||||
title=title,
|
||||
year=year,
|
||||
items=items,
|
||||
author_surname=author_surname,
|
||||
)
|
||||
if strict_match:
|
||||
return strict_match
|
||||
return _best_candidate_doi_relaxed(
|
||||
title=title,
|
||||
year=year,
|
||||
items=items,
|
||||
author_surname=author_surname,
|
||||
)
|
||||
|
||||
|
||||
def _works_client(email: str | None) -> Works:
|
||||
if email:
|
||||
etiquette = Etiquette(settings.app_name, "0.1.0", "https://scholarr.local", email)
|
||||
return Works(etiquette=etiquette)
|
||||
return Works()
|
||||
|
||||
|
||||
def _fetch_items_sync(
|
||||
*,
|
||||
query: str,
|
||||
author: str | None,
|
||||
date_range: tuple[str, str] | None,
|
||||
max_rows: int,
|
||||
email: str | None,
|
||||
min_interval_seconds: float,
|
||||
) -> list[dict]:
|
||||
_rate_limit_wait(min_interval_seconds)
|
||||
works = _works_client(email)
|
||||
params = {"bibliographic": query}
|
||||
if author:
|
||||
params["author"] = author
|
||||
request = works.query(**params)
|
||||
if date_range is not None:
|
||||
from_date, until_date = date_range
|
||||
request = request.filter(from_pub_date=from_date, until_pub_date=until_date)
|
||||
request = request.select(["DOI", "title", "issued", "score", "author"])
|
||||
items: list[dict] = []
|
||||
for entry in request:
|
||||
if isinstance(entry, dict):
|
||||
items.append(entry)
|
||||
if len(items) >= max(max_rows, 1):
|
||||
break
|
||||
return items
|
||||
|
||||
|
||||
async def _fetch_items(
|
||||
*,
|
||||
query: str,
|
||||
author: str | None,
|
||||
date_range: tuple[str, str] | None,
|
||||
max_rows: int,
|
||||
email: str | None,
|
||||
) -> list[dict]:
|
||||
timeout = max(float(settings.crossref_timeout_seconds), 0.5)
|
||||
try:
|
||||
return await asyncio.wait_for(
|
||||
asyncio.to_thread(
|
||||
_fetch_items_sync,
|
||||
query=query,
|
||||
author=author,
|
||||
date_range=date_range,
|
||||
max_rows=max_rows,
|
||||
email=email,
|
||||
min_interval_seconds=settings.crossref_min_interval_seconds,
|
||||
),
|
||||
timeout=timeout,
|
||||
)
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
|
||||
async def discover_doi_for_publication(
|
||||
*,
|
||||
item: PublicationListItem | UnreadPublicationItem,
|
||||
max_rows: int = 10,
|
||||
email: str | None = None,
|
||||
) -> str | None:
|
||||
title = (item.title or "").strip()
|
||||
query = _normalized_query(title)
|
||||
if not query:
|
||||
return None
|
||||
author = _query_author(item.scholar_label)
|
||||
author_surname = _author_surname(item.scholar_label)
|
||||
for date_range in _query_filters(item.year):
|
||||
items = await _fetch_items(
|
||||
query=query,
|
||||
author=author,
|
||||
date_range=date_range,
|
||||
max_rows=max_rows,
|
||||
email=email,
|
||||
)
|
||||
doi = _best_candidate_doi(
|
||||
title=title,
|
||||
year=item.year,
|
||||
items=items,
|
||||
author_surname=author_surname,
|
||||
)
|
||||
if doi:
|
||||
logger.debug("crossref.doi_discovered", extra={"event": "crossref.doi_discovered"})
|
||||
return doi
|
||||
return None
|
||||
|
|
@ -1,5 +0,0 @@
|
|||
from app.services.domains.dbops.application import run_publication_link_repair
|
||||
from app.services.domains.dbops.integrity import collect_integrity_report
|
||||
from app.services.domains.dbops.query import list_repair_jobs
|
||||
|
||||
__all__ = ["collect_integrity_report", "list_repair_jobs", "run_publication_link_repair"]
|
||||
|
|
@ -1,364 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import delete, exists, func, select, update
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.models import DataRepairJob, IngestionQueueItem, Publication, ScholarProfile, ScholarPublication
|
||||
|
||||
|
||||
REPAIR_STATUS_PLANNED = "planned"
|
||||
REPAIR_STATUS_RUNNING = "running"
|
||||
REPAIR_STATUS_COMPLETED = "completed"
|
||||
REPAIR_STATUS_FAILED = "failed"
|
||||
SCOPE_MODE_SINGLE_USER = "single_user"
|
||||
SCOPE_MODE_ALL_USERS = "all_users"
|
||||
|
||||
|
||||
def _utcnow() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def _normalize_scope_mode(scope_mode: str) -> str:
|
||||
normalized = scope_mode.strip().lower()
|
||||
if normalized in {SCOPE_MODE_SINGLE_USER, SCOPE_MODE_ALL_USERS}:
|
||||
return normalized
|
||||
raise ValueError("Unknown scope mode.")
|
||||
|
||||
|
||||
def _scope_user_id(*, scope_mode: str, user_id: int | None) -> int | None:
|
||||
if scope_mode == SCOPE_MODE_SINGLE_USER:
|
||||
if user_id is None:
|
||||
raise ValueError("user_id is required when scope_mode=single_user.")
|
||||
return int(user_id)
|
||||
if user_id is not None:
|
||||
raise ValueError("user_id must be omitted when scope_mode=all_users.")
|
||||
return None
|
||||
|
||||
|
||||
def _scope_payload(
|
||||
*,
|
||||
scope_mode: str,
|
||||
user_id: int | None,
|
||||
target_scholar_profile_ids: list[int],
|
||||
orphan_gc: bool,
|
||||
) -> dict[str, Any]:
|
||||
payload: dict[str, Any] = {
|
||||
"scope_mode": scope_mode,
|
||||
"scholar_profile_ids": [int(value) for value in target_scholar_profile_ids],
|
||||
"gc_orphan_publications": bool(orphan_gc),
|
||||
}
|
||||
if user_id is not None:
|
||||
payload["user_id"] = int(user_id)
|
||||
return payload
|
||||
|
||||
|
||||
async def _target_scholar_profile_ids(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
scope_mode: str,
|
||||
user_id: int | None,
|
||||
scholar_profile_ids: list[int] | None,
|
||||
) -> list[int]:
|
||||
stmt = select(ScholarProfile.id)
|
||||
if scope_mode == SCOPE_MODE_SINGLE_USER:
|
||||
stmt = stmt.where(ScholarProfile.user_id == user_id)
|
||||
if scholar_profile_ids:
|
||||
normalized_ids = [int(value) for value in scholar_profile_ids]
|
||||
stmt = stmt.where(ScholarProfile.id.in_(normalized_ids))
|
||||
result = await db_session.execute(stmt.order_by(ScholarProfile.id.asc()))
|
||||
ids = [int(row[0]) for row in result.all()]
|
||||
if not ids:
|
||||
raise ValueError("No target scholar profiles found for the requested scope.")
|
||||
return ids
|
||||
|
||||
|
||||
async def _count_scope(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int | None,
|
||||
target_scholar_profile_ids: list[int],
|
||||
) -> dict[str, int]:
|
||||
links_result = await db_session.execute(
|
||||
select(func.count())
|
||||
.select_from(ScholarPublication)
|
||||
.where(ScholarPublication.scholar_profile_id.in_(target_scholar_profile_ids))
|
||||
)
|
||||
queue_stmt = (
|
||||
select(func.count())
|
||||
.select_from(IngestionQueueItem)
|
||||
.where(IngestionQueueItem.scholar_profile_id.in_(target_scholar_profile_ids))
|
||||
)
|
||||
if user_id is not None:
|
||||
queue_stmt = queue_stmt.where(IngestionQueueItem.user_id == user_id)
|
||||
queue_result = await db_session.execute(queue_stmt)
|
||||
return {
|
||||
"target_scholar_count": len(target_scholar_profile_ids),
|
||||
"links_in_scope": int(links_result.scalar_one() or 0),
|
||||
"queue_items_in_scope": int(queue_result.scalar_one() or 0),
|
||||
}
|
||||
|
||||
|
||||
async def _count_orphan_publications(db_session: AsyncSession) -> int:
|
||||
stmt = (
|
||||
select(func.count())
|
||||
.select_from(Publication)
|
||||
.where(
|
||||
~exists(
|
||||
select(1).where(ScholarPublication.publication_id == Publication.id)
|
||||
)
|
||||
)
|
||||
)
|
||||
result = await db_session.execute(stmt)
|
||||
return int(result.scalar_one() or 0)
|
||||
|
||||
|
||||
async def _create_job(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
requested_by: str | None,
|
||||
scope: dict[str, Any],
|
||||
dry_run: bool,
|
||||
) -> DataRepairJob:
|
||||
job = DataRepairJob(
|
||||
job_name="repair_publication_links",
|
||||
requested_by=(requested_by or "").strip() or None,
|
||||
scope=scope,
|
||||
dry_run=dry_run,
|
||||
status=REPAIR_STATUS_PLANNED,
|
||||
summary={},
|
||||
)
|
||||
db_session.add(job)
|
||||
await db_session.flush()
|
||||
return job
|
||||
|
||||
|
||||
def _job_summary(
|
||||
*,
|
||||
counts: dict[str, int],
|
||||
dry_run: bool,
|
||||
links_deleted: int,
|
||||
queue_items_deleted: int,
|
||||
scholars_reset: int,
|
||||
orphan_publications_before: int,
|
||||
orphan_publications_deleted: int,
|
||||
) -> dict[str, Any]:
|
||||
return {
|
||||
**counts,
|
||||
"dry_run": bool(dry_run),
|
||||
"links_deleted": int(links_deleted),
|
||||
"queue_items_deleted": int(queue_items_deleted),
|
||||
"scholars_reset": int(scholars_reset),
|
||||
"orphan_publications_before": int(orphan_publications_before),
|
||||
"orphan_publications_deleted": int(orphan_publications_deleted),
|
||||
}
|
||||
|
||||
|
||||
async def _delete_links_for_targets(db_session: AsyncSession, *, target_scholar_profile_ids: list[int]) -> int:
|
||||
result = await db_session.execute(
|
||||
delete(ScholarPublication).where(
|
||||
ScholarPublication.scholar_profile_id.in_(target_scholar_profile_ids)
|
||||
)
|
||||
)
|
||||
return int(result.rowcount or 0)
|
||||
|
||||
|
||||
async def _delete_queue_for_targets(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int | None,
|
||||
target_scholar_profile_ids: list[int],
|
||||
) -> int:
|
||||
stmt = delete(IngestionQueueItem).where(
|
||||
IngestionQueueItem.scholar_profile_id.in_(target_scholar_profile_ids)
|
||||
)
|
||||
if user_id is not None:
|
||||
stmt = stmt.where(IngestionQueueItem.user_id == user_id)
|
||||
result = await db_session.execute(stmt)
|
||||
return int(result.rowcount or 0)
|
||||
|
||||
|
||||
async def _reset_scholar_tracking_state(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int | None,
|
||||
target_scholar_profile_ids: list[int],
|
||||
) -> int:
|
||||
stmt = update(ScholarProfile).where(ScholarProfile.id.in_(target_scholar_profile_ids))
|
||||
if user_id is not None:
|
||||
stmt = stmt.where(ScholarProfile.user_id == user_id)
|
||||
result = await db_session.execute(
|
||||
stmt.values(
|
||||
baseline_completed=False,
|
||||
last_initial_page_fingerprint_sha256=None,
|
||||
last_initial_page_checked_at=None,
|
||||
last_run_dt=None,
|
||||
last_run_status=None,
|
||||
)
|
||||
)
|
||||
return int(result.rowcount or 0)
|
||||
|
||||
|
||||
async def _delete_orphan_publications(db_session: AsyncSession) -> int:
|
||||
result = await db_session.execute(
|
||||
delete(Publication).where(
|
||||
~exists(select(1).where(ScholarPublication.publication_id == Publication.id))
|
||||
)
|
||||
)
|
||||
return int(result.rowcount or 0)
|
||||
|
||||
|
||||
async def _mutation_counts(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int | None,
|
||||
target_ids: list[int],
|
||||
dry_run: bool,
|
||||
gc_orphan_publications: bool,
|
||||
) -> tuple[int, int, int, int]:
|
||||
if dry_run:
|
||||
return 0, 0, 0, 0
|
||||
links_deleted = await _delete_links_for_targets(
|
||||
db_session,
|
||||
target_scholar_profile_ids=target_ids,
|
||||
)
|
||||
queue_deleted = await _delete_queue_for_targets(
|
||||
db_session,
|
||||
user_id=user_id,
|
||||
target_scholar_profile_ids=target_ids,
|
||||
)
|
||||
scholars_reset = await _reset_scholar_tracking_state(
|
||||
db_session,
|
||||
user_id=user_id,
|
||||
target_scholar_profile_ids=target_ids,
|
||||
)
|
||||
orphan_deleted = 0
|
||||
if gc_orphan_publications:
|
||||
orphan_deleted = await _delete_orphan_publications(db_session)
|
||||
return links_deleted, queue_deleted, scholars_reset, orphan_deleted
|
||||
|
||||
|
||||
def _result_payload(*, job: DataRepairJob, scope: dict[str, Any], summary: dict[str, Any]) -> dict[str, Any]:
|
||||
return {
|
||||
"job_id": int(job.id),
|
||||
"status": job.status,
|
||||
"scope": scope,
|
||||
"summary": summary,
|
||||
}
|
||||
|
||||
|
||||
async def _complete_job(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
job: DataRepairJob,
|
||||
summary: dict[str, Any],
|
||||
scope: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
job.summary = summary
|
||||
job.status = REPAIR_STATUS_COMPLETED
|
||||
job.finished_at = _utcnow()
|
||||
await db_session.commit()
|
||||
return _result_payload(job=job, scope=scope, summary=summary)
|
||||
|
||||
|
||||
async def _fail_job(db_session: AsyncSession, *, job: DataRepairJob, error: Exception) -> None:
|
||||
await db_session.rollback()
|
||||
job.status = REPAIR_STATUS_FAILED
|
||||
job.error_text = str(error)
|
||||
job.finished_at = _utcnow()
|
||||
db_session.add(job)
|
||||
await db_session.commit()
|
||||
|
||||
|
||||
async def _prepare_repair_job(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
scope_mode: str,
|
||||
user_id: int | None,
|
||||
scholar_profile_ids: list[int] | None,
|
||||
dry_run: bool,
|
||||
gc_orphan_publications: bool,
|
||||
requested_by: str | None,
|
||||
) -> tuple[int | None, list[int], dict[str, Any], DataRepairJob]:
|
||||
normalized_scope = _normalize_scope_mode(scope_mode)
|
||||
scope_user_id = _scope_user_id(scope_mode=normalized_scope, user_id=user_id)
|
||||
target_ids = await _target_scholar_profile_ids(
|
||||
db_session,
|
||||
scope_mode=normalized_scope,
|
||||
user_id=scope_user_id,
|
||||
scholar_profile_ids=scholar_profile_ids,
|
||||
)
|
||||
scope = _scope_payload(
|
||||
scope_mode=normalized_scope,
|
||||
user_id=scope_user_id,
|
||||
target_scholar_profile_ids=target_ids,
|
||||
orphan_gc=gc_orphan_publications,
|
||||
)
|
||||
job = await _create_job(db_session, requested_by=requested_by, scope=scope, dry_run=dry_run)
|
||||
job.status = REPAIR_STATUS_RUNNING
|
||||
job.started_at = _utcnow()
|
||||
return scope_user_id, target_ids, scope, job
|
||||
|
||||
|
||||
async def _build_repair_summary(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
scope_user_id: int | None,
|
||||
target_ids: list[int],
|
||||
dry_run: bool,
|
||||
gc_orphan_publications: bool,
|
||||
) -> dict[str, Any]:
|
||||
counts = await _count_scope(db_session, user_id=scope_user_id, target_scholar_profile_ids=target_ids)
|
||||
orphan_before = await _count_orphan_publications(db_session)
|
||||
links_deleted, queue_deleted, scholars_reset, orphan_deleted = await _mutation_counts(
|
||||
db_session,
|
||||
user_id=scope_user_id,
|
||||
target_ids=target_ids,
|
||||
dry_run=dry_run,
|
||||
gc_orphan_publications=gc_orphan_publications,
|
||||
)
|
||||
return _job_summary(
|
||||
counts=counts,
|
||||
dry_run=dry_run,
|
||||
links_deleted=links_deleted,
|
||||
queue_items_deleted=queue_deleted,
|
||||
scholars_reset=scholars_reset,
|
||||
orphan_publications_before=orphan_before,
|
||||
orphan_publications_deleted=orphan_deleted,
|
||||
)
|
||||
|
||||
|
||||
async def run_publication_link_repair(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
scope_mode: str = SCOPE_MODE_SINGLE_USER,
|
||||
user_id: int | None = None,
|
||||
scholar_profile_ids: list[int] | None = None,
|
||||
dry_run: bool = True,
|
||||
gc_orphan_publications: bool = False,
|
||||
requested_by: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
scope_user_id, target_ids, scope, job = await _prepare_repair_job(
|
||||
db_session,
|
||||
scope_mode=scope_mode,
|
||||
user_id=user_id,
|
||||
scholar_profile_ids=scholar_profile_ids,
|
||||
dry_run=dry_run,
|
||||
gc_orphan_publications=gc_orphan_publications,
|
||||
requested_by=requested_by,
|
||||
)
|
||||
|
||||
try:
|
||||
summary = await _build_repair_summary(
|
||||
db_session,
|
||||
scope_user_id=scope_user_id,
|
||||
target_ids=target_ids,
|
||||
dry_run=dry_run,
|
||||
gc_orphan_publications=gc_orphan_publications,
|
||||
)
|
||||
return await _complete_job(db_session, job=job, summary=summary, scope=scope)
|
||||
except Exception as exc:
|
||||
await _fail_job(db_session, job=job, error=exc)
|
||||
raise
|
||||
|
|
@ -1,175 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import func, select, text
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.models import Publication
|
||||
|
||||
INTEGRITY_CHECK_DEFS = (
|
||||
(
|
||||
"legacy_cluster_id_format",
|
||||
"warning",
|
||||
"Publications with non-namespaced cluster IDs.",
|
||||
),
|
||||
(
|
||||
"negative_citation_count",
|
||||
"failure",
|
||||
"Publications with negative citation counts.",
|
||||
),
|
||||
(
|
||||
"orphan_publications",
|
||||
"warning",
|
||||
"Publications with no scholar links.",
|
||||
),
|
||||
(
|
||||
"orphan_scholar_publication_links",
|
||||
"failure",
|
||||
"Link rows missing parent scholar/publication.",
|
||||
),
|
||||
(
|
||||
"duplicate_fingerprint_keys",
|
||||
"failure",
|
||||
"Duplicate publication fingerprint keys.",
|
||||
),
|
||||
(
|
||||
"duplicate_cluster_ids",
|
||||
"failure",
|
||||
"Duplicate non-null publication cluster IDs.",
|
||||
),
|
||||
(
|
||||
"missing_pdf_url",
|
||||
"metric",
|
||||
"Publications without a resolved PDF URL.",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
async def _legacy_cluster_id_count(db_session: AsyncSession) -> int:
|
||||
stmt = (
|
||||
select(func.count())
|
||||
.select_from(Publication)
|
||||
.where(Publication.cluster_id.is_not(None))
|
||||
.where(~Publication.cluster_id.like("cfv:%"))
|
||||
.where(~Publication.cluster_id.like("cluster:%"))
|
||||
)
|
||||
result = await db_session.execute(stmt)
|
||||
return int(result.scalar_one() or 0)
|
||||
|
||||
|
||||
async def _negative_citation_count(db_session: AsyncSession) -> int:
|
||||
result = await db_session.execute(
|
||||
select(func.count()).select_from(Publication).where(Publication.citation_count < 0)
|
||||
)
|
||||
return int(result.scalar_one() or 0)
|
||||
|
||||
|
||||
async def _missing_pdf_url_count(db_session: AsyncSession) -> int:
|
||||
result = await db_session.execute(
|
||||
select(func.count()).select_from(Publication).where(Publication.pdf_url.is_(None))
|
||||
)
|
||||
return int(result.scalar_one() or 0)
|
||||
|
||||
|
||||
async def _count_from_sql(db_session: AsyncSession, *, sql: str) -> int:
|
||||
result = await db_session.execute(text(sql))
|
||||
return int(result.scalar_one() or 0)
|
||||
|
||||
|
||||
def _issues_for_severity(*, checks: list[dict[str, Any]], severity: str) -> list[str]:
|
||||
return [row["name"] for row in checks if row["severity"] == severity and row["count"] > 0]
|
||||
|
||||
|
||||
def _check_row(*, name: str, count: int, severity: str, message: str) -> dict[str, Any]:
|
||||
return {
|
||||
"name": name,
|
||||
"count": int(count),
|
||||
"severity": severity,
|
||||
"message": message,
|
||||
}
|
||||
|
||||
|
||||
async def _collect_counts(db_session: AsyncSession) -> dict[str, int]:
|
||||
orphan_publications = await _count_from_sql(
|
||||
db_session,
|
||||
sql=(
|
||||
"SELECT count(*) FROM publications p "
|
||||
"LEFT JOIN scholar_publications sp ON sp.publication_id = p.id "
|
||||
"WHERE sp.publication_id IS NULL"
|
||||
),
|
||||
)
|
||||
orphan_links = await _count_from_sql(
|
||||
db_session,
|
||||
sql=(
|
||||
"SELECT count(*) FROM scholar_publications sp "
|
||||
"LEFT JOIN publications p ON p.id = sp.publication_id "
|
||||
"LEFT JOIN scholar_profiles s ON s.id = sp.scholar_profile_id "
|
||||
"WHERE p.id IS NULL OR s.id IS NULL"
|
||||
),
|
||||
)
|
||||
duplicate_fingerprints = await _count_from_sql(
|
||||
db_session,
|
||||
sql=(
|
||||
"SELECT count(*) FROM ("
|
||||
"SELECT fingerprint_sha256 FROM publications "
|
||||
"GROUP BY fingerprint_sha256 HAVING count(*) > 1"
|
||||
") dup"
|
||||
),
|
||||
)
|
||||
duplicate_cluster_ids = await _count_from_sql(
|
||||
db_session,
|
||||
sql=(
|
||||
"SELECT count(*) FROM ("
|
||||
"SELECT cluster_id FROM publications "
|
||||
"WHERE cluster_id IS NOT NULL "
|
||||
"GROUP BY cluster_id HAVING count(*) > 1"
|
||||
") dup"
|
||||
),
|
||||
)
|
||||
return {
|
||||
"legacy_cluster_id_format": await _legacy_cluster_id_count(db_session),
|
||||
"negative_citation_count": await _negative_citation_count(db_session),
|
||||
"orphan_publications": orphan_publications,
|
||||
"orphan_scholar_publication_links": orphan_links,
|
||||
"duplicate_fingerprint_keys": duplicate_fingerprints,
|
||||
"duplicate_cluster_ids": duplicate_cluster_ids,
|
||||
"missing_pdf_url": await _missing_pdf_url_count(db_session),
|
||||
}
|
||||
|
||||
|
||||
def _build_checks(*, counts: dict[str, int]) -> list[dict[str, Any]]:
|
||||
return [
|
||||
_check_row(
|
||||
name=name,
|
||||
count=counts[name],
|
||||
severity=severity,
|
||||
message=message,
|
||||
)
|
||||
for name, severity, message in INTEGRITY_CHECK_DEFS
|
||||
]
|
||||
|
||||
|
||||
def _status_from_issues(*, failures: list[str], warnings: list[str]) -> str:
|
||||
status = "ok"
|
||||
if failures:
|
||||
status = "failed"
|
||||
elif warnings:
|
||||
status = "warning"
|
||||
return status
|
||||
|
||||
|
||||
async def collect_integrity_report(db_session: AsyncSession) -> dict[str, Any]:
|
||||
counts = await _collect_counts(db_session)
|
||||
checks = _build_checks(counts=counts)
|
||||
failures = _issues_for_severity(checks=checks, severity="failure")
|
||||
warnings = _issues_for_severity(checks=checks, severity="warning")
|
||||
|
||||
return {
|
||||
"status": _status_from_issues(failures=failures, warnings=warnings),
|
||||
"checked_at": datetime.now(timezone.utc).isoformat(),
|
||||
"failures": failures,
|
||||
"warnings": warnings,
|
||||
"checks": checks,
|
||||
}
|
||||
|
|
@ -1,22 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.models import DataRepairJob
|
||||
|
||||
|
||||
def _bounded_limit(limit: int) -> int:
|
||||
return max(1, min(int(limit), 200))
|
||||
|
||||
|
||||
async def list_repair_jobs(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
limit: int = 50,
|
||||
) -> list[DataRepairJob]:
|
||||
bounded = _bounded_limit(limit)
|
||||
result = await db_session.execute(
|
||||
select(DataRepairJob).order_by(DataRepairJob.created_at.desc()).limit(bounded)
|
||||
)
|
||||
return list(result.scalars())
|
||||
|
|
@ -1,23 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from urllib.parse import unquote
|
||||
|
||||
DOI_RE = re.compile(r"10\.\d{4,9}/[-._;()/:A-Z0-9]+", re.I)
|
||||
|
||||
|
||||
def normalize_doi(value: str | None) -> str | None:
|
||||
if not value:
|
||||
return None
|
||||
match = DOI_RE.search(unquote(value))
|
||||
if not match:
|
||||
return None
|
||||
return match.group(0).rstrip(" .;,)").lower()
|
||||
|
||||
|
||||
def first_doi_from_texts(*values: str | None) -> str | None:
|
||||
for value in values:
|
||||
doi = normalize_doi(value)
|
||||
if doi:
|
||||
return doi
|
||||
return None
|
||||
|
|
@ -1 +0,0 @@
|
|||
from __future__ import annotations
|
||||
File diff suppressed because it is too large
Load diff
|
|
@ -1,31 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from app.services.domains.scholar.parser import ParseState
|
||||
|
||||
TITLE_ALNUM_RE = re.compile(r"[^a-z0-9]+")
|
||||
WORD_RE = re.compile(r"[a-z0-9]+")
|
||||
HTML_TAG_RE = re.compile(r"<[^>]+>", re.S)
|
||||
SPACE_RE = re.compile(r"\s+")
|
||||
|
||||
FAILED_STATES = {
|
||||
ParseState.BLOCKED_OR_CAPTCHA.value,
|
||||
ParseState.LAYOUT_CHANGED.value,
|
||||
ParseState.NETWORK_ERROR.value,
|
||||
"ingestion_error",
|
||||
}
|
||||
|
||||
FAILURE_BUCKET_BLOCKED = "blocked_or_captcha"
|
||||
FAILURE_BUCKET_NETWORK = "network_error"
|
||||
FAILURE_BUCKET_LAYOUT = "layout_changed"
|
||||
FAILURE_BUCKET_INGESTION = "ingestion_error"
|
||||
FAILURE_BUCKET_OTHER = "other_failure"
|
||||
|
||||
RUN_LOCK_NAMESPACE = 8217
|
||||
RESUMABLE_PARTIAL_REASONS = {
|
||||
"max_pages_reached",
|
||||
"pagination_cursor_stalled",
|
||||
}
|
||||
RESUMABLE_PARTIAL_REASON_PREFIXES = ("page_state_network_error",)
|
||||
INITIAL_PAGE_FINGERPRINT_MAX_PUBLICATIONS = 30
|
||||
|
|
@ -1,175 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from typing import Any
|
||||
from urllib.parse import urljoin
|
||||
|
||||
from app.services.domains.ingestion.constants import (
|
||||
HTML_TAG_RE,
|
||||
INITIAL_PAGE_FINGERPRINT_MAX_PUBLICATIONS,
|
||||
SPACE_RE,
|
||||
TITLE_ALNUM_RE,
|
||||
WORD_RE,
|
||||
)
|
||||
from app.services.domains.scholar.parser import ParseState, ParsedProfilePage, PublicationCandidate
|
||||
|
||||
|
||||
def normalize_title(value: str) -> str:
|
||||
lowered = value.lower()
|
||||
return TITLE_ALNUM_RE.sub("", lowered)
|
||||
|
||||
|
||||
def _first_author_last_name(authors_text: str | None) -> str:
|
||||
if not authors_text:
|
||||
return ""
|
||||
first_author = authors_text.split(",", maxsplit=1)[0].strip().lower()
|
||||
words = WORD_RE.findall(first_author)
|
||||
if not words:
|
||||
return ""
|
||||
return words[-1]
|
||||
|
||||
|
||||
def _first_venue_word(venue_text: str | None) -> str:
|
||||
if not venue_text:
|
||||
return ""
|
||||
words = WORD_RE.findall(venue_text.lower())
|
||||
if not words:
|
||||
return ""
|
||||
return words[0]
|
||||
|
||||
|
||||
def build_publication_fingerprint(candidate: PublicationCandidate) -> str:
|
||||
canonical = "|".join(
|
||||
[
|
||||
normalize_title(candidate.title),
|
||||
str(candidate.year) if candidate.year is not None else "",
|
||||
_first_author_last_name(candidate.authors_text),
|
||||
_first_venue_word(candidate.venue_text),
|
||||
]
|
||||
)
|
||||
return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def build_initial_page_fingerprint(parsed_page: ParsedProfilePage) -> str | None:
|
||||
if parsed_page.state not in {ParseState.OK, ParseState.NO_RESULTS}:
|
||||
return None
|
||||
|
||||
normalized_rows: list[dict[str, Any]] = []
|
||||
for publication in parsed_page.publications[:INITIAL_PAGE_FINGERPRINT_MAX_PUBLICATIONS]:
|
||||
normalized_rows.append(
|
||||
{
|
||||
"cluster_id": publication.cluster_id or "",
|
||||
"title_normalized": normalize_title(publication.title),
|
||||
"year": publication.year,
|
||||
"citation_count": publication.citation_count,
|
||||
}
|
||||
)
|
||||
|
||||
payload = {
|
||||
"state": parsed_page.state.value,
|
||||
"articles_range": parsed_page.articles_range or "",
|
||||
"has_show_more_button": parsed_page.has_show_more_button,
|
||||
"profile_name": parsed_page.profile_name or "",
|
||||
"publications": normalized_rows,
|
||||
}
|
||||
canonical = json.dumps(
|
||||
payload,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
ensure_ascii=True,
|
||||
)
|
||||
return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def build_publication_url(path_or_url: str | None) -> str | None:
|
||||
if not path_or_url:
|
||||
return None
|
||||
return urljoin("https://scholar.google.com", path_or_url)
|
||||
|
||||
|
||||
def _next_cstart_value(*, articles_range: str | None, fallback: int) -> int:
|
||||
if articles_range:
|
||||
numbers = re.findall(r"\d+", articles_range)
|
||||
if len(numbers) >= 2:
|
||||
try:
|
||||
return int(numbers[1])
|
||||
except ValueError:
|
||||
pass
|
||||
return int(fallback)
|
||||
|
||||
|
||||
def _title_tokens(value: str) -> set[str]:
|
||||
"""Extract normalized word tokens for fuzzy title comparison."""
|
||||
return set(WORD_RE.findall(value.lower()))
|
||||
|
||||
|
||||
def fuzzy_titles_match(
|
||||
title_a: str,
|
||||
title_b: str,
|
||||
*,
|
||||
threshold: float = 0.85,
|
||||
) -> bool:
|
||||
"""Return True if two titles are near-duplicates by token-level Jaccard similarity.
|
||||
|
||||
A threshold of 0.85 catches common academic duplicate patterns:
|
||||
differences in punctuation, minor word variations, subtitle changes.
|
||||
"""
|
||||
tokens_a = _title_tokens(title_a)
|
||||
tokens_b = _title_tokens(title_b)
|
||||
if not tokens_a or not tokens_b:
|
||||
return False
|
||||
intersection = tokens_a & tokens_b
|
||||
union = tokens_a | tokens_b
|
||||
return (len(intersection) / len(union)) >= threshold
|
||||
|
||||
|
||||
def _dedupe_publication_candidates(
|
||||
publications: list[PublicationCandidate],
|
||||
) -> list[PublicationCandidate]:
|
||||
deduped: list[PublicationCandidate] = []
|
||||
seen: set[str] = set()
|
||||
seen_titles: list[tuple[str, int]] = [] # (normalized_title, index into deduped)
|
||||
|
||||
for publication in publications:
|
||||
if publication.cluster_id:
|
||||
identity = f"cluster:{publication.cluster_id}"
|
||||
else:
|
||||
identity = "|".join(
|
||||
[
|
||||
"fallback",
|
||||
normalize_title(publication.title),
|
||||
str(publication.year) if publication.year is not None else "",
|
||||
_first_author_last_name(publication.authors_text),
|
||||
_first_venue_word(publication.venue_text),
|
||||
]
|
||||
)
|
||||
if identity in seen:
|
||||
continue
|
||||
|
||||
# Fuzzy title check — catch near-identical titles not caught by exact fingerprint
|
||||
norm_title = normalize_title(publication.title)
|
||||
is_fuzzy_dup = False
|
||||
for existing_title, _idx in seen_titles:
|
||||
if fuzzy_titles_match(norm_title, existing_title):
|
||||
is_fuzzy_dup = True
|
||||
break
|
||||
if is_fuzzy_dup:
|
||||
continue
|
||||
|
||||
seen.add(identity)
|
||||
seen_titles.append((norm_title, len(deduped)))
|
||||
deduped.append(publication)
|
||||
return deduped
|
||||
|
||||
|
||||
def _build_body_excerpt(body: str, *, max_chars: int = 220) -> str | None:
|
||||
if not body:
|
||||
return None
|
||||
flattened = SPACE_RE.sub(" ", HTML_TAG_RE.sub(" ", body)).strip()
|
||||
if not flattened:
|
||||
return None
|
||||
if len(flattened) <= max_chars:
|
||||
return flattened
|
||||
return f"{flattened[:max_chars - 1]}..."
|
||||
|
|
@ -1,348 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.models import IngestionQueueItem, QueueItemStatus
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ContinuationQueueJob:
|
||||
id: int
|
||||
user_id: int
|
||||
scholar_profile_id: int
|
||||
resume_cstart: int
|
||||
reason: str
|
||||
status: str
|
||||
attempt_count: int
|
||||
next_attempt_dt: datetime
|
||||
|
||||
|
||||
ACTIVE_QUEUE_STATUSES: tuple[str, ...] = (
|
||||
QueueItemStatus.QUEUED.value,
|
||||
QueueItemStatus.RETRYING.value,
|
||||
)
|
||||
|
||||
|
||||
def normalize_cstart(value: int | None) -> int:
|
||||
if value is None:
|
||||
return 0
|
||||
return max(0, int(value))
|
||||
|
||||
|
||||
def compute_backoff_seconds(*, base_seconds: int, attempt_count: int, max_seconds: int) -> int:
|
||||
base = max(1, int(base_seconds))
|
||||
attempts = max(1, int(attempt_count))
|
||||
maximum = max(base, int(max_seconds))
|
||||
seconds = base * (2 ** max(0, attempts - 1))
|
||||
return min(seconds, maximum)
|
||||
|
||||
|
||||
async def _get_item_for_user_scholar(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
scholar_profile_id: int,
|
||||
) -> IngestionQueueItem | None:
|
||||
result = await db_session.execute(
|
||||
select(IngestionQueueItem).where(
|
||||
IngestionQueueItem.user_id == user_id,
|
||||
IngestionQueueItem.scholar_profile_id == scholar_profile_id,
|
||||
)
|
||||
)
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
|
||||
def _build_queue_item(
|
||||
*,
|
||||
now: datetime,
|
||||
user_id: int,
|
||||
scholar_profile_id: int,
|
||||
normalized_cstart: int,
|
||||
reason: str,
|
||||
run_id: int | None,
|
||||
next_attempt_dt: datetime,
|
||||
) -> IngestionQueueItem:
|
||||
return IngestionQueueItem(
|
||||
user_id=user_id,
|
||||
scholar_profile_id=scholar_profile_id,
|
||||
resume_cstart=normalized_cstart,
|
||||
reason=reason,
|
||||
status=QueueItemStatus.QUEUED.value,
|
||||
attempt_count=0,
|
||||
next_attempt_dt=next_attempt_dt,
|
||||
last_run_id=run_id,
|
||||
last_error=None,
|
||||
dropped_reason=None,
|
||||
dropped_at=None,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
|
||||
|
||||
def _update_existing_queue_item(
|
||||
*,
|
||||
item: IngestionQueueItem,
|
||||
now: datetime,
|
||||
normalized_cstart: int,
|
||||
reason: str,
|
||||
run_id: int | None,
|
||||
next_attempt_dt: datetime,
|
||||
) -> None:
|
||||
item.resume_cstart = normalized_cstart
|
||||
item.reason = reason
|
||||
if item.status == QueueItemStatus.DROPPED.value:
|
||||
item.attempt_count = 0
|
||||
item.status = QueueItemStatus.QUEUED.value
|
||||
item.next_attempt_dt = next_attempt_dt
|
||||
item.last_run_id = run_id
|
||||
item.last_error = None
|
||||
item.dropped_reason = None
|
||||
item.dropped_at = None
|
||||
item.updated_at = now
|
||||
|
||||
|
||||
async def upsert_job(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
scholar_profile_id: int,
|
||||
resume_cstart: int,
|
||||
reason: str,
|
||||
run_id: int | None,
|
||||
delay_seconds: int,
|
||||
) -> IngestionQueueItem:
|
||||
now = datetime.now(timezone.utc)
|
||||
next_attempt_dt = now + timedelta(seconds=max(0, int(delay_seconds)))
|
||||
item = await _get_item_for_user_scholar(
|
||||
db_session,
|
||||
user_id=user_id,
|
||||
scholar_profile_id=scholar_profile_id,
|
||||
)
|
||||
normalized_cstart = normalize_cstart(resume_cstart)
|
||||
if item is None:
|
||||
item = _build_queue_item(
|
||||
now=now,
|
||||
user_id=user_id,
|
||||
scholar_profile_id=scholar_profile_id,
|
||||
normalized_cstart=normalized_cstart,
|
||||
reason=reason,
|
||||
run_id=run_id,
|
||||
next_attempt_dt=next_attempt_dt,
|
||||
)
|
||||
db_session.add(item)
|
||||
return item
|
||||
|
||||
_update_existing_queue_item(
|
||||
item=item,
|
||||
now=now,
|
||||
normalized_cstart=normalized_cstart,
|
||||
reason=reason,
|
||||
run_id=run_id,
|
||||
next_attempt_dt=next_attempt_dt,
|
||||
)
|
||||
return item
|
||||
|
||||
|
||||
async def clear_job_for_scholar(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
scholar_profile_id: int,
|
||||
) -> bool:
|
||||
result = await db_session.execute(
|
||||
select(IngestionQueueItem).where(
|
||||
IngestionQueueItem.user_id == user_id,
|
||||
IngestionQueueItem.scholar_profile_id == scholar_profile_id,
|
||||
)
|
||||
)
|
||||
item = result.scalar_one_or_none()
|
||||
if item is None:
|
||||
return False
|
||||
await db_session.delete(item)
|
||||
return True
|
||||
|
||||
|
||||
async def delete_job_by_id(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
job_id: int,
|
||||
) -> bool:
|
||||
result = await db_session.execute(
|
||||
select(IngestionQueueItem).where(IngestionQueueItem.id == job_id)
|
||||
)
|
||||
item = result.scalar_one_or_none()
|
||||
if item is None:
|
||||
return False
|
||||
await db_session.delete(item)
|
||||
return True
|
||||
|
||||
|
||||
async def list_due_jobs(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
now: datetime,
|
||||
limit: int,
|
||||
) -> list[ContinuationQueueJob]:
|
||||
result = await db_session.execute(
|
||||
select(IngestionQueueItem)
|
||||
.where(
|
||||
IngestionQueueItem.next_attempt_dt <= now,
|
||||
IngestionQueueItem.status.in_(ACTIVE_QUEUE_STATUSES),
|
||||
)
|
||||
.order_by(
|
||||
IngestionQueueItem.next_attempt_dt.asc(),
|
||||
IngestionQueueItem.id.asc(),
|
||||
)
|
||||
.limit(limit)
|
||||
)
|
||||
rows = list(result.scalars().all())
|
||||
jobs: list[ContinuationQueueJob] = []
|
||||
for row in rows:
|
||||
jobs.append(
|
||||
ContinuationQueueJob(
|
||||
id=int(row.id),
|
||||
user_id=int(row.user_id),
|
||||
scholar_profile_id=int(row.scholar_profile_id),
|
||||
resume_cstart=normalize_cstart(row.resume_cstart),
|
||||
reason=row.reason,
|
||||
status=row.status,
|
||||
attempt_count=int(row.attempt_count),
|
||||
next_attempt_dt=row.next_attempt_dt,
|
||||
)
|
||||
)
|
||||
return jobs
|
||||
|
||||
|
||||
async def increment_attempt_count(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
job_id: int,
|
||||
) -> IngestionQueueItem | None:
|
||||
now = datetime.now(timezone.utc)
|
||||
result = await db_session.execute(
|
||||
select(IngestionQueueItem).where(IngestionQueueItem.id == job_id)
|
||||
)
|
||||
item = result.scalar_one_or_none()
|
||||
if item is None:
|
||||
return None
|
||||
item.attempt_count = int(item.attempt_count or 0) + 1
|
||||
item.updated_at = now
|
||||
return item
|
||||
|
||||
|
||||
async def reset_attempt_count(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
job_id: int,
|
||||
) -> IngestionQueueItem | None:
|
||||
now = datetime.now(timezone.utc)
|
||||
result = await db_session.execute(
|
||||
select(IngestionQueueItem).where(IngestionQueueItem.id == job_id)
|
||||
)
|
||||
item = result.scalar_one_or_none()
|
||||
if item is None:
|
||||
return None
|
||||
item.attempt_count = 0
|
||||
item.updated_at = now
|
||||
return item
|
||||
|
||||
|
||||
async def reschedule_job(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
job_id: int,
|
||||
delay_seconds: int,
|
||||
reason: str,
|
||||
error: str | None = None,
|
||||
) -> IngestionQueueItem | None:
|
||||
now = datetime.now(timezone.utc)
|
||||
result = await db_session.execute(
|
||||
select(IngestionQueueItem).where(IngestionQueueItem.id == job_id)
|
||||
)
|
||||
item = result.scalar_one_or_none()
|
||||
if item is None:
|
||||
return None
|
||||
item.next_attempt_dt = now + timedelta(seconds=max(1, int(delay_seconds)))
|
||||
item.status = QueueItemStatus.QUEUED.value
|
||||
item.reason = reason
|
||||
item.last_error = error
|
||||
item.dropped_reason = None
|
||||
item.dropped_at = None
|
||||
item.updated_at = now
|
||||
return item
|
||||
|
||||
|
||||
async def mark_retrying(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
job_id: int,
|
||||
reason: str | None = None,
|
||||
) -> IngestionQueueItem | None:
|
||||
now = datetime.now(timezone.utc)
|
||||
result = await db_session.execute(
|
||||
select(IngestionQueueItem).where(IngestionQueueItem.id == job_id)
|
||||
)
|
||||
item = result.scalar_one_or_none()
|
||||
if item is None:
|
||||
return None
|
||||
if item.status == QueueItemStatus.DROPPED.value:
|
||||
return item
|
||||
item.status = QueueItemStatus.RETRYING.value
|
||||
if reason:
|
||||
item.reason = reason
|
||||
item.updated_at = now
|
||||
return item
|
||||
|
||||
|
||||
async def mark_dropped(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
job_id: int,
|
||||
reason: str,
|
||||
error: str | None = None,
|
||||
) -> IngestionQueueItem | None:
|
||||
now = datetime.now(timezone.utc)
|
||||
result = await db_session.execute(
|
||||
select(IngestionQueueItem).where(IngestionQueueItem.id == job_id)
|
||||
)
|
||||
item = result.scalar_one_or_none()
|
||||
if item is None:
|
||||
return None
|
||||
item.status = QueueItemStatus.DROPPED.value
|
||||
item.reason = "dropped"
|
||||
item.dropped_reason = reason
|
||||
item.dropped_at = now
|
||||
if error is not None:
|
||||
item.last_error = error
|
||||
item.updated_at = now
|
||||
return item
|
||||
|
||||
|
||||
async def mark_queued_now(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
job_id: int,
|
||||
reason: str,
|
||||
reset_attempt_count: bool = False,
|
||||
) -> IngestionQueueItem | None:
|
||||
now = datetime.now(timezone.utc)
|
||||
result = await db_session.execute(
|
||||
select(IngestionQueueItem).where(IngestionQueueItem.id == job_id)
|
||||
)
|
||||
item = result.scalar_one_or_none()
|
||||
if item is None:
|
||||
return None
|
||||
item.status = QueueItemStatus.QUEUED.value
|
||||
item.reason = reason
|
||||
item.next_attempt_dt = now
|
||||
if reset_attempt_count:
|
||||
item.attempt_count = 0
|
||||
item.last_error = None
|
||||
item.dropped_reason = None
|
||||
item.dropped_at = None
|
||||
item.updated_at = now
|
||||
return item
|
||||
|
|
@ -1,282 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any
|
||||
|
||||
from app.db.models import UserSetting
|
||||
|
||||
COOLDOWN_REASON_BLOCKED_FAILURE_THRESHOLD = "blocked_failure_threshold_exceeded"
|
||||
COOLDOWN_REASON_NETWORK_FAILURE_THRESHOLD = "network_failure_threshold_exceeded"
|
||||
|
||||
_COUNTER_CONSECUTIVE_BLOCKED_RUNS = "consecutive_blocked_runs"
|
||||
_COUNTER_CONSECUTIVE_NETWORK_RUNS = "consecutive_network_runs"
|
||||
_COUNTER_COOLDOWN_ENTRY_COUNT = "cooldown_entry_count"
|
||||
_COUNTER_BLOCKED_START_COUNT = "blocked_start_count"
|
||||
_COUNTER_LAST_BLOCKED_FAILURE_COUNT = "last_blocked_failure_count"
|
||||
_COUNTER_LAST_NETWORK_FAILURE_COUNT = "last_network_failure_count"
|
||||
_COUNTER_LAST_EVALUATED_RUN_ID = "last_evaluated_run_id"
|
||||
|
||||
|
||||
def _utcnow() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def _safe_int(value: Any, default: int = 0) -> int:
|
||||
try:
|
||||
return int(value)
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
|
||||
|
||||
def _safe_optional_int(value: Any) -> int | None:
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
return int(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _normalize_datetime(value: datetime | None) -> datetime | None:
|
||||
if value is None:
|
||||
return None
|
||||
if value.tzinfo is None:
|
||||
return value.replace(tzinfo=timezone.utc)
|
||||
return value.astimezone(timezone.utc)
|
||||
|
||||
|
||||
def _state_dict(settings: UserSetting) -> dict[str, Any]:
|
||||
state = settings.scrape_safety_state
|
||||
if isinstance(state, dict):
|
||||
return state
|
||||
return {}
|
||||
|
||||
|
||||
def _counters_from_state(settings: UserSetting) -> dict[str, Any]:
|
||||
state = _state_dict(settings)
|
||||
return {
|
||||
_COUNTER_CONSECUTIVE_BLOCKED_RUNS: max(
|
||||
0,
|
||||
_safe_int(state.get(_COUNTER_CONSECUTIVE_BLOCKED_RUNS), 0),
|
||||
),
|
||||
_COUNTER_CONSECUTIVE_NETWORK_RUNS: max(
|
||||
0,
|
||||
_safe_int(state.get(_COUNTER_CONSECUTIVE_NETWORK_RUNS), 0),
|
||||
),
|
||||
_COUNTER_COOLDOWN_ENTRY_COUNT: max(
|
||||
0,
|
||||
_safe_int(state.get(_COUNTER_COOLDOWN_ENTRY_COUNT), 0),
|
||||
),
|
||||
_COUNTER_BLOCKED_START_COUNT: max(
|
||||
0,
|
||||
_safe_int(state.get(_COUNTER_BLOCKED_START_COUNT), 0),
|
||||
),
|
||||
_COUNTER_LAST_BLOCKED_FAILURE_COUNT: max(
|
||||
0,
|
||||
_safe_int(state.get(_COUNTER_LAST_BLOCKED_FAILURE_COUNT), 0),
|
||||
),
|
||||
_COUNTER_LAST_NETWORK_FAILURE_COUNT: max(
|
||||
0,
|
||||
_safe_int(state.get(_COUNTER_LAST_NETWORK_FAILURE_COUNT), 0),
|
||||
),
|
||||
_COUNTER_LAST_EVALUATED_RUN_ID: _safe_optional_int(
|
||||
state.get(_COUNTER_LAST_EVALUATED_RUN_ID),
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def _cooldown_reason_label(reason: str | None) -> str | None:
|
||||
if reason == COOLDOWN_REASON_BLOCKED_FAILURE_THRESHOLD:
|
||||
return "Blocked responses exceeded safety threshold"
|
||||
if reason == COOLDOWN_REASON_NETWORK_FAILURE_THRESHOLD:
|
||||
return "Network failures exceeded safety threshold"
|
||||
return None
|
||||
|
||||
|
||||
def _recommended_action(reason: str | None) -> str | None:
|
||||
if reason == COOLDOWN_REASON_BLOCKED_FAILURE_THRESHOLD:
|
||||
return (
|
||||
"Google Scholar appears to be blocking requests. Wait for cooldown to expire, "
|
||||
"increase request delay, and avoid repeated manual retries."
|
||||
)
|
||||
if reason == COOLDOWN_REASON_NETWORK_FAILURE_THRESHOLD:
|
||||
return (
|
||||
"Network failures crossed the threshold. Verify connectivity and retry after cooldown."
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def is_cooldown_active(
|
||||
settings: UserSetting,
|
||||
*,
|
||||
now_utc: datetime | None = None,
|
||||
) -> bool:
|
||||
now = now_utc or _utcnow()
|
||||
cooldown_until = _normalize_datetime(settings.scrape_cooldown_until)
|
||||
if cooldown_until is None:
|
||||
return False
|
||||
return cooldown_until > now
|
||||
|
||||
|
||||
def clear_expired_cooldown(
|
||||
settings: UserSetting,
|
||||
*,
|
||||
now_utc: datetime | None = None,
|
||||
) -> bool:
|
||||
now = now_utc or _utcnow()
|
||||
cooldown_until = _normalize_datetime(settings.scrape_cooldown_until)
|
||||
if cooldown_until is None:
|
||||
return False
|
||||
if cooldown_until > now:
|
||||
return False
|
||||
settings.scrape_cooldown_until = None
|
||||
settings.scrape_cooldown_reason = None
|
||||
return True
|
||||
|
||||
|
||||
def register_cooldown_blocked_start(
|
||||
settings: UserSetting,
|
||||
*,
|
||||
now_utc: datetime | None = None,
|
||||
) -> dict[str, Any]:
|
||||
now = now_utc or _utcnow()
|
||||
counters = _counters_from_state(settings)
|
||||
counters[_COUNTER_BLOCKED_START_COUNT] = int(counters[_COUNTER_BLOCKED_START_COUNT]) + 1
|
||||
settings.scrape_safety_state = counters
|
||||
return get_safety_state_payload(settings, now_utc=now)
|
||||
|
||||
|
||||
def _update_run_counters(
|
||||
*,
|
||||
counters: dict[str, Any],
|
||||
run_id: int,
|
||||
blocked_failure_count: int,
|
||||
network_failure_count: int,
|
||||
) -> tuple[int, int]:
|
||||
bounded_blocked_failures = max(0, int(blocked_failure_count))
|
||||
bounded_network_failures = max(0, int(network_failure_count))
|
||||
counters[_COUNTER_LAST_BLOCKED_FAILURE_COUNT] = bounded_blocked_failures
|
||||
counters[_COUNTER_LAST_NETWORK_FAILURE_COUNT] = bounded_network_failures
|
||||
counters[_COUNTER_LAST_EVALUATED_RUN_ID] = int(run_id)
|
||||
counters[_COUNTER_CONSECUTIVE_BLOCKED_RUNS] = (
|
||||
int(counters[_COUNTER_CONSECUTIVE_BLOCKED_RUNS]) + 1
|
||||
if bounded_blocked_failures > 0
|
||||
else 0
|
||||
)
|
||||
counters[_COUNTER_CONSECUTIVE_NETWORK_RUNS] = (
|
||||
int(counters[_COUNTER_CONSECUTIVE_NETWORK_RUNS]) + 1
|
||||
if bounded_network_failures > 0
|
||||
else 0
|
||||
)
|
||||
return bounded_blocked_failures, bounded_network_failures
|
||||
|
||||
|
||||
def _resolve_cooldown_trigger(
|
||||
*,
|
||||
blocked_failures: int,
|
||||
network_failures: int,
|
||||
blocked_failure_threshold: int,
|
||||
network_failure_threshold: int,
|
||||
blocked_cooldown_seconds: int,
|
||||
network_cooldown_seconds: int,
|
||||
) -> tuple[str | None, int]:
|
||||
if blocked_failures >= max(1, int(blocked_failure_threshold)):
|
||||
return COOLDOWN_REASON_BLOCKED_FAILURE_THRESHOLD, max(60, int(blocked_cooldown_seconds))
|
||||
if network_failures >= max(1, int(network_failure_threshold)):
|
||||
return COOLDOWN_REASON_NETWORK_FAILURE_THRESHOLD, max(60, int(network_cooldown_seconds))
|
||||
return None, 0
|
||||
|
||||
|
||||
def _apply_cooldown_decision(
|
||||
*,
|
||||
settings: UserSetting,
|
||||
counters: dict[str, Any],
|
||||
now: datetime,
|
||||
reason: str | None,
|
||||
cooldown_seconds: int,
|
||||
) -> None:
|
||||
if reason is None:
|
||||
clear_expired_cooldown(settings, now_utc=now)
|
||||
return
|
||||
settings.scrape_cooldown_reason = reason
|
||||
settings.scrape_cooldown_until = now + timedelta(seconds=max(60, int(cooldown_seconds)))
|
||||
counters[_COUNTER_COOLDOWN_ENTRY_COUNT] = int(counters[_COUNTER_COOLDOWN_ENTRY_COUNT]) + 1
|
||||
|
||||
|
||||
def apply_run_safety_outcome(
|
||||
settings: UserSetting,
|
||||
*,
|
||||
run_id: int,
|
||||
blocked_failure_count: int,
|
||||
network_failure_count: int,
|
||||
blocked_failure_threshold: int,
|
||||
network_failure_threshold: int,
|
||||
blocked_cooldown_seconds: int,
|
||||
network_cooldown_seconds: int,
|
||||
now_utc: datetime | None = None,
|
||||
) -> tuple[dict[str, Any], str | None]:
|
||||
now = now_utc or _utcnow()
|
||||
counters = _counters_from_state(settings)
|
||||
blocked_failures, network_failures = _update_run_counters(
|
||||
counters=counters,
|
||||
run_id=run_id,
|
||||
blocked_failure_count=blocked_failure_count,
|
||||
network_failure_count=network_failure_count,
|
||||
)
|
||||
reason, cooldown_seconds = _resolve_cooldown_trigger(
|
||||
blocked_failures=blocked_failures,
|
||||
network_failures=network_failures,
|
||||
blocked_failure_threshold=blocked_failure_threshold,
|
||||
network_failure_threshold=network_failure_threshold,
|
||||
blocked_cooldown_seconds=blocked_cooldown_seconds,
|
||||
network_cooldown_seconds=network_cooldown_seconds,
|
||||
)
|
||||
_apply_cooldown_decision(
|
||||
settings=settings,
|
||||
counters=counters,
|
||||
now=now,
|
||||
reason=reason,
|
||||
cooldown_seconds=cooldown_seconds,
|
||||
)
|
||||
settings.scrape_safety_state = counters
|
||||
return get_safety_state_payload(settings, now_utc=now), reason
|
||||
|
||||
|
||||
def get_safety_state_payload(
|
||||
settings: UserSetting,
|
||||
*,
|
||||
now_utc: datetime | None = None,
|
||||
) -> dict[str, Any]:
|
||||
now = now_utc or _utcnow()
|
||||
cooldown_until = _normalize_datetime(settings.scrape_cooldown_until)
|
||||
cooldown_active = bool(cooldown_until is not None and cooldown_until > now)
|
||||
cooldown_remaining_seconds = 0
|
||||
if cooldown_active and cooldown_until is not None:
|
||||
cooldown_remaining_seconds = max(0, int((cooldown_until - now).total_seconds()))
|
||||
|
||||
reason = settings.scrape_cooldown_reason if cooldown_active else None
|
||||
|
||||
return {
|
||||
"cooldown_active": cooldown_active,
|
||||
"cooldown_reason": reason,
|
||||
"cooldown_reason_label": _cooldown_reason_label(reason),
|
||||
"cooldown_until": cooldown_until,
|
||||
"cooldown_remaining_seconds": cooldown_remaining_seconds,
|
||||
"recommended_action": _recommended_action(reason),
|
||||
"counters": _counters_from_state(settings),
|
||||
}
|
||||
|
||||
|
||||
def get_safety_event_context(
|
||||
settings: UserSetting,
|
||||
*,
|
||||
now_utc: datetime | None = None,
|
||||
) -> dict[str, Any]:
|
||||
payload = get_safety_state_payload(settings, now_utc=now_utc)
|
||||
return {
|
||||
"cooldown_active": bool(payload.get("cooldown_active")),
|
||||
"cooldown_reason": payload.get("cooldown_reason"),
|
||||
"cooldown_until": payload.get("cooldown_until"),
|
||||
"cooldown_remaining_seconds": int(payload.get("cooldown_remaining_seconds") or 0),
|
||||
"safety_counters": payload.get("counters", {}),
|
||||
}
|
||||
|
|
@ -1,622 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta, timezone
|
||||
import logging
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.models import (
|
||||
CrawlRun,
|
||||
QueueItemStatus,
|
||||
RunTriggerType,
|
||||
ScholarProfile,
|
||||
User,
|
||||
UserSetting,
|
||||
)
|
||||
from app.db.session import get_session_factory
|
||||
from app.services.domains.ingestion import queue as queue_service
|
||||
from app.services.domains.ingestion.application import (
|
||||
RunAlreadyInProgressError,
|
||||
RunBlockedBySafetyPolicyError,
|
||||
ScholarIngestionService,
|
||||
)
|
||||
from app.services.domains.settings import application as user_settings_service
|
||||
from app.services.domains.scholar.source import LiveScholarSource
|
||||
from app.settings import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _request_delay_floor_seconds() -> int:
|
||||
return user_settings_service.resolve_request_delay_minimum(
|
||||
settings.ingestion_min_request_delay_seconds
|
||||
)
|
||||
|
||||
|
||||
def _effective_request_delay_seconds(value: int | None) -> int:
|
||||
floor = _request_delay_floor_seconds()
|
||||
try:
|
||||
parsed = int(value) if value is not None else floor
|
||||
except (TypeError, ValueError):
|
||||
parsed = floor
|
||||
return max(floor, parsed)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _AutoRunCandidate:
|
||||
user_id: int
|
||||
run_interval_minutes: int
|
||||
request_delay_seconds: int
|
||||
cooldown_until: datetime | None
|
||||
cooldown_reason: str | None
|
||||
|
||||
|
||||
class SchedulerService:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
enabled: bool,
|
||||
tick_seconds: int,
|
||||
network_error_retries: int,
|
||||
retry_backoff_seconds: float,
|
||||
max_pages_per_scholar: int,
|
||||
page_size: int,
|
||||
continuation_queue_enabled: bool,
|
||||
continuation_base_delay_seconds: int,
|
||||
continuation_max_delay_seconds: int,
|
||||
continuation_max_attempts: int,
|
||||
queue_batch_size: int,
|
||||
) -> None:
|
||||
self._enabled = enabled
|
||||
self._tick_seconds = max(5, int(tick_seconds))
|
||||
self._network_error_retries = max(0, int(network_error_retries))
|
||||
self._retry_backoff_seconds = max(0.0, float(retry_backoff_seconds))
|
||||
self._max_pages_per_scholar = max(1, int(max_pages_per_scholar))
|
||||
self._page_size = max(1, int(page_size))
|
||||
self._continuation_queue_enabled = bool(continuation_queue_enabled)
|
||||
self._continuation_base_delay_seconds = max(1, int(continuation_base_delay_seconds))
|
||||
self._continuation_max_delay_seconds = max(
|
||||
self._continuation_base_delay_seconds,
|
||||
int(continuation_max_delay_seconds),
|
||||
)
|
||||
self._continuation_max_attempts = max(1, int(continuation_max_attempts))
|
||||
self._queue_batch_size = max(1, int(queue_batch_size))
|
||||
self._task: asyncio.Task[None] | None = None
|
||||
self._source = LiveScholarSource()
|
||||
|
||||
async def start(self) -> None:
|
||||
if not self._enabled:
|
||||
logger.info(
|
||||
"scheduler.disabled",
|
||||
extra={
|
||||
"event": "scheduler.disabled",
|
||||
},
|
||||
)
|
||||
return
|
||||
if self._task is not None:
|
||||
return
|
||||
self._task = asyncio.create_task(self._run_loop(), name="scholarr-scheduler")
|
||||
logger.info(
|
||||
"scheduler.started",
|
||||
extra={
|
||||
"event": "scheduler.started",
|
||||
"tick_seconds": self._tick_seconds,
|
||||
"network_error_retries": self._network_error_retries,
|
||||
"retry_backoff_seconds": self._retry_backoff_seconds,
|
||||
"max_pages_per_scholar": self._max_pages_per_scholar,
|
||||
"page_size": self._page_size,
|
||||
"continuation_queue_enabled": self._continuation_queue_enabled,
|
||||
"continuation_base_delay_seconds": self._continuation_base_delay_seconds,
|
||||
"continuation_max_delay_seconds": self._continuation_max_delay_seconds,
|
||||
"continuation_max_attempts": self._continuation_max_attempts,
|
||||
"queue_batch_size": self._queue_batch_size,
|
||||
},
|
||||
)
|
||||
|
||||
async def stop(self) -> None:
|
||||
if self._task is None:
|
||||
return
|
||||
self._task.cancel()
|
||||
try:
|
||||
await self._task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
finally:
|
||||
self._task = None
|
||||
logger.info("scheduler.stopped", extra={"event": "scheduler.stopped"})
|
||||
|
||||
async def _run_loop(self) -> None:
|
||||
while True:
|
||||
try:
|
||||
await self._tick_once()
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"scheduler.tick_failed",
|
||||
extra={
|
||||
"event": "scheduler.tick_failed",
|
||||
},
|
||||
)
|
||||
await asyncio.sleep(float(self._tick_seconds))
|
||||
|
||||
async def _tick_once(self) -> None:
|
||||
if self._continuation_queue_enabled:
|
||||
await self._drain_continuation_queue()
|
||||
|
||||
await self._drain_pdf_queue()
|
||||
|
||||
candidates = await self._load_candidates()
|
||||
if not candidates:
|
||||
return
|
||||
now = datetime.now(timezone.utc)
|
||||
for candidate in candidates:
|
||||
if not await self._is_due(candidate, now=now):
|
||||
continue
|
||||
await self._run_candidate(candidate)
|
||||
|
||||
async def _load_candidate_rows(self) -> list[tuple]:
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as session:
|
||||
result = await session.execute(
|
||||
select(
|
||||
UserSetting.user_id,
|
||||
UserSetting.run_interval_minutes,
|
||||
UserSetting.request_delay_seconds,
|
||||
UserSetting.scrape_cooldown_until,
|
||||
UserSetting.scrape_cooldown_reason,
|
||||
)
|
||||
.join(User, User.id == UserSetting.user_id)
|
||||
.where(User.is_active.is_(True), UserSetting.auto_run_enabled.is_(True))
|
||||
.order_by(UserSetting.user_id.asc())
|
||||
)
|
||||
return result.all()
|
||||
|
||||
@staticmethod
|
||||
def _candidate_from_row(row: tuple, *, now_utc: datetime) -> _AutoRunCandidate | None:
|
||||
user_id, run_interval_minutes, request_delay_seconds, cooldown_until, cooldown_reason = row
|
||||
if cooldown_until is not None and cooldown_until.tzinfo is None:
|
||||
cooldown_until = cooldown_until.replace(tzinfo=timezone.utc)
|
||||
if cooldown_until is not None and cooldown_until > now_utc:
|
||||
logger.info(
|
||||
"scheduler.run_skipped_safety_cooldown_precheck",
|
||||
extra={
|
||||
"event": "scheduler.run_skipped_safety_cooldown_precheck",
|
||||
"user_id": int(user_id),
|
||||
"reason": cooldown_reason,
|
||||
"cooldown_until": cooldown_until,
|
||||
"cooldown_remaining_seconds": int((cooldown_until - now_utc).total_seconds()),
|
||||
"metric_name": "scheduler_run_skipped_safety_cooldown_total",
|
||||
"metric_value": 1,
|
||||
},
|
||||
)
|
||||
return None
|
||||
return _AutoRunCandidate(
|
||||
user_id=int(user_id),
|
||||
run_interval_minutes=int(run_interval_minutes),
|
||||
request_delay_seconds=_effective_request_delay_seconds(request_delay_seconds),
|
||||
cooldown_until=cooldown_until,
|
||||
cooldown_reason=(str(cooldown_reason).strip() if cooldown_reason else None),
|
||||
)
|
||||
|
||||
async def _load_candidates(self) -> list[_AutoRunCandidate]:
|
||||
if not settings.ingestion_automation_allowed:
|
||||
return []
|
||||
rows = await self._load_candidate_rows()
|
||||
now_utc = datetime.now(timezone.utc)
|
||||
candidates: list[_AutoRunCandidate] = []
|
||||
for row in rows:
|
||||
candidate = self._candidate_from_row(row, now_utc=now_utc)
|
||||
if candidate is not None:
|
||||
candidates.append(candidate)
|
||||
return candidates
|
||||
|
||||
async def _is_due(self, candidate: _AutoRunCandidate, *, now: datetime) -> bool:
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as session:
|
||||
result = await session.execute(
|
||||
select(CrawlRun.start_dt)
|
||||
.where(
|
||||
CrawlRun.user_id == candidate.user_id,
|
||||
)
|
||||
.order_by(CrawlRun.start_dt.desc(), CrawlRun.id.desc())
|
||||
.limit(1)
|
||||
)
|
||||
last_run = result.scalar_one_or_none()
|
||||
|
||||
if last_run is None:
|
||||
return True
|
||||
|
||||
next_due_dt = last_run + timedelta(
|
||||
minutes=candidate.run_interval_minutes
|
||||
)
|
||||
return now >= next_due_dt
|
||||
|
||||
async def _run_candidate_ingestion(
|
||||
self,
|
||||
*,
|
||||
candidate: _AutoRunCandidate,
|
||||
):
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as session:
|
||||
ingestion = ScholarIngestionService(source=self._source)
|
||||
try:
|
||||
return await ingestion.run_for_user(
|
||||
session,
|
||||
user_id=candidate.user_id,
|
||||
trigger_type=RunTriggerType.SCHEDULED,
|
||||
request_delay_seconds=candidate.request_delay_seconds,
|
||||
network_error_retries=self._network_error_retries,
|
||||
retry_backoff_seconds=self._retry_backoff_seconds,
|
||||
max_pages_per_scholar=self._max_pages_per_scholar,
|
||||
page_size=self._page_size,
|
||||
auto_queue_continuations=self._continuation_queue_enabled,
|
||||
queue_delay_seconds=self._continuation_base_delay_seconds,
|
||||
alert_blocked_failure_threshold=settings.ingestion_alert_blocked_failure_threshold,
|
||||
alert_network_failure_threshold=settings.ingestion_alert_network_failure_threshold,
|
||||
alert_retry_scheduled_threshold=settings.ingestion_alert_retry_scheduled_threshold,
|
||||
)
|
||||
except RunAlreadyInProgressError:
|
||||
await session.rollback()
|
||||
logger.info("scheduler.run_skipped_locked", extra={"event": "scheduler.run_skipped_locked", "user_id": candidate.user_id})
|
||||
return None
|
||||
except RunBlockedBySafetyPolicyError as exc:
|
||||
await session.rollback()
|
||||
logger.info(
|
||||
"scheduler.run_skipped_safety_cooldown",
|
||||
extra={
|
||||
"event": "scheduler.run_skipped_safety_cooldown",
|
||||
"user_id": candidate.user_id,
|
||||
"reason": exc.safety_state.get("cooldown_reason"),
|
||||
"cooldown_until": exc.safety_state.get("cooldown_until"),
|
||||
"cooldown_remaining_seconds": exc.safety_state.get("cooldown_remaining_seconds"),
|
||||
"metric_name": "scheduler_run_skipped_safety_cooldown_total",
|
||||
"metric_value": 1,
|
||||
},
|
||||
)
|
||||
return None
|
||||
except Exception:
|
||||
await session.rollback()
|
||||
logger.exception("scheduler.run_failed", extra={"event": "scheduler.run_failed", "user_id": candidate.user_id})
|
||||
return None
|
||||
|
||||
async def _run_candidate(self, candidate: _AutoRunCandidate) -> None:
|
||||
run_summary = await self._run_candidate_ingestion(candidate=candidate)
|
||||
if run_summary is None:
|
||||
return
|
||||
logger.info(
|
||||
"scheduler.run_completed",
|
||||
extra={
|
||||
"event": "scheduler.run_completed",
|
||||
"user_id": candidate.user_id,
|
||||
"run_id": run_summary.crawl_run_id,
|
||||
"status": run_summary.status.value,
|
||||
"scholar_count": run_summary.scholar_count,
|
||||
"new_publication_count": run_summary.new_publication_count,
|
||||
},
|
||||
)
|
||||
|
||||
async def _drain_continuation_queue(self) -> None:
|
||||
now = datetime.now(timezone.utc)
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as session:
|
||||
jobs = await queue_service.list_due_jobs(
|
||||
session,
|
||||
now=now,
|
||||
limit=self._queue_batch_size,
|
||||
)
|
||||
for job in jobs:
|
||||
await self._run_queue_job(job)
|
||||
|
||||
async def _drop_queue_job_if_max_attempts(
|
||||
self,
|
||||
job: queue_service.ContinuationQueueJob,
|
||||
) -> bool:
|
||||
if job.attempt_count < self._continuation_max_attempts:
|
||||
return False
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as session:
|
||||
dropped = await queue_service.mark_dropped(
|
||||
session,
|
||||
job_id=job.id,
|
||||
reason="max_attempts_reached",
|
||||
)
|
||||
await session.commit()
|
||||
if dropped is not None:
|
||||
logger.warning(
|
||||
"scheduler.queue_item_dropped_max_attempts",
|
||||
extra={
|
||||
"event": "scheduler.queue_item_dropped_max_attempts",
|
||||
"queue_item_id": job.id,
|
||||
"user_id": job.user_id,
|
||||
"scholar_profile_id": job.scholar_profile_id,
|
||||
"attempt_count": job.attempt_count,
|
||||
"max_attempts": self._continuation_max_attempts,
|
||||
},
|
||||
)
|
||||
return True
|
||||
|
||||
async def _mark_queue_job_retrying(
|
||||
self,
|
||||
job: queue_service.ContinuationQueueJob,
|
||||
) -> bool:
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as session:
|
||||
queue_item = await queue_service.mark_retrying(session, job_id=job.id)
|
||||
await session.commit()
|
||||
if queue_item is None:
|
||||
return False
|
||||
if queue_item.status == QueueItemStatus.DROPPED.value:
|
||||
return False
|
||||
return True
|
||||
|
||||
async def _queue_job_has_available_scholar(
|
||||
self,
|
||||
job: queue_service.ContinuationQueueJob,
|
||||
) -> bool:
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as session:
|
||||
scholar_result = await session.execute(
|
||||
select(ScholarProfile.id).where(
|
||||
ScholarProfile.user_id == job.user_id,
|
||||
ScholarProfile.id == job.scholar_profile_id,
|
||||
ScholarProfile.is_enabled.is_(True),
|
||||
)
|
||||
)
|
||||
scholar_id = scholar_result.scalar_one_or_none()
|
||||
if scholar_id is not None:
|
||||
return True
|
||||
dropped = await queue_service.mark_dropped(
|
||||
session,
|
||||
job_id=job.id,
|
||||
reason="scholar_unavailable",
|
||||
)
|
||||
await session.commit()
|
||||
if dropped is not None:
|
||||
logger.info(
|
||||
"scheduler.queue_item_dropped_scholar_unavailable",
|
||||
extra={
|
||||
"event": "scheduler.queue_item_dropped_scholar_unavailable",
|
||||
"queue_item_id": job.id,
|
||||
"user_id": job.user_id,
|
||||
"scholar_profile_id": job.scholar_profile_id,
|
||||
},
|
||||
)
|
||||
return False
|
||||
|
||||
async def _reschedule_queue_job_lock_active(self, job: queue_service.ContinuationQueueJob) -> None:
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as recovery_session:
|
||||
await queue_service.reschedule_job(
|
||||
recovery_session,
|
||||
job_id=job.id,
|
||||
delay_seconds=max(self._tick_seconds, 15),
|
||||
reason="user_run_lock_active",
|
||||
error="run_already_in_progress",
|
||||
)
|
||||
await recovery_session.commit()
|
||||
logger.info(
|
||||
"scheduler.queue_item_deferred_lock",
|
||||
extra={"event": "scheduler.queue_item_deferred_lock", "queue_item_id": job.id, "user_id": job.user_id},
|
||||
)
|
||||
|
||||
async def _reschedule_queue_job_safety_cooldown(
|
||||
self,
|
||||
job: queue_service.ContinuationQueueJob,
|
||||
exc: RunBlockedBySafetyPolicyError,
|
||||
) -> None:
|
||||
cooldown_remaining_seconds = max(
|
||||
self._tick_seconds,
|
||||
int(exc.safety_state.get("cooldown_remaining_seconds") or 0),
|
||||
)
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as recovery_session:
|
||||
await queue_service.reschedule_job(
|
||||
recovery_session,
|
||||
job_id=job.id,
|
||||
delay_seconds=max(self._tick_seconds, cooldown_remaining_seconds),
|
||||
reason="scrape_safety_cooldown",
|
||||
error=str(exc.message),
|
||||
)
|
||||
await recovery_session.commit()
|
||||
logger.info(
|
||||
"scheduler.queue_item_deferred_safety_cooldown",
|
||||
extra={
|
||||
"event": "scheduler.queue_item_deferred_safety_cooldown",
|
||||
"queue_item_id": job.id,
|
||||
"user_id": job.user_id,
|
||||
"reason": exc.safety_state.get("cooldown_reason"),
|
||||
"cooldown_remaining_seconds": cooldown_remaining_seconds,
|
||||
"metric_name": "scheduler_queue_item_deferred_safety_cooldown_total",
|
||||
"metric_value": 1,
|
||||
},
|
||||
)
|
||||
|
||||
async def _reschedule_queue_job_after_exception(
|
||||
self,
|
||||
job: queue_service.ContinuationQueueJob,
|
||||
*,
|
||||
exc: Exception,
|
||||
) -> None:
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as recovery_session:
|
||||
queue_item = await queue_service.increment_attempt_count(recovery_session, job_id=job.id)
|
||||
if queue_item is None:
|
||||
await recovery_session.commit()
|
||||
return
|
||||
if int(queue_item.attempt_count) >= self._continuation_max_attempts:
|
||||
await queue_service.mark_dropped(
|
||||
recovery_session,
|
||||
job_id=job.id,
|
||||
reason="scheduler_exception_max_attempts",
|
||||
error=str(exc),
|
||||
)
|
||||
await recovery_session.commit()
|
||||
logger.warning(
|
||||
"scheduler.queue_item_dropped_after_exception",
|
||||
extra={"event": "scheduler.queue_item_dropped_after_exception", "queue_item_id": job.id, "user_id": job.user_id, "attempt_count": queue_item.attempt_count},
|
||||
)
|
||||
return
|
||||
delay_seconds = queue_service.compute_backoff_seconds(
|
||||
base_seconds=self._continuation_base_delay_seconds,
|
||||
attempt_count=int(queue_item.attempt_count),
|
||||
max_seconds=self._continuation_max_delay_seconds,
|
||||
)
|
||||
await queue_service.reschedule_job(
|
||||
recovery_session,
|
||||
job_id=job.id,
|
||||
delay_seconds=delay_seconds,
|
||||
reason="scheduler_exception",
|
||||
error=str(exc),
|
||||
)
|
||||
await recovery_session.commit()
|
||||
logger.exception("scheduler.queue_item_run_failed", extra={"event": "scheduler.queue_item_run_failed", "queue_item_id": job.id, "user_id": job.user_id})
|
||||
|
||||
async def _run_ingestion_for_queue_job(
|
||||
self,
|
||||
job: queue_service.ContinuationQueueJob,
|
||||
):
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as session:
|
||||
request_delay_seconds = await self._load_request_delay_for_user(session, user_id=job.user_id)
|
||||
ingestion = ScholarIngestionService(source=self._source)
|
||||
try:
|
||||
return await ingestion.run_for_user(
|
||||
session,
|
||||
user_id=job.user_id,
|
||||
trigger_type=RunTriggerType.SCHEDULED,
|
||||
request_delay_seconds=request_delay_seconds,
|
||||
network_error_retries=self._network_error_retries,
|
||||
retry_backoff_seconds=self._retry_backoff_seconds,
|
||||
rate_limit_retries=settings.ingestion_rate_limit_retries,
|
||||
rate_limit_backoff_seconds=settings.ingestion_rate_limit_backoff_seconds,
|
||||
max_pages_per_scholar=self._max_pages_per_scholar,
|
||||
page_size=self._page_size,
|
||||
scholar_profile_ids={job.scholar_profile_id},
|
||||
start_cstart_by_scholar_id={job.scholar_profile_id: job.resume_cstart},
|
||||
auto_queue_continuations=self._continuation_queue_enabled,
|
||||
queue_delay_seconds=self._continuation_base_delay_seconds,
|
||||
alert_blocked_failure_threshold=settings.ingestion_alert_blocked_failure_threshold,
|
||||
alert_network_failure_threshold=settings.ingestion_alert_network_failure_threshold,
|
||||
alert_retry_scheduled_threshold=settings.ingestion_alert_retry_scheduled_threshold,
|
||||
)
|
||||
except RunAlreadyInProgressError:
|
||||
await session.rollback()
|
||||
await self._reschedule_queue_job_lock_active(job)
|
||||
except RunBlockedBySafetyPolicyError as exc:
|
||||
await session.rollback()
|
||||
await self._reschedule_queue_job_safety_cooldown(job, exc)
|
||||
except Exception as exc:
|
||||
await session.rollback()
|
||||
await self._reschedule_queue_job_after_exception(job, exc=exc)
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _log_queue_item_resolved(
|
||||
*,
|
||||
event_name: str,
|
||||
job: queue_service.ContinuationQueueJob,
|
||||
run_summary,
|
||||
attempt_count: int | None = None,
|
||||
delay_seconds: int | None = None,
|
||||
) -> None:
|
||||
payload = {
|
||||
"event": event_name,
|
||||
"queue_item_id": job.id,
|
||||
"user_id": job.user_id,
|
||||
"run_id": run_summary.crawl_run_id,
|
||||
"status": run_summary.status.value,
|
||||
}
|
||||
if attempt_count is not None:
|
||||
payload["attempt_count"] = attempt_count
|
||||
if delay_seconds is not None:
|
||||
payload["delay_seconds"] = delay_seconds
|
||||
logger.info(event_name, extra=payload)
|
||||
|
||||
async def _finalize_queue_job_after_run(self, job: queue_service.ContinuationQueueJob, run_summary) -> None:
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as session:
|
||||
if int(run_summary.failed_count) <= 0:
|
||||
queue_item = await queue_service.reset_attempt_count(session, job_id=job.id)
|
||||
await session.commit()
|
||||
if queue_item is None:
|
||||
self._log_queue_item_resolved(event_name="scheduler.queue_item_resolved", job=job, run_summary=run_summary)
|
||||
return
|
||||
self._log_queue_item_resolved(
|
||||
event_name="scheduler.queue_item_progressed",
|
||||
job=job,
|
||||
run_summary=run_summary,
|
||||
attempt_count=int(queue_item.attempt_count),
|
||||
)
|
||||
return
|
||||
queue_item = await queue_service.increment_attempt_count(session, job_id=job.id)
|
||||
if queue_item is None:
|
||||
await session.commit()
|
||||
self._log_queue_item_resolved(event_name="scheduler.queue_item_resolved", job=job, run_summary=run_summary)
|
||||
return
|
||||
if int(queue_item.attempt_count) >= self._continuation_max_attempts:
|
||||
await queue_service.mark_dropped(session, job_id=job.id, reason="max_attempts_after_run")
|
||||
await session.commit()
|
||||
logger.warning(
|
||||
"scheduler.queue_item_dropped_max_attempts_after_run",
|
||||
extra={"event": "scheduler.queue_item_dropped_max_attempts_after_run", "queue_item_id": job.id, "user_id": job.user_id, "attempt_count": queue_item.attempt_count, "run_id": run_summary.crawl_run_id, "status": run_summary.status.value},
|
||||
)
|
||||
return
|
||||
delay_seconds = queue_service.compute_backoff_seconds(base_seconds=self._continuation_base_delay_seconds, attempt_count=int(queue_item.attempt_count), max_seconds=self._continuation_max_delay_seconds)
|
||||
await queue_service.reschedule_job(session, job_id=job.id, delay_seconds=delay_seconds, reason=queue_item.reason, error=queue_item.last_error)
|
||||
await session.commit()
|
||||
self._log_queue_item_resolved(
|
||||
event_name="scheduler.queue_item_rescheduled_failed",
|
||||
job=job,
|
||||
run_summary=run_summary,
|
||||
attempt_count=int(queue_item.attempt_count),
|
||||
delay_seconds=delay_seconds,
|
||||
)
|
||||
|
||||
async def _run_queue_job(self, job: queue_service.ContinuationQueueJob) -> None:
|
||||
if await self._drop_queue_job_if_max_attempts(job):
|
||||
return
|
||||
if not await self._mark_queue_job_retrying(job):
|
||||
return
|
||||
if not await self._queue_job_has_available_scholar(job):
|
||||
return
|
||||
run_summary = await self._run_ingestion_for_queue_job(job)
|
||||
if run_summary is None:
|
||||
return
|
||||
await self._finalize_queue_job_after_run(job, run_summary)
|
||||
|
||||
async def _drain_pdf_queue(self) -> None:
|
||||
from app.services.domains.publications.pdf_queue import drain_ready_jobs
|
||||
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as session:
|
||||
try:
|
||||
processed = await drain_ready_jobs(
|
||||
session,
|
||||
limit=settings.scheduler_pdf_queue_batch_size,
|
||||
max_attempts=settings.pdf_auto_retry_max_attempts,
|
||||
)
|
||||
if processed > 0:
|
||||
logger.info("scheduler.pdf_queue_drain_completed", extra={
|
||||
"event": "scheduler.pdf_queue_drain_completed",
|
||||
"processed_count": processed,
|
||||
})
|
||||
except Exception:
|
||||
logger.exception("scheduler.pdf_queue_drain_failed", extra={
|
||||
"event": "scheduler.pdf_queue_drain_failed",
|
||||
})
|
||||
|
||||
async def _load_request_delay_for_user(
|
||||
self,
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
) -> int:
|
||||
result = await db_session.execute(
|
||||
select(UserSetting.request_delay_seconds).where(UserSetting.user_id == user_id)
|
||||
)
|
||||
delay = result.scalar_one_or_none()
|
||||
return _effective_request_delay_seconds(delay)
|
||||
|
|
@ -1,110 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from app.db.models import RunStatus
|
||||
from app.services.domains.scholar.parser import ParsedProfilePage, PublicationCandidate
|
||||
from app.services.domains.scholar.source import FetchResult
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RunExecutionSummary:
|
||||
crawl_run_id: int
|
||||
status: RunStatus
|
||||
scholar_count: int
|
||||
succeeded_count: int
|
||||
failed_count: int
|
||||
partial_count: int
|
||||
new_publication_count: int
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PagedParseResult:
|
||||
fetch_result: FetchResult
|
||||
parsed_page: ParsedProfilePage
|
||||
first_page_fetch_result: FetchResult
|
||||
first_page_parsed_page: ParsedProfilePage
|
||||
first_page_fingerprint_sha256: str | None
|
||||
publications: list[PublicationCandidate]
|
||||
attempt_log: list[dict[str, Any]]
|
||||
page_logs: list[dict[str, Any]]
|
||||
pages_fetched: int
|
||||
pages_attempted: int
|
||||
has_more_remaining: bool
|
||||
pagination_truncated_reason: str | None
|
||||
continuation_cstart: int | None
|
||||
skipped_no_change: bool
|
||||
discovered_publication_count: int
|
||||
|
||||
|
||||
@dataclass
|
||||
class RunProgress:
|
||||
succeeded_count: int = 0
|
||||
failed_count: int = 0
|
||||
partial_count: int = 0
|
||||
scholar_results: list[dict[str, Any]] = field(default_factory=list)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ScholarProcessingOutcome:
|
||||
result_entry: dict[str, Any]
|
||||
succeeded_count_delta: int
|
||||
failed_count_delta: int
|
||||
partial_count_delta: int
|
||||
discovered_publication_count: int
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RunFailureSummary:
|
||||
failed_state_counts: dict[str, int]
|
||||
failed_reason_counts: dict[str, int]
|
||||
scrape_failure_counts: dict[str, int]
|
||||
retries_scheduled_count: int
|
||||
scholars_with_retries_count: int
|
||||
retry_exhausted_count: int
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RunAlertSummary:
|
||||
blocked_failure_count: int
|
||||
network_failure_count: int
|
||||
blocked_failure_threshold: int
|
||||
network_failure_threshold: int
|
||||
retry_scheduled_threshold: int
|
||||
alert_flags: dict[str, bool]
|
||||
|
||||
|
||||
@dataclass
|
||||
class PagedLoopState:
|
||||
fetch_result: FetchResult
|
||||
parsed_page: ParsedProfilePage
|
||||
attempt_log: list[dict[str, Any]]
|
||||
page_logs: list[dict[str, Any]]
|
||||
publications: list[PublicationCandidate]
|
||||
pages_fetched: int
|
||||
pages_attempted: int
|
||||
current_cstart: int
|
||||
next_cstart: int
|
||||
has_more_remaining: bool = False
|
||||
pagination_truncated_reason: str | None = None
|
||||
continuation_cstart: int | None = None
|
||||
discovered_publication_count: int = 0
|
||||
|
||||
|
||||
class RunAlreadyInProgressError(RuntimeError):
|
||||
"""Raised when a run lock for a user is already held by another process."""
|
||||
|
||||
|
||||
class RunBlockedBySafetyPolicyError(RuntimeError):
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
code: str,
|
||||
message: str,
|
||||
safety_state: dict[str, Any],
|
||||
) -> None:
|
||||
super().__init__(message)
|
||||
self.code = code
|
||||
self.message = message
|
||||
self.safety_state = safety_state
|
||||
|
|
@ -1,5 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
|
@ -1,135 +0,0 @@
|
|||
import asyncio
|
||||
import logging
|
||||
from typing import Any, Mapping
|
||||
|
||||
import httpx
|
||||
from tenacity import (
|
||||
retry,
|
||||
retry_if_exception_type,
|
||||
stop_after_attempt,
|
||||
wait_exponential,
|
||||
)
|
||||
|
||||
from app.services.domains.openalex.types import OpenAlexWork
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
OPENALEX_BASE_URL = "https://api.openalex.org"
|
||||
|
||||
|
||||
class OpenAlexClientError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class OpenAlexRateLimitError(OpenAlexClientError):
|
||||
pass
|
||||
|
||||
|
||||
class OpenAlexClient:
|
||||
def __init__(
|
||||
self,
|
||||
api_key: str | None = None,
|
||||
mailto: str | None = None,
|
||||
timeout: float = 10.0,
|
||||
) -> None:
|
||||
self.api_key = api_key
|
||||
self.mailto = mailto
|
||||
self.timeout = timeout
|
||||
|
||||
@property
|
||||
def _base_params(self) -> dict[str, str]:
|
||||
params = {}
|
||||
if self.mailto:
|
||||
params["mailto"] = self.mailto
|
||||
if self.api_key:
|
||||
params["api_key"] = self.api_key
|
||||
return params
|
||||
|
||||
@retry(
|
||||
retry=retry_if_exception_type((httpx.NetworkError, httpx.TimeoutException, OpenAlexRateLimitError)),
|
||||
stop=stop_after_attempt(3),
|
||||
wait=wait_exponential(multiplier=1, min=2, max=10),
|
||||
reraise=True,
|
||||
)
|
||||
async def get_work_by_doi(self, doi: str) -> OpenAlexWork | None:
|
||||
"""Fetch a single work by DOI directly."""
|
||||
clean_doi = doi.replace("https://doi.org/", "")
|
||||
if not clean_doi:
|
||||
return None
|
||||
|
||||
url = f"{OPENALEX_BASE_URL}/works/{clean_doi}"
|
||||
|
||||
headers = {}
|
||||
if self.mailto:
|
||||
headers["User-Agent"] = f"scholar-scraper/1.0 (mailto:{self.mailto})"
|
||||
else:
|
||||
headers["User-Agent"] = "scholar-scraper/1.0"
|
||||
|
||||
async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True, headers=headers) as client:
|
||||
response = await client.get(url, params=self._base_params)
|
||||
|
||||
if response.status_code == 404:
|
||||
return None
|
||||
if response.status_code == 429:
|
||||
raise OpenAlexRateLimitError("Rate limit exceeded fetching OpenAlex work by DOI")
|
||||
if response.status_code >= 400:
|
||||
logger.warning("OpenAlex API error: %s %s", response.status_code, response.text)
|
||||
raise OpenAlexClientError(f"API Error {response.status_code}")
|
||||
|
||||
data = response.json()
|
||||
return OpenAlexWork.from_api_dict(data)
|
||||
|
||||
@retry(
|
||||
retry=retry_if_exception_type((httpx.NetworkError, httpx.TimeoutException, OpenAlexRateLimitError)),
|
||||
stop=stop_after_attempt(3),
|
||||
wait=wait_exponential(multiplier=1, min=2, max=10),
|
||||
reraise=True,
|
||||
)
|
||||
async def get_works_by_filter(
|
||||
self,
|
||||
filters: dict[str, str],
|
||||
limit: int = 50,
|
||||
) -> list[OpenAlexWork]:
|
||||
"""
|
||||
Fetch works using the ?filter= query parameter.
|
||||
Supports fetching multiple records by joining filters with | (OR logic).
|
||||
"""
|
||||
if not filters:
|
||||
return []
|
||||
|
||||
# Example: {"doi": "10.foo|10.bar", "title.search": "query"}
|
||||
filter_str = ",".join(f"{k}:{v}" for k, v in filters.items())
|
||||
|
||||
params = self._base_params.copy()
|
||||
params["filter"] = filter_str
|
||||
params["per-page"] = str(limit)
|
||||
|
||||
url = f"{OPENALEX_BASE_URL}/works"
|
||||
|
||||
headers = {}
|
||||
if self.mailto:
|
||||
headers["User-Agent"] = f"scholar-scraper/1.0 (mailto:{self.mailto})"
|
||||
else:
|
||||
headers["User-Agent"] = "scholar-scraper/1.0"
|
||||
|
||||
async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True, headers=headers) as client:
|
||||
response = await client.get(url, params=params)
|
||||
|
||||
if response.status_code == 429:
|
||||
raise OpenAlexRateLimitError("Rate limit exceeded fetching OpenAlex works list")
|
||||
if response.status_code >= 400:
|
||||
logger.warning("OpenAlex API error (filters=%s): %s %s", filters, response.status_code, response.text)
|
||||
raise OpenAlexClientError(f"API Error {response.status_code}")
|
||||
|
||||
data = response.json()
|
||||
results = data.get("results") or []
|
||||
|
||||
parsed_works = []
|
||||
for raw_work in results:
|
||||
try:
|
||||
parsed_works.append(OpenAlexWork.from_api_dict(raw_work))
|
||||
except Exception as e:
|
||||
logger.warning("Failed to parse OpenAlex raw dict: %s", e)
|
||||
continue
|
||||
|
||||
return parsed_works
|
||||
|
|
@ -1,108 +0,0 @@
|
|||
import logging
|
||||
import re
|
||||
|
||||
from rapidfuzz import fuzz
|
||||
|
||||
from app.services.domains.openalex.types import OpenAlexWork
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# A minimum similarity score out of 100 for a title to be considered a match candidate.
|
||||
TITLE_MATCH_THRESHOLD = 90.0
|
||||
# The margin within the top score where a secondary tiebreaker (author/year) is necessary.
|
||||
TIEBREAKER_MARGIN = 5.0
|
||||
|
||||
|
||||
def _clean_string(s: str | None) -> str:
|
||||
if not s:
|
||||
return ""
|
||||
# Strip non-alphanumeric (keep spaces), lowercase, and collapse whitespace
|
||||
cleaned = re.sub(r"[^a-z0-9\s]", " ", s.lower())
|
||||
return " ".join(cleaned.split())
|
||||
|
||||
|
||||
def _author_overlap_score(target_authors: str | None, candidate_authors: list[str]) -> bool:
|
||||
if not target_authors or not candidate_authors:
|
||||
return False
|
||||
|
||||
target_clean = _clean_string(target_authors)
|
||||
if not target_clean:
|
||||
return False
|
||||
|
||||
for candidate in candidate_authors:
|
||||
cand_clean = _clean_string(candidate)
|
||||
if cand_clean and (cand_clean in target_clean or target_clean in cand_clean):
|
||||
return True
|
||||
# Alternatively check rapidfuzz token_set_ratio
|
||||
if cand_clean and fuzz.token_set_ratio(target_clean, cand_clean) > 80:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def find_best_match(
|
||||
target_title: str,
|
||||
target_year: int | None,
|
||||
target_authors: str | None,
|
||||
candidates: list[OpenAlexWork],
|
||||
) -> OpenAlexWork | None:
|
||||
"""
|
||||
Finds the best matching OpenAlexWork from a list of candidates, prioritizing title similarity (>90%)
|
||||
with year and author overlap as tiebreakers for close candidates.
|
||||
"""
|
||||
if not target_title or not candidates:
|
||||
return None
|
||||
|
||||
clean_target = _clean_string(target_title)
|
||||
if not clean_target:
|
||||
return None
|
||||
|
||||
scored_candidates: list[tuple[float, OpenAlexWork]] = []
|
||||
|
||||
for cand in candidates:
|
||||
if not cand.title:
|
||||
continue
|
||||
|
||||
clean_cand = _clean_string(cand.title)
|
||||
|
||||
# Primary sort: string similarity ratio
|
||||
score = fuzz.ratio(clean_target, clean_cand)
|
||||
|
||||
if score >= TITLE_MATCH_THRESHOLD:
|
||||
scored_candidates.append((score, cand))
|
||||
|
||||
if not scored_candidates:
|
||||
return None
|
||||
|
||||
# Sort descending by score
|
||||
scored_candidates.sort(key=lambda x: x[0], reverse=True)
|
||||
|
||||
best_score = scored_candidates[0][0]
|
||||
|
||||
# Extract all candidates within the tiebreaker margin
|
||||
top_scored_candidates = [
|
||||
(score, cand) for score, cand in scored_candidates
|
||||
if best_score - score <= TIEBREAKER_MARGIN
|
||||
]
|
||||
|
||||
if len(top_scored_candidates) == 1:
|
||||
return top_scored_candidates[0][1]
|
||||
|
||||
# We have a tie or near-tie. Use year and author overlap to break the tie.
|
||||
# Score candidates: +1 for year match (within 1 year), +1 for author overlap
|
||||
tiebreaker_scores: list[tuple[int, float, OpenAlexWork]] = []
|
||||
|
||||
for original_score, cand in top_scored_candidates:
|
||||
tb_score = 0
|
||||
if target_year is not None and cand.publication_year is not None:
|
||||
if abs(target_year - cand.publication_year) <= 1:
|
||||
tb_score += 1
|
||||
|
||||
candidate_author_names = [a.display_name for a in cand.authors if a.display_name]
|
||||
if _author_overlap_score(target_authors, candidate_author_names):
|
||||
tb_score += 1
|
||||
|
||||
tiebreaker_scores.append((tb_score, original_score, cand))
|
||||
|
||||
tiebreaker_scores.sort(key=lambda x: (x[0], x[1]), reverse=True)
|
||||
return tiebreaker_scores[0][2]
|
||||
|
|
@ -1,71 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from typing import Any, Mapping
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class OpenAlexAuthor:
|
||||
openalex_id: str | None
|
||||
display_name: str | None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class OpenAlexWork:
|
||||
openalex_id: str
|
||||
doi: str | None
|
||||
pmid: str | None
|
||||
pmcid: str | None
|
||||
title: str | None
|
||||
publication_year: int | None
|
||||
cited_by_count: int
|
||||
is_oa: bool
|
||||
oa_url: str | None
|
||||
authors: list[OpenAlexAuthor] = field(default_factory=list)
|
||||
raw_data: Mapping[str, Any] = field(default_factory=dict, repr=False)
|
||||
|
||||
@classmethod
|
||||
def from_api_dict(cls, data: Mapping[str, Any]) -> OpenAlexWork:
|
||||
ids = data.get("ids") or {}
|
||||
|
||||
# Extract DOI without the https://doi.org/ prefix
|
||||
doi = ids.get("doi")
|
||||
if doi and doi.startswith("https://doi.org/"):
|
||||
doi = doi[16:]
|
||||
|
||||
# Extract PMID without the url prefix
|
||||
pmid = ids.get("pmid")
|
||||
if pmid and pmid.startswith("https://pubmed.ncbi.nlm.nih.gov/"):
|
||||
pmid = pmid[32:]
|
||||
|
||||
# Extract PMCID without the url prefix
|
||||
pmcid = ids.get("pmcid")
|
||||
if pmcid and pmcid.startswith("https://www.ncbi.nlm.nih.gov/pmc/articles/"):
|
||||
pmcid = pmcid[42:]
|
||||
|
||||
open_access = data.get("open_access") or {}
|
||||
|
||||
authors = []
|
||||
for authorship in data.get("authorships") or []:
|
||||
author_data = authorship.get("author") or {}
|
||||
authors.append(
|
||||
OpenAlexAuthor(
|
||||
openalex_id=author_data.get("id"),
|
||||
display_name=author_data.get("display_name"),
|
||||
)
|
||||
)
|
||||
|
||||
return cls(
|
||||
openalex_id=data.get("id", ""),
|
||||
doi=doi,
|
||||
pmid=pmid,
|
||||
pmcid=pmcid,
|
||||
title=data.get("title"),
|
||||
publication_year=data.get("publication_year"),
|
||||
cited_by_count=data.get("cited_by_count", 0),
|
||||
is_oa=bool(open_access.get("is_oa")),
|
||||
oa_url=open_access.get("oa_url"),
|
||||
authors=authors,
|
||||
raw_data=dict(data),
|
||||
)
|
||||
|
|
@ -1 +0,0 @@
|
|||
from app.services.domains.portability.application import *
|
||||
|
|
@ -1,67 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.models import Publication
|
||||
from app.services.domains.portability.constants import (
|
||||
EXPORT_SCHEMA_VERSION,
|
||||
MAX_IMPORT_PUBLICATIONS,
|
||||
MAX_IMPORT_SCHOLARS,
|
||||
)
|
||||
from app.services.domains.portability.exporting import export_user_data
|
||||
from app.services.domains.portability.normalize import _validate_import_sizes
|
||||
from app.services.domains.portability.publication_import import (
|
||||
_build_imported_publication_input,
|
||||
_initialize_import_counters,
|
||||
_upsert_imported_publication,
|
||||
)
|
||||
from app.services.domains.portability.scholar_import import _upsert_imported_scholars
|
||||
from app.services.domains.portability.types import ImportExportError, ImportedPublicationInput
|
||||
|
||||
|
||||
async def import_user_data(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
scholars: list[dict[str, Any]],
|
||||
publications: list[dict[str, Any]],
|
||||
) -> dict[str, int]:
|
||||
_validate_import_sizes(scholars=scholars, publications=publications)
|
||||
scholar_map, counters = await _upsert_imported_scholars(
|
||||
db_session,
|
||||
user_id=user_id,
|
||||
scholars=scholars,
|
||||
)
|
||||
cluster_cache: dict[str, Publication | None] = {}
|
||||
fingerprint_cache: dict[str, Publication | None] = {}
|
||||
_initialize_import_counters(counters)
|
||||
for item in publications:
|
||||
parsed_item = _build_imported_publication_input(
|
||||
item=item,
|
||||
scholar_map=scholar_map,
|
||||
)
|
||||
if parsed_item is None:
|
||||
counters["skipped_records"] += 1
|
||||
continue
|
||||
await _upsert_imported_publication(
|
||||
db_session,
|
||||
payload=parsed_item,
|
||||
cluster_cache=cluster_cache,
|
||||
fingerprint_cache=fingerprint_cache,
|
||||
counters=counters,
|
||||
)
|
||||
await db_session.commit()
|
||||
return counters
|
||||
|
||||
|
||||
__all__ = [
|
||||
"EXPORT_SCHEMA_VERSION",
|
||||
"MAX_IMPORT_SCHOLARS",
|
||||
"MAX_IMPORT_PUBLICATIONS",
|
||||
"ImportExportError",
|
||||
"ImportedPublicationInput",
|
||||
"export_user_data",
|
||||
"import_user_data",
|
||||
]
|
||||
|
|
@ -1,9 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
EXPORT_SCHEMA_VERSION = 1
|
||||
MAX_IMPORT_SCHOLARS = 10_000
|
||||
MAX_IMPORT_PUBLICATIONS = 100_000
|
||||
WORD_RE = re.compile(r"[a-z0-9]+")
|
||||
SHA256_RE = re.compile(r"^[0-9a-f]{64}$")
|
||||
|
|
@ -1,91 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.models import Publication, ScholarProfile, ScholarPublication
|
||||
from app.services.domains.portability.constants import EXPORT_SCHEMA_VERSION
|
||||
|
||||
|
||||
def _exported_at_iso() -> str:
|
||||
return datetime.now(timezone.utc).replace(microsecond=0).isoformat()
|
||||
|
||||
|
||||
def _serialize_export_scholar(profile: ScholarProfile) -> dict[str, Any]:
|
||||
return {
|
||||
"scholar_id": profile.scholar_id,
|
||||
"display_name": profile.display_name,
|
||||
"is_enabled": bool(profile.is_enabled),
|
||||
"profile_image_override_url": profile.profile_image_override_url,
|
||||
}
|
||||
|
||||
|
||||
def _serialize_export_publication(row: tuple[Any, ...]) -> dict[str, Any]:
|
||||
(
|
||||
scholar_id,
|
||||
cluster_id,
|
||||
fingerprint_sha256,
|
||||
title_raw,
|
||||
year,
|
||||
citation_count,
|
||||
author_text,
|
||||
venue_text,
|
||||
pub_url,
|
||||
pdf_url,
|
||||
is_read,
|
||||
) = row
|
||||
return {
|
||||
"scholar_id": scholar_id,
|
||||
"cluster_id": cluster_id,
|
||||
"fingerprint_sha256": fingerprint_sha256,
|
||||
"title": title_raw,
|
||||
"year": year,
|
||||
"citation_count": int(citation_count or 0),
|
||||
"author_text": author_text,
|
||||
"venue_text": venue_text,
|
||||
"pub_url": pub_url,
|
||||
"pdf_url": pdf_url,
|
||||
"is_read": bool(is_read),
|
||||
}
|
||||
|
||||
|
||||
async def export_user_data(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
) -> dict[str, Any]:
|
||||
scholars_result = await db_session.execute(
|
||||
select(ScholarProfile)
|
||||
.where(ScholarProfile.user_id == user_id)
|
||||
.order_by(ScholarProfile.id.asc())
|
||||
)
|
||||
publication_result = await db_session.execute(
|
||||
select(
|
||||
ScholarProfile.scholar_id,
|
||||
Publication.cluster_id,
|
||||
Publication.fingerprint_sha256,
|
||||
Publication.title_raw,
|
||||
Publication.year,
|
||||
Publication.citation_count,
|
||||
Publication.author_text,
|
||||
Publication.venue_text,
|
||||
Publication.pub_url,
|
||||
Publication.pdf_url,
|
||||
ScholarPublication.is_read,
|
||||
)
|
||||
.join(ScholarPublication, ScholarPublication.scholar_profile_id == ScholarProfile.id)
|
||||
.join(Publication, Publication.id == ScholarPublication.publication_id)
|
||||
.where(ScholarProfile.user_id == user_id)
|
||||
.order_by(ScholarPublication.created_at.desc(), Publication.id.desc())
|
||||
)
|
||||
scholars = [_serialize_export_scholar(profile) for profile in scholars_result.scalars().all()]
|
||||
publications = [_serialize_export_publication(row) for row in publication_result.all()]
|
||||
return {
|
||||
"schema_version": EXPORT_SCHEMA_VERSION,
|
||||
"exported_at": _exported_at_iso(),
|
||||
"scholars": scholars,
|
||||
"publications": publications,
|
||||
}
|
||||
|
|
@ -1,109 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
from typing import Any
|
||||
|
||||
from app.services.domains.ingestion.application import normalize_title
|
||||
from app.services.domains.portability.constants import (
|
||||
MAX_IMPORT_PUBLICATIONS,
|
||||
MAX_IMPORT_SCHOLARS,
|
||||
SHA256_RE,
|
||||
WORD_RE,
|
||||
)
|
||||
from app.services.domains.portability.types import ImportExportError
|
||||
|
||||
|
||||
def _normalize_optional_text(value: Any) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
normalized = str(value).strip()
|
||||
return normalized or None
|
||||
|
||||
|
||||
def _normalize_optional_year(value: Any) -> int | None:
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
year = int(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
if year < 1500 or year > 3000:
|
||||
return None
|
||||
return year
|
||||
|
||||
|
||||
def _normalize_citation_count(value: Any) -> int:
|
||||
try:
|
||||
parsed = int(value)
|
||||
except (TypeError, ValueError):
|
||||
return 0
|
||||
return max(0, parsed)
|
||||
|
||||
|
||||
def _first_author_last_name(authors_text: str | None) -> str:
|
||||
if not authors_text:
|
||||
return ""
|
||||
first_author = authors_text.split(",", maxsplit=1)[0].strip().lower()
|
||||
words = WORD_RE.findall(first_author)
|
||||
if not words:
|
||||
return ""
|
||||
return words[-1]
|
||||
|
||||
|
||||
def _first_venue_word(venue_text: str | None) -> str:
|
||||
if not venue_text:
|
||||
return ""
|
||||
words = WORD_RE.findall(venue_text.lower())
|
||||
if not words:
|
||||
return ""
|
||||
return words[0]
|
||||
|
||||
|
||||
def _build_fingerprint(
|
||||
*,
|
||||
title: str,
|
||||
year: int | None,
|
||||
author_text: str | None,
|
||||
venue_text: str | None,
|
||||
) -> str:
|
||||
canonical = "|".join(
|
||||
[
|
||||
normalize_title(title),
|
||||
str(year) if year is not None else "",
|
||||
_first_author_last_name(author_text),
|
||||
_first_venue_word(venue_text),
|
||||
]
|
||||
)
|
||||
return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _resolve_fingerprint(
|
||||
*,
|
||||
title: str,
|
||||
year: int | None,
|
||||
author_text: str | None,
|
||||
venue_text: str | None,
|
||||
provided_fingerprint: Any,
|
||||
) -> str:
|
||||
normalized = _normalize_optional_text(provided_fingerprint)
|
||||
if normalized and SHA256_RE.fullmatch(normalized.lower()):
|
||||
return normalized.lower()
|
||||
return _build_fingerprint(
|
||||
title=title,
|
||||
year=year,
|
||||
author_text=author_text,
|
||||
venue_text=venue_text,
|
||||
)
|
||||
|
||||
|
||||
def _validate_import_sizes(
|
||||
*,
|
||||
scholars: list[dict[str, Any]],
|
||||
publications: list[dict[str, Any]],
|
||||
) -> None:
|
||||
if len(scholars) > MAX_IMPORT_SCHOLARS:
|
||||
raise ImportExportError(f"Import exceeds max scholars ({MAX_IMPORT_SCHOLARS}).")
|
||||
if len(publications) > MAX_IMPORT_PUBLICATIONS:
|
||||
raise ImportExportError(
|
||||
f"Import exceeds max publications ({MAX_IMPORT_PUBLICATIONS})."
|
||||
)
|
||||
|
|
@ -1,370 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.models import Publication, ScholarProfile, ScholarPublication
|
||||
from app.services.domains.ingestion.application import build_publication_url, normalize_title
|
||||
from app.services.domains.doi.normalize import normalize_doi
|
||||
from app.services.domains.portability.normalize import (
|
||||
_normalize_citation_count,
|
||||
_normalize_optional_text,
|
||||
_normalize_optional_year,
|
||||
_resolve_fingerprint,
|
||||
)
|
||||
from app.services.domains.portability.types import ImportedPublicationInput
|
||||
|
||||
|
||||
async def _find_publication_by_cluster(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
cluster_id: str,
|
||||
) -> Publication | None:
|
||||
result = await db_session.execute(
|
||||
select(Publication).where(Publication.cluster_id == cluster_id)
|
||||
)
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
|
||||
async def _find_publication_by_fingerprint(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
fingerprint_sha256: str,
|
||||
) -> Publication | None:
|
||||
result = await db_session.execute(
|
||||
select(Publication).where(Publication.fingerprint_sha256 == fingerprint_sha256)
|
||||
)
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
|
||||
async def _find_linked_publication_by_title(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
scholar_profile_id: int,
|
||||
title: str,
|
||||
) -> Publication | None:
|
||||
normalized_title = normalize_title(title)
|
||||
result = await db_session.execute(
|
||||
select(Publication)
|
||||
.join(
|
||||
ScholarPublication,
|
||||
ScholarPublication.publication_id == Publication.id,
|
||||
)
|
||||
.where(
|
||||
ScholarPublication.scholar_profile_id == scholar_profile_id,
|
||||
Publication.title_normalized == normalized_title,
|
||||
)
|
||||
.order_by(Publication.id.asc())
|
||||
.limit(1)
|
||||
)
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
|
||||
def _apply_imported_publication_values(
|
||||
*,
|
||||
publication: Publication,
|
||||
title: str,
|
||||
year: int | None,
|
||||
citation_count: int,
|
||||
author_text: str | None,
|
||||
venue_text: str | None,
|
||||
pub_url: str | None,
|
||||
pdf_url: str | None,
|
||||
cluster_id: str | None,
|
||||
) -> bool:
|
||||
updated = False
|
||||
if cluster_id and publication.cluster_id != cluster_id:
|
||||
publication.cluster_id = cluster_id
|
||||
updated = True
|
||||
if publication.title_raw != title:
|
||||
publication.title_raw = title
|
||||
publication.title_normalized = normalize_title(title)
|
||||
updated = True
|
||||
if publication.year != year:
|
||||
publication.year = year
|
||||
updated = True
|
||||
if int(publication.citation_count or 0) != citation_count:
|
||||
publication.citation_count = citation_count
|
||||
updated = True
|
||||
if publication.author_text != author_text:
|
||||
publication.author_text = author_text
|
||||
updated = True
|
||||
if publication.venue_text != venue_text:
|
||||
publication.venue_text = venue_text
|
||||
updated = True
|
||||
if pub_url and publication.pub_url != pub_url:
|
||||
publication.pub_url = pub_url
|
||||
updated = True
|
||||
if pdf_url and publication.pdf_url != pdf_url:
|
||||
publication.pdf_url = pdf_url
|
||||
updated = True
|
||||
return updated
|
||||
|
||||
|
||||
def _new_publication(
|
||||
*,
|
||||
cluster_id: str | None,
|
||||
fingerprint_sha256: str,
|
||||
title: str,
|
||||
year: int | None,
|
||||
citation_count: int,
|
||||
author_text: str | None,
|
||||
venue_text: str | None,
|
||||
pub_url: str | None,
|
||||
pdf_url: str | None,
|
||||
) -> Publication:
|
||||
return Publication(
|
||||
cluster_id=cluster_id,
|
||||
fingerprint_sha256=fingerprint_sha256,
|
||||
title_raw=title,
|
||||
title_normalized=normalize_title(title),
|
||||
year=year,
|
||||
citation_count=citation_count,
|
||||
author_text=author_text,
|
||||
venue_text=venue_text,
|
||||
pub_url=pub_url,
|
||||
pdf_url=pdf_url,
|
||||
)
|
||||
|
||||
|
||||
async def _resolve_publication_for_import(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
scholar_profile_id: int,
|
||||
title: str,
|
||||
cluster_id: str | None,
|
||||
fingerprint_sha256: str,
|
||||
cluster_cache: dict[str, Publication | None],
|
||||
fingerprint_cache: dict[str, Publication | None],
|
||||
) -> Publication | None:
|
||||
if cluster_id:
|
||||
if cluster_id not in cluster_cache:
|
||||
cluster_cache[cluster_id] = await _find_publication_by_cluster(
|
||||
db_session,
|
||||
cluster_id=cluster_id,
|
||||
)
|
||||
if cluster_cache[cluster_id] is not None:
|
||||
return cluster_cache[cluster_id]
|
||||
if fingerprint_sha256 not in fingerprint_cache:
|
||||
fingerprint_cache[fingerprint_sha256] = await _find_publication_by_fingerprint(
|
||||
db_session,
|
||||
fingerprint_sha256=fingerprint_sha256,
|
||||
)
|
||||
if fingerprint_cache[fingerprint_sha256] is not None:
|
||||
return fingerprint_cache[fingerprint_sha256]
|
||||
return await _find_linked_publication_by_title(
|
||||
db_session,
|
||||
scholar_profile_id=scholar_profile_id,
|
||||
title=title,
|
||||
)
|
||||
|
||||
|
||||
async def _upsert_scholar_publication_link(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
scholar_profile_id: int,
|
||||
publication_id: int,
|
||||
is_read: bool,
|
||||
) -> tuple[bool, bool]:
|
||||
result = await db_session.execute(
|
||||
select(ScholarPublication).where(
|
||||
ScholarPublication.scholar_profile_id == scholar_profile_id,
|
||||
ScholarPublication.publication_id == publication_id,
|
||||
)
|
||||
)
|
||||
link = result.scalar_one_or_none()
|
||||
if link is None:
|
||||
db_session.add(
|
||||
ScholarPublication(
|
||||
scholar_profile_id=scholar_profile_id,
|
||||
publication_id=publication_id,
|
||||
is_read=bool(is_read),
|
||||
)
|
||||
)
|
||||
return True, False
|
||||
if bool(link.is_read) == bool(is_read):
|
||||
return False, False
|
||||
link.is_read = bool(is_read)
|
||||
return False, True
|
||||
|
||||
|
||||
def _initialize_import_counters(counters: dict[str, int]) -> None:
|
||||
counters.update(
|
||||
{
|
||||
"publications_created": 0,
|
||||
"publications_updated": 0,
|
||||
"links_created": 0,
|
||||
"links_updated": 0,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _build_imported_publication_input(
|
||||
*,
|
||||
item: dict[str, Any],
|
||||
scholar_map: dict[str, ScholarProfile],
|
||||
) -> ImportedPublicationInput | None:
|
||||
scholar_id = _normalize_optional_text(item.get("scholar_id"))
|
||||
title = _normalize_optional_text(item.get("title"))
|
||||
if not scholar_id or not title:
|
||||
return None
|
||||
|
||||
profile = scholar_map.get(scholar_id)
|
||||
if profile is None:
|
||||
return None
|
||||
|
||||
year = _normalize_optional_year(item.get("year"))
|
||||
author_text = _normalize_optional_text(item.get("author_text"))
|
||||
venue_text = _normalize_optional_text(item.get("venue_text"))
|
||||
return ImportedPublicationInput(
|
||||
profile=profile,
|
||||
title=title,
|
||||
year=year,
|
||||
citation_count=_normalize_citation_count(item.get("citation_count")),
|
||||
author_text=author_text,
|
||||
venue_text=venue_text,
|
||||
cluster_id=_normalize_optional_text(item.get("cluster_id")),
|
||||
pub_url=build_publication_url(_normalize_optional_text(item.get("pub_url"))),
|
||||
doi=normalize_doi(_normalize_optional_text(item.get("doi"))),
|
||||
pdf_url=build_publication_url(_normalize_optional_text(item.get("pdf_url"))),
|
||||
fingerprint=_resolve_fingerprint(
|
||||
title=title,
|
||||
year=year,
|
||||
author_text=author_text,
|
||||
venue_text=venue_text,
|
||||
provided_fingerprint=item.get("fingerprint_sha256"),
|
||||
),
|
||||
is_read=bool(item.get("is_read", False)),
|
||||
)
|
||||
|
||||
|
||||
def _update_link_counters(
|
||||
*,
|
||||
counters: dict[str, int],
|
||||
link_created: bool,
|
||||
link_updated: bool,
|
||||
) -> None:
|
||||
if link_created:
|
||||
counters["links_created"] += 1
|
||||
if link_updated:
|
||||
counters["links_updated"] += 1
|
||||
|
||||
|
||||
def _cache_resolved_publication(
|
||||
*,
|
||||
publication: Publication,
|
||||
cluster_id: str | None,
|
||||
fingerprint_sha256: str,
|
||||
cluster_cache: dict[str, Publication | None],
|
||||
fingerprint_cache: dict[str, Publication | None],
|
||||
) -> None:
|
||||
if cluster_id:
|
||||
cluster_cache[cluster_id] = publication
|
||||
fingerprint_cache[fingerprint_sha256] = publication
|
||||
|
||||
|
||||
async def _create_import_publication(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
payload: ImportedPublicationInput,
|
||||
) -> Publication:
|
||||
publication = _new_publication(
|
||||
cluster_id=payload.cluster_id,
|
||||
fingerprint_sha256=payload.fingerprint,
|
||||
title=payload.title,
|
||||
year=payload.year,
|
||||
citation_count=payload.citation_count,
|
||||
author_text=payload.author_text,
|
||||
venue_text=payload.venue_text,
|
||||
pub_url=payload.pub_url,
|
||||
pdf_url=payload.pdf_url,
|
||||
)
|
||||
db_session.add(publication)
|
||||
await db_session.flush()
|
||||
return publication
|
||||
|
||||
|
||||
def _update_import_publication(
|
||||
*,
|
||||
publication: Publication,
|
||||
payload: ImportedPublicationInput,
|
||||
) -> bool:
|
||||
return _apply_imported_publication_values(
|
||||
publication=publication,
|
||||
title=payload.title,
|
||||
year=payload.year,
|
||||
citation_count=payload.citation_count,
|
||||
author_text=payload.author_text,
|
||||
venue_text=payload.venue_text,
|
||||
pub_url=payload.pub_url,
|
||||
pdf_url=payload.pdf_url,
|
||||
cluster_id=payload.cluster_id,
|
||||
)
|
||||
|
||||
|
||||
async def _upsert_publication_entity(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
payload: ImportedPublicationInput,
|
||||
cluster_cache: dict[str, Publication | None],
|
||||
fingerprint_cache: dict[str, Publication | None],
|
||||
) -> tuple[Publication, bool, bool]:
|
||||
publication = await _resolve_publication_for_import(
|
||||
db_session,
|
||||
scholar_profile_id=int(payload.profile.id),
|
||||
title=payload.title,
|
||||
cluster_id=payload.cluster_id,
|
||||
fingerprint_sha256=payload.fingerprint,
|
||||
cluster_cache=cluster_cache,
|
||||
fingerprint_cache=fingerprint_cache,
|
||||
)
|
||||
created = False
|
||||
updated = False
|
||||
if publication is None:
|
||||
publication = await _create_import_publication(db_session, payload=payload)
|
||||
created = True
|
||||
else:
|
||||
updated = _update_import_publication(publication=publication, payload=payload)
|
||||
|
||||
_cache_resolved_publication(
|
||||
publication=publication,
|
||||
cluster_id=payload.cluster_id,
|
||||
fingerprint_sha256=payload.fingerprint,
|
||||
cluster_cache=cluster_cache,
|
||||
fingerprint_cache=fingerprint_cache,
|
||||
)
|
||||
return publication, created, updated
|
||||
|
||||
|
||||
async def _upsert_imported_publication(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
payload: ImportedPublicationInput,
|
||||
cluster_cache: dict[str, Publication | None],
|
||||
fingerprint_cache: dict[str, Publication | None],
|
||||
counters: dict[str, int],
|
||||
) -> None:
|
||||
publication, created, updated = await _upsert_publication_entity(
|
||||
db_session,
|
||||
payload=payload,
|
||||
cluster_cache=cluster_cache,
|
||||
fingerprint_cache=fingerprint_cache,
|
||||
)
|
||||
if created:
|
||||
counters["publications_created"] += 1
|
||||
if updated:
|
||||
counters["publications_updated"] += 1
|
||||
|
||||
link_created, link_updated = await _upsert_scholar_publication_link(
|
||||
db_session,
|
||||
scholar_profile_id=int(payload.profile.id),
|
||||
publication_id=int(publication.id),
|
||||
is_read=payload.is_read,
|
||||
)
|
||||
_update_link_counters(
|
||||
counters=counters,
|
||||
link_created=link_created,
|
||||
link_updated=link_updated,
|
||||
)
|
||||
|
|
@ -1,107 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.models import ScholarProfile
|
||||
from app.services.domains.portability.normalize import _normalize_optional_text
|
||||
from app.services.domains.scholars import application as scholar_service
|
||||
|
||||
|
||||
async def _load_user_scholar_map(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
) -> dict[str, ScholarProfile]:
|
||||
result = await db_session.execute(
|
||||
select(ScholarProfile).where(ScholarProfile.user_id == user_id)
|
||||
)
|
||||
profiles = list(result.scalars().all())
|
||||
return {profile.scholar_id: profile for profile in profiles}
|
||||
|
||||
|
||||
def _apply_imported_scholar_values(
|
||||
*,
|
||||
profile: ScholarProfile,
|
||||
display_name: str | None,
|
||||
profile_image_override_url: str | None,
|
||||
is_enabled: bool,
|
||||
) -> bool:
|
||||
updated = False
|
||||
if display_name and profile.display_name != display_name:
|
||||
profile.display_name = display_name
|
||||
updated = True
|
||||
if profile.profile_image_override_url != profile_image_override_url:
|
||||
profile.profile_image_override_url = profile_image_override_url
|
||||
updated = True
|
||||
if bool(profile.is_enabled) != bool(is_enabled):
|
||||
profile.is_enabled = bool(is_enabled)
|
||||
updated = True
|
||||
return updated
|
||||
|
||||
|
||||
def _new_scholar_profile(
|
||||
*,
|
||||
user_id: int,
|
||||
scholar_id: str,
|
||||
display_name: str | None,
|
||||
profile_image_override_url: str | None,
|
||||
is_enabled: bool,
|
||||
) -> ScholarProfile:
|
||||
return ScholarProfile(
|
||||
user_id=user_id,
|
||||
scholar_id=scholar_id,
|
||||
display_name=display_name,
|
||||
profile_image_override_url=profile_image_override_url,
|
||||
is_enabled=bool(is_enabled),
|
||||
)
|
||||
|
||||
|
||||
async def _upsert_imported_scholars(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
scholars: list[dict[str, Any]],
|
||||
) -> tuple[dict[str, ScholarProfile], dict[str, int]]:
|
||||
scholar_map = await _load_user_scholar_map(db_session, user_id=user_id)
|
||||
counters = {"scholars_created": 0, "scholars_updated": 0, "skipped_records": 0}
|
||||
for item in scholars:
|
||||
try:
|
||||
scholar_id = scholar_service.validate_scholar_id(str(item["scholar_id"]))
|
||||
display_name = scholar_service.normalize_display_name(
|
||||
str(item.get("display_name") or "")
|
||||
)
|
||||
override_url = scholar_service.normalize_profile_image_url(
|
||||
_normalize_optional_text(item.get("profile_image_override_url"))
|
||||
)
|
||||
except (KeyError, scholar_service.ScholarServiceError):
|
||||
counters["skipped_records"] += 1
|
||||
continue
|
||||
|
||||
is_enabled = bool(item.get("is_enabled", True))
|
||||
existing = scholar_map.get(scholar_id)
|
||||
if existing is None:
|
||||
profile = _new_scholar_profile(
|
||||
user_id=user_id,
|
||||
scholar_id=scholar_id,
|
||||
display_name=display_name,
|
||||
profile_image_override_url=override_url,
|
||||
is_enabled=is_enabled,
|
||||
)
|
||||
db_session.add(profile)
|
||||
scholar_map[scholar_id] = profile
|
||||
counters["scholars_created"] += 1
|
||||
continue
|
||||
|
||||
if _apply_imported_scholar_values(
|
||||
profile=existing,
|
||||
display_name=display_name,
|
||||
profile_image_override_url=override_url,
|
||||
is_enabled=is_enabled,
|
||||
):
|
||||
counters["scholars_updated"] += 1
|
||||
|
||||
await db_session.flush()
|
||||
return scholar_map, counters
|
||||
|
|
@ -1,25 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
from app.db.models import ScholarProfile
|
||||
|
||||
|
||||
class ImportExportError(ValueError):
|
||||
"""Raised when import/export payload constraints are violated."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ImportedPublicationInput:
|
||||
profile: ScholarProfile
|
||||
title: str
|
||||
year: int | None
|
||||
citation_count: int
|
||||
author_text: str | None
|
||||
venue_text: str | None
|
||||
cluster_id: str | None
|
||||
pub_url: str | None
|
||||
doi: str | None
|
||||
pdf_url: str | None
|
||||
fingerprint: str
|
||||
is_read: bool
|
||||
|
|
@ -1,17 +0,0 @@
|
|||
from app.services.domains.publication_identifiers.application import (
|
||||
DisplayIdentifier,
|
||||
derive_display_identifier_from_values,
|
||||
overlay_pdf_queue_items_with_display_identifiers,
|
||||
overlay_publication_items_with_display_identifiers,
|
||||
sync_identifiers_for_publication_fields,
|
||||
sync_identifiers_for_publication_resolution,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"DisplayIdentifier",
|
||||
"derive_display_identifier_from_values",
|
||||
"overlay_pdf_queue_items_with_display_identifiers",
|
||||
"overlay_publication_items_with_display_identifiers",
|
||||
"sync_identifiers_for_publication_fields",
|
||||
"sync_identifiers_for_publication_resolution",
|
||||
]
|
||||
|
|
@ -1,433 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from dataclasses import replace
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.models import Publication, PublicationIdentifier
|
||||
from app.services.domains.doi.normalize import normalize_doi
|
||||
from app.services.domains.publication_identifiers.normalize import (
|
||||
normalize_arxiv_id,
|
||||
normalize_pmcid,
|
||||
normalize_pmid,
|
||||
)
|
||||
from app.services.domains.publication_identifiers.types import (
|
||||
DisplayIdentifier,
|
||||
IdentifierCandidate,
|
||||
IdentifierKind,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from app.services.domains.publications.pdf_queue import PdfQueueListItem
|
||||
from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem
|
||||
|
||||
CONFIDENCE_HIGH = 0.98
|
||||
CONFIDENCE_MEDIUM = 0.9
|
||||
CONFIDENCE_LOW = 0.6
|
||||
CONFIDENCE_FALLBACK = 0.4
|
||||
PRIORITY_DOI = 400
|
||||
PRIORITY_ARXIV = 300
|
||||
PRIORITY_PMCID = 200
|
||||
PRIORITY_PMID = 100
|
||||
|
||||
|
||||
def derive_display_identifier_from_values(
|
||||
*,
|
||||
doi: str | None,
|
||||
pub_url: str | None = None,
|
||||
pdf_url: str | None = None,
|
||||
) -> DisplayIdentifier | None:
|
||||
candidates = _fallback_candidates_from_values(doi=doi, pub_url=pub_url, pdf_url=pdf_url)
|
||||
return _best_display_identifier(candidates)
|
||||
|
||||
|
||||
def _fallback_candidates_from_values(
|
||||
*,
|
||||
doi: str | None,
|
||||
pub_url: str | None,
|
||||
pdf_url: str | None,
|
||||
) -> list[IdentifierCandidate]:
|
||||
values = [value for value in [pub_url, pdf_url] if value]
|
||||
candidates = []
|
||||
if doi:
|
||||
normalized_doi = normalize_doi(doi)
|
||||
if normalized_doi:
|
||||
candidates.append(_candidate(IdentifierKind.DOI, doi, normalized_doi, "legacy_doi", CONFIDENCE_HIGH, pub_url))
|
||||
candidates.extend(_url_identifier_candidates(values=values, source="legacy_urls"))
|
||||
return _dedup_candidates(candidates)
|
||||
|
||||
|
||||
def _url_identifier_candidates(*, values: list[str], source: str) -> list[IdentifierCandidate]:
|
||||
candidates: list[IdentifierCandidate] = []
|
||||
for value in values:
|
||||
candidates.extend(_url_candidates_for_value(value=value, source=source))
|
||||
return candidates
|
||||
|
||||
|
||||
def _url_candidates_for_value(*, value: str, source: str) -> list[IdentifierCandidate]:
|
||||
candidates: list[IdentifierCandidate] = []
|
||||
arxiv = normalize_arxiv_id(value)
|
||||
if arxiv:
|
||||
candidates.append(_candidate(IdentifierKind.ARXIV, value, arxiv, source, CONFIDENCE_MEDIUM, value))
|
||||
pmcid = normalize_pmcid(value)
|
||||
if pmcid:
|
||||
candidates.append(_candidate(IdentifierKind.PMCID, value, pmcid, source, CONFIDENCE_LOW, value))
|
||||
pmid = normalize_pmid(value)
|
||||
if pmid:
|
||||
candidates.append(_candidate(IdentifierKind.PMID, value, pmid, source, CONFIDENCE_FALLBACK, value))
|
||||
return candidates
|
||||
|
||||
|
||||
def _candidate(
|
||||
kind: IdentifierKind,
|
||||
value_raw: str,
|
||||
value_normalized: str,
|
||||
source: str,
|
||||
confidence_score: float,
|
||||
evidence_url: str | None,
|
||||
) -> IdentifierCandidate:
|
||||
return IdentifierCandidate(
|
||||
kind=kind,
|
||||
value_raw=value_raw,
|
||||
value_normalized=value_normalized,
|
||||
source=source,
|
||||
confidence_score=float(confidence_score),
|
||||
evidence_url=evidence_url,
|
||||
)
|
||||
|
||||
|
||||
def _dedup_candidates(candidates: list[IdentifierCandidate]) -> list[IdentifierCandidate]:
|
||||
deduped: dict[tuple[str, str], IdentifierCandidate] = {}
|
||||
for candidate in candidates:
|
||||
key = (candidate.kind.value, candidate.value_normalized)
|
||||
current = deduped.get(key)
|
||||
if current is None or candidate.confidence_score > current.confidence_score:
|
||||
deduped[key] = candidate
|
||||
return list(deduped.values())
|
||||
|
||||
|
||||
async def sync_identifiers_for_publication_fields(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
publication: Publication,
|
||||
) -> None:
|
||||
candidates = _publication_field_candidates(publication)
|
||||
await _upsert_publication_candidates(db_session, publication_id=int(publication.id), candidates=candidates)
|
||||
|
||||
|
||||
async def discover_and_sync_identifiers_for_publication(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
publication: Publication,
|
||||
scholar_label: str,
|
||||
) -> None:
|
||||
await sync_identifiers_for_publication_fields(db_session, publication=publication)
|
||||
|
||||
existing_doi = await _existing_identifier_by_kind(
|
||||
db_session,
|
||||
publication_id=int(publication.id),
|
||||
kind=IdentifierKind.DOI.value,
|
||||
)
|
||||
if existing_doi is not None:
|
||||
return
|
||||
|
||||
from app.services.domains.crossref import application as crossref_service
|
||||
from app.services.domains.publications.types import UnreadPublicationItem
|
||||
|
||||
item = UnreadPublicationItem(
|
||||
publication_id=int(publication.id),
|
||||
scholar_profile_id=0,
|
||||
scholar_label=scholar_label,
|
||||
title=str(publication.title_raw or ""),
|
||||
year=publication.year,
|
||||
citation_count=publication.citation_count,
|
||||
venue_text=publication.venue_text,
|
||||
pub_url=publication.pub_url,
|
||||
pdf_url=publication.pdf_url,
|
||||
)
|
||||
|
||||
discovered_doi = await crossref_service.discover_doi_for_publication(item=item)
|
||||
if discovered_doi:
|
||||
normalized_doi = normalize_doi(discovered_doi)
|
||||
if normalized_doi:
|
||||
candidate = _candidate(
|
||||
IdentifierKind.DOI,
|
||||
discovered_doi,
|
||||
normalized_doi,
|
||||
"crossref_api",
|
||||
CONFIDENCE_MEDIUM,
|
||||
None,
|
||||
)
|
||||
await _upsert_publication_candidate(db_session, publication_id=int(publication.id), candidate=candidate)
|
||||
|
||||
existing_arxiv = await _existing_identifier_by_kind(
|
||||
db_session,
|
||||
publication_id=int(publication.id),
|
||||
kind=IdentifierKind.ARXIV.value,
|
||||
)
|
||||
if existing_arxiv is None:
|
||||
from app.services.domains.arxiv import application as arxiv_service
|
||||
discovered_arxiv = await arxiv_service.discover_arxiv_id_for_publication(item=item)
|
||||
if discovered_arxiv:
|
||||
normalized_arxiv = normalize_arxiv_id(discovered_arxiv)
|
||||
if normalized_arxiv:
|
||||
candidate = _candidate(
|
||||
IdentifierKind.ARXIV,
|
||||
discovered_arxiv,
|
||||
normalized_arxiv,
|
||||
"arxiv_api",
|
||||
CONFIDENCE_MEDIUM,
|
||||
None,
|
||||
)
|
||||
await _upsert_publication_candidate(db_session, publication_id=int(publication.id), candidate=candidate)
|
||||
|
||||
|
||||
def _publication_field_candidates(publication: Publication) -> list[IdentifierCandidate]:
|
||||
return _fallback_candidates_from_values(
|
||||
doi=None,
|
||||
pub_url=publication.pub_url,
|
||||
pdf_url=publication.pdf_url,
|
||||
)
|
||||
|
||||
|
||||
async def sync_identifiers_for_publication_resolution(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
publication: Publication,
|
||||
source: str | None,
|
||||
) -> None:
|
||||
candidates = _publication_field_candidates(publication)
|
||||
rewritten = [_candidate_with_source(candidate, source=source) for candidate in candidates]
|
||||
await _upsert_publication_candidates(db_session, publication_id=int(publication.id), candidates=rewritten)
|
||||
|
||||
|
||||
def _candidate_with_source(candidate: IdentifierCandidate, *, source: str | None) -> IdentifierCandidate:
|
||||
if not source:
|
||||
return candidate
|
||||
return IdentifierCandidate(
|
||||
kind=candidate.kind,
|
||||
value_raw=candidate.value_raw,
|
||||
value_normalized=candidate.value_normalized,
|
||||
source=source,
|
||||
confidence_score=candidate.confidence_score,
|
||||
evidence_url=candidate.evidence_url,
|
||||
)
|
||||
|
||||
|
||||
async def _upsert_publication_candidates(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
publication_id: int,
|
||||
candidates: list[IdentifierCandidate],
|
||||
) -> None:
|
||||
for candidate in _dedup_candidates(candidates):
|
||||
await _upsert_publication_candidate(db_session, publication_id=publication_id, candidate=candidate)
|
||||
|
||||
|
||||
async def _upsert_publication_candidate(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
publication_id: int,
|
||||
candidate: IdentifierCandidate,
|
||||
) -> None:
|
||||
existing = await _existing_identifier(
|
||||
db_session,
|
||||
publication_id=publication_id,
|
||||
kind=candidate.kind.value,
|
||||
value_normalized=candidate.value_normalized,
|
||||
)
|
||||
if existing is None:
|
||||
db_session.add(_new_identifier_row(publication_id=publication_id, candidate=candidate))
|
||||
return
|
||||
_merge_identifier_row(existing, candidate=candidate)
|
||||
|
||||
|
||||
async def _existing_identifier(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
publication_id: int,
|
||||
kind: str,
|
||||
value_normalized: str,
|
||||
) -> PublicationIdentifier | None:
|
||||
result = await db_session.execute(
|
||||
select(PublicationIdentifier).where(
|
||||
PublicationIdentifier.publication_id == publication_id,
|
||||
PublicationIdentifier.kind == kind,
|
||||
PublicationIdentifier.value_normalized == value_normalized,
|
||||
)
|
||||
)
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
|
||||
async def _existing_identifier_by_kind(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
publication_id: int,
|
||||
kind: str,
|
||||
) -> PublicationIdentifier | None:
|
||||
result = await db_session.execute(
|
||||
select(PublicationIdentifier).where(
|
||||
PublicationIdentifier.publication_id == publication_id,
|
||||
PublicationIdentifier.kind == kind,
|
||||
).order_by(PublicationIdentifier.confidence_score.desc()).limit(1)
|
||||
)
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
|
||||
def _new_identifier_row(
|
||||
*,
|
||||
publication_id: int,
|
||||
candidate: IdentifierCandidate,
|
||||
) -> PublicationIdentifier:
|
||||
return PublicationIdentifier(
|
||||
publication_id=publication_id,
|
||||
kind=candidate.kind.value,
|
||||
value_raw=candidate.value_raw,
|
||||
value_normalized=candidate.value_normalized,
|
||||
source=candidate.source,
|
||||
confidence_score=candidate.confidence_score,
|
||||
evidence_url=candidate.evidence_url,
|
||||
)
|
||||
|
||||
|
||||
def _merge_identifier_row(existing: PublicationIdentifier, *, candidate: IdentifierCandidate) -> None:
|
||||
if candidate.confidence_score >= float(existing.confidence_score):
|
||||
existing.value_raw = candidate.value_raw
|
||||
existing.source = candidate.source
|
||||
existing.confidence_score = candidate.confidence_score
|
||||
if candidate.evidence_url:
|
||||
existing.evidence_url = candidate.evidence_url
|
||||
|
||||
|
||||
async def overlay_publication_items_with_display_identifiers(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
items: list[PublicationListItem],
|
||||
) -> list[PublicationListItem]:
|
||||
if not items:
|
||||
return []
|
||||
mapping = await _display_identifier_map(db_session, publication_ids=[item.publication_id for item in items])
|
||||
return [_overlay_publication_item(item, mapping.get(item.publication_id)) for item in items]
|
||||
|
||||
|
||||
def _overlay_publication_item(
|
||||
item: PublicationListItem,
|
||||
display_identifier: DisplayIdentifier | None,
|
||||
) -> PublicationListItem:
|
||||
fallback = display_identifier or derive_display_identifier_from_values(doi=None, pub_url=item.pub_url, pdf_url=item.pdf_url)
|
||||
return replace(item, display_identifier=fallback)
|
||||
|
||||
|
||||
async def overlay_pdf_queue_items_with_display_identifiers(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
items: list[PdfQueueListItem],
|
||||
) -> list[PdfQueueListItem]:
|
||||
if not items:
|
||||
return []
|
||||
mapping = await _display_identifier_map(db_session, publication_ids=[item.publication_id for item in items])
|
||||
return [_overlay_queue_item(item, mapping.get(item.publication_id)) for item in items]
|
||||
|
||||
|
||||
def _overlay_queue_item(
|
||||
item: PdfQueueListItem,
|
||||
display_identifier: DisplayIdentifier | None,
|
||||
) -> PdfQueueListItem:
|
||||
fallback = display_identifier or derive_display_identifier_from_values(doi=None, pdf_url=item.pdf_url)
|
||||
return replace(item, display_identifier=fallback)
|
||||
|
||||
|
||||
async def _display_identifier_map(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
publication_ids: list[int],
|
||||
) -> dict[int, DisplayIdentifier]:
|
||||
normalized_ids = sorted({int(value) for value in publication_ids if int(value) > 0})
|
||||
if not normalized_ids:
|
||||
return {}
|
||||
result = await db_session.execute(
|
||||
select(PublicationIdentifier).where(PublicationIdentifier.publication_id.in_(normalized_ids))
|
||||
)
|
||||
rows = list(result.scalars().all())
|
||||
return _best_display_identifier_map(rows)
|
||||
|
||||
|
||||
def _best_display_identifier_map(rows: list[PublicationIdentifier]) -> dict[int, DisplayIdentifier]:
|
||||
grouped: dict[int, list[IdentifierCandidate]] = {}
|
||||
for row in rows:
|
||||
grouped.setdefault(int(row.publication_id), []).append(_candidate_from_row(row))
|
||||
return {
|
||||
publication_id: display
|
||||
for publication_id, display in (
|
||||
(publication_id, _best_display_identifier(candidates))
|
||||
for publication_id, candidates in grouped.items()
|
||||
)
|
||||
if display is not None
|
||||
}
|
||||
|
||||
|
||||
def _candidate_from_row(row: PublicationIdentifier) -> IdentifierCandidate:
|
||||
return IdentifierCandidate(
|
||||
kind=IdentifierKind(str(row.kind)),
|
||||
value_raw=str(row.value_raw),
|
||||
value_normalized=str(row.value_normalized),
|
||||
source=str(row.source),
|
||||
confidence_score=float(row.confidence_score),
|
||||
evidence_url=row.evidence_url,
|
||||
)
|
||||
|
||||
|
||||
def _best_display_identifier(candidates: list[IdentifierCandidate]) -> DisplayIdentifier | None:
|
||||
if not candidates:
|
||||
return None
|
||||
ordered = sorted(candidates, key=_display_sort_key, reverse=True)
|
||||
return _display_identifier_from_candidate(ordered[0])
|
||||
|
||||
|
||||
def _display_sort_key(candidate: IdentifierCandidate) -> tuple[int, float]:
|
||||
return (_kind_priority(candidate.kind), float(candidate.confidence_score))
|
||||
|
||||
|
||||
def _kind_priority(kind: IdentifierKind) -> int:
|
||||
if kind == IdentifierKind.DOI:
|
||||
return PRIORITY_DOI
|
||||
if kind == IdentifierKind.ARXIV:
|
||||
return PRIORITY_ARXIV
|
||||
if kind == IdentifierKind.PMCID:
|
||||
return PRIORITY_PMCID
|
||||
return PRIORITY_PMID
|
||||
|
||||
|
||||
def _display_identifier_from_candidate(candidate: IdentifierCandidate) -> DisplayIdentifier:
|
||||
value = candidate.value_normalized
|
||||
return DisplayIdentifier(
|
||||
kind=candidate.kind.value,
|
||||
value=value,
|
||||
label=_display_label(candidate.kind, value),
|
||||
url=_identifier_url(candidate.kind, value),
|
||||
confidence_score=float(candidate.confidence_score),
|
||||
)
|
||||
|
||||
|
||||
def _display_label(kind: IdentifierKind, value: str) -> str:
|
||||
if kind == IdentifierKind.DOI:
|
||||
return f"DOI: {value}"
|
||||
if kind == IdentifierKind.ARXIV:
|
||||
return f"arXiv: {value}"
|
||||
if kind == IdentifierKind.PMCID:
|
||||
return f"PMCID: {value}"
|
||||
return f"PMID: {value}"
|
||||
|
||||
|
||||
def _identifier_url(kind: IdentifierKind, value: str) -> str | None:
|
||||
if kind == IdentifierKind.DOI:
|
||||
return f"https://doi.org/{value}"
|
||||
if kind == IdentifierKind.ARXIV:
|
||||
return f"https://arxiv.org/abs/{value}"
|
||||
if kind == IdentifierKind.PMCID:
|
||||
return f"https://pmc.ncbi.nlm.nih.gov/articles/{value}/"
|
||||
if kind == IdentifierKind.PMID:
|
||||
return f"https://pubmed.ncbi.nlm.nih.gov/{value}/"
|
||||
return None
|
||||
|
|
@ -1,76 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from app.services.domains.doi.normalize import normalize_doi
|
||||
from app.services.domains.publication_identifiers.types import IdentifierKind
|
||||
|
||||
ARXIV_ABS_RE = re.compile(r"\barxiv:\s*([a-z-]+/\d{7}|\d{4}\.\d{4,5})(v\d+)?\b", re.I)
|
||||
ARXIV_PATH_RE = re.compile(r"^/(?:abs|pdf|html|ps|format)/([a-z-]+/\d{7}|\d{4}\.\d{4,5})(v\d+)?(?:\.pdf)?/?$", re.I)
|
||||
PMCID_RE = re.compile(r"\b(PMC\d+)\b", re.I)
|
||||
PUBMED_PATH_RE = re.compile(r"^/(\d+)/?$")
|
||||
|
||||
|
||||
def normalize_identifier(kind: IdentifierKind, value: str | None) -> str | None:
|
||||
if kind == IdentifierKind.DOI:
|
||||
return normalize_doi(value)
|
||||
if kind == IdentifierKind.ARXIV:
|
||||
return normalize_arxiv_id(value)
|
||||
if kind == IdentifierKind.PMCID:
|
||||
return normalize_pmcid(value)
|
||||
if kind == IdentifierKind.PMID:
|
||||
return normalize_pmid(value)
|
||||
return None
|
||||
|
||||
|
||||
def normalize_arxiv_id(value: str | None) -> str | None:
|
||||
text = (value or "").strip()
|
||||
if not text:
|
||||
return None
|
||||
parsed = urlparse(text)
|
||||
if parsed.scheme in {"http", "https"} and "arxiv.org" in parsed.netloc.lower():
|
||||
return _arxiv_from_path(parsed.path)
|
||||
match = ARXIV_ABS_RE.search(text)
|
||||
if not match:
|
||||
return None
|
||||
version = (match.group(2) or "").lower()
|
||||
return f"{match.group(1).lower()}{version}"
|
||||
|
||||
|
||||
def _arxiv_from_path(path: str) -> str | None:
|
||||
match = ARXIV_PATH_RE.match(path or "")
|
||||
if not match:
|
||||
return None
|
||||
version = (match.group(2) or "").lower()
|
||||
return f"{match.group(1).lower()}{version}"
|
||||
|
||||
|
||||
def normalize_pmcid(value: str | None) -> str | None:
|
||||
text = (value or "").strip()
|
||||
if not text:
|
||||
return None
|
||||
parsed = urlparse(text)
|
||||
if parsed.scheme in {"http", "https"} and "ncbi.nlm.nih.gov" in parsed.netloc.lower():
|
||||
return _first_match(PMCID_RE, parsed.path)
|
||||
return _first_match(PMCID_RE, text)
|
||||
|
||||
|
||||
def normalize_pmid(value: str | None) -> str | None:
|
||||
text = (value or "").strip()
|
||||
if not text:
|
||||
return None
|
||||
parsed = urlparse(text)
|
||||
if parsed.scheme in {"http", "https"} and "pubmed.ncbi.nlm.nih.gov" in parsed.netloc.lower():
|
||||
match = PUBMED_PATH_RE.match(parsed.path or "")
|
||||
if not match:
|
||||
return None
|
||||
return match.group(1)
|
||||
return None
|
||||
|
||||
|
||||
def _first_match(pattern: re.Pattern[str], value: str) -> str | None:
|
||||
match = pattern.search(value)
|
||||
if not match:
|
||||
return None
|
||||
return match.group(1).upper()
|
||||
|
|
@ -1,30 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from enum import StrEnum
|
||||
|
||||
|
||||
class IdentifierKind(StrEnum):
|
||||
DOI = "doi"
|
||||
ARXIV = "arxiv"
|
||||
PMCID = "pmcid"
|
||||
PMID = "pmid"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DisplayIdentifier:
|
||||
kind: str
|
||||
value: str
|
||||
label: str
|
||||
url: str | None
|
||||
confidence_score: float
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class IdentifierCandidate:
|
||||
kind: IdentifierKind
|
||||
value_raw: str
|
||||
value_normalized: str
|
||||
source: str
|
||||
confidence_score: float
|
||||
evidence_url: str | None
|
||||
|
|
@ -1 +0,0 @@
|
|||
from app.services.domains.publications.application import *
|
||||
|
|
@ -1,74 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from app.services.domains.publications.counts import (
|
||||
count_for_user,
|
||||
count_favorite_for_user,
|
||||
count_latest_for_user,
|
||||
count_unread_for_user,
|
||||
)
|
||||
from app.services.domains.publications.listing import (
|
||||
list_for_user,
|
||||
retry_pdf_for_user,
|
||||
list_unread_for_user,
|
||||
)
|
||||
from app.services.domains.publications.enrichment import (
|
||||
hydrate_pdf_enrichment_state,
|
||||
schedule_missing_pdf_enrichment_for_user,
|
||||
schedule_retry_pdf_enrichment_for_row,
|
||||
)
|
||||
from app.services.domains.publications.modes import (
|
||||
MODE_ALL,
|
||||
MODE_LATEST,
|
||||
MODE_NEW,
|
||||
MODE_UNREAD,
|
||||
resolve_publication_view_mode,
|
||||
)
|
||||
from app.services.domains.publications.queries import (
|
||||
get_latest_completed_run_id_for_user,
|
||||
get_publication_item_for_user,
|
||||
publications_query,
|
||||
)
|
||||
from app.services.domains.publications.read_state import (
|
||||
mark_all_unread_as_read_for_user,
|
||||
mark_selected_as_read_for_user,
|
||||
set_publication_favorite_for_user,
|
||||
)
|
||||
from app.services.domains.publications.pdf_queue import (
|
||||
count_pdf_queue_items,
|
||||
enqueue_all_missing_pdf_jobs,
|
||||
enqueue_retry_pdf_job_for_publication_id,
|
||||
list_pdf_queue_page,
|
||||
list_pdf_queue_items,
|
||||
)
|
||||
from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem
|
||||
|
||||
__all__ = [
|
||||
"MODE_ALL",
|
||||
"MODE_UNREAD",
|
||||
"MODE_LATEST",
|
||||
"MODE_NEW",
|
||||
"PublicationListItem",
|
||||
"UnreadPublicationItem",
|
||||
"resolve_publication_view_mode",
|
||||
"get_latest_completed_run_id_for_user",
|
||||
"publications_query",
|
||||
"get_publication_item_for_user",
|
||||
"list_for_user",
|
||||
"list_unread_for_user",
|
||||
"retry_pdf_for_user",
|
||||
"hydrate_pdf_enrichment_state",
|
||||
"schedule_retry_pdf_enrichment_for_row",
|
||||
"list_pdf_queue_items",
|
||||
"list_pdf_queue_page",
|
||||
"count_pdf_queue_items",
|
||||
"enqueue_all_missing_pdf_jobs",
|
||||
"enqueue_retry_pdf_job_for_publication_id",
|
||||
"schedule_missing_pdf_enrichment_for_user",
|
||||
"count_for_user",
|
||||
"count_favorite_for_user",
|
||||
"count_unread_for_user",
|
||||
"count_latest_for_user",
|
||||
"mark_all_unread_as_read_for_user",
|
||||
"mark_selected_as_read_for_user",
|
||||
"set_publication_favorite_for_user",
|
||||
]
|
||||
|
|
@ -1,90 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.models import ScholarProfile, ScholarPublication
|
||||
from app.services.domains.publications.modes import (
|
||||
MODE_ALL,
|
||||
MODE_LATEST,
|
||||
MODE_UNREAD,
|
||||
resolve_publication_view_mode,
|
||||
)
|
||||
from app.services.domains.publications.queries import get_latest_completed_run_id_for_user
|
||||
|
||||
|
||||
async def count_for_user(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
mode: str = MODE_ALL,
|
||||
scholar_profile_id: int | None = None,
|
||||
favorite_only: bool = False,
|
||||
) -> int:
|
||||
resolved_mode = resolve_publication_view_mode(mode)
|
||||
latest_run_id = await get_latest_completed_run_id_for_user(db_session, user_id=user_id)
|
||||
stmt = (
|
||||
select(func.count())
|
||||
.select_from(ScholarPublication)
|
||||
.join(ScholarProfile, ScholarProfile.id == ScholarPublication.scholar_profile_id)
|
||||
.where(ScholarProfile.user_id == user_id)
|
||||
)
|
||||
if scholar_profile_id is not None:
|
||||
stmt = stmt.where(ScholarProfile.id == scholar_profile_id)
|
||||
if favorite_only:
|
||||
stmt = stmt.where(ScholarPublication.is_favorite.is_(True))
|
||||
if resolved_mode == MODE_UNREAD:
|
||||
stmt = stmt.where(ScholarPublication.is_read.is_(False))
|
||||
if resolved_mode == MODE_LATEST:
|
||||
if latest_run_id is None:
|
||||
return 0
|
||||
stmt = stmt.where(ScholarPublication.first_seen_run_id == latest_run_id)
|
||||
result = await db_session.execute(stmt)
|
||||
return int(result.scalar_one() or 0)
|
||||
|
||||
|
||||
async def count_unread_for_user(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
scholar_profile_id: int | None = None,
|
||||
favorite_only: bool = False,
|
||||
) -> int:
|
||||
return await count_for_user(
|
||||
db_session,
|
||||
user_id=user_id,
|
||||
mode=MODE_UNREAD,
|
||||
scholar_profile_id=scholar_profile_id,
|
||||
favorite_only=favorite_only,
|
||||
)
|
||||
|
||||
|
||||
async def count_latest_for_user(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
scholar_profile_id: int | None = None,
|
||||
favorite_only: bool = False,
|
||||
) -> int:
|
||||
return await count_for_user(
|
||||
db_session,
|
||||
user_id=user_id,
|
||||
mode=MODE_LATEST,
|
||||
scholar_profile_id=scholar_profile_id,
|
||||
favorite_only=favorite_only,
|
||||
)
|
||||
|
||||
|
||||
async def count_favorite_for_user(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
scholar_profile_id: int | None = None,
|
||||
) -> int:
|
||||
return await count_for_user(
|
||||
db_session,
|
||||
user_id=user_id,
|
||||
mode=MODE_ALL,
|
||||
scholar_profile_id=scholar_profile_id,
|
||||
favorite_only=True,
|
||||
)
|
||||
|
|
@ -1,73 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.services.domains.publications.pdf_queue import (
|
||||
enqueue_missing_pdf_jobs,
|
||||
enqueue_retry_pdf_job,
|
||||
overlay_pdf_job_state,
|
||||
)
|
||||
from app.services.domains.publications.types import PublicationListItem
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def schedule_missing_pdf_enrichment_for_user(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
request_email: str | None,
|
||||
items: list[PublicationListItem],
|
||||
max_items: int,
|
||||
) -> int:
|
||||
queued_ids = await enqueue_missing_pdf_jobs(
|
||||
db_session,
|
||||
user_id=user_id,
|
||||
request_email=request_email,
|
||||
rows=items,
|
||||
max_items=max_items,
|
||||
)
|
||||
logger.info(
|
||||
"publications.enrichment.scheduled",
|
||||
extra={
|
||||
"event": "publications.enrichment.scheduled",
|
||||
"user_id": user_id,
|
||||
"publication_count": len(queued_ids),
|
||||
},
|
||||
)
|
||||
return len(queued_ids)
|
||||
|
||||
|
||||
async def schedule_retry_pdf_enrichment_for_row(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
request_email: str | None,
|
||||
item: PublicationListItem,
|
||||
) -> bool:
|
||||
queued = await enqueue_retry_pdf_job(
|
||||
db_session,
|
||||
user_id=user_id,
|
||||
request_email=request_email,
|
||||
row=item,
|
||||
)
|
||||
logger.info(
|
||||
"publications.enrichment.retry_scheduled",
|
||||
extra={
|
||||
"event": "publications.enrichment.retry_scheduled",
|
||||
"user_id": user_id,
|
||||
"publication_id": item.publication_id,
|
||||
"queued": queued,
|
||||
},
|
||||
)
|
||||
return queued
|
||||
|
||||
|
||||
async def hydrate_pdf_enrichment_state(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
items: list[PublicationListItem],
|
||||
) -> list[PublicationListItem]:
|
||||
return await overlay_pdf_job_state(db_session, rows=items)
|
||||
|
|
@ -1,93 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.services.domains.publication_identifiers import application as identifier_service
|
||||
from app.services.domains.publications.modes import (
|
||||
MODE_ALL,
|
||||
MODE_UNREAD,
|
||||
resolve_publication_view_mode,
|
||||
)
|
||||
from app.services.domains.publications.queries import (
|
||||
get_latest_completed_run_id_for_user,
|
||||
get_publication_item_for_user,
|
||||
publication_list_item_from_row,
|
||||
publications_query,
|
||||
unread_item_from_row,
|
||||
)
|
||||
from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem
|
||||
|
||||
|
||||
async def list_for_user(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
mode: str = MODE_ALL,
|
||||
scholar_profile_id: int | None = None,
|
||||
favorite_only: bool = False,
|
||||
limit: int = 300,
|
||||
offset: int = 0,
|
||||
) -> list[PublicationListItem]:
|
||||
resolved_mode = resolve_publication_view_mode(mode)
|
||||
latest_run_id = await get_latest_completed_run_id_for_user(db_session, user_id=user_id)
|
||||
result = await db_session.execute(
|
||||
publications_query(
|
||||
user_id=user_id,
|
||||
mode=resolved_mode,
|
||||
latest_run_id=latest_run_id,
|
||||
scholar_profile_id=scholar_profile_id,
|
||||
favorite_only=favorite_only,
|
||||
limit=limit,
|
||||
offset=offset,
|
||||
)
|
||||
)
|
||||
rows = [
|
||||
publication_list_item_from_row(row, latest_run_id=latest_run_id)
|
||||
for row in result.all()
|
||||
]
|
||||
return await identifier_service.overlay_publication_items_with_display_identifiers(
|
||||
db_session,
|
||||
items=rows,
|
||||
)
|
||||
|
||||
|
||||
async def retry_pdf_for_user(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
scholar_profile_id: int,
|
||||
publication_id: int,
|
||||
) -> PublicationListItem | None:
|
||||
item = await get_publication_item_for_user(
|
||||
db_session,
|
||||
user_id=user_id,
|
||||
scholar_profile_id=scholar_profile_id,
|
||||
publication_id=publication_id,
|
||||
)
|
||||
if item is None:
|
||||
return None
|
||||
hydrated = await identifier_service.overlay_publication_items_with_display_identifiers(
|
||||
db_session,
|
||||
items=[item],
|
||||
)
|
||||
return hydrated[0] if hydrated else item
|
||||
|
||||
|
||||
async def list_unread_for_user(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
limit: int = 100,
|
||||
) -> list[UnreadPublicationItem]:
|
||||
result = await db_session.execute(
|
||||
publications_query(
|
||||
user_id=user_id,
|
||||
mode=MODE_UNREAD,
|
||||
latest_run_id=None,
|
||||
scholar_profile_id=None,
|
||||
favorite_only=False,
|
||||
limit=limit,
|
||||
offset=0,
|
||||
)
|
||||
)
|
||||
return [unread_item_from_row(row) for row in result.all()]
|
||||
|
|
@ -1,14 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
MODE_ALL = "all"
|
||||
MODE_UNREAD = "unread"
|
||||
MODE_LATEST = "latest"
|
||||
MODE_NEW = "new" # compatibility alias for MODE_LATEST
|
||||
|
||||
|
||||
def resolve_publication_view_mode(value: str | None) -> str:
|
||||
if value == MODE_UNREAD:
|
||||
return MODE_UNREAD
|
||||
if value in {MODE_LATEST, MODE_NEW}:
|
||||
return MODE_LATEST
|
||||
return MODE_ALL
|
||||
|
|
@ -1,915 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
import logging
|
||||
|
||||
from sqlalchemy import Select, func, literal, or_, select, union_all
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.models import (
|
||||
Publication,
|
||||
PublicationPdfJob,
|
||||
PublicationPdfJobEvent,
|
||||
ScholarProfile,
|
||||
ScholarPublication,
|
||||
User,
|
||||
)
|
||||
from app.db.session import get_session_factory
|
||||
from app.services.domains.publication_identifiers import application as identifier_service
|
||||
from app.services.domains.publication_identifiers.types import DisplayIdentifier
|
||||
from app.services.domains.publications.pdf_resolution_pipeline import (
|
||||
resolve_publication_pdf_outcome_for_row,
|
||||
)
|
||||
from app.services.domains.publications.types import PublicationListItem
|
||||
from app.services.domains.unpaywall.application import (
|
||||
FAILURE_RESOLUTION_EXCEPTION,
|
||||
OaResolutionOutcome,
|
||||
)
|
||||
from app.settings import settings
|
||||
|
||||
PDF_STATUS_UNTRACKED = "untracked"
|
||||
PDF_STATUS_QUEUED = "queued"
|
||||
PDF_STATUS_RUNNING = "running"
|
||||
PDF_STATUS_RESOLVED = "resolved"
|
||||
PDF_STATUS_FAILED = "failed"
|
||||
|
||||
PDF_EVENT_QUEUED = "queued"
|
||||
PDF_EVENT_ATTEMPT_STARTED = "attempt_started"
|
||||
PDF_EVENT_RESOLVED = "resolved"
|
||||
PDF_EVENT_FAILED = "failed"
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
_scheduled_tasks: set[asyncio.Task[None]] = set()
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PdfQueueListItem:
|
||||
publication_id: int
|
||||
title: str
|
||||
pdf_url: str | None
|
||||
status: str
|
||||
attempt_count: int
|
||||
last_failure_reason: str | None
|
||||
last_failure_detail: str | None
|
||||
last_source: str | None
|
||||
requested_by_user_id: int | None
|
||||
requested_by_email: str | None
|
||||
queued_at: datetime | None
|
||||
last_attempt_at: datetime | None
|
||||
resolved_at: datetime | None
|
||||
updated_at: datetime
|
||||
display_identifier: DisplayIdentifier | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PdfRequeueResult:
|
||||
publication_exists: bool
|
||||
queued: bool
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PdfBulkQueueResult:
|
||||
requested_count: int
|
||||
queued_count: int
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PdfQueuePage:
|
||||
items: list[PdfQueueListItem]
|
||||
total_count: int
|
||||
limit: int
|
||||
offset: int
|
||||
|
||||
|
||||
def _utcnow() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def _publication_ids(rows: list[PublicationListItem]) -> list[int]:
|
||||
return sorted({row.publication_id for row in rows})
|
||||
|
||||
|
||||
def _status_from_job(row: PublicationListItem, job: PublicationPdfJob | None) -> str:
|
||||
if row.pdf_url:
|
||||
return PDF_STATUS_RESOLVED
|
||||
if job is None:
|
||||
return PDF_STATUS_UNTRACKED
|
||||
return job.status
|
||||
|
||||
|
||||
def _item_from_row_and_job(
|
||||
row: PublicationListItem,
|
||||
job: PublicationPdfJob | None,
|
||||
) -> PublicationListItem:
|
||||
return PublicationListItem(
|
||||
publication_id=row.publication_id,
|
||||
scholar_profile_id=row.scholar_profile_id,
|
||||
scholar_label=row.scholar_label,
|
||||
title=row.title,
|
||||
year=row.year,
|
||||
citation_count=row.citation_count,
|
||||
venue_text=row.venue_text,
|
||||
pub_url=row.pub_url,
|
||||
pdf_url=row.pdf_url,
|
||||
is_read=row.is_read,
|
||||
is_favorite=row.is_favorite,
|
||||
first_seen_at=row.first_seen_at,
|
||||
is_new_in_latest_run=row.is_new_in_latest_run,
|
||||
pdf_status=_status_from_job(row, job),
|
||||
pdf_attempt_count=int(job.attempt_count) if job is not None else 0,
|
||||
pdf_failure_reason=job.last_failure_reason if job is not None else None,
|
||||
pdf_failure_detail=job.last_failure_detail if job is not None else None,
|
||||
display_identifier=row.display_identifier,
|
||||
)
|
||||
|
||||
|
||||
def _queueable_rows(
|
||||
rows: list[PublicationListItem],
|
||||
*,
|
||||
max_items: int,
|
||||
) -> list[PublicationListItem]:
|
||||
bounded = max(0, int(max_items))
|
||||
if bounded == 0:
|
||||
return []
|
||||
candidates = [row for row in rows if not row.pdf_url]
|
||||
return candidates[:bounded]
|
||||
|
||||
|
||||
def _bounded_limit(limit: int, *, max_value: int = 500) -> int:
|
||||
return max(1, min(int(limit), max_value))
|
||||
|
||||
|
||||
def _bounded_offset(offset: int) -> int:
|
||||
return max(int(offset), 0)
|
||||
|
||||
|
||||
def _auto_retry_interval_seconds() -> int:
|
||||
return max(int(settings.pdf_auto_retry_interval_seconds), 1)
|
||||
|
||||
|
||||
def _auto_retry_first_interval_seconds() -> int:
|
||||
return max(int(settings.pdf_auto_retry_first_interval_seconds), 1)
|
||||
|
||||
|
||||
def _auto_retry_max_attempts() -> int:
|
||||
return max(int(settings.pdf_auto_retry_max_attempts), 1)
|
||||
|
||||
|
||||
def _retry_interval_seconds_for_attempt_count(attempt_count: int) -> int:
|
||||
if int(attempt_count) <= 1:
|
||||
return _auto_retry_first_interval_seconds()
|
||||
return _auto_retry_interval_seconds()
|
||||
|
||||
|
||||
def _cooldown_active(
|
||||
*,
|
||||
last_attempt_at: datetime | None,
|
||||
attempt_count: int,
|
||||
) -> bool:
|
||||
if last_attempt_at is None:
|
||||
return False
|
||||
elapsed = (_utcnow() - last_attempt_at).total_seconds()
|
||||
return elapsed < _retry_interval_seconds_for_attempt_count(int(attempt_count))
|
||||
|
||||
|
||||
def _can_enqueue_job(
|
||||
job: PublicationPdfJob | None,
|
||||
*,
|
||||
force_retry: bool,
|
||||
) -> bool:
|
||||
if job is None:
|
||||
return True
|
||||
if job.status in {PDF_STATUS_QUEUED, PDF_STATUS_RUNNING}:
|
||||
return False
|
||||
if force_retry:
|
||||
return job.status in {PDF_STATUS_FAILED, PDF_STATUS_RESOLVED, PDF_STATUS_UNTRACKED}
|
||||
if job.status == PDF_STATUS_RESOLVED:
|
||||
return False
|
||||
if int(job.attempt_count) >= _auto_retry_max_attempts():
|
||||
return False
|
||||
if _cooldown_active(
|
||||
last_attempt_at=job.last_attempt_at,
|
||||
attempt_count=int(job.attempt_count),
|
||||
):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _event_row(
|
||||
*,
|
||||
publication_id: int,
|
||||
user_id: int | None,
|
||||
event_type: str,
|
||||
status: str | None,
|
||||
source: str | None = None,
|
||||
failure_reason: str | None = None,
|
||||
message: str | None = None,
|
||||
) -> PublicationPdfJobEvent:
|
||||
return PublicationPdfJobEvent(
|
||||
publication_id=publication_id,
|
||||
user_id=user_id,
|
||||
event_type=event_type,
|
||||
status=status,
|
||||
source=source,
|
||||
failure_reason=failure_reason,
|
||||
message=message,
|
||||
)
|
||||
|
||||
|
||||
def _queued_job(
|
||||
*,
|
||||
publication_id: int,
|
||||
user_id: int,
|
||||
) -> PublicationPdfJob:
|
||||
now = _utcnow()
|
||||
return PublicationPdfJob(
|
||||
publication_id=publication_id,
|
||||
status=PDF_STATUS_QUEUED,
|
||||
queued_at=now,
|
||||
last_requested_by_user_id=user_id,
|
||||
)
|
||||
|
||||
|
||||
def _mark_job_queued(job: PublicationPdfJob, *, user_id: int) -> None:
|
||||
now = _utcnow()
|
||||
job.status = PDF_STATUS_QUEUED
|
||||
job.queued_at = now
|
||||
job.last_requested_by_user_id = user_id
|
||||
job.last_failure_reason = None
|
||||
job.last_failure_detail = None
|
||||
job.last_source = None
|
||||
|
||||
|
||||
def _state_map(jobs: list[PublicationPdfJob]) -> dict[int, PublicationPdfJob]:
|
||||
return {int(job.publication_id): job for job in jobs}
|
||||
|
||||
|
||||
async def _jobs_for_publication_ids(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
publication_ids: list[int],
|
||||
) -> dict[int, PublicationPdfJob]:
|
||||
if not publication_ids:
|
||||
return {}
|
||||
result = await db_session.execute(
|
||||
select(PublicationPdfJob).where(PublicationPdfJob.publication_id.in_(publication_ids))
|
||||
)
|
||||
return _state_map(list(result.scalars()))
|
||||
|
||||
|
||||
async def overlay_pdf_job_state(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
rows: list[PublicationListItem],
|
||||
) -> list[PublicationListItem]:
|
||||
if not rows:
|
||||
return []
|
||||
jobs = await _jobs_for_publication_ids(
|
||||
db_session,
|
||||
publication_ids=_publication_ids(rows),
|
||||
)
|
||||
return [_item_from_row_and_job(row, jobs.get(row.publication_id)) for row in rows]
|
||||
|
||||
|
||||
async def _enqueue_rows(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
rows: list[PublicationListItem],
|
||||
force_retry: bool,
|
||||
) -> list[PublicationListItem]:
|
||||
if not rows:
|
||||
return []
|
||||
queued: list[PublicationListItem] = []
|
||||
jobs = await _jobs_for_publication_ids(
|
||||
db_session,
|
||||
publication_ids=_publication_ids(rows),
|
||||
)
|
||||
for row in rows:
|
||||
job = jobs.get(row.publication_id)
|
||||
if not _can_enqueue_job(job, force_retry=force_retry):
|
||||
continue
|
||||
if job is None:
|
||||
job = _queued_job(publication_id=row.publication_id, user_id=user_id)
|
||||
jobs[row.publication_id] = job
|
||||
db_session.add(job)
|
||||
else:
|
||||
_mark_job_queued(job, user_id=user_id)
|
||||
db_session.add(
|
||||
_event_row(
|
||||
publication_id=row.publication_id,
|
||||
user_id=user_id,
|
||||
event_type=PDF_EVENT_QUEUED,
|
||||
status=PDF_STATUS_QUEUED,
|
||||
)
|
||||
)
|
||||
queued.append(row)
|
||||
if queued:
|
||||
await db_session.commit()
|
||||
return queued
|
||||
|
||||
|
||||
def _register_task(task: asyncio.Task[None]) -> None:
|
||||
_scheduled_tasks.add(task)
|
||||
|
||||
|
||||
def _drop_finished_task(task: asyncio.Task[None]) -> None:
|
||||
_scheduled_tasks.discard(task)
|
||||
try:
|
||||
task.result()
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"publications.pdf_queue.task_failed",
|
||||
extra={"event": "publications.pdf_queue.task_failed"},
|
||||
)
|
||||
|
||||
|
||||
async def _mark_attempt_started(
|
||||
*,
|
||||
publication_id: int,
|
||||
user_id: int,
|
||||
) -> None:
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as db_session:
|
||||
job = await db_session.get(PublicationPdfJob, publication_id)
|
||||
if job is None:
|
||||
job = _queued_job(publication_id=publication_id, user_id=user_id)
|
||||
db_session.add(job)
|
||||
job.status = PDF_STATUS_RUNNING
|
||||
job.last_attempt_at = _utcnow()
|
||||
job.attempt_count = int(job.attempt_count) + 1
|
||||
db_session.add(
|
||||
_event_row(
|
||||
publication_id=publication_id,
|
||||
user_id=user_id,
|
||||
event_type=PDF_EVENT_ATTEMPT_STARTED,
|
||||
status=PDF_STATUS_RUNNING,
|
||||
)
|
||||
)
|
||||
await db_session.commit()
|
||||
|
||||
|
||||
def _failed_outcome(
|
||||
*,
|
||||
row: PublicationListItem,
|
||||
) -> OaResolutionOutcome:
|
||||
return OaResolutionOutcome(
|
||||
publication_id=row.publication_id,
|
||||
doi=None,
|
||||
pdf_url=None,
|
||||
failure_reason=FAILURE_RESOLUTION_EXCEPTION,
|
||||
source=None,
|
||||
used_crossref=False,
|
||||
)
|
||||
|
||||
|
||||
async def _fetch_outcome_for_row(
|
||||
*,
|
||||
row: PublicationListItem,
|
||||
request_email: str | None,
|
||||
) -> OaResolutionOutcome:
|
||||
pipeline_result = await resolve_publication_pdf_outcome_for_row(
|
||||
row=row,
|
||||
request_email=request_email,
|
||||
)
|
||||
outcome = pipeline_result.outcome
|
||||
if outcome is not None:
|
||||
return outcome
|
||||
return _failed_outcome(row=row)
|
||||
|
||||
|
||||
def _apply_publication_update(
|
||||
publication: Publication,
|
||||
*,
|
||||
pdf_url: str | None,
|
||||
) -> None:
|
||||
if pdf_url and publication.pdf_url != pdf_url:
|
||||
publication.pdf_url = pdf_url
|
||||
|
||||
|
||||
def _apply_job_outcome(job: PublicationPdfJob, *, outcome: OaResolutionOutcome) -> None:
|
||||
job.last_source = outcome.source
|
||||
if outcome.pdf_url:
|
||||
job.status = PDF_STATUS_RESOLVED
|
||||
job.resolved_at = _utcnow()
|
||||
job.last_failure_reason = None
|
||||
job.last_failure_detail = None
|
||||
return
|
||||
job.status = PDF_STATUS_FAILED
|
||||
job.last_failure_reason = outcome.failure_reason
|
||||
job.last_failure_detail = outcome.failure_reason
|
||||
|
||||
|
||||
def _result_event(outcome: OaResolutionOutcome) -> tuple[str, str]:
|
||||
if outcome.pdf_url:
|
||||
return PDF_EVENT_RESOLVED, PDF_STATUS_RESOLVED
|
||||
return PDF_EVENT_FAILED, PDF_STATUS_FAILED
|
||||
|
||||
|
||||
async def _persist_outcome(
|
||||
*,
|
||||
publication_id: int,
|
||||
user_id: int,
|
||||
outcome: OaResolutionOutcome,
|
||||
) -> None:
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as db_session:
|
||||
publication = await db_session.get(Publication, publication_id)
|
||||
job = await db_session.get(PublicationPdfJob, publication_id)
|
||||
if publication is None or job is None:
|
||||
return
|
||||
_apply_publication_update(publication, pdf_url=outcome.pdf_url)
|
||||
await identifier_service.sync_identifiers_for_publication_resolution(
|
||||
db_session,
|
||||
publication=publication,
|
||||
source=outcome.source,
|
||||
)
|
||||
_apply_job_outcome(job, outcome=outcome)
|
||||
event_type, status = _result_event(outcome)
|
||||
db_session.add(
|
||||
_event_row(
|
||||
publication_id=publication_id,
|
||||
user_id=user_id,
|
||||
event_type=event_type,
|
||||
status=status,
|
||||
source=outcome.source,
|
||||
failure_reason=outcome.failure_reason,
|
||||
message=outcome.failure_reason,
|
||||
)
|
||||
)
|
||||
await db_session.commit()
|
||||
|
||||
|
||||
async def _resolve_publication_row(
|
||||
*,
|
||||
user_id: int,
|
||||
request_email: str | None,
|
||||
row: PublicationListItem,
|
||||
) -> None:
|
||||
await _mark_attempt_started(publication_id=row.publication_id, user_id=user_id)
|
||||
try:
|
||||
outcome = await _fetch_outcome_for_row(row=row, request_email=request_email)
|
||||
except Exception as exc: # pragma: no cover - defensive network boundary
|
||||
logger.warning(
|
||||
"publications.pdf_queue.resolve_failed",
|
||||
extra={
|
||||
"event": "publications.pdf_queue.resolve_failed",
|
||||
"publication_id": row.publication_id,
|
||||
"error": str(exc),
|
||||
},
|
||||
)
|
||||
outcome = _failed_outcome(row=row)
|
||||
await _persist_outcome(
|
||||
publication_id=row.publication_id,
|
||||
user_id=user_id,
|
||||
outcome=outcome,
|
||||
)
|
||||
|
||||
|
||||
async def _run_resolution_task(
|
||||
*,
|
||||
user_id: int,
|
||||
request_email: str | None,
|
||||
rows: list[PublicationListItem],
|
||||
) -> None:
|
||||
for row in rows:
|
||||
await _resolve_publication_row(
|
||||
user_id=user_id,
|
||||
request_email=request_email,
|
||||
row=row,
|
||||
)
|
||||
|
||||
|
||||
def _schedule_rows(
|
||||
*,
|
||||
user_id: int,
|
||||
request_email: str | None,
|
||||
rows: list[PublicationListItem],
|
||||
) -> None:
|
||||
if not rows:
|
||||
return
|
||||
task = asyncio.create_task(
|
||||
_run_resolution_task(
|
||||
user_id=user_id,
|
||||
request_email=request_email,
|
||||
rows=rows,
|
||||
)
|
||||
)
|
||||
_register_task(task)
|
||||
task.add_done_callback(_drop_finished_task)
|
||||
|
||||
|
||||
async def enqueue_missing_pdf_jobs(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
request_email: str | None,
|
||||
rows: list[PublicationListItem],
|
||||
max_items: int,
|
||||
) -> list[int]:
|
||||
queueable = _queueable_rows(rows, max_items=max_items)
|
||||
queued_rows = await _enqueue_rows(
|
||||
db_session,
|
||||
user_id=user_id,
|
||||
rows=queueable,
|
||||
force_retry=False,
|
||||
)
|
||||
_schedule_rows(user_id=user_id, request_email=request_email, rows=queued_rows)
|
||||
return [row.publication_id for row in queued_rows]
|
||||
|
||||
|
||||
async def enqueue_retry_pdf_job(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
request_email: str | None,
|
||||
row: PublicationListItem,
|
||||
) -> bool:
|
||||
queued_rows = await _enqueue_rows(
|
||||
db_session,
|
||||
user_id=user_id,
|
||||
rows=[row],
|
||||
force_retry=True,
|
||||
)
|
||||
_schedule_rows(user_id=user_id, request_email=request_email, rows=queued_rows)
|
||||
return bool(queued_rows)
|
||||
|
||||
|
||||
def _retry_item_label(display_name: str | None, scholar_id: str | None) -> str:
|
||||
return str(display_name or scholar_id or "unknown")
|
||||
|
||||
|
||||
def _retry_item_from_publication(
|
||||
publication: Publication,
|
||||
*,
|
||||
link_row: tuple | None,
|
||||
) -> PublicationListItem:
|
||||
if link_row is None:
|
||||
scholar_profile_id = 0
|
||||
scholar_label = "unknown"
|
||||
is_read = True
|
||||
first_seen_at = publication.created_at or _utcnow()
|
||||
else:
|
||||
scholar_profile_id = int(link_row[0])
|
||||
scholar_label = _retry_item_label(link_row[1], link_row[2])
|
||||
is_read = bool(link_row[3])
|
||||
first_seen_at = link_row[4] or publication.created_at or _utcnow()
|
||||
return PublicationListItem(
|
||||
publication_id=int(publication.id),
|
||||
scholar_profile_id=scholar_profile_id,
|
||||
scholar_label=scholar_label,
|
||||
title=publication.title_raw,
|
||||
year=publication.year,
|
||||
citation_count=int(publication.citation_count or 0),
|
||||
venue_text=publication.venue_text,
|
||||
pub_url=publication.pub_url,
|
||||
pdf_url=publication.pdf_url,
|
||||
is_read=is_read,
|
||||
first_seen_at=first_seen_at,
|
||||
is_new_in_latest_run=False,
|
||||
)
|
||||
|
||||
|
||||
async def _retry_item_for_publication_id(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
publication_id: int,
|
||||
) -> PublicationListItem | None:
|
||||
publication = await db_session.get(Publication, publication_id)
|
||||
if publication is None:
|
||||
return None
|
||||
result = await db_session.execute(
|
||||
select(
|
||||
ScholarProfile.id,
|
||||
ScholarProfile.display_name,
|
||||
ScholarProfile.scholar_id,
|
||||
ScholarPublication.is_read,
|
||||
ScholarPublication.created_at,
|
||||
)
|
||||
.join(ScholarProfile, ScholarProfile.id == ScholarPublication.scholar_profile_id)
|
||||
.where(ScholarPublication.publication_id == publication_id)
|
||||
.order_by(ScholarPublication.created_at.asc())
|
||||
.limit(1)
|
||||
)
|
||||
return _retry_item_from_publication(publication, link_row=result.one_or_none())
|
||||
|
||||
|
||||
async def enqueue_retry_pdf_job_for_publication_id(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
request_email: str | None,
|
||||
publication_id: int,
|
||||
) -> PdfRequeueResult:
|
||||
row = await _retry_item_for_publication_id(
|
||||
db_session,
|
||||
publication_id=publication_id,
|
||||
)
|
||||
if row is None:
|
||||
return PdfRequeueResult(publication_exists=False, queued=False)
|
||||
queued = await enqueue_retry_pdf_job(
|
||||
db_session,
|
||||
user_id=user_id,
|
||||
request_email=request_email,
|
||||
row=row,
|
||||
)
|
||||
return PdfRequeueResult(publication_exists=True, queued=queued)
|
||||
|
||||
|
||||
def _queue_candidate_from_publication(publication: Publication) -> PublicationListItem:
|
||||
return PublicationListItem(
|
||||
publication_id=int(publication.id),
|
||||
scholar_profile_id=0,
|
||||
scholar_label="admin",
|
||||
title=publication.title_raw,
|
||||
year=publication.year,
|
||||
citation_count=int(publication.citation_count or 0),
|
||||
venue_text=publication.venue_text,
|
||||
pub_url=publication.pub_url,
|
||||
pdf_url=publication.pdf_url,
|
||||
is_read=True,
|
||||
first_seen_at=publication.created_at or _utcnow(),
|
||||
is_new_in_latest_run=False,
|
||||
)
|
||||
|
||||
|
||||
async def _missing_pdf_candidates(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
limit: int,
|
||||
) -> list[PublicationListItem]:
|
||||
bounded_limit = max(1, min(int(limit), 5000))
|
||||
result = await db_session.execute(
|
||||
select(Publication)
|
||||
.outerjoin(PublicationPdfJob, PublicationPdfJob.publication_id == Publication.id)
|
||||
.where(Publication.pdf_url.is_(None))
|
||||
.where(
|
||||
or_(
|
||||
PublicationPdfJob.publication_id.is_(None),
|
||||
PublicationPdfJob.status.notin_([PDF_STATUS_QUEUED, PDF_STATUS_RUNNING]),
|
||||
)
|
||||
)
|
||||
.order_by(Publication.updated_at.desc(), Publication.id.desc())
|
||||
.limit(bounded_limit)
|
||||
)
|
||||
return [
|
||||
_queue_candidate_from_publication(publication)
|
||||
for publication in result.scalars()
|
||||
]
|
||||
|
||||
|
||||
async def enqueue_all_missing_pdf_jobs(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
request_email: str | None,
|
||||
limit: int = 1000,
|
||||
) -> PdfBulkQueueResult:
|
||||
candidates = await _missing_pdf_candidates(db_session, limit=limit)
|
||||
queued_rows = await _enqueue_rows(
|
||||
db_session,
|
||||
user_id=user_id,
|
||||
rows=candidates,
|
||||
force_retry=True,
|
||||
)
|
||||
_schedule_rows(user_id=user_id, request_email=request_email, rows=queued_rows)
|
||||
return PdfBulkQueueResult(
|
||||
requested_count=len(candidates),
|
||||
queued_count=len(queued_rows),
|
||||
)
|
||||
|
||||
|
||||
def _tracked_queue_select_base(*, status: str | None) -> Select[tuple]:
|
||||
stmt = (
|
||||
select(
|
||||
PublicationPdfJob.publication_id,
|
||||
Publication.title_raw,
|
||||
Publication.pdf_url,
|
||||
PublicationPdfJob.status,
|
||||
PublicationPdfJob.attempt_count,
|
||||
PublicationPdfJob.last_failure_reason,
|
||||
PublicationPdfJob.last_failure_detail,
|
||||
PublicationPdfJob.last_source,
|
||||
PublicationPdfJob.last_requested_by_user_id,
|
||||
User.email,
|
||||
PublicationPdfJob.queued_at,
|
||||
PublicationPdfJob.last_attempt_at,
|
||||
PublicationPdfJob.resolved_at,
|
||||
PublicationPdfJob.updated_at,
|
||||
)
|
||||
.join(Publication, Publication.id == PublicationPdfJob.publication_id)
|
||||
.outerjoin(User, User.id == PublicationPdfJob.last_requested_by_user_id)
|
||||
)
|
||||
if status:
|
||||
stmt = stmt.where(PublicationPdfJob.status == status)
|
||||
return stmt
|
||||
|
||||
|
||||
def _tracked_queue_select(*, limit: int, offset: int, status: str | None) -> Select[tuple]:
|
||||
return (
|
||||
_tracked_queue_select_base(status=status)
|
||||
.order_by(PublicationPdfJob.updated_at.desc())
|
||||
.limit(_bounded_limit(limit))
|
||||
.offset(_bounded_offset(offset))
|
||||
)
|
||||
|
||||
|
||||
def _untracked_queue_select_base() -> Select[tuple]:
|
||||
return (
|
||||
select(
|
||||
Publication.id,
|
||||
Publication.title_raw,
|
||||
Publication.pdf_url,
|
||||
literal(PDF_STATUS_UNTRACKED),
|
||||
literal(0),
|
||||
literal(None),
|
||||
literal(None),
|
||||
literal(None),
|
||||
literal(None),
|
||||
literal(None),
|
||||
literal(None),
|
||||
literal(None),
|
||||
literal(None),
|
||||
Publication.updated_at,
|
||||
)
|
||||
.outerjoin(PublicationPdfJob, PublicationPdfJob.publication_id == Publication.id)
|
||||
.where(Publication.pdf_url.is_(None))
|
||||
.where(PublicationPdfJob.publication_id.is_(None))
|
||||
)
|
||||
|
||||
|
||||
def _untracked_queue_select(*, limit: int, offset: int) -> Select[tuple]:
|
||||
return (
|
||||
_untracked_queue_select_base()
|
||||
.order_by(Publication.updated_at.desc(), Publication.id.desc())
|
||||
.limit(_bounded_limit(limit))
|
||||
.offset(_bounded_offset(offset))
|
||||
)
|
||||
|
||||
|
||||
def _all_queue_select(*, limit: int, offset: int) -> Select[tuple]:
|
||||
union_stmt = union_all(
|
||||
_tracked_queue_select_base(status=None),
|
||||
_untracked_queue_select_base(),
|
||||
).subquery()
|
||||
return (
|
||||
select(union_stmt)
|
||||
.order_by(union_stmt.c.updated_at.desc())
|
||||
.limit(_bounded_limit(limit))
|
||||
.offset(_bounded_offset(offset))
|
||||
)
|
||||
|
||||
|
||||
def _tracked_queue_count_select(*, status: str | None) -> Select[tuple]:
|
||||
stmt = select(func.count()).select_from(PublicationPdfJob)
|
||||
if status:
|
||||
stmt = stmt.where(PublicationPdfJob.status == status)
|
||||
return stmt
|
||||
|
||||
|
||||
def _untracked_queue_count_select() -> Select[tuple]:
|
||||
return (
|
||||
select(func.count())
|
||||
.select_from(Publication)
|
||||
.outerjoin(PublicationPdfJob, PublicationPdfJob.publication_id == Publication.id)
|
||||
.where(Publication.pdf_url.is_(None))
|
||||
.where(PublicationPdfJob.publication_id.is_(None))
|
||||
)
|
||||
|
||||
|
||||
def _queue_item_from_row(row: tuple) -> PdfQueueListItem:
|
||||
return PdfQueueListItem(
|
||||
publication_id=int(row[0]),
|
||||
title=str(row[1] or ""),
|
||||
pdf_url=row[2],
|
||||
status=str(row[3] or PDF_STATUS_UNTRACKED),
|
||||
attempt_count=int(row[4] or 0),
|
||||
last_failure_reason=row[5],
|
||||
last_failure_detail=row[6],
|
||||
last_source=row[7],
|
||||
requested_by_user_id=int(row[8]) if row[8] is not None else None,
|
||||
requested_by_email=row[9],
|
||||
queued_at=row[10],
|
||||
last_attempt_at=row[11],
|
||||
resolved_at=row[12],
|
||||
updated_at=row[13],
|
||||
)
|
||||
|
||||
|
||||
async def _hydrated_queue_items(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
rows: list[tuple],
|
||||
) -> list[PdfQueueListItem]:
|
||||
items = [_queue_item_from_row(row) for row in rows]
|
||||
return await identifier_service.overlay_pdf_queue_items_with_display_identifiers(
|
||||
db_session,
|
||||
items=items,
|
||||
)
|
||||
|
||||
|
||||
async def list_pdf_queue_items(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
limit: int = 100,
|
||||
offset: int = 0,
|
||||
status: str | None = None,
|
||||
) -> list[PdfQueueListItem]:
|
||||
bounded_limit = _bounded_limit(limit)
|
||||
bounded_offset = _bounded_offset(offset)
|
||||
normalized_status = (status or "").strip().lower() or None
|
||||
if normalized_status == PDF_STATUS_UNTRACKED:
|
||||
result = await db_session.execute(
|
||||
_untracked_queue_select(
|
||||
limit=bounded_limit,
|
||||
offset=bounded_offset,
|
||||
)
|
||||
)
|
||||
return await _hydrated_queue_items(db_session, rows=list(result.all()))
|
||||
if normalized_status is None:
|
||||
result = await db_session.execute(
|
||||
_all_queue_select(
|
||||
limit=bounded_limit,
|
||||
offset=bounded_offset,
|
||||
)
|
||||
)
|
||||
return await _hydrated_queue_items(db_session, rows=list(result.all()))
|
||||
result = await db_session.execute(
|
||||
_tracked_queue_select(
|
||||
limit=bounded_limit,
|
||||
offset=bounded_offset,
|
||||
status=normalized_status,
|
||||
)
|
||||
)
|
||||
return await _hydrated_queue_items(db_session, rows=list(result.all()))
|
||||
|
||||
|
||||
async def count_pdf_queue_items(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
status: str | None = None,
|
||||
) -> int:
|
||||
normalized_status = (status or "").strip().lower() or None
|
||||
if normalized_status == PDF_STATUS_UNTRACKED:
|
||||
result = await db_session.execute(_untracked_queue_count_select())
|
||||
return int(result.scalar_one() or 0)
|
||||
tracked_result = await db_session.execute(
|
||||
_tracked_queue_count_select(status=normalized_status)
|
||||
)
|
||||
tracked_count = int(tracked_result.scalar_one() or 0)
|
||||
if normalized_status is not None:
|
||||
return tracked_count
|
||||
untracked_result = await db_session.execute(_untracked_queue_count_select())
|
||||
untracked_count = int(untracked_result.scalar_one() or 0)
|
||||
return tracked_count + untracked_count
|
||||
|
||||
|
||||
async def list_pdf_queue_page(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
limit: int = 100,
|
||||
offset: int = 0,
|
||||
status: str | None = None,
|
||||
) -> PdfQueuePage:
|
||||
bounded_limit = _bounded_limit(limit)
|
||||
bounded_offset = _bounded_offset(offset)
|
||||
items = await list_pdf_queue_items(
|
||||
db_session,
|
||||
limit=bounded_limit,
|
||||
offset=bounded_offset,
|
||||
status=status,
|
||||
)
|
||||
total_count = await count_pdf_queue_items(
|
||||
db_session,
|
||||
status=status,
|
||||
)
|
||||
return PdfQueuePage(
|
||||
items=items,
|
||||
total_count=total_count,
|
||||
limit=bounded_limit,
|
||||
offset=bounded_offset,
|
||||
)
|
||||
|
||||
|
||||
async def drain_ready_jobs(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
limit: int,
|
||||
max_attempts: int,
|
||||
) -> int:
|
||||
result = await db_session.execute(
|
||||
select(User.id).where(User.is_active.is_(True)).order_by(User.id.asc()).limit(1)
|
||||
)
|
||||
system_user_id = result.scalar_one_or_none()
|
||||
if system_user_id is None:
|
||||
return 0
|
||||
|
||||
bulk_result = await enqueue_all_missing_pdf_jobs(
|
||||
db_session,
|
||||
user_id=system_user_id,
|
||||
request_email=settings.unpaywall_email,
|
||||
limit=limit,
|
||||
)
|
||||
return bulk_result.queued_count
|
||||
|
|
@ -1,108 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from app.services.domains.publications.types import PublicationListItem
|
||||
from app.services.domains.unpaywall.application import OaResolutionOutcome, resolve_publication_oa_outcomes
|
||||
from app.settings import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PipelineOutcome:
|
||||
outcome: OaResolutionOutcome | None
|
||||
scholar_candidates: Any | None # Kept for backward compatibility with calling signatures
|
||||
|
||||
|
||||
async def resolve_publication_pdf_outcome_for_row(
|
||||
*,
|
||||
row: PublicationListItem,
|
||||
request_email: str | None,
|
||||
) -> PipelineOutcome:
|
||||
# 1. OpenAlex OA
|
||||
openalex_outcome = await _openalex_outcome(row, request_email=request_email)
|
||||
if openalex_outcome and openalex_outcome.pdf_url:
|
||||
return PipelineOutcome(openalex_outcome, None)
|
||||
|
||||
# 2. arXiv
|
||||
arxiv_outcome = await _arxiv_outcome(row, request_email=request_email)
|
||||
if arxiv_outcome and arxiv_outcome.pdf_url:
|
||||
return PipelineOutcome(arxiv_outcome, None)
|
||||
|
||||
# 3. Unpaywall (which falls back to Crossref)
|
||||
oa_outcome = await _oa_outcome(row=row, request_email=request_email)
|
||||
return PipelineOutcome(oa_outcome, None)
|
||||
|
||||
|
||||
async def _openalex_outcome(row: PublicationListItem, request_email: str | None) -> OaResolutionOutcome | None:
|
||||
from app.services.domains.openalex.client import OpenAlexClient
|
||||
from app.services.domains.openalex.matching import find_best_match
|
||||
|
||||
if not row.title:
|
||||
return None
|
||||
|
||||
import re
|
||||
safe_title = re.sub(r"[^\w\s]", " ", row.title)
|
||||
safe_title = " ".join(safe_title.split())
|
||||
if not safe_title:
|
||||
return None
|
||||
|
||||
client = OpenAlexClient(api_key=settings.openalex_api_key, mailto=request_email or settings.crossref_api_mailto)
|
||||
try:
|
||||
openalex_works = await client.get_works_by_filter({"title.search": safe_title}, limit=5)
|
||||
match = find_best_match(
|
||||
target_title=row.title,
|
||||
target_year=row.year,
|
||||
target_authors=row.scholar_label,
|
||||
candidates=openalex_works,
|
||||
)
|
||||
if match and match.oa_url:
|
||||
return OaResolutionOutcome(
|
||||
publication_id=row.publication_id,
|
||||
doi=match.doi,
|
||||
pdf_url=match.oa_url,
|
||||
failure_reason=None,
|
||||
source="openalex",
|
||||
used_crossref=False,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"publications.pdf_resolution.openalex_failed",
|
||||
extra={"event": "publications.pdf_resolution.openalex_failed", "error": str(exc)},
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
async def _arxiv_outcome(row: PublicationListItem, request_email: str | None) -> OaResolutionOutcome | None:
|
||||
from app.services.domains.arxiv.application import discover_arxiv_id_for_publication
|
||||
|
||||
try:
|
||||
arxiv_id = await discover_arxiv_id_for_publication(item=row, request_email=request_email)
|
||||
if arxiv_id:
|
||||
pdf_url = f"https://arxiv.org/pdf/{arxiv_id}.pdf"
|
||||
return OaResolutionOutcome(
|
||||
publication_id=row.publication_id,
|
||||
doi=None,
|
||||
pdf_url=pdf_url,
|
||||
failure_reason=None,
|
||||
source="arxiv",
|
||||
used_crossref=False,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"publications.pdf_resolution.arxiv_failed",
|
||||
extra={"event": "publications.pdf_resolution.arxiv_failed", "error": str(exc)},
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
async def _oa_outcome(
|
||||
*,
|
||||
row: PublicationListItem,
|
||||
request_email: str | None,
|
||||
) -> OaResolutionOutcome | None:
|
||||
outcomes = await resolve_publication_oa_outcomes([row], request_email=request_email)
|
||||
return outcomes.get(row.publication_id)
|
||||
|
|
@ -1,203 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from sqlalchemy import Select, func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.models import CrawlRun, Publication, RunStatus, ScholarProfile, ScholarPublication
|
||||
from app.services.domains.publications.modes import MODE_LATEST, MODE_UNREAD
|
||||
from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem
|
||||
|
||||
|
||||
def _normalized_citation_count(value: object) -> int:
|
||||
try:
|
||||
return int(value or 0)
|
||||
except (TypeError, ValueError):
|
||||
return 0
|
||||
|
||||
|
||||
async def get_latest_completed_run_id_for_user(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
) -> int | None:
|
||||
result = await db_session.execute(
|
||||
select(func.max(CrawlRun.id)).where(
|
||||
CrawlRun.user_id == user_id,
|
||||
CrawlRun.status != RunStatus.RUNNING,
|
||||
)
|
||||
)
|
||||
latest_run_id = result.scalar_one_or_none()
|
||||
return int(latest_run_id) if latest_run_id is not None else None
|
||||
|
||||
|
||||
def publications_query(
|
||||
*,
|
||||
user_id: int,
|
||||
mode: str,
|
||||
latest_run_id: int | None,
|
||||
scholar_profile_id: int | None,
|
||||
favorite_only: bool,
|
||||
limit: int,
|
||||
offset: int = 0,
|
||||
) -> Select[tuple]:
|
||||
scholar_label = ScholarProfile.display_name
|
||||
stmt = (
|
||||
select(
|
||||
Publication.id,
|
||||
ScholarProfile.id,
|
||||
scholar_label,
|
||||
ScholarProfile.scholar_id,
|
||||
Publication.title_raw,
|
||||
Publication.year,
|
||||
Publication.citation_count,
|
||||
Publication.venue_text,
|
||||
Publication.pub_url,
|
||||
Publication.pdf_url,
|
||||
ScholarPublication.is_read,
|
||||
ScholarPublication.is_favorite,
|
||||
ScholarPublication.first_seen_run_id,
|
||||
ScholarPublication.created_at,
|
||||
)
|
||||
.join(ScholarPublication, ScholarPublication.publication_id == Publication.id)
|
||||
.join(ScholarProfile, ScholarProfile.id == ScholarPublication.scholar_profile_id)
|
||||
.where(ScholarProfile.user_id == user_id)
|
||||
.order_by(ScholarPublication.created_at.desc(), Publication.id.desc())
|
||||
.offset(max(int(offset), 0))
|
||||
.limit(limit)
|
||||
)
|
||||
if scholar_profile_id is not None:
|
||||
stmt = stmt.where(ScholarProfile.id == scholar_profile_id)
|
||||
if favorite_only:
|
||||
stmt = stmt.where(ScholarPublication.is_favorite.is_(True))
|
||||
if mode == MODE_UNREAD:
|
||||
stmt = stmt.where(ScholarPublication.is_read.is_(False))
|
||||
if mode == MODE_LATEST:
|
||||
if latest_run_id is None:
|
||||
return stmt.where(False)
|
||||
stmt = stmt.where(ScholarPublication.first_seen_run_id == latest_run_id)
|
||||
return stmt
|
||||
|
||||
|
||||
def publication_query_for_user(
|
||||
*,
|
||||
user_id: int,
|
||||
scholar_profile_id: int,
|
||||
publication_id: int,
|
||||
) -> Select[tuple]:
|
||||
return (
|
||||
select(
|
||||
Publication.id,
|
||||
ScholarProfile.id,
|
||||
ScholarProfile.display_name,
|
||||
ScholarProfile.scholar_id,
|
||||
Publication.title_raw,
|
||||
Publication.year,
|
||||
Publication.citation_count,
|
||||
Publication.venue_text,
|
||||
Publication.pub_url,
|
||||
Publication.pdf_url,
|
||||
ScholarPublication.is_read,
|
||||
ScholarPublication.is_favorite,
|
||||
ScholarPublication.first_seen_run_id,
|
||||
ScholarPublication.created_at,
|
||||
)
|
||||
.join(ScholarPublication, ScholarPublication.publication_id == Publication.id)
|
||||
.join(ScholarProfile, ScholarProfile.id == ScholarPublication.scholar_profile_id)
|
||||
.where(
|
||||
ScholarProfile.user_id == user_id,
|
||||
ScholarProfile.id == scholar_profile_id,
|
||||
Publication.id == publication_id,
|
||||
)
|
||||
.limit(1)
|
||||
)
|
||||
|
||||
|
||||
async def get_publication_item_for_user(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
scholar_profile_id: int,
|
||||
publication_id: int,
|
||||
) -> PublicationListItem | None:
|
||||
latest_run_id = await get_latest_completed_run_id_for_user(db_session, user_id=user_id)
|
||||
result = await db_session.execute(
|
||||
publication_query_for_user(
|
||||
user_id=user_id,
|
||||
scholar_profile_id=scholar_profile_id,
|
||||
publication_id=publication_id,
|
||||
)
|
||||
)
|
||||
row = result.one_or_none()
|
||||
if row is None:
|
||||
return None
|
||||
return publication_list_item_from_row(row, latest_run_id=latest_run_id)
|
||||
|
||||
|
||||
def publication_list_item_from_row(
|
||||
row: tuple,
|
||||
*,
|
||||
latest_run_id: int | None,
|
||||
) -> PublicationListItem:
|
||||
(
|
||||
publication_id,
|
||||
scholar_profile_id,
|
||||
display_name,
|
||||
scholar_id,
|
||||
title_raw,
|
||||
year,
|
||||
citation_count,
|
||||
venue_text,
|
||||
pub_url,
|
||||
pdf_url,
|
||||
is_read,
|
||||
is_favorite,
|
||||
first_seen_run_id,
|
||||
created_at,
|
||||
) = row
|
||||
return PublicationListItem(
|
||||
publication_id=int(publication_id),
|
||||
scholar_profile_id=int(scholar_profile_id),
|
||||
scholar_label=(display_name or scholar_id),
|
||||
title=title_raw,
|
||||
year=year,
|
||||
citation_count=_normalized_citation_count(citation_count),
|
||||
venue_text=venue_text,
|
||||
pub_url=pub_url,
|
||||
pdf_url=pdf_url,
|
||||
is_read=bool(is_read),
|
||||
is_favorite=bool(is_favorite),
|
||||
first_seen_at=created_at,
|
||||
is_new_in_latest_run=(
|
||||
latest_run_id is not None and int(first_seen_run_id or 0) == latest_run_id
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def unread_item_from_row(row: tuple) -> UnreadPublicationItem:
|
||||
(
|
||||
publication_id,
|
||||
scholar_profile_id,
|
||||
display_name,
|
||||
scholar_id,
|
||||
title_raw,
|
||||
year,
|
||||
citation_count,
|
||||
venue_text,
|
||||
pub_url,
|
||||
pdf_url,
|
||||
_is_read,
|
||||
_is_favorite,
|
||||
_first_seen_run_id,
|
||||
_created_at,
|
||||
) = row
|
||||
return UnreadPublicationItem(
|
||||
publication_id=int(publication_id),
|
||||
scholar_profile_id=int(scholar_profile_id),
|
||||
scholar_label=(display_name or scholar_id),
|
||||
title=title_raw,
|
||||
year=year,
|
||||
citation_count=_normalized_citation_count(citation_count),
|
||||
venue_text=venue_text,
|
||||
pub_url=pub_url,
|
||||
pdf_url=pdf_url,
|
||||
)
|
||||
|
|
@ -1,94 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from sqlalchemy import select, tuple_, update
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.models import ScholarProfile, ScholarPublication
|
||||
|
||||
|
||||
def _normalized_selection_pairs(selections: list[tuple[int, int]]) -> set[tuple[int, int]]:
|
||||
pairs: set[tuple[int, int]] = set()
|
||||
for scholar_profile_id, publication_id in selections:
|
||||
normalized = (int(scholar_profile_id), int(publication_id))
|
||||
if normalized[0] <= 0 or normalized[1] <= 0:
|
||||
continue
|
||||
pairs.add(normalized)
|
||||
return pairs
|
||||
|
||||
|
||||
def _scoped_scholar_ids_query(*, user_id: int):
|
||||
return (
|
||||
select(ScholarProfile.id)
|
||||
.where(ScholarProfile.user_id == user_id)
|
||||
.scalar_subquery()
|
||||
)
|
||||
|
||||
|
||||
async def mark_all_unread_as_read_for_user(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
) -> int:
|
||||
scholar_ids = _scoped_scholar_ids_query(user_id=user_id)
|
||||
stmt = (
|
||||
update(ScholarPublication)
|
||||
.where(
|
||||
ScholarPublication.scholar_profile_id.in_(scholar_ids),
|
||||
ScholarPublication.is_read.is_(False),
|
||||
)
|
||||
.values(is_read=True)
|
||||
)
|
||||
result = await db_session.execute(stmt)
|
||||
await db_session.commit()
|
||||
return int(result.rowcount or 0)
|
||||
|
||||
|
||||
async def mark_selected_as_read_for_user(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
selections: list[tuple[int, int]],
|
||||
) -> int:
|
||||
normalized_pairs = _normalized_selection_pairs(selections)
|
||||
if not normalized_pairs:
|
||||
return 0
|
||||
|
||||
scholar_ids = _scoped_scholar_ids_query(user_id=user_id)
|
||||
stmt = (
|
||||
update(ScholarPublication)
|
||||
.where(
|
||||
ScholarPublication.scholar_profile_id.in_(scholar_ids),
|
||||
tuple_(
|
||||
ScholarPublication.scholar_profile_id,
|
||||
ScholarPublication.publication_id,
|
||||
).in_(list(normalized_pairs)),
|
||||
ScholarPublication.is_read.is_(False),
|
||||
)
|
||||
.values(is_read=True)
|
||||
)
|
||||
result = await db_session.execute(stmt)
|
||||
await db_session.commit()
|
||||
return int(result.rowcount or 0)
|
||||
|
||||
|
||||
async def set_publication_favorite_for_user(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
scholar_profile_id: int,
|
||||
publication_id: int,
|
||||
is_favorite: bool,
|
||||
) -> int:
|
||||
scholar_ids = _scoped_scholar_ids_query(user_id=user_id)
|
||||
stmt = (
|
||||
update(ScholarPublication)
|
||||
.where(
|
||||
ScholarPublication.scholar_profile_id.in_(scholar_ids),
|
||||
ScholarPublication.scholar_profile_id == int(scholar_profile_id),
|
||||
ScholarPublication.publication_id == int(publication_id),
|
||||
)
|
||||
.values(is_favorite=bool(is_favorite))
|
||||
)
|
||||
result = await db_session.execute(stmt)
|
||||
await db_session.commit()
|
||||
return int(result.rowcount or 0)
|
||||
|
|
@ -1,41 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
|
||||
from app.services.domains.publication_identifiers.types import DisplayIdentifier
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PublicationListItem:
|
||||
publication_id: int
|
||||
scholar_profile_id: int
|
||||
scholar_label: str
|
||||
title: str
|
||||
year: int | None
|
||||
citation_count: int
|
||||
venue_text: str | None
|
||||
pub_url: str | None
|
||||
pdf_url: str | None
|
||||
is_read: bool
|
||||
first_seen_at: datetime
|
||||
is_new_in_latest_run: bool
|
||||
is_favorite: bool = False
|
||||
pdf_status: str = "untracked"
|
||||
pdf_attempt_count: int = 0
|
||||
pdf_failure_reason: str | None = None
|
||||
pdf_failure_detail: str | None = None
|
||||
display_identifier: DisplayIdentifier | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class UnreadPublicationItem:
|
||||
publication_id: int
|
||||
scholar_profile_id: int
|
||||
scholar_label: str
|
||||
title: str
|
||||
year: int | None
|
||||
citation_count: int
|
||||
venue_text: str | None
|
||||
pub_url: str | None
|
||||
pdf_url: str | None
|
||||
|
|
@ -1 +0,0 @@
|
|||
from app.services.domains.runs.application import *
|
||||
|
|
@ -1,45 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from app.services.domains.runs.queue_service import (
|
||||
clear_queue_item_for_user,
|
||||
drop_queue_item_for_user,
|
||||
get_queue_item_for_user,
|
||||
list_queue_items_for_user,
|
||||
queue_status_counts_for_user,
|
||||
retry_queue_item_for_user,
|
||||
)
|
||||
from app.services.domains.runs.runs_service import (
|
||||
get_manual_run_by_idempotency_key,
|
||||
get_run_for_user,
|
||||
list_recent_runs_for_user,
|
||||
list_runs_for_user,
|
||||
)
|
||||
from app.services.domains.runs.summary import extract_run_summary
|
||||
from app.services.domains.runs.types import (
|
||||
QUEUE_STATUS_DROPPED,
|
||||
QUEUE_STATUS_QUEUED,
|
||||
QUEUE_STATUS_RETRYING,
|
||||
QueueClearResult,
|
||||
QueueListItem,
|
||||
QueueTransitionError,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"QUEUE_STATUS_QUEUED",
|
||||
"QUEUE_STATUS_RETRYING",
|
||||
"QUEUE_STATUS_DROPPED",
|
||||
"QueueListItem",
|
||||
"QueueClearResult",
|
||||
"QueueTransitionError",
|
||||
"extract_run_summary",
|
||||
"list_recent_runs_for_user",
|
||||
"list_runs_for_user",
|
||||
"get_run_for_user",
|
||||
"get_manual_run_by_idempotency_key",
|
||||
"list_queue_items_for_user",
|
||||
"get_queue_item_for_user",
|
||||
"retry_queue_item_for_user",
|
||||
"drop_queue_item_for_user",
|
||||
"clear_queue_item_for_user",
|
||||
"queue_status_counts_for_user",
|
||||
]
|
||||
|
|
@ -1,59 +0,0 @@
|
|||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
from typing import Any, AsyncGenerator, Dict, Set
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
class RunEventPublisher:
|
||||
def __init__(self) -> None:
|
||||
# Maps run_id to a set of subscriber queues
|
||||
self._subscribers: Dict[int, Set[asyncio.Queue]] = {}
|
||||
|
||||
def subscribe(self, run_id: int) -> asyncio.Queue:
|
||||
if run_id not in self._subscribers:
|
||||
self._subscribers[run_id] = set()
|
||||
queue: asyncio.Queue[Any] = asyncio.Queue()
|
||||
self._subscribers[run_id].add(queue)
|
||||
logger.debug(f"New subscriber for run {run_id}. Total: {len(self._subscribers[run_id])}")
|
||||
return queue
|
||||
|
||||
def unsubscribe(self, run_id: int, queue: asyncio.Queue) -> None:
|
||||
if run_id in self._subscribers:
|
||||
self._subscribers[run_id].discard(queue)
|
||||
if not self._subscribers[run_id]:
|
||||
self._subscribers.pop(run_id, None)
|
||||
|
||||
async def publish(self, run_id: int, event_type: str, data: dict[str, Any]) -> None:
|
||||
if run_id not in self._subscribers:
|
||||
return
|
||||
|
||||
message = {
|
||||
"type": event_type,
|
||||
"data": data
|
||||
}
|
||||
|
||||
# Fan-out to all active subscribers for this run
|
||||
for queue in list(self._subscribers[run_id]):
|
||||
try:
|
||||
queue.put_nowait(message)
|
||||
except asyncio.QueueFull:
|
||||
logger.warning(f"Subscriber queue full for run {run_id}, dropping message")
|
||||
|
||||
run_events = RunEventPublisher()
|
||||
|
||||
async def event_generator(run_id: int) -> AsyncGenerator[str, None]:
|
||||
queue = run_events.subscribe(run_id)
|
||||
try:
|
||||
while True:
|
||||
# Wait for a new event
|
||||
message = await queue.get()
|
||||
# Server-Sent Events format: "event: <type>\ndata: <json>\n\n"
|
||||
event_type = message["type"]
|
||||
data_str = json.dumps(message["data"])
|
||||
yield f"event: {event_type}\ndata: {data_str}\n\n"
|
||||
except asyncio.CancelledError:
|
||||
logger.debug(f"Client disconnected from SSE stream for run {run_id}")
|
||||
raise
|
||||
finally:
|
||||
run_events.unsubscribe(run_id, queue)
|
||||
|
|
@ -1,70 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from sqlalchemy import and_, select
|
||||
|
||||
from app.db.models import IngestionQueueItem, ScholarProfile
|
||||
from app.services.domains.runs.types import QueueListItem
|
||||
|
||||
|
||||
def queue_item_columns() -> tuple:
|
||||
return (
|
||||
IngestionQueueItem.id,
|
||||
IngestionQueueItem.scholar_profile_id,
|
||||
ScholarProfile.display_name,
|
||||
ScholarProfile.scholar_id,
|
||||
IngestionQueueItem.status,
|
||||
IngestionQueueItem.reason,
|
||||
IngestionQueueItem.dropped_reason,
|
||||
IngestionQueueItem.attempt_count,
|
||||
IngestionQueueItem.resume_cstart,
|
||||
IngestionQueueItem.next_attempt_dt,
|
||||
IngestionQueueItem.updated_at,
|
||||
IngestionQueueItem.last_error,
|
||||
IngestionQueueItem.last_run_id,
|
||||
)
|
||||
|
||||
|
||||
def queue_item_select(*, user_id: int):
|
||||
return (
|
||||
select(*queue_item_columns())
|
||||
.join(
|
||||
ScholarProfile,
|
||||
and_(
|
||||
ScholarProfile.id == IngestionQueueItem.scholar_profile_id,
|
||||
ScholarProfile.user_id == IngestionQueueItem.user_id,
|
||||
),
|
||||
)
|
||||
.where(IngestionQueueItem.user_id == user_id)
|
||||
)
|
||||
|
||||
|
||||
def queue_list_item_from_row(row: tuple) -> QueueListItem:
|
||||
(
|
||||
item_id,
|
||||
scholar_profile_id,
|
||||
display_name,
|
||||
scholar_id,
|
||||
status,
|
||||
reason,
|
||||
dropped_reason,
|
||||
attempt_count,
|
||||
resume_cstart,
|
||||
next_attempt_dt,
|
||||
updated_at,
|
||||
last_error,
|
||||
last_run_id,
|
||||
) = row
|
||||
return QueueListItem(
|
||||
id=int(item_id),
|
||||
scholar_profile_id=int(scholar_profile_id),
|
||||
scholar_label=(display_name or scholar_id),
|
||||
status=str(status),
|
||||
reason=str(reason),
|
||||
dropped_reason=dropped_reason,
|
||||
attempt_count=int(attempt_count or 0),
|
||||
resume_cstart=int(resume_cstart or 0),
|
||||
next_attempt_dt=next_attempt_dt,
|
||||
updated_at=updated_at,
|
||||
last_error=last_error,
|
||||
last_run_id=int(last_run_id) if last_run_id is not None else None,
|
||||
)
|
||||
|
|
@ -1,182 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from sqlalchemy import case, func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.models import IngestionQueueItem
|
||||
from app.services.domains.ingestion import queue as queue_mutations
|
||||
from app.services.domains.runs.queue_queries import queue_item_select, queue_list_item_from_row
|
||||
from app.services.domains.runs.types import (
|
||||
QUEUE_STATUS_DROPPED,
|
||||
QUEUE_STATUS_QUEUED,
|
||||
QUEUE_STATUS_RETRYING,
|
||||
QueueClearResult,
|
||||
QueueListItem,
|
||||
QueueTransitionError,
|
||||
)
|
||||
|
||||
|
||||
async def list_queue_items_for_user(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
limit: int = 200,
|
||||
) -> list[QueueListItem]:
|
||||
result = await db_session.execute(
|
||||
queue_item_select(user_id=user_id)
|
||||
.order_by(
|
||||
case((IngestionQueueItem.status == QUEUE_STATUS_DROPPED, 1), else_=0).asc(),
|
||||
IngestionQueueItem.next_attempt_dt.asc(),
|
||||
IngestionQueueItem.id.asc(),
|
||||
)
|
||||
.limit(limit)
|
||||
)
|
||||
return [queue_list_item_from_row(row) for row in result.all()]
|
||||
|
||||
|
||||
async def get_queue_item_for_user(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
queue_item_id: int,
|
||||
) -> QueueListItem | None:
|
||||
result = await db_session.execute(
|
||||
queue_item_select(user_id=user_id)
|
||||
.where(IngestionQueueItem.id == queue_item_id)
|
||||
.limit(1)
|
||||
)
|
||||
row = result.one_or_none()
|
||||
if row is None:
|
||||
return None
|
||||
return queue_list_item_from_row(row)
|
||||
|
||||
|
||||
async def retry_queue_item_for_user(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
queue_item_id: int,
|
||||
) -> QueueListItem | None:
|
||||
result = await db_session.execute(
|
||||
select(IngestionQueueItem).where(
|
||||
IngestionQueueItem.id == queue_item_id,
|
||||
IngestionQueueItem.user_id == user_id,
|
||||
)
|
||||
)
|
||||
item = result.scalar_one_or_none()
|
||||
if item is None:
|
||||
return None
|
||||
if item.status == QUEUE_STATUS_QUEUED:
|
||||
raise QueueTransitionError(
|
||||
code="queue_item_already_queued",
|
||||
message="Queue item is already queued.",
|
||||
current_status=item.status,
|
||||
)
|
||||
if item.status == QUEUE_STATUS_RETRYING:
|
||||
raise QueueTransitionError(
|
||||
code="queue_item_retrying",
|
||||
message="Queue item is currently retrying.",
|
||||
current_status=item.status,
|
||||
)
|
||||
|
||||
await queue_mutations.mark_queued_now(
|
||||
db_session,
|
||||
job_id=item.id,
|
||||
reason="manual_retry",
|
||||
reset_attempt_count=(item.status == QUEUE_STATUS_DROPPED),
|
||||
)
|
||||
await db_session.commit()
|
||||
return await get_queue_item_for_user(
|
||||
db_session,
|
||||
user_id=user_id,
|
||||
queue_item_id=queue_item_id,
|
||||
)
|
||||
|
||||
|
||||
async def drop_queue_item_for_user(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
queue_item_id: int,
|
||||
) -> QueueListItem | None:
|
||||
result = await db_session.execute(
|
||||
select(IngestionQueueItem).where(
|
||||
IngestionQueueItem.id == queue_item_id,
|
||||
IngestionQueueItem.user_id == user_id,
|
||||
)
|
||||
)
|
||||
item = result.scalar_one_or_none()
|
||||
if item is None:
|
||||
return None
|
||||
if item.status == QUEUE_STATUS_DROPPED:
|
||||
raise QueueTransitionError(
|
||||
code="queue_item_already_dropped",
|
||||
message="Queue item is already dropped.",
|
||||
current_status=item.status,
|
||||
)
|
||||
|
||||
await queue_mutations.mark_dropped(
|
||||
db_session,
|
||||
job_id=int(item.id),
|
||||
reason="manual_drop",
|
||||
)
|
||||
await db_session.commit()
|
||||
return await get_queue_item_for_user(
|
||||
db_session,
|
||||
user_id=user_id,
|
||||
queue_item_id=queue_item_id,
|
||||
)
|
||||
|
||||
|
||||
async def clear_queue_item_for_user(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
queue_item_id: int,
|
||||
) -> QueueClearResult | None:
|
||||
result = await db_session.execute(
|
||||
select(IngestionQueueItem).where(
|
||||
IngestionQueueItem.id == queue_item_id,
|
||||
IngestionQueueItem.user_id == user_id,
|
||||
)
|
||||
)
|
||||
item = result.scalar_one_or_none()
|
||||
if item is None:
|
||||
return None
|
||||
if item.status != QUEUE_STATUS_DROPPED:
|
||||
raise QueueTransitionError(
|
||||
code="queue_item_not_dropped",
|
||||
message="Queue item can only be cleared after it is dropped.",
|
||||
current_status=item.status,
|
||||
)
|
||||
|
||||
item_id = int(item.id)
|
||||
previous_status = str(item.status)
|
||||
deleted = await queue_mutations.delete_job_by_id(db_session, job_id=item_id)
|
||||
await db_session.commit()
|
||||
if not deleted:
|
||||
return None
|
||||
return QueueClearResult(queue_item_id=item_id, previous_status=previous_status)
|
||||
|
||||
|
||||
async def queue_status_counts_for_user(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
) -> dict[str, int]:
|
||||
result = await db_session.execute(
|
||||
select(
|
||||
IngestionQueueItem.status,
|
||||
func.count(IngestionQueueItem.id),
|
||||
)
|
||||
.where(IngestionQueueItem.user_id == user_id)
|
||||
.group_by(IngestionQueueItem.status)
|
||||
)
|
||||
counts: dict[str, int] = {
|
||||
QUEUE_STATUS_QUEUED: 0,
|
||||
QUEUE_STATUS_RETRYING: 0,
|
||||
QUEUE_STATUS_DROPPED: 0,
|
||||
}
|
||||
for status, count in result.all():
|
||||
counts[str(status)] = int(count or 0)
|
||||
return counts
|
||||
|
|
@ -1,79 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from sqlalchemy import or_, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.models import CrawlRun, RunStatus, RunTriggerType
|
||||
|
||||
|
||||
async def list_recent_runs_for_user(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
limit: int = 20,
|
||||
) -> list[CrawlRun]:
|
||||
result = await db_session.execute(
|
||||
select(CrawlRun)
|
||||
.where(CrawlRun.user_id == user_id)
|
||||
.order_by(CrawlRun.start_dt.desc(), CrawlRun.id.desc())
|
||||
.limit(limit)
|
||||
)
|
||||
return list(result.scalars().all())
|
||||
|
||||
|
||||
async def list_runs_for_user(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
limit: int = 100,
|
||||
failed_only: bool = False,
|
||||
) -> list[CrawlRun]:
|
||||
stmt = (
|
||||
select(CrawlRun)
|
||||
.where(CrawlRun.user_id == user_id)
|
||||
.order_by(CrawlRun.start_dt.desc(), CrawlRun.id.desc())
|
||||
.limit(limit)
|
||||
)
|
||||
if failed_only:
|
||||
stmt = stmt.where(
|
||||
CrawlRun.status.in_([RunStatus.FAILED, RunStatus.PARTIAL_FAILURE])
|
||||
)
|
||||
result = await db_session.execute(stmt)
|
||||
return list(result.scalars().all())
|
||||
|
||||
|
||||
async def get_run_for_user(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
run_id: int,
|
||||
) -> CrawlRun | None:
|
||||
result = await db_session.execute(
|
||||
select(CrawlRun).where(
|
||||
CrawlRun.user_id == user_id,
|
||||
CrawlRun.id == run_id,
|
||||
)
|
||||
)
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
|
||||
async def get_manual_run_by_idempotency_key(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
idempotency_key: str,
|
||||
) -> CrawlRun | None:
|
||||
result = await db_session.execute(
|
||||
select(CrawlRun)
|
||||
.where(
|
||||
CrawlRun.user_id == user_id,
|
||||
CrawlRun.trigger_type == RunTriggerType.MANUAL,
|
||||
or_(
|
||||
CrawlRun.idempotency_key == idempotency_key,
|
||||
CrawlRun.error_log["meta"]["idempotency_key"].astext == idempotency_key,
|
||||
),
|
||||
)
|
||||
.order_by(CrawlRun.start_dt.desc(), CrawlRun.id.desc())
|
||||
.limit(1)
|
||||
)
|
||||
return result.scalar_one_or_none()
|
||||
|
|
@ -1,76 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
|
||||
def _safe_int(value: object, default: int = 0) -> int:
|
||||
try:
|
||||
return int(value)
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
|
||||
|
||||
def _summary_dict(error_log: object) -> dict[str, Any]:
|
||||
if not isinstance(error_log, dict):
|
||||
return {}
|
||||
summary = error_log.get("summary")
|
||||
if not isinstance(summary, dict):
|
||||
return {}
|
||||
return summary
|
||||
|
||||
|
||||
def _summary_int_dict(summary: dict[str, Any], key: str) -> dict[str, int]:
|
||||
value = summary.get(key)
|
||||
if not isinstance(value, dict):
|
||||
return {}
|
||||
return {
|
||||
str(item_key): _safe_int(item_value, 0)
|
||||
for item_key, item_value in value.items()
|
||||
if isinstance(item_key, str)
|
||||
}
|
||||
|
||||
|
||||
def _summary_bool_dict(summary: dict[str, Any], key: str) -> dict[str, bool]:
|
||||
value = summary.get(key)
|
||||
if not isinstance(value, dict):
|
||||
return {}
|
||||
return {
|
||||
str(item_key): bool(item_value)
|
||||
for item_key, item_value in value.items()
|
||||
if isinstance(item_key, str)
|
||||
}
|
||||
|
||||
|
||||
def _retry_counts(summary: dict[str, Any]) -> dict[str, int]:
|
||||
retry_counts = summary.get("retry_counts")
|
||||
if not isinstance(retry_counts, dict):
|
||||
retry_counts = {}
|
||||
return {
|
||||
"retries_scheduled_count": _safe_int(
|
||||
retry_counts.get("retries_scheduled_count", 0),
|
||||
0,
|
||||
),
|
||||
"scholars_with_retries_count": _safe_int(
|
||||
retry_counts.get("scholars_with_retries_count", 0),
|
||||
0,
|
||||
),
|
||||
"retry_exhausted_count": _safe_int(
|
||||
retry_counts.get("retry_exhausted_count", 0),
|
||||
0,
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def extract_run_summary(error_log: object) -> dict[str, Any]:
|
||||
summary = _summary_dict(error_log)
|
||||
return {
|
||||
"succeeded_count": _safe_int(summary.get("succeeded_count", 0)),
|
||||
"failed_count": _safe_int(summary.get("failed_count", 0)),
|
||||
"partial_count": _safe_int(summary.get("partial_count", 0)),
|
||||
"failed_state_counts": _summary_int_dict(summary, "failed_state_counts"),
|
||||
"failed_reason_counts": _summary_int_dict(summary, "failed_reason_counts"),
|
||||
"scrape_failure_counts": _summary_int_dict(summary, "scrape_failure_counts"),
|
||||
"retry_counts": _retry_counts(summary),
|
||||
"alert_thresholds": _summary_int_dict(summary, "alert_thresholds"),
|
||||
"alert_flags": _summary_bool_dict(summary, "alert_flags"),
|
||||
}
|
||||
|
|
@ -1,38 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
|
||||
QUEUE_STATUS_QUEUED = "queued"
|
||||
QUEUE_STATUS_RETRYING = "retrying"
|
||||
QUEUE_STATUS_DROPPED = "dropped"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class QueueListItem:
|
||||
id: int
|
||||
scholar_profile_id: int
|
||||
scholar_label: str
|
||||
status: str
|
||||
reason: str
|
||||
dropped_reason: str | None
|
||||
attempt_count: int
|
||||
resume_cstart: int
|
||||
next_attempt_dt: datetime | None
|
||||
updated_at: datetime
|
||||
last_error: str | None
|
||||
last_run_id: int | None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class QueueClearResult:
|
||||
queue_item_id: int
|
||||
previous_status: str
|
||||
|
||||
|
||||
class QueueTransitionError(RuntimeError):
|
||||
def __init__(self, *, code: str, message: str, current_status: str) -> None:
|
||||
super().__init__(message)
|
||||
self.code = code
|
||||
self.message = message
|
||||
self.current_status = current_status
|
||||
|
|
@ -1 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
|
@ -1,202 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from html.parser import HTMLParser
|
||||
from typing import Any
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
|
||||
from app.services.domains.scholar.parser_constants import AUTHOR_SEARCH_MARKER_KEYS
|
||||
from app.services.domains.scholar.parser_types import ScholarSearchCandidate
|
||||
from app.services.domains.scholar.parser_utils import (
|
||||
attr_class,
|
||||
attr_href,
|
||||
attr_src,
|
||||
build_absolute_scholar_url,
|
||||
normalize_space,
|
||||
)
|
||||
|
||||
|
||||
def parse_scholar_id_from_href(href: str | None) -> str | None:
|
||||
if not href:
|
||||
return None
|
||||
parsed = urlparse(href)
|
||||
query = parse_qs(parsed.query)
|
||||
user_values = query.get("user")
|
||||
if not user_values:
|
||||
return None
|
||||
candidate = user_values[0].strip()
|
||||
return candidate or None
|
||||
|
||||
|
||||
def _extract_verified_email_domain(value: str | None) -> str | None:
|
||||
if not value:
|
||||
return None
|
||||
match = re.search(r"verified email at\s+(.+)$", value.strip(), re.I)
|
||||
if not match:
|
||||
return None
|
||||
domain = normalize_space(match.group(1))
|
||||
return domain or None
|
||||
|
||||
|
||||
class ScholarAuthorSearchParser(HTMLParser):
|
||||
def __init__(self) -> None:
|
||||
super().__init__(convert_charrefs=True)
|
||||
self.candidates: list[ScholarSearchCandidate] = []
|
||||
self._candidate: dict[str, Any] | None = None
|
||||
|
||||
def _begin_candidate(self) -> None:
|
||||
self._candidate = {
|
||||
"depth": 1,
|
||||
"name_href": None,
|
||||
"name_parts": [],
|
||||
"aff_depth": 0,
|
||||
"aff_parts": [],
|
||||
"name_depth": 0,
|
||||
"eml_depth": 0,
|
||||
"eml_parts": [],
|
||||
"cby_depth": 0,
|
||||
"cby_parts": [],
|
||||
"interest_depth": 0,
|
||||
"interest_parts": [],
|
||||
"interests": [],
|
||||
"image_src": None,
|
||||
}
|
||||
|
||||
def _increment_capture_depths(self) -> None:
|
||||
if self._candidate is None:
|
||||
return
|
||||
for key in ("name_depth", "aff_depth", "eml_depth", "cby_depth", "interest_depth"):
|
||||
if self._candidate[key] > 0:
|
||||
self._candidate[key] += 1
|
||||
|
||||
def _finalize_candidate(self) -> None:
|
||||
if self._candidate is None:
|
||||
return
|
||||
|
||||
name = normalize_space("".join(self._candidate["name_parts"]))
|
||||
scholar_id = parse_scholar_id_from_href(self._candidate["name_href"])
|
||||
if not name or not scholar_id:
|
||||
return
|
||||
|
||||
affiliation = normalize_space("".join(self._candidate["aff_parts"])) or None
|
||||
email_domain = _extract_verified_email_domain(
|
||||
normalize_space("".join(self._candidate["eml_parts"])) or None
|
||||
)
|
||||
cited_by_text = normalize_space("".join(self._candidate["cby_parts"]))
|
||||
cited_by_match = re.search(r"\d+", cited_by_text)
|
||||
cited_by_count = int(cited_by_match.group(0)) if cited_by_match else None
|
||||
|
||||
seen_interests: set[str] = set()
|
||||
interests: list[str] = []
|
||||
for interest in self._candidate["interests"]:
|
||||
normalized = normalize_space(interest)
|
||||
if not normalized or normalized in seen_interests:
|
||||
continue
|
||||
seen_interests.add(normalized)
|
||||
interests.append(normalized)
|
||||
|
||||
profile_url = build_absolute_scholar_url(self._candidate["name_href"])
|
||||
if not profile_url:
|
||||
profile_url = (
|
||||
"https://scholar.google.com/citations"
|
||||
f"?hl=en&user={scholar_id}"
|
||||
)
|
||||
|
||||
self.candidates.append(
|
||||
ScholarSearchCandidate(
|
||||
scholar_id=scholar_id,
|
||||
display_name=name,
|
||||
affiliation=affiliation,
|
||||
email_domain=email_domain,
|
||||
cited_by_count=cited_by_count,
|
||||
interests=interests,
|
||||
profile_url=profile_url,
|
||||
profile_image_url=build_absolute_scholar_url(self._candidate["image_src"]),
|
||||
)
|
||||
)
|
||||
|
||||
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
||||
classes = attr_class(attrs)
|
||||
|
||||
if self._candidate is None:
|
||||
if tag == "div" and "gsc_1usr" in classes:
|
||||
self._begin_candidate()
|
||||
return
|
||||
|
||||
self._candidate["depth"] += 1
|
||||
self._increment_capture_depths()
|
||||
|
||||
if tag == "a" and "gs_ai_name" in classes:
|
||||
self._candidate["name_depth"] = 1
|
||||
self._candidate["name_href"] = attr_href(attrs)
|
||||
return
|
||||
|
||||
if tag == "div" and "gs_ai_aff" in classes:
|
||||
self._candidate["aff_depth"] = 1
|
||||
return
|
||||
|
||||
if tag == "div" and "gs_ai_eml" in classes:
|
||||
self._candidate["eml_depth"] = 1
|
||||
return
|
||||
|
||||
if tag == "div" and "gs_ai_cby" in classes:
|
||||
self._candidate["cby_depth"] = 1
|
||||
return
|
||||
|
||||
if tag == "a" and "gs_ai_one_int" in classes:
|
||||
self._candidate["interest_depth"] = 1
|
||||
self._candidate["interest_parts"] = []
|
||||
return
|
||||
|
||||
if tag == "img" and self._candidate["image_src"] is None:
|
||||
self._candidate["image_src"] = attr_src(attrs)
|
||||
|
||||
def handle_data(self, data: str) -> None:
|
||||
if self._candidate is None:
|
||||
return
|
||||
if self._candidate["name_depth"] > 0:
|
||||
self._candidate["name_parts"].append(data)
|
||||
if self._candidate["aff_depth"] > 0:
|
||||
self._candidate["aff_parts"].append(data)
|
||||
if self._candidate["eml_depth"] > 0:
|
||||
self._candidate["eml_parts"].append(data)
|
||||
if self._candidate["cby_depth"] > 0:
|
||||
self._candidate["cby_parts"].append(data)
|
||||
if self._candidate["interest_depth"] > 0:
|
||||
self._candidate["interest_parts"].append(data)
|
||||
|
||||
def _decrement_capture_depth(self, key: str) -> bool:
|
||||
if self._candidate is None:
|
||||
return False
|
||||
if self._candidate[key] <= 0:
|
||||
return False
|
||||
self._candidate[key] -= 1
|
||||
return self._candidate[key] == 0
|
||||
|
||||
def handle_endtag(self, _tag: str) -> None:
|
||||
if self._candidate is None:
|
||||
return
|
||||
|
||||
interest_closed = self._decrement_capture_depth("interest_depth")
|
||||
self._decrement_capture_depth("name_depth")
|
||||
self._decrement_capture_depth("aff_depth")
|
||||
self._decrement_capture_depth("eml_depth")
|
||||
self._decrement_capture_depth("cby_depth")
|
||||
|
||||
if interest_closed:
|
||||
interest_text = normalize_space("".join(self._candidate["interest_parts"]))
|
||||
if interest_text:
|
||||
self._candidate["interests"].append(interest_text)
|
||||
self._candidate["interest_parts"] = []
|
||||
|
||||
self._candidate["depth"] -= 1
|
||||
if self._candidate["depth"] > 0:
|
||||
return
|
||||
|
||||
self._finalize_candidate()
|
||||
self._candidate = None
|
||||
|
||||
|
||||
def count_author_search_markers(html: str) -> dict[str, int]:
|
||||
lowered = html.lower()
|
||||
return {key: lowered.count(key.lower()) for key in AUTHOR_SEARCH_MARKER_KEYS}
|
||||
|
|
@ -1,165 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from app.services.domains.scholar.author_rows import (
|
||||
ScholarAuthorSearchParser,
|
||||
count_author_search_markers,
|
||||
parse_scholar_id_from_href,
|
||||
)
|
||||
from app.services.domains.scholar.parser_constants import SCRIPT_STYLE_RE
|
||||
from app.services.domains.scholar.parser_types import (
|
||||
ParseState,
|
||||
ParsedAuthorSearchPage,
|
||||
ParsedProfilePage,
|
||||
PublicationCandidate,
|
||||
ScholarDomInvariantError,
|
||||
ScholarMalformedDataError,
|
||||
ScholarParserError,
|
||||
ScholarSearchCandidate,
|
||||
)
|
||||
from app.services.domains.scholar.parser_utils import (
|
||||
strip_tags,
|
||||
)
|
||||
from app.services.domains.scholar.profile_rows import (
|
||||
ScholarRowParser,
|
||||
count_markers,
|
||||
extract_articles_range,
|
||||
extract_profile_image_url,
|
||||
extract_profile_name,
|
||||
extract_rows,
|
||||
has_operation_error_banner,
|
||||
has_show_more_button,
|
||||
parse_citation_count,
|
||||
parse_cluster_id_from_href,
|
||||
parse_publications,
|
||||
parse_year,
|
||||
)
|
||||
from app.services.domains.scholar.source import FetchResult
|
||||
from app.services.domains.scholar.state_detection import (
|
||||
classify_block_or_captcha_reason,
|
||||
classify_network_error_reason,
|
||||
detect_author_search_state,
|
||||
detect_state,
|
||||
)
|
||||
|
||||
|
||||
def _raise_dom_error(code: str, message: str) -> None:
|
||||
raise ScholarDomInvariantError(code=code, message=message)
|
||||
|
||||
|
||||
def _assert_profile_dom_invariants(
|
||||
*,
|
||||
fetch_result: FetchResult,
|
||||
marker_counts: dict[str, int],
|
||||
publications: list[PublicationCandidate],
|
||||
warnings: list[str],
|
||||
has_show_more_button_flag: bool,
|
||||
articles_range: str | None,
|
||||
) -> None:
|
||||
if fetch_result.status_code is None:
|
||||
return
|
||||
final_url = (fetch_result.final_url or "").lower()
|
||||
if "accounts.google.com" in final_url or "sorry/index" in final_url:
|
||||
return
|
||||
if any(code.startswith("layout_") for code in warnings):
|
||||
reason = next(code for code in warnings if code.startswith("layout_"))
|
||||
_raise_dom_error(reason, f"Detected layout warning: {reason}")
|
||||
if has_show_more_button_flag and not articles_range:
|
||||
_raise_dom_error(
|
||||
"layout_show_more_without_articles_range",
|
||||
"Show-more control exists without an articles range marker.",
|
||||
)
|
||||
if marker_counts.get("gsc_a_tr", 0) > 0 and marker_counts.get("gsc_a_at", 0) <= 0:
|
||||
_raise_dom_error(
|
||||
"layout_missing_publication_title_anchor",
|
||||
"Publication rows were present but title anchors were absent.",
|
||||
)
|
||||
if not publications:
|
||||
has_profile_markers = marker_counts.get("gsc_prf_in", 0) > 0
|
||||
has_table_markers = marker_counts.get("gsc_a_tr", 0) > 0 or marker_counts.get("gsc_a_at", 0) > 0
|
||||
if not has_profile_markers and not has_table_markers:
|
||||
_raise_dom_error("layout_markers_missing", "Expected scholar profile markers were absent.")
|
||||
for publication in publications:
|
||||
if not publication.title.strip():
|
||||
raise ScholarMalformedDataError(
|
||||
code="malformed_publication_title",
|
||||
message="Encountered a publication candidate with an empty title.",
|
||||
)
|
||||
if publication.citation_count is not None and int(publication.citation_count) < 0:
|
||||
raise ScholarMalformedDataError(
|
||||
code="malformed_publication_negative_citations",
|
||||
message="Encountered a publication candidate with negative citations.",
|
||||
)
|
||||
|
||||
|
||||
def parse_profile_page(fetch_result: FetchResult) -> ParsedProfilePage:
|
||||
publications, warnings = parse_publications(fetch_result.body)
|
||||
marker_counts = count_markers(fetch_result.body)
|
||||
visible_text = strip_tags(SCRIPT_STYLE_RE.sub(" ", fetch_result.body)).lower()
|
||||
|
||||
show_more = has_show_more_button(fetch_result.body)
|
||||
operation_error_banner = has_operation_error_banner(fetch_result.body)
|
||||
articles_range = extract_articles_range(fetch_result.body)
|
||||
|
||||
if show_more:
|
||||
warnings.append("possible_partial_page_show_more_present")
|
||||
if operation_error_banner:
|
||||
warnings.append("operation_error_banner_present")
|
||||
|
||||
warnings = sorted(set(warnings))
|
||||
_assert_profile_dom_invariants(
|
||||
fetch_result=fetch_result,
|
||||
marker_counts=marker_counts,
|
||||
publications=publications,
|
||||
warnings=warnings,
|
||||
has_show_more_button_flag=show_more,
|
||||
articles_range=articles_range,
|
||||
)
|
||||
|
||||
state, state_reason = detect_state(
|
||||
fetch_result,
|
||||
publications,
|
||||
marker_counts,
|
||||
warnings=warnings,
|
||||
has_show_more_button_flag=show_more,
|
||||
articles_range=articles_range,
|
||||
visible_text=visible_text,
|
||||
)
|
||||
|
||||
return ParsedProfilePage(
|
||||
state=state,
|
||||
state_reason=state_reason,
|
||||
profile_name=extract_profile_name(fetch_result.body),
|
||||
profile_image_url=extract_profile_image_url(fetch_result.body),
|
||||
publications=publications,
|
||||
marker_counts=marker_counts,
|
||||
warnings=warnings,
|
||||
has_show_more_button=show_more,
|
||||
has_operation_error_banner=operation_error_banner,
|
||||
articles_range=articles_range,
|
||||
)
|
||||
|
||||
|
||||
def parse_author_search_page(fetch_result: FetchResult) -> ParsedAuthorSearchPage:
|
||||
parser = ScholarAuthorSearchParser()
|
||||
parser.feed(fetch_result.body)
|
||||
|
||||
marker_counts = count_author_search_markers(fetch_result.body)
|
||||
visible_text = strip_tags(SCRIPT_STYLE_RE.sub(" ", fetch_result.body)).lower()
|
||||
warnings: list[str] = []
|
||||
if not parser.candidates:
|
||||
warnings.append("no_author_candidates_detected")
|
||||
|
||||
state, state_reason = detect_author_search_state(
|
||||
fetch_result,
|
||||
parser.candidates,
|
||||
marker_counts,
|
||||
visible_text=visible_text,
|
||||
)
|
||||
|
||||
return ParsedAuthorSearchPage(
|
||||
state=state,
|
||||
state_reason=state_reason,
|
||||
candidates=parser.candidates,
|
||||
marker_counts=marker_counts,
|
||||
warnings=warnings,
|
||||
)
|
||||
|
|
@ -1,87 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
BLOCKED_KEYWORDS = [
|
||||
"unusual traffic",
|
||||
"sorry/index",
|
||||
"not a robot",
|
||||
"our systems have detected",
|
||||
"automated queries",
|
||||
"recaptcha",
|
||||
"captcha",
|
||||
]
|
||||
|
||||
NO_RESULTS_KEYWORDS = [
|
||||
"didn't match any articles",
|
||||
"did not match any articles",
|
||||
"no articles",
|
||||
"no documents",
|
||||
]
|
||||
|
||||
NO_AUTHOR_RESULTS_KEYWORDS = [
|
||||
"didn't match any user profiles",
|
||||
"did not match any user profiles",
|
||||
"didn't match any scholars",
|
||||
"did not match any scholars",
|
||||
"no user profiles",
|
||||
]
|
||||
|
||||
MARKER_KEYS = [
|
||||
"gsc_a_tr",
|
||||
"gsc_a_at",
|
||||
"gsc_a_ac",
|
||||
"gsc_a_h",
|
||||
"gsc_a_y",
|
||||
"gs_gray",
|
||||
"gsc_prf_in",
|
||||
"gsc_rsb_st",
|
||||
]
|
||||
|
||||
AUTHOR_SEARCH_MARKER_KEYS = [
|
||||
"gsc_1usr",
|
||||
"gs_ai_name",
|
||||
"gs_ai_aff",
|
||||
"gs_ai_eml",
|
||||
"gs_ai_cby",
|
||||
"gs_ai_one_int",
|
||||
]
|
||||
|
||||
NETWORK_DNS_ERROR_KEYWORDS = [
|
||||
"temporary failure in name resolution",
|
||||
"name or service not known",
|
||||
"nodename nor servname provided",
|
||||
"getaddrinfo failed",
|
||||
]
|
||||
|
||||
NETWORK_TIMEOUT_KEYWORDS = [
|
||||
"timed out",
|
||||
"timeout",
|
||||
]
|
||||
|
||||
NETWORK_TLS_ERROR_KEYWORDS = [
|
||||
"ssl",
|
||||
"tls",
|
||||
"certificate verify failed",
|
||||
]
|
||||
|
||||
TAG_RE = re.compile(r"<[^>]+>", re.S)
|
||||
SCRIPT_STYLE_RE = re.compile(r"<(script|style)\b[^>]*>.*?</\1>", re.I | re.S)
|
||||
SHOW_MORE_BUTTON_RE = re.compile(
|
||||
r"<button\b[^>]*\bid\s*=\s*['\"]gsc_bpf_more['\"][^>]*>",
|
||||
re.I | re.S,
|
||||
)
|
||||
|
||||
PROFILE_ROW_PARSER_DIRECT_MARKERS = (
|
||||
"gs_ggs",
|
||||
"gs_ggsd",
|
||||
"gs_ggsa",
|
||||
"gs_or_ggsm",
|
||||
)
|
||||
|
||||
PROFILE_ROW_DIRECT_LABEL_TOKENS = (
|
||||
"pdf",
|
||||
"[pdf]",
|
||||
"full text",
|
||||
"download",
|
||||
)
|
||||
|
|
@ -1,75 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from enum import StrEnum
|
||||
|
||||
|
||||
class ParseState(StrEnum):
|
||||
OK = "ok"
|
||||
NO_RESULTS = "no_results"
|
||||
BLOCKED_OR_CAPTCHA = "blocked_or_captcha"
|
||||
LAYOUT_CHANGED = "layout_changed"
|
||||
NETWORK_ERROR = "network_error"
|
||||
|
||||
|
||||
class ScholarParserError(RuntimeError):
|
||||
code: str
|
||||
|
||||
def __init__(self, *, code: str, message: str) -> None:
|
||||
super().__init__(message)
|
||||
self.code = code
|
||||
|
||||
|
||||
class ScholarDomInvariantError(ScholarParserError):
|
||||
pass
|
||||
|
||||
|
||||
class ScholarMalformedDataError(ScholarParserError):
|
||||
pass
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PublicationCandidate:
|
||||
title: str
|
||||
title_url: str | None
|
||||
cluster_id: str | None
|
||||
year: int | None
|
||||
citation_count: int | None
|
||||
authors_text: str | None
|
||||
venue_text: str | None
|
||||
pdf_url: str | None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ScholarSearchCandidate:
|
||||
scholar_id: str
|
||||
display_name: str
|
||||
affiliation: str | None
|
||||
email_domain: str | None
|
||||
cited_by_count: int | None
|
||||
interests: list[str]
|
||||
profile_url: str
|
||||
profile_image_url: str | None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ParsedProfilePage:
|
||||
state: ParseState
|
||||
state_reason: str
|
||||
profile_name: str | None
|
||||
profile_image_url: str | None
|
||||
publications: list[PublicationCandidate]
|
||||
marker_counts: dict[str, int]
|
||||
warnings: list[str]
|
||||
has_show_more_button: bool
|
||||
has_operation_error_banner: bool
|
||||
articles_range: str | None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ParsedAuthorSearchPage:
|
||||
state: ParseState
|
||||
state_reason: str
|
||||
candidates: list[ScholarSearchCandidate]
|
||||
marker_counts: dict[str, int]
|
||||
warnings: list[str]
|
||||
|
|
@ -1,42 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from html import unescape
|
||||
|
||||
from app.services.domains.scholar.parser_constants import TAG_RE
|
||||
|
||||
|
||||
def normalize_space(value: str) -> str:
|
||||
return " ".join(unescape(value).split())
|
||||
|
||||
|
||||
def strip_tags(value: str) -> str:
|
||||
return normalize_space(TAG_RE.sub(" ", value))
|
||||
|
||||
|
||||
def attr_class(attrs: list[tuple[str, str | None]]) -> str:
|
||||
for name, raw_value in attrs:
|
||||
if name.lower() == "class":
|
||||
return raw_value or ""
|
||||
return ""
|
||||
|
||||
|
||||
def attr_href(attrs: list[tuple[str, str | None]]) -> str | None:
|
||||
for name, raw_value in attrs:
|
||||
if name.lower() == "href":
|
||||
return raw_value
|
||||
return None
|
||||
|
||||
|
||||
def attr_src(attrs: list[tuple[str, str | None]]) -> str | None:
|
||||
for name, raw_value in attrs:
|
||||
if name.lower() == "src":
|
||||
return raw_value
|
||||
return None
|
||||
|
||||
|
||||
def build_absolute_scholar_url(path_or_url: str | None) -> str | None:
|
||||
if not path_or_url:
|
||||
return None
|
||||
from urllib.parse import urljoin
|
||||
|
||||
return urljoin("https://scholar.google.com", path_or_url)
|
||||
|
|
@ -1,270 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from html.parser import HTMLParser
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
|
||||
from app.services.domains.scholar.parser_constants import (
|
||||
MARKER_KEYS,
|
||||
SHOW_MORE_BUTTON_RE,
|
||||
)
|
||||
from app.services.domains.scholar.parser_types import PublicationCandidate
|
||||
from app.services.domains.scholar.parser_utils import (
|
||||
attr_class,
|
||||
attr_href,
|
||||
attr_src,
|
||||
build_absolute_scholar_url,
|
||||
normalize_space,
|
||||
strip_tags,
|
||||
)
|
||||
|
||||
|
||||
class ScholarRowParser(HTMLParser):
|
||||
def __init__(self) -> None:
|
||||
super().__init__(convert_charrefs=True)
|
||||
self.title_href: str | None = None
|
||||
self.title_parts: list[str] = []
|
||||
self.citation_parts: list[str] = []
|
||||
self.year_parts: list[str] = []
|
||||
self.gray_texts: list[str] = []
|
||||
|
||||
self._title_depth = 0
|
||||
self._citation_depth = 0
|
||||
self._year_depth = 0
|
||||
self._gray_stack: list[dict[str, object]] = []
|
||||
|
||||
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
||||
if self._title_depth > 0:
|
||||
self._title_depth += 1
|
||||
if self._citation_depth > 0:
|
||||
self._citation_depth += 1
|
||||
if self._year_depth > 0:
|
||||
self._year_depth += 1
|
||||
if self._gray_stack:
|
||||
self._gray_stack[-1]["depth"] += 1
|
||||
|
||||
classes = attr_class(attrs)
|
||||
|
||||
if tag == "a" and "gsc_a_at" in classes:
|
||||
self._title_depth = 1
|
||||
self.title_href = attr_href(attrs)
|
||||
return
|
||||
|
||||
if tag == "a" and "gsc_a_ac" in classes:
|
||||
self._citation_depth = 1
|
||||
return
|
||||
|
||||
if tag in {"span", "a"} and ("gsc_a_h" in classes or "gsc_a_y" in classes):
|
||||
self._year_depth = 1
|
||||
return
|
||||
|
||||
if tag == "div" and "gs_gray" in classes:
|
||||
self._gray_stack.append({"depth": 1, "parts": []})
|
||||
return
|
||||
|
||||
def handle_data(self, data: str) -> None:
|
||||
if self._title_depth > 0:
|
||||
self.title_parts.append(data)
|
||||
if self._citation_depth > 0:
|
||||
self.citation_parts.append(data)
|
||||
if self._year_depth > 0:
|
||||
self.year_parts.append(data)
|
||||
if self._gray_stack:
|
||||
self._gray_stack[-1]["parts"].append(data)
|
||||
|
||||
def handle_endtag(self, tag: str) -> None:
|
||||
if self._title_depth > 0:
|
||||
self._title_depth -= 1
|
||||
if self._citation_depth > 0:
|
||||
self._citation_depth -= 1
|
||||
if self._year_depth > 0:
|
||||
self._year_depth -= 1
|
||||
if self._gray_stack:
|
||||
self._gray_stack[-1]["depth"] -= 1
|
||||
if self._gray_stack[-1]["depth"] == 0:
|
||||
text_value = normalize_space("".join(self._gray_stack[-1]["parts"]))
|
||||
if text_value:
|
||||
self.gray_texts.append(text_value)
|
||||
self._gray_stack.pop()
|
||||
|
||||
|
||||
def extract_rows(html: str) -> list[str]:
|
||||
pattern = re.compile(
|
||||
r"<tr\b(?=[^>]*\bclass\s*=\s*['\"][^'\"]*\bgsc_a_tr\b[^'\"]*['\"])[^>]*>(.*?)</tr>",
|
||||
re.I | re.S,
|
||||
)
|
||||
return [match.group(1) for match in pattern.finditer(html)]
|
||||
|
||||
|
||||
def parse_cluster_id_from_href(href: str | None) -> str | None:
|
||||
if not href:
|
||||
return None
|
||||
parsed = urlparse(href)
|
||||
query = parse_qs(parsed.query)
|
||||
|
||||
citation_for_view = query.get("citation_for_view")
|
||||
if citation_for_view:
|
||||
token = citation_for_view[0].strip()
|
||||
if token:
|
||||
return f"cfv:{token}"
|
||||
|
||||
cluster = query.get("cluster")
|
||||
if cluster:
|
||||
token = cluster[0].strip()
|
||||
if token:
|
||||
return f"cluster:{token}"
|
||||
return None
|
||||
|
||||
|
||||
def parse_year(parts: list[str]) -> int | None:
|
||||
text = normalize_space(" ".join(parts))
|
||||
match = re.search(r"\b(19|20)\d{2}\b", text)
|
||||
if not match:
|
||||
return None
|
||||
try:
|
||||
return int(match.group(0))
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def parse_citation_count(parts: list[str]) -> int | None:
|
||||
text = normalize_space(" ".join(parts))
|
||||
if not text:
|
||||
return 0
|
||||
digits = re.sub(r"\D+", "", text)
|
||||
if not digits:
|
||||
return None
|
||||
return int(digits)
|
||||
|
||||
|
||||
def _parse_publication_row(row_html: str) -> tuple[PublicationCandidate | None, list[str]]:
|
||||
parser = ScholarRowParser()
|
||||
parser.feed(row_html)
|
||||
warnings: list[str] = []
|
||||
title = normalize_space("".join(parser.title_parts))
|
||||
if not title:
|
||||
warnings.append("row_missing_title")
|
||||
return None, warnings
|
||||
if not parser.title_href:
|
||||
warnings.append("row_missing_title_href")
|
||||
|
||||
citation_text = normalize_space(" ".join(parser.citation_parts))
|
||||
citation_count = parse_citation_count(parser.citation_parts)
|
||||
if citation_text and citation_count is None:
|
||||
warnings.append("layout_row_citation_unparseable")
|
||||
|
||||
year_text = normalize_space(" ".join(parser.year_parts))
|
||||
year = parse_year(parser.year_parts)
|
||||
if year_text and year is None:
|
||||
warnings.append("layout_row_year_unparseable")
|
||||
|
||||
authors_text = parser.gray_texts[0] if len(parser.gray_texts) > 0 else None
|
||||
venue_text = parser.gray_texts[1] if len(parser.gray_texts) > 1 else None
|
||||
return (
|
||||
PublicationCandidate(
|
||||
title=title,
|
||||
title_url=parser.title_href,
|
||||
cluster_id=parse_cluster_id_from_href(parser.title_href),
|
||||
year=year,
|
||||
citation_count=citation_count,
|
||||
authors_text=authors_text,
|
||||
venue_text=venue_text,
|
||||
pdf_url=None,
|
||||
),
|
||||
warnings,
|
||||
)
|
||||
|
||||
|
||||
def parse_publications(html: str) -> tuple[list[PublicationCandidate], list[str]]:
|
||||
rows = extract_rows(html)
|
||||
warnings: list[str] = []
|
||||
publications: list[PublicationCandidate] = []
|
||||
|
||||
for row_html in rows:
|
||||
publication, row_warnings = _parse_publication_row(row_html)
|
||||
warnings.extend(row_warnings)
|
||||
if publication is None:
|
||||
continue
|
||||
publications.append(publication)
|
||||
|
||||
if not rows:
|
||||
warnings.append("no_rows_detected")
|
||||
if rows and not publications:
|
||||
warnings.append("layout_all_rows_unparseable")
|
||||
|
||||
return publications, sorted(set(warnings))
|
||||
|
||||
|
||||
def extract_profile_name(html: str) -> str | None:
|
||||
pattern = re.compile(
|
||||
r"<[^>]*\bid\s*=\s*['\"]gsc_prf_in['\"][^>]*>(.*?)</[^>]+>",
|
||||
re.I | re.S,
|
||||
)
|
||||
match = pattern.search(html)
|
||||
if not match:
|
||||
return None
|
||||
value = strip_tags(match.group(1))
|
||||
return value or None
|
||||
|
||||
|
||||
def extract_profile_image_url(html: str) -> str | None:
|
||||
og_image_pattern = re.compile(
|
||||
r"<meta[^>]+property=['\"]og:image['\"][^>]+content=['\"]([^'\"]+)['\"][^>]*>",
|
||||
re.I | re.S,
|
||||
)
|
||||
og_match = og_image_pattern.search(html)
|
||||
if og_match:
|
||||
value = normalize_space(og_match.group(1))
|
||||
absolute = build_absolute_scholar_url(value)
|
||||
if absolute:
|
||||
return absolute
|
||||
|
||||
image_pattern = re.compile(
|
||||
r"<img[^>]*\bid=['\"]gsc_prf_pup-img['\"][^>]*\bsrc=['\"]([^'\"]+)['\"][^>]*>",
|
||||
re.I | re.S,
|
||||
)
|
||||
image_match = image_pattern.search(html)
|
||||
if not image_match:
|
||||
return None
|
||||
|
||||
value = normalize_space(image_match.group(1))
|
||||
return build_absolute_scholar_url(value)
|
||||
|
||||
|
||||
def extract_articles_range(html: str) -> str | None:
|
||||
pattern = re.compile(
|
||||
r"<[^>]*\bid\s*=\s*['\"]gsc_a_nn['\"][^>]*>(.*?)</[^>]+>",
|
||||
re.I | re.S,
|
||||
)
|
||||
match = pattern.search(html)
|
||||
if not match:
|
||||
return None
|
||||
value = strip_tags(match.group(1))
|
||||
return value or None
|
||||
|
||||
|
||||
def has_show_more_button(html: str) -> bool:
|
||||
match = SHOW_MORE_BUTTON_RE.search(html)
|
||||
if match is None:
|
||||
return False
|
||||
|
||||
button_tag = match.group(0).lower()
|
||||
if "disabled" in button_tag:
|
||||
return False
|
||||
if 'aria-disabled="true"' in button_tag or "aria-disabled='true'" in button_tag:
|
||||
return False
|
||||
if "gs_dis" in button_tag:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def has_operation_error_banner(html: str) -> bool:
|
||||
lowered = html.lower()
|
||||
if "id=\"gsc_a_err\"" not in lowered and "id='gsc_a_err'" not in lowered:
|
||||
return False
|
||||
return "can't perform the operation now" in lowered or "cannot perform the operation now" in lowered
|
||||
|
||||
|
||||
def count_markers(html: str) -> dict[str, int]:
|
||||
lowered = html.lower()
|
||||
return {key: lowered.count(key.lower()) for key in MARKER_KEYS}
|
||||
|
|
@ -1,18 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import time
|
||||
|
||||
_REQUEST_LOCK = asyncio.Lock()
|
||||
_LAST_REQUEST_AT = 0.0
|
||||
|
||||
|
||||
async def wait_for_scholar_slot(*, min_interval_seconds: float) -> None:
|
||||
global _LAST_REQUEST_AT
|
||||
interval = max(float(min_interval_seconds), 0.0)
|
||||
async with _REQUEST_LOCK:
|
||||
elapsed = time.monotonic() - _LAST_REQUEST_AT
|
||||
remaining = interval - elapsed
|
||||
if remaining > 0:
|
||||
await asyncio.sleep(remaining)
|
||||
_LAST_REQUEST_AT = time.monotonic()
|
||||
|
|
@ -1,235 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import random
|
||||
from dataclasses import dataclass
|
||||
from typing import Protocol
|
||||
from urllib.error import HTTPError, URLError
|
||||
from urllib.parse import urlencode
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
SCHOLAR_PROFILE_URL = "https://scholar.google.com/citations"
|
||||
DEFAULT_PAGE_SIZE = 100
|
||||
|
||||
DEFAULT_USER_AGENTS = [
|
||||
(
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
|
||||
),
|
||||
(
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 14_7) AppleWebKit/605.1.15 "
|
||||
"(KHTML, like Gecko) Version/18.1 Safari/605.1.15"
|
||||
),
|
||||
(
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:131.0) "
|
||||
"Gecko/20100101 Firefox/131.0"
|
||||
),
|
||||
]
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FetchResult:
|
||||
requested_url: str
|
||||
status_code: int | None
|
||||
final_url: str | None
|
||||
body: str
|
||||
error: str | None
|
||||
|
||||
|
||||
class ScholarSource(Protocol):
|
||||
async def fetch_profile_html(self, scholar_id: str) -> FetchResult:
|
||||
...
|
||||
|
||||
async def fetch_profile_page_html(
|
||||
self,
|
||||
scholar_id: str,
|
||||
*,
|
||||
cstart: int,
|
||||
pagesize: int,
|
||||
) -> FetchResult:
|
||||
...
|
||||
|
||||
async def fetch_author_search_html(
|
||||
self,
|
||||
query: str,
|
||||
*,
|
||||
start: int,
|
||||
) -> FetchResult:
|
||||
...
|
||||
|
||||
|
||||
class LiveScholarSource:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
timeout_seconds: float = 25.0,
|
||||
user_agents: list[str] | None = None,
|
||||
) -> None:
|
||||
self._timeout_seconds = timeout_seconds
|
||||
self._user_agents = user_agents or DEFAULT_USER_AGENTS
|
||||
|
||||
async def fetch_profile_html(self, scholar_id: str) -> FetchResult:
|
||||
return await self.fetch_profile_page_html(
|
||||
scholar_id,
|
||||
cstart=0,
|
||||
pagesize=DEFAULT_PAGE_SIZE,
|
||||
)
|
||||
|
||||
async def fetch_profile_page_html(
|
||||
self,
|
||||
scholar_id: str,
|
||||
*,
|
||||
cstart: int,
|
||||
pagesize: int = DEFAULT_PAGE_SIZE,
|
||||
) -> FetchResult:
|
||||
requested_url = _build_profile_url(
|
||||
scholar_id=scholar_id,
|
||||
cstart=cstart,
|
||||
pagesize=pagesize,
|
||||
)
|
||||
logger.debug(
|
||||
"scholar_source.fetch_started",
|
||||
extra={
|
||||
"event": "scholar_source.fetch_started",
|
||||
"scholar_id": scholar_id,
|
||||
"requested_url": requested_url,
|
||||
"cstart": cstart,
|
||||
"pagesize": pagesize,
|
||||
},
|
||||
)
|
||||
return await asyncio.to_thread(self._fetch_sync, requested_url)
|
||||
|
||||
async def fetch_author_search_html(
|
||||
self,
|
||||
query: str,
|
||||
*,
|
||||
start: int = 0,
|
||||
) -> FetchResult:
|
||||
requested_url = _build_author_search_url(
|
||||
query=query,
|
||||
start=start,
|
||||
)
|
||||
logger.debug(
|
||||
"scholar_source.search_fetch_started",
|
||||
extra={
|
||||
"event": "scholar_source.search_fetch_started",
|
||||
"query": query,
|
||||
"requested_url": requested_url,
|
||||
"start": start,
|
||||
},
|
||||
)
|
||||
return await asyncio.to_thread(self._fetch_sync, requested_url)
|
||||
|
||||
async def fetch_publication_html(self, publication_url: str) -> FetchResult:
|
||||
logger.debug(
|
||||
"scholar_source.publication_fetch_started",
|
||||
extra={
|
||||
"event": "scholar_source.publication_fetch_started",
|
||||
"requested_url": publication_url,
|
||||
},
|
||||
)
|
||||
return await asyncio.to_thread(self._fetch_sync, publication_url)
|
||||
|
||||
def _build_request(self, requested_url: str) -> Request:
|
||||
return Request(
|
||||
requested_url,
|
||||
headers={
|
||||
"User-Agent": random.choice(self._user_agents),
|
||||
"Accept": "text/html,application/xhtml+xml",
|
||||
"Accept-Language": "en-US,en;q=0.9",
|
||||
"Connection": "close",
|
||||
},
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _http_error_body(exc: HTTPError) -> str:
|
||||
try:
|
||||
return exc.read().decode("utf-8", errors="replace")
|
||||
except Exception:
|
||||
return ""
|
||||
|
||||
@staticmethod
|
||||
def _network_error_result(requested_url: str, exc: URLError) -> FetchResult:
|
||||
logger.warning(
|
||||
"scholar_source.fetch_network_error",
|
||||
extra={"event": "scholar_source.fetch_network_error", "requested_url": requested_url},
|
||||
)
|
||||
return FetchResult(
|
||||
requested_url=requested_url,
|
||||
status_code=None,
|
||||
final_url=None,
|
||||
body="",
|
||||
error=str(exc),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _http_error_result(requested_url: str, exc: HTTPError) -> FetchResult:
|
||||
logger.warning(
|
||||
"scholar_source.fetch_http_error",
|
||||
extra={
|
||||
"event": "scholar_source.fetch_http_error",
|
||||
"requested_url": requested_url,
|
||||
"status_code": exc.code,
|
||||
},
|
||||
)
|
||||
return FetchResult(
|
||||
requested_url=requested_url,
|
||||
status_code=exc.code,
|
||||
final_url=exc.geturl(),
|
||||
body=LiveScholarSource._http_error_body(exc),
|
||||
error=str(exc),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _success_result(requested_url: str, response) -> FetchResult:
|
||||
body = response.read().decode("utf-8", errors="replace")
|
||||
status_code = getattr(response, "status", 200)
|
||||
logger.debug(
|
||||
"scholar_source.fetch_succeeded",
|
||||
extra={
|
||||
"event": "scholar_source.fetch_succeeded",
|
||||
"requested_url": requested_url,
|
||||
"status_code": status_code,
|
||||
},
|
||||
)
|
||||
return FetchResult(
|
||||
requested_url=requested_url,
|
||||
status_code=status_code,
|
||||
final_url=response.geturl(),
|
||||
body=body,
|
||||
error=None,
|
||||
)
|
||||
|
||||
def _fetch_sync(self, requested_url: str) -> FetchResult:
|
||||
request = self._build_request(requested_url)
|
||||
|
||||
try:
|
||||
with urlopen(request, timeout=self._timeout_seconds) as response:
|
||||
return self._success_result(requested_url, response)
|
||||
except HTTPError as exc:
|
||||
return self._http_error_result(requested_url, exc)
|
||||
except URLError as exc:
|
||||
return self._network_error_result(requested_url, exc)
|
||||
|
||||
|
||||
def _build_profile_url(*, scholar_id: str, cstart: int, pagesize: int) -> str:
|
||||
query: dict[str, int | str] = {"hl": "en", "user": scholar_id}
|
||||
if cstart > 0:
|
||||
query["cstart"] = int(cstart)
|
||||
if pagesize > 0:
|
||||
query["pagesize"] = int(pagesize)
|
||||
return f"{SCHOLAR_PROFILE_URL}?{urlencode(query)}"
|
||||
|
||||
|
||||
def _build_author_search_url(*, query: str, start: int) -> str:
|
||||
params: dict[str, int | str] = {
|
||||
"hl": "en",
|
||||
"view_op": "search_authors",
|
||||
"mauthors": query,
|
||||
}
|
||||
if start > 0:
|
||||
params["astart"] = int(start)
|
||||
return f"{SCHOLAR_PROFILE_URL}?{urlencode(params)}"
|
||||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Add a link
Reference in a new issue