First product

This commit is contained in:
Justin Visser 2026-02-17 14:51:25 +01:00
parent 778da9e2fc
commit 4433d7d2c4
157 changed files with 23975 additions and 0 deletions

2
app/api/__init__.py Normal file
View file

@ -0,0 +1,2 @@
from __future__ import annotations

36
app/api/deps.py Normal file
View file

@ -0,0 +1,36 @@
from __future__ import annotations
from fastapi import Depends, Request
from sqlalchemy.ext.asyncio import AsyncSession
from app.api.errors import ApiException
from app.db.models import User
from app.db.session import get_db_session
from app.web import common as web_common
async def get_api_current_user(
request: Request,
db_session: AsyncSession = Depends(get_db_session),
) -> User:
current_user = await web_common.get_authenticated_user(request, db_session)
if current_user is None:
raise ApiException(
status_code=401,
code="auth_required",
message="Authentication required.",
)
return current_user
async def get_api_admin_user(
current_user: User = Depends(get_api_current_user),
) -> User:
if not current_user.is_admin:
raise ApiException(
status_code=403,
code="forbidden",
message="Admin access required.",
)
return current_user

85
app/api/errors.py Normal file
View file

@ -0,0 +1,85 @@
from __future__ import annotations
from typing import Any
from fastapi import FastAPI, HTTPException, Request
from fastapi.exception_handlers import (
http_exception_handler as fastapi_http_exception_handler,
)
from fastapi.exception_handlers import (
request_validation_exception_handler as fastapi_validation_exception_handler,
)
from fastapi.exceptions import RequestValidationError
from app.api.responses import error_response
def _is_api_path(path: str) -> bool:
return path.startswith("/api/")
def _status_code_to_error_code(status_code: int) -> str:
mapping = {
400: "bad_request",
401: "auth_required",
403: "forbidden",
404: "not_found",
409: "conflict",
422: "validation_error",
429: "rate_limited",
500: "internal_error",
}
return mapping.get(status_code, "error")
class ApiException(Exception):
def __init__(
self,
*,
status_code: int,
code: str,
message: str,
details: Any | None = None,
) -> None:
super().__init__(message)
self.status_code = status_code
self.code = code
self.message = message
self.details = details
def register_api_exception_handlers(app: FastAPI) -> None:
@app.exception_handler(ApiException)
async def _handle_api_exception(request: Request, exc: ApiException):
return error_response(
request,
status_code=exc.status_code,
code=exc.code,
message=exc.message,
details=exc.details,
)
@app.exception_handler(HTTPException)
async def _handle_http_exception(request: Request, exc: HTTPException):
if not _is_api_path(request.url.path):
return await fastapi_http_exception_handler(request, exc)
return error_response(
request,
status_code=exc.status_code,
code=_status_code_to_error_code(exc.status_code),
message=str(exc.detail) if exc.detail is not None else "Request failed.",
details=exc.detail if isinstance(exc.detail, (dict, list)) else None,
)
@app.exception_handler(RequestValidationError)
async def _handle_validation_exception(request: Request, exc: RequestValidationError):
if not _is_api_path(request.url.path):
return await fastapi_validation_exception_handler(request, exc)
return error_response(
request,
status_code=422,
code="validation_error",
message="Request validation failed.",
details=exc.errors(),
)

64
app/api/responses.py Normal file
View file

@ -0,0 +1,64 @@
from __future__ import annotations
from typing import Any
from fastapi.encoders import jsonable_encoder
from starlette.requests import Request
from starlette.responses import JSONResponse
def _request_id(request: Request) -> str | None:
request_state = getattr(request, "state", None)
if request_state is None:
return None
return getattr(request_state, "request_id", None)
def success_payload(
request: Request,
*,
data: Any,
) -> dict[str, Any]:
return {
"data": data,
"meta": {
"request_id": _request_id(request),
},
}
def success_response(
request: Request,
*,
data: Any,
status_code: int = 200,
) -> JSONResponse:
payload = success_payload(request, data=data)
return JSONResponse(
status_code=status_code,
content=jsonable_encoder(payload),
)
def error_response(
request: Request,
*,
status_code: int,
code: str,
message: str,
details: Any | None = None,
) -> JSONResponse:
payload = {
"error": {
"code": code,
"message": message,
"details": details,
},
"meta": {
"request_id": _request_id(request),
},
}
return JSONResponse(
status_code=status_code,
content=jsonable_encoder(payload),
)

13
app/api/router.py Normal file
View file

@ -0,0 +1,13 @@
from __future__ import annotations
from fastapi import APIRouter
from app.api.routers import admin, auth, publications, runs, scholars, settings
router = APIRouter(prefix="/api/v1")
router.include_router(auth.router)
router.include_router(admin.router)
router.include_router(scholars.router)
router.include_router(settings.router)
router.include_router(runs.router)
router.include_router(publications.router)

View file

@ -0,0 +1,2 @@
from __future__ import annotations

202
app/api/routers/admin.py Normal file
View file

@ -0,0 +1,202 @@
from __future__ import annotations
import logging
from fastapi import APIRouter, Depends, Request
from sqlalchemy.ext.asyncio import AsyncSession
from app.api.deps import get_api_admin_user
from app.api.errors import ApiException
from app.api.responses import success_payload
from app.api.schemas import (
AdminResetPasswordRequest,
AdminUserActiveUpdateRequest,
AdminUserCreateRequest,
AdminUserEnvelope,
AdminUsersListEnvelope,
MessageEnvelope,
)
from app.auth.deps import get_auth_service
from app.auth.service import AuthService
from app.db.models import User
from app.db.session import get_db_session
from app.services import users as user_service
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/admin/users", tags=["api-admin-users"])
def _serialize_user(user: User) -> dict[str, object]:
return {
"id": int(user.id),
"email": user.email,
"is_active": bool(user.is_active),
"is_admin": bool(user.is_admin),
"created_at": user.created_at,
"updated_at": user.updated_at,
}
@router.get(
"",
response_model=AdminUsersListEnvelope,
)
async def list_users(
request: Request,
db_session: AsyncSession = Depends(get_db_session),
admin_user: User = Depends(get_api_admin_user),
):
users = await user_service.list_users(db_session)
logger.info(
"api.admin.users_listed",
extra={
"event": "api.admin.users_listed",
"admin_user_id": int(admin_user.id),
"user_count": len(users),
},
)
return success_payload(
request,
data={
"users": [_serialize_user(user) for user in users],
},
)
@router.post(
"",
response_model=AdminUserEnvelope,
status_code=201,
)
async def create_user(
payload: AdminUserCreateRequest,
request: Request,
db_session: AsyncSession = Depends(get_db_session),
auth_service: AuthService = Depends(get_auth_service),
admin_user: User = Depends(get_api_admin_user),
):
try:
validated_email = user_service.validate_email(payload.email)
validated_password = user_service.validate_password(payload.password)
created_user = await user_service.create_user(
db_session,
email=validated_email,
password_hash=auth_service.hash_password(validated_password),
is_admin=bool(payload.is_admin),
)
except user_service.UserServiceError as exc:
raise ApiException(
status_code=400,
code="invalid_user_input",
message=str(exc),
) from exc
logger.info(
"api.admin.user_created",
extra={
"event": "api.admin.user_created",
"admin_user_id": int(admin_user.id),
"target_user_id": int(created_user.id),
"target_is_admin": bool(created_user.is_admin),
},
)
return success_payload(
request,
data=_serialize_user(created_user),
)
@router.patch(
"/{user_id}/active",
response_model=AdminUserEnvelope,
)
async def set_user_active(
user_id: int,
payload: AdminUserActiveUpdateRequest,
request: Request,
db_session: AsyncSession = Depends(get_db_session),
admin_user: User = Depends(get_api_admin_user),
):
target_user = await user_service.get_user_by_id(db_session, user_id)
if target_user is None:
raise ApiException(
status_code=404,
code="user_not_found",
message="User not found.",
)
if (
int(target_user.id) == int(admin_user.id)
and bool(target_user.is_active)
and not bool(payload.is_active)
):
raise ApiException(
status_code=400,
code="cannot_deactivate_self",
message="You cannot deactivate your own account.",
)
updated_user = await user_service.set_user_active(
db_session,
user=target_user,
is_active=bool(payload.is_active),
)
logger.info(
"api.admin.user_active_updated",
extra={
"event": "api.admin.user_active_updated",
"admin_user_id": int(admin_user.id),
"target_user_id": int(updated_user.id),
"is_active": bool(updated_user.is_active),
},
)
return success_payload(
request,
data=_serialize_user(updated_user),
)
@router.post(
"/{user_id}/reset-password",
response_model=MessageEnvelope,
)
async def reset_user_password(
user_id: int,
payload: AdminResetPasswordRequest,
request: Request,
db_session: AsyncSession = Depends(get_db_session),
auth_service: AuthService = Depends(get_auth_service),
admin_user: User = Depends(get_api_admin_user),
):
target_user = await user_service.get_user_by_id(db_session, user_id)
if target_user is None:
raise ApiException(
status_code=404,
code="user_not_found",
message="User not found.",
)
try:
validated_password = user_service.validate_password(payload.new_password)
except user_service.UserServiceError as exc:
raise ApiException(
status_code=400,
code="invalid_password",
message=str(exc),
) from exc
await user_service.set_user_password_hash(
db_session,
user=target_user,
password_hash=auth_service.hash_password(validated_password),
)
logger.info(
"api.admin.user_password_reset",
extra={
"event": "api.admin.user_password_reset",
"admin_user_id": int(admin_user.id),
"target_user_id": int(target_user.id),
},
)
return success_payload(
request,
data={"message": f"Password reset: {target_user.email}"},
)

230
app/api/routers/auth.py Normal file
View file

@ -0,0 +1,230 @@
from __future__ import annotations
import logging
from fastapi import APIRouter, Depends, Request
from sqlalchemy.ext.asyncio import AsyncSession
from app.api.errors import ApiException
from app.api.deps import get_api_current_user
from app.api.responses import success_payload
from app.api.schemas import (
AuthMeEnvelope,
ChangePasswordRequest,
CsrfBootstrapEnvelope,
LoginEnvelope,
LoginRequest,
MessageEnvelope,
)
from app.auth.deps import get_auth_service, get_login_rate_limiter
from app.auth.rate_limit import SlidingWindowRateLimiter
from app.auth.service import AuthService
from app.auth.session import set_session_user
from app.db.models import User
from app.db.session import get_db_session
from app.security.csrf import ensure_csrf_token
from app.services import users as user_service
from app.web import common as web_common
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/auth", tags=["api-auth"])
@router.post(
"/login",
response_model=LoginEnvelope,
)
async def login(
payload: LoginRequest,
request: Request,
db_session: AsyncSession = Depends(get_db_session),
auth_service: AuthService = Depends(get_auth_service),
rate_limiter: SlidingWindowRateLimiter = Depends(get_login_rate_limiter),
):
limiter_key = web_common.login_rate_limit_key(request, payload.email)
decision = rate_limiter.check(limiter_key)
normalized_email = payload.email.strip().lower()
if not decision.allowed:
logger.warning(
"api.auth.login_rate_limited",
extra={
"event": "api.auth.login_rate_limited",
"email": normalized_email,
"retry_after_seconds": decision.retry_after_seconds,
},
)
raise ApiException(
status_code=429,
code="rate_limited",
message="Too many login attempts. Please try again later.",
details={"retry_after_seconds": decision.retry_after_seconds},
)
user = await auth_service.authenticate_user(
db_session,
email=payload.email,
password=payload.password,
)
if user is None:
rate_limiter.record_failure(limiter_key)
logger.info(
"api.auth.login_failed",
extra={
"event": "api.auth.login_failed",
"email": normalized_email,
},
)
raise ApiException(
status_code=401,
code="invalid_credentials",
message="Invalid email or password.",
)
rate_limiter.reset(limiter_key)
set_session_user(
request,
user_id=user.id,
email=user.email,
is_admin=user.is_admin,
)
logger.info(
"api.auth.login_succeeded",
extra={
"event": "api.auth.login_succeeded",
"user_id": user.id,
"is_admin": user.is_admin,
},
)
return success_payload(
request,
data={
"authenticated": True,
"csrf_token": ensure_csrf_token(request),
"user": {
"id": int(user.id),
"email": user.email,
"is_admin": bool(user.is_admin),
"is_active": bool(user.is_active),
},
},
)
@router.get(
"/me",
response_model=AuthMeEnvelope,
)
async def get_current_session(
request: Request,
current_user: User = Depends(get_api_current_user),
):
return success_payload(
request,
data={
"authenticated": True,
"csrf_token": ensure_csrf_token(request),
"user": {
"id": int(current_user.id),
"email": current_user.email,
"is_admin": bool(current_user.is_admin),
"is_active": bool(current_user.is_active),
},
},
)
@router.get(
"/csrf",
response_model=CsrfBootstrapEnvelope,
)
async def get_csrf_bootstrap(
request: Request,
db_session: AsyncSession = Depends(get_db_session),
):
current_user = await web_common.get_authenticated_user(request, db_session)
return success_payload(
request,
data={
"csrf_token": ensure_csrf_token(request),
"authenticated": current_user is not None,
},
)
@router.post(
"/change-password",
response_model=MessageEnvelope,
)
async def change_password(
payload: ChangePasswordRequest,
request: Request,
current_user: User = Depends(get_api_current_user),
db_session: AsyncSession = Depends(get_db_session),
auth_service: AuthService = Depends(get_auth_service),
):
if not auth_service.verify_password(
password_hash=current_user.password_hash,
password=payload.current_password,
):
raise ApiException(
status_code=400,
code="invalid_current_password",
message="Current password is incorrect.",
)
if payload.new_password != payload.confirm_password:
raise ApiException(
status_code=400,
code="password_confirmation_mismatch",
message="New password and confirmation do not match.",
)
try:
validated_password = user_service.validate_password(payload.new_password)
except user_service.UserServiceError as exc:
raise ApiException(
status_code=400,
code="invalid_password",
message=str(exc),
) from exc
await user_service.set_user_password_hash(
db_session,
user=current_user,
password_hash=auth_service.hash_password(validated_password),
)
logger.info(
"api.auth.password_changed",
extra={
"event": "api.auth.password_changed",
"user_id": int(current_user.id),
},
)
return success_payload(
request,
data={"message": "Password updated successfully."},
)
@router.post(
"/logout",
response_model=MessageEnvelope,
)
async def logout(
request: Request,
db_session: AsyncSession = Depends(get_db_session),
):
current_user = await web_common.get_authenticated_user(request, db_session)
web_common.invalidate_session(request)
logger.info(
"api.auth.logout",
extra={
"event": "api.auth.logout",
"user_id": int(current_user.id) if current_user is not None else None,
},
)
return success_payload(
request,
data={
"message": "Logged out.",
},
)

View file

@ -0,0 +1,123 @@
from __future__ import annotations
import logging
from typing import Literal
from fastapi import APIRouter, Depends, Query, Request
from sqlalchemy.ext.asyncio import AsyncSession
from app.api.deps import get_api_current_user
from app.api.errors import ApiException
from app.api.responses import success_payload
from app.api.schemas import MarkAllReadEnvelope, PublicationsListEnvelope
from app.db.models import User
from app.db.session import get_db_session
from app.services import publications as publication_service
from app.services import scholars as scholar_service
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/publications", tags=["api-publications"])
@router.get(
"",
response_model=PublicationsListEnvelope,
)
async def list_publications(
request: Request,
mode: Literal["all", "new"] | None = Query(default=None),
scholar_profile_id: int | None = Query(default=None, ge=1),
limit: int = Query(default=300, ge=1, le=1000),
db_session: AsyncSession = Depends(get_db_session),
current_user: User = Depends(get_api_current_user),
):
resolved_mode = publication_service.resolve_mode(mode)
selected_scholar_id = scholar_profile_id
if selected_scholar_id is not None:
selected_profile = await scholar_service.get_user_scholar_by_id(
db_session,
user_id=current_user.id,
scholar_profile_id=selected_scholar_id,
)
if selected_profile is None:
raise ApiException(
status_code=404,
code="scholar_not_found",
message="Scholar filter not found.",
)
publications = await publication_service.list_for_user(
db_session,
user_id=current_user.id,
mode=resolved_mode,
scholar_profile_id=selected_scholar_id,
limit=limit,
)
new_count = await publication_service.count_for_user(
db_session,
user_id=current_user.id,
mode=publication_service.MODE_NEW,
scholar_profile_id=selected_scholar_id,
)
total_count = await publication_service.count_for_user(
db_session,
user_id=current_user.id,
mode=publication_service.MODE_ALL,
scholar_profile_id=selected_scholar_id,
)
return success_payload(
request,
data={
"mode": resolved_mode,
"selected_scholar_profile_id": selected_scholar_id,
"new_count": new_count,
"total_count": total_count,
"publications": [
{
"publication_id": item.publication_id,
"scholar_profile_id": item.scholar_profile_id,
"scholar_label": item.scholar_label,
"title": item.title,
"year": item.year,
"citation_count": item.citation_count,
"venue_text": item.venue_text,
"pub_url": item.pub_url,
"is_read": item.is_read,
"first_seen_at": item.first_seen_at,
"is_new_in_latest_run": item.is_new_in_latest_run,
}
for item in publications
],
},
)
@router.post(
"/mark-all-read",
response_model=MarkAllReadEnvelope,
)
async def mark_all_publications_read(
request: Request,
db_session: AsyncSession = Depends(get_db_session),
current_user: User = Depends(get_api_current_user),
):
updated_count = await publication_service.mark_all_unread_as_read_for_user(
db_session,
user_id=current_user.id,
)
logger.info(
"api.publications.mark_all_read",
extra={
"event": "api.publications.mark_all_read",
"user_id": current_user.id,
"updated_count": updated_count,
},
)
return success_payload(
request,
data={
"message": "Marked all unread publications as read.",
"updated_count": updated_count,
},
)

560
app/api/routers/runs.py Normal file
View file

@ -0,0 +1,560 @@
from __future__ import annotations
import logging
import re
from typing import Any
from fastapi import APIRouter, Depends, Query, Request
from sqlalchemy.ext.asyncio import AsyncSession
from app.api.deps import get_api_current_user
from app.api.errors import ApiException
from app.api.responses import success_payload
from app.api.schemas import (
ManualRunEnvelope,
QueueClearEnvelope,
QueueItemEnvelope,
QueueListEnvelope,
RunDetailEnvelope,
RunsListEnvelope,
)
from app.db.models import RunStatus, RunTriggerType, User
from app.db.session import get_db_session
from app.services import ingestion as ingestion_service
from app.services import runs as run_service
from app.services import user_settings as user_settings_service
from app.settings import settings
from app.web.deps import get_ingestion_service
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/runs", tags=["api-runs"])
IDEMPOTENCY_HEADER = "Idempotency-Key"
IDEMPOTENCY_MAX_LENGTH = 128
IDEMPOTENCY_PATTERN = re.compile(r"^[A-Za-z0-9._:-]{8,128}$")
def _int_value(value: Any, default: int = 0) -> int:
try:
return int(value)
except (TypeError, ValueError):
return default
def _str_value(value: Any) -> str | None:
if value is None:
return None
text = str(value).strip()
return text if text else None
def _bool_value(value: Any, default: bool = False) -> bool:
if isinstance(value, bool):
return value
if isinstance(value, str):
lowered = value.strip().lower()
if lowered in {"1", "true", "yes", "on"}:
return True
if lowered in {"0", "false", "no", "off"}:
return False
return default
def _normalize_idempotency_key(raw_value: str | None) -> str | None:
if raw_value is None:
return None
candidate = raw_value.strip()
if not candidate:
return None
if len(candidate) > IDEMPOTENCY_MAX_LENGTH or not IDEMPOTENCY_PATTERN.match(candidate):
raise ApiException(
status_code=400,
code="invalid_idempotency_key",
message=(
"Invalid Idempotency-Key. Use 8-128 characters from: "
"A-Z a-z 0-9 . _ : -"
),
)
return candidate
def _serialize_run(run) -> dict[str, Any]:
summary = run_service.extract_run_summary(run.error_log)
return {
"id": int(run.id),
"trigger_type": run.trigger_type.value,
"status": run.status.value,
"start_dt": run.start_dt,
"end_dt": run.end_dt,
"scholar_count": int(run.scholar_count or 0),
"new_publication_count": int(run.new_pub_count or 0),
"failed_count": int(summary["failed_count"]),
"partial_count": int(summary["partial_count"]),
}
def _serialize_queue_item(item) -> dict[str, Any]:
return {
"id": int(item.id),
"scholar_profile_id": int(item.scholar_profile_id),
"scholar_label": item.scholar_label,
"status": item.status,
"reason": item.reason,
"dropped_reason": item.dropped_reason,
"attempt_count": int(item.attempt_count),
"resume_cstart": int(item.resume_cstart),
"next_attempt_dt": item.next_attempt_dt,
"updated_at": item.updated_at,
"last_error": item.last_error,
"last_run_id": item.last_run_id,
}
def _normalize_attempt_log(value: Any) -> list[dict[str, Any]]:
if not isinstance(value, list):
return []
normalized: list[dict[str, Any]] = []
for item in value:
if not isinstance(item, dict):
continue
normalized.append(
{
"attempt": _int_value(item.get("attempt"), 0),
"cstart": _int_value(item.get("cstart"), 0),
"state": _str_value(item.get("state")),
"state_reason": _str_value(item.get("state_reason")),
"status_code": (
_int_value(item.get("status_code"))
if item.get("status_code") is not None
else None
),
"fetch_error": _str_value(item.get("fetch_error")),
}
)
return normalized
def _normalize_page_logs(value: Any) -> list[dict[str, Any]]:
if not isinstance(value, list):
return []
normalized: list[dict[str, Any]] = []
for item in value:
if not isinstance(item, dict):
continue
warning_codes = item.get("warning_codes")
normalized.append(
{
"page": _int_value(item.get("page"), 0),
"cstart": _int_value(item.get("cstart"), 0),
"state": _str_value(item.get("state")) or "unknown",
"state_reason": _str_value(item.get("state_reason")),
"status_code": (
_int_value(item.get("status_code"))
if item.get("status_code") is not None
else None
),
"publication_count": _int_value(item.get("publication_count"), 0),
"attempt_count": _int_value(item.get("attempt_count"), 0),
"has_show_more_button": _bool_value(item.get("has_show_more_button"), False),
"articles_range": _str_value(item.get("articles_range")),
"warning_codes": [
str(code)
for code in (warning_codes if isinstance(warning_codes, list) else [])
],
}
)
return normalized
def _normalize_debug(value: Any) -> dict[str, Any] | None:
if not isinstance(value, dict):
return None
marker_counts = value.get("marker_counts_nonzero")
warning_codes = value.get("warning_codes")
return {
"status_code": (
_int_value(value.get("status_code"))
if value.get("status_code") is not None
else None
),
"final_url": _str_value(value.get("final_url")),
"fetch_error": _str_value(value.get("fetch_error")),
"body_sha256": _str_value(value.get("body_sha256")),
"body_length": (
_int_value(value.get("body_length"))
if value.get("body_length") is not None
else None
),
"has_show_more_button": (
_bool_value(value.get("has_show_more_button"), False)
if value.get("has_show_more_button") is not None
else None
),
"articles_range": _str_value(value.get("articles_range")),
"state_reason": _str_value(value.get("state_reason")),
"warning_codes": [
str(code)
for code in (warning_codes if isinstance(warning_codes, list) else [])
],
"marker_counts_nonzero": {
str(key): _int_value(count, 0)
for key, count in (marker_counts.items() if isinstance(marker_counts, dict) else [])
},
"page_logs": _normalize_page_logs(value.get("page_logs")),
"attempt_log": _normalize_attempt_log(value.get("attempt_log")),
}
def _normalize_scholar_result(value: Any) -> dict[str, Any]:
if not isinstance(value, dict):
return {
"scholar_profile_id": 0,
"scholar_id": "unknown",
"state": "unknown",
"state_reason": None,
"outcome": "failed",
"attempt_count": 0,
"publication_count": 0,
"start_cstart": 0,
"continuation_cstart": None,
"continuation_enqueued": False,
"continuation_cleared": False,
"warnings": [],
"error": None,
"debug": None,
}
warnings = value.get("warnings")
return {
"scholar_profile_id": _int_value(value.get("scholar_profile_id"), 0),
"scholar_id": _str_value(value.get("scholar_id")) or "unknown",
"state": _str_value(value.get("state")) or "unknown",
"state_reason": _str_value(value.get("state_reason")),
"outcome": _str_value(value.get("outcome")) or "failed",
"attempt_count": _int_value(value.get("attempt_count"), 0),
"publication_count": _int_value(value.get("publication_count"), 0),
"start_cstart": _int_value(value.get("start_cstart"), 0),
"continuation_cstart": (
_int_value(value.get("continuation_cstart"))
if value.get("continuation_cstart") is not None
else None
),
"continuation_enqueued": _bool_value(value.get("continuation_enqueued"), False),
"continuation_cleared": _bool_value(value.get("continuation_cleared"), False),
"warnings": [str(item) for item in (warnings if isinstance(warnings, list) else [])],
"error": _str_value(value.get("error")),
"debug": _normalize_debug(value.get("debug")),
}
def _manual_run_payload_from_run(
run,
*,
idempotency_key: str | None,
reused_existing_run: bool,
) -> dict[str, Any]:
summary = run_service.extract_run_summary(run.error_log)
return {
"run_id": int(run.id),
"status": run.status.value,
"scholar_count": int(run.scholar_count or 0),
"succeeded_count": int(summary["succeeded_count"]),
"failed_count": int(summary["failed_count"]),
"partial_count": int(summary["partial_count"]),
"new_publication_count": int(run.new_pub_count or 0),
"reused_existing_run": reused_existing_run,
"idempotency_key": idempotency_key,
}
@router.get(
"",
response_model=RunsListEnvelope,
)
async def list_runs(
request: Request,
failed_only: bool = Query(default=False),
limit: int = Query(default=100, ge=1, le=500),
db_session: AsyncSession = Depends(get_db_session),
current_user: User = Depends(get_api_current_user),
):
runs = await run_service.list_runs_for_user(
db_session,
user_id=current_user.id,
limit=limit,
failed_only=failed_only,
)
return success_payload(
request,
data={
"runs": [_serialize_run(run) for run in runs],
},
)
@router.get(
"/{run_id}",
response_model=RunDetailEnvelope,
)
async def get_run(
run_id: int,
request: Request,
db_session: AsyncSession = Depends(get_db_session),
current_user: User = Depends(get_api_current_user),
):
run = await run_service.get_run_for_user(
db_session,
user_id=current_user.id,
run_id=run_id,
)
if run is None:
raise ApiException(
status_code=404,
code="run_not_found",
message="Run not found.",
)
error_log = run.error_log if isinstance(run.error_log, dict) else {}
scholar_results = error_log.get("scholar_results")
if not isinstance(scholar_results, list):
scholar_results = []
return success_payload(
request,
data={
"run": _serialize_run(run),
"summary": run_service.extract_run_summary(error_log),
"scholar_results": [_normalize_scholar_result(item) for item in scholar_results],
},
)
@router.post(
"/manual",
response_model=ManualRunEnvelope,
)
async def run_manual(
request: Request,
db_session: AsyncSession = Depends(get_db_session),
current_user: User = Depends(get_api_current_user),
ingest_service: ingestion_service.ScholarIngestionService = Depends(get_ingestion_service),
):
idempotency_key = _normalize_idempotency_key(request.headers.get(IDEMPOTENCY_HEADER))
if idempotency_key is not None:
previous_run = await run_service.get_manual_run_by_idempotency_key(
db_session,
user_id=current_user.id,
idempotency_key=idempotency_key,
)
if previous_run is not None:
if previous_run.status == RunStatus.RUNNING:
raise ApiException(
status_code=409,
code="run_in_progress",
message="A run with this idempotency key is still in progress.",
details={
"run_id": int(previous_run.id),
"idempotency_key": idempotency_key,
},
)
return success_payload(
request,
data=_manual_run_payload_from_run(
previous_run,
idempotency_key=idempotency_key,
reused_existing_run=True,
),
)
user_settings = await user_settings_service.get_or_create_settings(
db_session,
user_id=current_user.id,
)
try:
run_summary = await ingest_service.run_for_user(
db_session,
user_id=current_user.id,
trigger_type=RunTriggerType.MANUAL,
request_delay_seconds=user_settings.request_delay_seconds,
network_error_retries=settings.ingestion_network_error_retries,
retry_backoff_seconds=settings.ingestion_retry_backoff_seconds,
max_pages_per_scholar=settings.ingestion_max_pages_per_scholar,
page_size=settings.ingestion_page_size,
auto_queue_continuations=settings.ingestion_continuation_queue_enabled,
queue_delay_seconds=settings.ingestion_continuation_base_delay_seconds,
)
except ingestion_service.RunAlreadyInProgressError as exc:
await db_session.rollback()
raise ApiException(
status_code=409,
code="run_in_progress",
message="A run is already in progress for this account.",
) from exc
except Exception as exc:
await db_session.rollback()
logger.exception(
"api.runs.manual_failed",
extra={
"event": "api.runs.manual_failed",
"user_id": current_user.id,
},
)
raise ApiException(
status_code=500,
code="manual_run_failed",
message="Manual run failed.",
) from exc
if idempotency_key is not None:
await run_service.set_manual_run_idempotency_key(
db_session,
user_id=current_user.id,
run_id=run_summary.crawl_run_id,
idempotency_key=idempotency_key,
)
return success_payload(
request,
data={
"run_id": run_summary.crawl_run_id,
"status": run_summary.status.value,
"scholar_count": run_summary.scholar_count,
"succeeded_count": run_summary.succeeded_count,
"failed_count": run_summary.failed_count,
"partial_count": run_summary.partial_count,
"new_publication_count": run_summary.new_publication_count,
"reused_existing_run": False,
"idempotency_key": idempotency_key,
},
)
@router.get(
"/queue/items",
response_model=QueueListEnvelope,
)
async def list_queue_items(
request: Request,
limit: int = Query(default=200, ge=1, le=500),
db_session: AsyncSession = Depends(get_db_session),
current_user: User = Depends(get_api_current_user),
):
items = await run_service.list_queue_items_for_user(
db_session,
user_id=current_user.id,
limit=limit,
)
return success_payload(
request,
data={
"queue_items": [_serialize_queue_item(item) for item in items],
},
)
@router.post(
"/queue/{queue_item_id}/retry",
response_model=QueueItemEnvelope,
)
async def retry_queue_item(
queue_item_id: int,
request: Request,
db_session: AsyncSession = Depends(get_db_session),
current_user: User = Depends(get_api_current_user),
):
try:
queue_item = await run_service.retry_queue_item_for_user(
db_session,
user_id=current_user.id,
queue_item_id=queue_item_id,
)
except run_service.QueueTransitionError as exc:
raise ApiException(
status_code=409,
code=exc.code,
message=exc.message,
details={"current_status": exc.current_status},
) from exc
if queue_item is None:
raise ApiException(
status_code=404,
code="queue_item_not_found",
message="Queue item not found.",
)
return success_payload(
request,
data=_serialize_queue_item(queue_item),
)
@router.post(
"/queue/{queue_item_id}/drop",
response_model=QueueItemEnvelope,
)
async def drop_queue_item(
queue_item_id: int,
request: Request,
db_session: AsyncSession = Depends(get_db_session),
current_user: User = Depends(get_api_current_user),
):
try:
dropped = await run_service.drop_queue_item_for_user(
db_session,
user_id=current_user.id,
queue_item_id=queue_item_id,
)
except run_service.QueueTransitionError as exc:
raise ApiException(
status_code=409,
code=exc.code,
message=exc.message,
details={"current_status": exc.current_status},
) from exc
if dropped is None:
raise ApiException(
status_code=404,
code="queue_item_not_found",
message="Queue item not found.",
)
return success_payload(
request,
data=_serialize_queue_item(dropped),
)
@router.delete(
"/queue/{queue_item_id}",
response_model=QueueClearEnvelope,
)
async def clear_queue_item(
queue_item_id: int,
request: Request,
db_session: AsyncSession = Depends(get_db_session),
current_user: User = Depends(get_api_current_user),
):
try:
deleted = await run_service.clear_queue_item_for_user(
db_session,
user_id=current_user.id,
queue_item_id=queue_item_id,
)
except run_service.QueueTransitionError as exc:
raise ApiException(
status_code=409,
code=exc.code,
message=exc.message,
details={"current_status": exc.current_status},
) from exc
if deleted is None:
raise ApiException(
status_code=404,
code="queue_item_not_found",
message="Queue item not found.",
)
return success_payload(
request,
data={
"queue_item_id": deleted.queue_item_id,
"previous_status": deleted.previous_status,
"status": "cleared",
"message": "Queue item cleared.",
},
)

169
app/api/routers/scholars.py Normal file
View file

@ -0,0 +1,169 @@
from __future__ import annotations
import logging
from fastapi import APIRouter, Depends, Request
from sqlalchemy.ext.asyncio import AsyncSession
from app.api.deps import get_api_current_user
from app.api.errors import ApiException
from app.api.responses import success_payload
from app.api.schemas import (
MessageEnvelope,
ScholarCreateRequest,
ScholarEnvelope,
ScholarsListEnvelope,
)
from app.db.models import User
from app.db.session import get_db_session
from app.services import scholars as scholar_service
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/scholars", tags=["api-scholars"])
def _serialize_scholar(profile) -> dict[str, object]:
return {
"id": int(profile.id),
"scholar_id": profile.scholar_id,
"display_name": profile.display_name,
"is_enabled": bool(profile.is_enabled),
"baseline_completed": bool(profile.baseline_completed),
"last_run_dt": profile.last_run_dt,
"last_run_status": (
profile.last_run_status.value if profile.last_run_status is not None else None
),
}
@router.get(
"",
response_model=ScholarsListEnvelope,
)
async def list_scholars(
request: Request,
db_session: AsyncSession = Depends(get_db_session),
current_user: User = Depends(get_api_current_user),
):
scholars = await scholar_service.list_scholars_for_user(
db_session,
user_id=current_user.id,
)
return success_payload(
request,
data={
"scholars": [_serialize_scholar(profile) for profile in scholars],
},
)
@router.post(
"",
response_model=ScholarEnvelope,
status_code=201,
)
async def create_scholar(
payload: ScholarCreateRequest,
request: Request,
db_session: AsyncSession = Depends(get_db_session),
current_user: User = Depends(get_api_current_user),
):
try:
created = await scholar_service.create_scholar_for_user(
db_session,
user_id=current_user.id,
scholar_id=payload.scholar_id,
display_name=payload.display_name or "",
)
except scholar_service.ScholarServiceError as exc:
raise ApiException(
status_code=400,
code="invalid_scholar",
message=str(exc),
) from exc
logger.info(
"api.scholars.created",
extra={
"event": "api.scholars.created",
"user_id": current_user.id,
"scholar_profile_id": created.id,
},
)
return success_payload(
request,
data=_serialize_scholar(created),
)
@router.patch(
"/{scholar_profile_id}/toggle",
response_model=ScholarEnvelope,
)
async def toggle_scholar(
scholar_profile_id: int,
request: Request,
db_session: AsyncSession = Depends(get_db_session),
current_user: User = Depends(get_api_current_user),
):
profile = await scholar_service.get_user_scholar_by_id(
db_session,
user_id=current_user.id,
scholar_profile_id=scholar_profile_id,
)
if profile is None:
raise ApiException(
status_code=404,
code="scholar_not_found",
message="Scholar not found.",
)
updated = await scholar_service.toggle_scholar_enabled(db_session, profile=profile)
logger.info(
"api.scholars.toggled",
extra={
"event": "api.scholars.toggled",
"user_id": current_user.id,
"scholar_profile_id": updated.id,
"is_enabled": updated.is_enabled,
},
)
return success_payload(
request,
data=_serialize_scholar(updated),
)
@router.delete(
"/{scholar_profile_id}",
response_model=MessageEnvelope,
)
async def delete_scholar(
scholar_profile_id: int,
request: Request,
db_session: AsyncSession = Depends(get_db_session),
current_user: User = Depends(get_api_current_user),
):
profile = await scholar_service.get_user_scholar_by_id(
db_session,
user_id=current_user.id,
scholar_profile_id=scholar_profile_id,
)
if profile is None:
raise ApiException(
status_code=404,
code="scholar_not_found",
message="Scholar not found.",
)
await scholar_service.delete_scholar(db_session, profile=profile)
logger.info(
"api.scholars.deleted",
extra={
"event": "api.scholars.deleted",
"user_id": current_user.id,
"scholar_profile_id": scholar_profile_id,
},
)
return success_payload(
request,
data={"message": "Scholar deleted."},
)

View file

@ -0,0 +1,96 @@
from __future__ import annotations
import logging
from fastapi import APIRouter, Depends, Request
from sqlalchemy.ext.asyncio import AsyncSession
from app.api.deps import get_api_current_user
from app.api.errors import ApiException
from app.api.responses import success_payload
from app.api.schemas import SettingsEnvelope, SettingsUpdateRequest
from app.db.models import User
from app.db.session import get_db_session
from app.services import user_settings as user_settings_service
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/settings", tags=["api-settings"])
def _serialize_settings(settings) -> dict[str, object]:
return {
"auto_run_enabled": bool(settings.auto_run_enabled),
"run_interval_minutes": int(settings.run_interval_minutes),
"request_delay_seconds": int(settings.request_delay_seconds),
}
@router.get(
"",
response_model=SettingsEnvelope,
)
async def get_settings(
request: Request,
db_session: AsyncSession = Depends(get_db_session),
current_user: User = Depends(get_api_current_user),
):
settings = await user_settings_service.get_or_create_settings(
db_session,
user_id=current_user.id,
)
return success_payload(
request,
data=_serialize_settings(settings),
)
@router.put(
"",
response_model=SettingsEnvelope,
)
async def update_settings(
payload: SettingsUpdateRequest,
request: Request,
db_session: AsyncSession = Depends(get_db_session),
current_user: User = Depends(get_api_current_user),
):
try:
parsed_interval = user_settings_service.parse_run_interval_minutes(
str(payload.run_interval_minutes)
)
parsed_delay = user_settings_service.parse_request_delay_seconds(
str(payload.request_delay_seconds)
)
except user_settings_service.UserSettingsServiceError as exc:
raise ApiException(
status_code=400,
code="invalid_settings",
message=str(exc),
) from exc
settings = await user_settings_service.get_or_create_settings(
db_session,
user_id=current_user.id,
)
updated = await user_settings_service.update_settings(
db_session,
settings=settings,
auto_run_enabled=bool(payload.auto_run_enabled),
run_interval_minutes=parsed_interval,
request_delay_seconds=parsed_delay,
)
logger.info(
"api.settings.updated",
extra={
"event": "api.settings.updated",
"user_id": current_user.id,
"auto_run_enabled": updated.auto_run_enabled,
"run_interval_minutes": updated.run_interval_minutes,
"request_delay_seconds": updated.request_delay_seconds,
},
)
return success_payload(
request,
data=_serialize_settings(updated),
)

456
app/api/schemas.py Normal file
View file

@ -0,0 +1,456 @@
from __future__ import annotations
from datetime import datetime
from typing import Any
from pydantic import BaseModel, ConfigDict, Field
class ApiMeta(BaseModel):
request_id: str | None = None
model_config = ConfigDict(extra="forbid")
class ApiErrorData(BaseModel):
code: str
message: str
details: Any | None = None
model_config = ConfigDict(extra="forbid")
class ApiErrorEnvelope(BaseModel):
error: ApiErrorData
meta: ApiMeta
model_config = ConfigDict(extra="forbid")
class MessageData(BaseModel):
message: str
model_config = ConfigDict(extra="forbid")
class MessageEnvelope(BaseModel):
data: MessageData
meta: ApiMeta
model_config = ConfigDict(extra="forbid")
class SessionUserData(BaseModel):
id: int
email: str
is_admin: bool
is_active: bool
model_config = ConfigDict(extra="forbid")
class AuthMeData(BaseModel):
authenticated: bool
csrf_token: str
user: SessionUserData
model_config = ConfigDict(extra="forbid")
class AuthMeEnvelope(BaseModel):
data: AuthMeData
meta: ApiMeta
model_config = ConfigDict(extra="forbid")
class CsrfBootstrapData(BaseModel):
csrf_token: str
authenticated: bool
model_config = ConfigDict(extra="forbid")
class CsrfBootstrapEnvelope(BaseModel):
data: CsrfBootstrapData
meta: ApiMeta
model_config = ConfigDict(extra="forbid")
class LoginRequest(BaseModel):
email: str
password: str
model_config = ConfigDict(extra="forbid")
class LoginData(BaseModel):
authenticated: bool
csrf_token: str
user: SessionUserData
model_config = ConfigDict(extra="forbid")
class LoginEnvelope(BaseModel):
data: LoginData
meta: ApiMeta
model_config = ConfigDict(extra="forbid")
class ChangePasswordRequest(BaseModel):
current_password: str
new_password: str
confirm_password: str
model_config = ConfigDict(extra="forbid")
class ScholarItemData(BaseModel):
id: int
scholar_id: str
display_name: str | None
is_enabled: bool
baseline_completed: bool
last_run_dt: datetime | None
last_run_status: str | None
model_config = ConfigDict(extra="forbid")
class ScholarsListData(BaseModel):
scholars: list[ScholarItemData]
model_config = ConfigDict(extra="forbid")
class ScholarsListEnvelope(BaseModel):
data: ScholarsListData
meta: ApiMeta
model_config = ConfigDict(extra="forbid")
class ScholarCreateRequest(BaseModel):
scholar_id: str
display_name: str | None = None
model_config = ConfigDict(extra="forbid")
class ScholarEnvelope(BaseModel):
data: ScholarItemData
meta: ApiMeta
model_config = ConfigDict(extra="forbid")
class RunListItemData(BaseModel):
id: int
trigger_type: str
status: str
start_dt: datetime
end_dt: datetime | None
scholar_count: int
new_publication_count: int
failed_count: int
partial_count: int
model_config = ConfigDict(extra="forbid")
class RunsListData(BaseModel):
runs: list[RunListItemData]
model_config = ConfigDict(extra="forbid")
class RunsListEnvelope(BaseModel):
data: RunsListData
meta: ApiMeta
model_config = ConfigDict(extra="forbid")
class RunSummaryData(BaseModel):
succeeded_count: int
failed_count: int
partial_count: int
failed_state_counts: dict[str, int] = Field(default_factory=dict)
failed_reason_counts: dict[str, int] = Field(default_factory=dict)
model_config = ConfigDict(extra="forbid")
class RunAttemptLogData(BaseModel):
attempt: int
cstart: int
state: str | None = None
state_reason: str | None = None
status_code: int | None = None
fetch_error: str | None = None
model_config = ConfigDict(extra="forbid")
class RunPageLogData(BaseModel):
page: int
cstart: int
state: str
state_reason: str | None = None
status_code: int | None = None
publication_count: int = 0
attempt_count: int = 0
has_show_more_button: bool = False
articles_range: str | None = None
warning_codes: list[str] = Field(default_factory=list)
model_config = ConfigDict(extra="forbid")
class RunDebugData(BaseModel):
status_code: int | None = None
final_url: str | None = None
fetch_error: str | None = None
body_sha256: str | None = None
body_length: int | None = None
has_show_more_button: bool | None = None
articles_range: str | None = None
state_reason: str | None = None
warning_codes: list[str] = Field(default_factory=list)
marker_counts_nonzero: dict[str, int] = Field(default_factory=dict)
page_logs: list[RunPageLogData] = Field(default_factory=list)
attempt_log: list[RunAttemptLogData] = Field(default_factory=list)
model_config = ConfigDict(extra="forbid")
class RunScholarResultData(BaseModel):
scholar_profile_id: int
scholar_id: str
state: str
state_reason: str | None = None
outcome: str
attempt_count: int = 0
publication_count: int = 0
start_cstart: int = 0
continuation_cstart: int | None = None
continuation_enqueued: bool = False
continuation_cleared: bool = False
warnings: list[str] = Field(default_factory=list)
error: str | None = None
debug: RunDebugData | None = None
model_config = ConfigDict(extra="forbid")
class RunDetailData(BaseModel):
run: RunListItemData
summary: RunSummaryData
scholar_results: list[RunScholarResultData]
model_config = ConfigDict(extra="forbid")
class RunDetailEnvelope(BaseModel):
data: RunDetailData
meta: ApiMeta
model_config = ConfigDict(extra="forbid")
class ManualRunData(BaseModel):
run_id: int
status: str
scholar_count: int
succeeded_count: int
failed_count: int
partial_count: int
new_publication_count: int
reused_existing_run: bool
idempotency_key: str | None = None
model_config = ConfigDict(extra="forbid")
class ManualRunEnvelope(BaseModel):
data: ManualRunData
meta: ApiMeta
model_config = ConfigDict(extra="forbid")
class QueueItemData(BaseModel):
id: int
scholar_profile_id: int
scholar_label: str
status: str
reason: str
dropped_reason: str | None
attempt_count: int
resume_cstart: int
next_attempt_dt: datetime | None
updated_at: datetime
last_error: str | None
last_run_id: int | None
model_config = ConfigDict(extra="forbid")
class QueueListData(BaseModel):
queue_items: list[QueueItemData]
model_config = ConfigDict(extra="forbid")
class QueueListEnvelope(BaseModel):
data: QueueListData
meta: ApiMeta
model_config = ConfigDict(extra="forbid")
class QueueItemEnvelope(BaseModel):
data: QueueItemData
meta: ApiMeta
model_config = ConfigDict(extra="forbid")
class QueueClearData(BaseModel):
queue_item_id: int
previous_status: str
status: str
message: str
model_config = ConfigDict(extra="forbid")
class QueueClearEnvelope(BaseModel):
data: QueueClearData
meta: ApiMeta
model_config = ConfigDict(extra="forbid")
class AdminUserData(BaseModel):
id: int
email: str
is_active: bool
is_admin: bool
created_at: datetime
updated_at: datetime
model_config = ConfigDict(extra="forbid")
class AdminUsersListData(BaseModel):
users: list[AdminUserData]
model_config = ConfigDict(extra="forbid")
class AdminUsersListEnvelope(BaseModel):
data: AdminUsersListData
meta: ApiMeta
model_config = ConfigDict(extra="forbid")
class AdminUserCreateRequest(BaseModel):
email: str
password: str
is_admin: bool = False
model_config = ConfigDict(extra="forbid")
class AdminUserActiveUpdateRequest(BaseModel):
is_active: bool
model_config = ConfigDict(extra="forbid")
class AdminUserEnvelope(BaseModel):
data: AdminUserData
meta: ApiMeta
model_config = ConfigDict(extra="forbid")
class AdminResetPasswordRequest(BaseModel):
new_password: str
model_config = ConfigDict(extra="forbid")
class SettingsData(BaseModel):
auto_run_enabled: bool
run_interval_minutes: int
request_delay_seconds: int
model_config = ConfigDict(extra="forbid")
class SettingsEnvelope(BaseModel):
data: SettingsData
meta: ApiMeta
model_config = ConfigDict(extra="forbid")
class SettingsUpdateRequest(BaseModel):
auto_run_enabled: bool
run_interval_minutes: int
request_delay_seconds: int
model_config = ConfigDict(extra="forbid")
class PublicationItemData(BaseModel):
publication_id: int
scholar_profile_id: int
scholar_label: str
title: str
year: int | None
citation_count: int
venue_text: str | None
pub_url: str | None
is_read: bool
first_seen_at: datetime
is_new_in_latest_run: bool
model_config = ConfigDict(extra="forbid")
class PublicationsListData(BaseModel):
mode: str
selected_scholar_profile_id: int | None
new_count: int
total_count: int
publications: list[PublicationItemData]
model_config = ConfigDict(extra="forbid")
class PublicationsListEnvelope(BaseModel):
data: PublicationsListData
meta: ApiMeta
model_config = ConfigDict(extra="forbid")
class MarkAllReadData(BaseModel):
message: str
updated_count: int
model_config = ConfigDict(extra="forbid")
class MarkAllReadEnvelope(BaseModel):
data: MarkAllReadData
meta: ApiMeta
model_config = ConfigDict(extra="forbid")