From 4433d7d2c4a1009b95cb41927176290e71d2384d Mon Sep 17 00:00:00 2001 From: Justin Visser Date: Tue, 17 Feb 2026 14:51:25 +0100 Subject: [PATCH] First product --- .dockerignore | 14 + .env.example | 45 + .github/workflows/ci.yml | 53 + .gitignore | 12 + .python-version | 2 + Dockerfile | 54 + README.md | 206 +++ alembic.ini | 37 + alembic/env.py | 66 + alembic/script.py.mako | 27 + alembic/versions/.gitkeep | 1 + .../20260216_0001_multi_user_schema.py | 353 +++++ .../20260216_0002_add_user_admin_flag.py | 34 + .../20260216_0003_add_ingestion_queue.py | 94 ++ .../20260217_0004_queue_status_fields.py | 96 ++ app/__init__.py | 1 + app/api/__init__.py | 2 + app/api/deps.py | 36 + app/api/errors.py | 85 ++ app/api/responses.py | 64 + app/api/router.py | 13 + app/api/routers/__init__.py | 2 + app/api/routers/admin.py | 202 +++ app/api/routers/auth.py | 230 ++++ app/api/routers/publications.py | 123 ++ app/api/routers/runs.py | 560 ++++++++ app/api/routers/scholars.py | 169 +++ app/api/routers/settings.py | 96 ++ app/api/schemas.py | 456 +++++++ app/auth/__init__.py | 2 + app/auth/deps.py | 27 + app/auth/rate_limit.py | 62 + app/auth/security.py | 19 + app/auth/service.py | 38 + app/auth/session.py | 50 + app/db/__init__.py | 1 + app/db/base.py | 18 + app/db/models.py | 292 +++++ app/db/session.py | 64 + app/logging_config.py | 183 +++ app/logging_context.py | 15 + app/main.py | 87 ++ app/presentation/__init__.py | 2 + app/presentation/dashboard.py | 108 ++ app/security/__init__.py | 2 + app/security/csrf.py | 135 ++ app/services/__init__.py | 2 + app/services/continuation_queue.py | 278 ++++ app/services/ingestion.py | 967 ++++++++++++++ app/services/publications.py | 297 +++++ app/services/runs.py | 464 +++++++ app/services/scheduler.py | 478 +++++++ app/services/scholar_parser.py | 370 ++++++ app/services/scholar_source.py | 171 +++ app/services/scholars.py | 98 ++ app/services/user_settings.py | 66 + app/services/users.py | 98 ++ app/settings.py | 84 ++ app/static/app.css | 501 +++++++ app/static/app.js | 76 ++ app/static/theme.css | 98 ++ app/static/theme.js | 56 + app/templates/account_password.html | 23 + app/templates/base.html | 66 + app/templates/dashboard/_run_controls.html | 23 + app/templates/dashboard/_run_history.html | 38 + .../dashboard/_unread_publications.html | 47 + app/templates/index.html | 27 + app/templates/login.html | 23 + app/templates/publications.html | 104 ++ app/templates/run_detail.html | 98 ++ app/templates/runs.html | 129 ++ app/templates/scholars.html | 99 ++ app/templates/settings.html | 44 + app/templates/users.html | 75 ++ app/theme.py | 24 + app/web/__init__.py | 2 + app/web/common.py | 140 ++ app/web/deps.py | 17 + app/web/middleware.py | 85 ++ app/web/routers/__init__.py | 2 + app/web/routers/admin.py | 209 +++ app/web/routers/auth.py | 214 +++ app/web/routers/dashboard.py | 167 +++ app/web/routers/health.py | 15 + app/web/routers/publications.py | 165 +++ app/web/routers/runs.py | 274 ++++ app/web/routers/scholars.py | 212 +++ app/web/routers/settings.py | 94 ++ docker-compose.yml | 68 + planning/mvp_implementation_reminders.md | 92 ++ planning/scholar_probe_tmp/README.md | 13 + .../profile_AAAAAAAAAAAA.html | 421 ++++++ .../profile_LZ5D_p4AAAAJ.html | 76 ++ .../profile_P1RwlvoAAAAJ.html | 76 ++ .../profile_RxmmtT8AAAAJ.html | 76 ++ .../profile_amIMrIEAAAAJ.html | 76 ++ .../fixtures/run_20260216T182029Z/robots.txt | 24 + .../profile_AAAAAAAAAAAA.html | 421 ++++++ .../profile_LZ5D_p4AAAAJ.html | 76 ++ .../profile_P1RwlvoAAAAJ.html | 76 ++ .../profile_RxmmtT8AAAAJ.html | 76 ++ .../profile_amIMrIEAAAAJ.html | 76 ++ .../fixtures/run_20260216T182157Z/robots.txt | 24 + .../profile_AAAAAAAAAAAA.html | 421 ++++++ .../profile_LZ5D_p4AAAAJ.html | 76 ++ .../profile_P1RwlvoAAAAJ.html | 76 ++ .../profile_RxmmtT8AAAAJ.html | 76 ++ .../profile_amIMrIEAAAAJ.html | 76 ++ .../fixtures/run_20260216T182334Z/robots.txt | 24 + .../notes/findings_phase1_input.md | 71 + .../probe_report_run_20260216T182029Z.json | 210 +++ .../probe_report_run_20260216T182029Z.md | 69 + .../probe_report_run_20260216T182142Z.json | 658 ++++++++++ .../probe_report_run_20260216T182142Z.md | 76 ++ .../probe_report_run_20260216T182157Z.json | 719 ++++++++++ .../probe_report_run_20260216T182157Z.md | 82 ++ .../probe_report_run_20260216T182303Z.json | 831 ++++++++++++ .../probe_report_run_20260216T182303Z.md | 83 ++ .../probe_report_run_20260216T182320Z.json | 681 ++++++++++ .../probe_report_run_20260216T182320Z.md | 78 ++ .../probe_report_run_20260216T182334Z.json | 742 +++++++++++ .../probe_report_run_20260216T182334Z.md | 84 ++ .../notes/seed_scholar_ids.txt | 6 + .../scripts/scholar_probe.py | 777 +++++++++++ pyproject.toml | 47 + scripts/bootstrap_admin.py | 126 ++ scripts/entrypoint.sh | 22 + scripts/smoke_compose.sh | 27 + scripts/wait_for_db.py | 68 + tests/__init__.py | 2 + tests/conftest.py | 171 +++ .../scholar/profile_ok_amIMrIEAAAAJ.html | 76 ++ .../regression/profile_AAAAAAAAAAAA.html | 421 ++++++ .../regression/profile_LZ5D_p4AAAAJ.html | 76 ++ .../regression/profile_P1RwlvoAAAAJ.html | 76 ++ tests/integration/__init__.py | 2 + tests/integration/helpers.py | 63 + .../integration/test_admin_user_management.py | 200 +++ tests/integration/test_api_v1.py | 518 ++++++++ tests/integration/test_auth_flow.py | 109 ++ .../integration/test_manual_ingestion_flow.py | 1151 +++++++++++++++++ tests/integration/test_migrations.py | 93 ++ tests/integration/test_multi_user_schema.py | 241 ++++ tests/integration/test_queue_ui.py | 227 ++++ tests/integration/test_tenant_routes.py | 197 +++ tests/smoke/test_db_connectivity.py | 23 + tests/smoke/test_migration_schema.py | 46 + tests/unit/test_auth.py | 204 +++ tests/unit/test_auth_primitives.py | 34 + tests/unit/test_dashboard_view_model.py | 88 ++ tests/unit/test_healthz.py | 25 + tests/unit/test_logging.py | 52 + tests/unit/test_pages.py | 54 + tests/unit/test_phase2_pages.py | 20 + tests/unit/test_scholar_parser.py | 240 ++++ uv.lock | 931 +++++++++++++ 157 files changed, 23975 insertions(+) create mode 100644 .dockerignore create mode 100644 .env.example create mode 100644 .github/workflows/ci.yml create mode 100644 .gitignore create mode 100644 .python-version create mode 100644 Dockerfile create mode 100644 README.md create mode 100644 alembic.ini create mode 100644 alembic/env.py create mode 100644 alembic/script.py.mako create mode 100644 alembic/versions/.gitkeep create mode 100644 alembic/versions/20260216_0001_multi_user_schema.py create mode 100644 alembic/versions/20260216_0002_add_user_admin_flag.py create mode 100644 alembic/versions/20260216_0003_add_ingestion_queue.py create mode 100644 alembic/versions/20260217_0004_queue_status_fields.py create mode 100644 app/__init__.py create mode 100644 app/api/__init__.py create mode 100644 app/api/deps.py create mode 100644 app/api/errors.py create mode 100644 app/api/responses.py create mode 100644 app/api/router.py create mode 100644 app/api/routers/__init__.py create mode 100644 app/api/routers/admin.py create mode 100644 app/api/routers/auth.py create mode 100644 app/api/routers/publications.py create mode 100644 app/api/routers/runs.py create mode 100644 app/api/routers/scholars.py create mode 100644 app/api/routers/settings.py create mode 100644 app/api/schemas.py create mode 100644 app/auth/__init__.py create mode 100644 app/auth/deps.py create mode 100644 app/auth/rate_limit.py create mode 100644 app/auth/security.py create mode 100644 app/auth/service.py create mode 100644 app/auth/session.py create mode 100644 app/db/__init__.py create mode 100644 app/db/base.py create mode 100644 app/db/models.py create mode 100644 app/db/session.py create mode 100644 app/logging_config.py create mode 100644 app/logging_context.py create mode 100644 app/main.py create mode 100644 app/presentation/__init__.py create mode 100644 app/presentation/dashboard.py create mode 100644 app/security/__init__.py create mode 100644 app/security/csrf.py create mode 100644 app/services/__init__.py create mode 100644 app/services/continuation_queue.py create mode 100644 app/services/ingestion.py create mode 100644 app/services/publications.py create mode 100644 app/services/runs.py create mode 100644 app/services/scheduler.py create mode 100644 app/services/scholar_parser.py create mode 100644 app/services/scholar_source.py create mode 100644 app/services/scholars.py create mode 100644 app/services/user_settings.py create mode 100644 app/services/users.py create mode 100644 app/settings.py create mode 100644 app/static/app.css create mode 100644 app/static/app.js create mode 100644 app/static/theme.css create mode 100644 app/static/theme.js create mode 100644 app/templates/account_password.html create mode 100644 app/templates/base.html create mode 100644 app/templates/dashboard/_run_controls.html create mode 100644 app/templates/dashboard/_run_history.html create mode 100644 app/templates/dashboard/_unread_publications.html create mode 100644 app/templates/index.html create mode 100644 app/templates/login.html create mode 100644 app/templates/publications.html create mode 100644 app/templates/run_detail.html create mode 100644 app/templates/runs.html create mode 100644 app/templates/scholars.html create mode 100644 app/templates/settings.html create mode 100644 app/templates/users.html create mode 100644 app/theme.py create mode 100644 app/web/__init__.py create mode 100644 app/web/common.py create mode 100644 app/web/deps.py create mode 100644 app/web/middleware.py create mode 100644 app/web/routers/__init__.py create mode 100644 app/web/routers/admin.py create mode 100644 app/web/routers/auth.py create mode 100644 app/web/routers/dashboard.py create mode 100644 app/web/routers/health.py create mode 100644 app/web/routers/publications.py create mode 100644 app/web/routers/runs.py create mode 100644 app/web/routers/scholars.py create mode 100644 app/web/routers/settings.py create mode 100644 docker-compose.yml create mode 100644 planning/mvp_implementation_reminders.md create mode 100644 planning/scholar_probe_tmp/README.md create mode 100644 planning/scholar_probe_tmp/fixtures/run_20260216T182029Z/profile_AAAAAAAAAAAA.html create mode 100644 planning/scholar_probe_tmp/fixtures/run_20260216T182029Z/profile_LZ5D_p4AAAAJ.html create mode 100644 planning/scholar_probe_tmp/fixtures/run_20260216T182029Z/profile_P1RwlvoAAAAJ.html create mode 100644 planning/scholar_probe_tmp/fixtures/run_20260216T182029Z/profile_RxmmtT8AAAAJ.html create mode 100644 planning/scholar_probe_tmp/fixtures/run_20260216T182029Z/profile_amIMrIEAAAAJ.html create mode 100644 planning/scholar_probe_tmp/fixtures/run_20260216T182029Z/robots.txt create mode 100644 planning/scholar_probe_tmp/fixtures/run_20260216T182157Z/profile_AAAAAAAAAAAA.html create mode 100644 planning/scholar_probe_tmp/fixtures/run_20260216T182157Z/profile_LZ5D_p4AAAAJ.html create mode 100644 planning/scholar_probe_tmp/fixtures/run_20260216T182157Z/profile_P1RwlvoAAAAJ.html create mode 100644 planning/scholar_probe_tmp/fixtures/run_20260216T182157Z/profile_RxmmtT8AAAAJ.html create mode 100644 planning/scholar_probe_tmp/fixtures/run_20260216T182157Z/profile_amIMrIEAAAAJ.html create mode 100644 planning/scholar_probe_tmp/fixtures/run_20260216T182157Z/robots.txt create mode 100644 planning/scholar_probe_tmp/fixtures/run_20260216T182334Z/profile_AAAAAAAAAAAA.html create mode 100644 planning/scholar_probe_tmp/fixtures/run_20260216T182334Z/profile_LZ5D_p4AAAAJ.html create mode 100644 planning/scholar_probe_tmp/fixtures/run_20260216T182334Z/profile_P1RwlvoAAAAJ.html create mode 100644 planning/scholar_probe_tmp/fixtures/run_20260216T182334Z/profile_RxmmtT8AAAAJ.html create mode 100644 planning/scholar_probe_tmp/fixtures/run_20260216T182334Z/profile_amIMrIEAAAAJ.html create mode 100644 planning/scholar_probe_tmp/fixtures/run_20260216T182334Z/robots.txt create mode 100644 planning/scholar_probe_tmp/notes/findings_phase1_input.md create mode 100644 planning/scholar_probe_tmp/notes/probe_report_run_20260216T182029Z.json create mode 100644 planning/scholar_probe_tmp/notes/probe_report_run_20260216T182029Z.md create mode 100644 planning/scholar_probe_tmp/notes/probe_report_run_20260216T182142Z.json create mode 100644 planning/scholar_probe_tmp/notes/probe_report_run_20260216T182142Z.md create mode 100644 planning/scholar_probe_tmp/notes/probe_report_run_20260216T182157Z.json create mode 100644 planning/scholar_probe_tmp/notes/probe_report_run_20260216T182157Z.md create mode 100644 planning/scholar_probe_tmp/notes/probe_report_run_20260216T182303Z.json create mode 100644 planning/scholar_probe_tmp/notes/probe_report_run_20260216T182303Z.md create mode 100644 planning/scholar_probe_tmp/notes/probe_report_run_20260216T182320Z.json create mode 100644 planning/scholar_probe_tmp/notes/probe_report_run_20260216T182320Z.md create mode 100644 planning/scholar_probe_tmp/notes/probe_report_run_20260216T182334Z.json create mode 100644 planning/scholar_probe_tmp/notes/probe_report_run_20260216T182334Z.md create mode 100644 planning/scholar_probe_tmp/notes/seed_scholar_ids.txt create mode 100755 planning/scholar_probe_tmp/scripts/scholar_probe.py create mode 100644 pyproject.toml create mode 100644 scripts/bootstrap_admin.py create mode 100755 scripts/entrypoint.sh create mode 100755 scripts/smoke_compose.sh create mode 100644 scripts/wait_for_db.py create mode 100644 tests/__init__.py create mode 100644 tests/conftest.py create mode 100644 tests/fixtures/scholar/profile_ok_amIMrIEAAAAJ.html create mode 100644 tests/fixtures/scholar/regression/profile_AAAAAAAAAAAA.html create mode 100644 tests/fixtures/scholar/regression/profile_LZ5D_p4AAAAJ.html create mode 100644 tests/fixtures/scholar/regression/profile_P1RwlvoAAAAJ.html create mode 100644 tests/integration/__init__.py create mode 100644 tests/integration/helpers.py create mode 100644 tests/integration/test_admin_user_management.py create mode 100644 tests/integration/test_api_v1.py create mode 100644 tests/integration/test_auth_flow.py create mode 100644 tests/integration/test_manual_ingestion_flow.py create mode 100644 tests/integration/test_migrations.py create mode 100644 tests/integration/test_multi_user_schema.py create mode 100644 tests/integration/test_queue_ui.py create mode 100644 tests/integration/test_tenant_routes.py create mode 100644 tests/smoke/test_db_connectivity.py create mode 100644 tests/smoke/test_migration_schema.py create mode 100644 tests/unit/test_auth.py create mode 100644 tests/unit/test_auth_primitives.py create mode 100644 tests/unit/test_dashboard_view_model.py create mode 100644 tests/unit/test_healthz.py create mode 100644 tests/unit/test_logging.py create mode 100644 tests/unit/test_pages.py create mode 100644 tests/unit/test_phase2_pages.py create mode 100644 tests/unit/test_scholar_parser.py create mode 100644 uv.lock diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..d32c9e5 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,14 @@ +.git +__pycache__ +.pytest_cache +.mypy_cache +.ruff_cache +.venv +.uv-cache +*.pyc +*.pyo +*.pyd +*.egg-info +*.log +.env +htmlcov diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..82480f1 --- /dev/null +++ b/.env.example @@ -0,0 +1,45 @@ +POSTGRES_DB=scholar +POSTGRES_USER=scholar +POSTGRES_PASSWORD=scholar +POSTGRES_PORT=5432 + +DATABASE_URL=postgresql+asyncpg://scholar:scholar@db:5432/scholar +# Optional override. If empty, tests derive "_test". +TEST_DATABASE_URL= + +APP_NAME=scholarr +APP_HOST=0.0.0.0 +APP_PORT=8000 +APP_HOST_PORT=8000 +APP_RELOAD=1 +MIGRATE_ON_START=1 + +SESSION_SECRET_KEY=replace-with-a-long-random-secret +SESSION_COOKIE_SECURE=0 +LOGIN_RATE_LIMIT_ATTEMPTS=5 +LOGIN_RATE_LIMIT_WINDOW_SECONDS=60 +LOG_LEVEL=INFO +LOG_FORMAT=console +LOG_REQUESTS=1 +LOG_UVICORN_ACCESS=0 +LOG_REQUEST_SKIP_PATHS=/healthz,/static/ +LOG_REDACT_FIELDS= +SCHEDULER_ENABLED=1 +SCHEDULER_TICK_SECONDS=60 +INGESTION_NETWORK_ERROR_RETRIES=1 +INGESTION_RETRY_BACKOFF_SECONDS=1.0 +INGESTION_MAX_PAGES_PER_SCHOLAR=30 +INGESTION_PAGE_SIZE=100 +INGESTION_CONTINUATION_QUEUE_ENABLED=1 +INGESTION_CONTINUATION_BASE_DELAY_SECONDS=120 +INGESTION_CONTINUATION_MAX_DELAY_SECONDS=3600 +INGESTION_CONTINUATION_MAX_ATTEMPTS=6 +SCHEDULER_QUEUE_BATCH_SIZE=10 + +BOOTSTRAP_ADMIN_ON_START=0 +BOOTSTRAP_ADMIN_EMAIL= +BOOTSTRAP_ADMIN_PASSWORD= +BOOTSTRAP_ADMIN_FORCE_PASSWORD=0 + +DB_WAIT_TIMEOUT_SECONDS=60 +DB_WAIT_INTERVAL_SECONDS=2 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..d5e6592 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,53 @@ +name: CI + +on: + pull_request: + push: + branches: + - main + +jobs: + test: + runs-on: ubuntu-latest + services: + postgres: + image: postgres:15 + env: + POSTGRES_DB: scholar + POSTGRES_USER: scholar + POSTGRES_PASSWORD: scholar + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U scholar -d scholar" + --health-interval 5s + --health-timeout 5s + --health-retries 20 + + env: + DATABASE_URL: postgresql+asyncpg://scholar:scholar@localhost:5432/scholar + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Setup uv + uses: astral-sh/setup-uv@v4 + + - name: Install dependencies + run: uv sync --extra dev + + - name: Migration smoke + run: uv run alembic upgrade head + + - name: Unit tests + run: uv run pytest tests/unit + + - name: Integration tests + run: uv run pytest -m integration + diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f0a8f4e --- /dev/null +++ b/.gitignore @@ -0,0 +1,12 @@ +__pycache__/ +*.py[cod] +*.egg-info/ +.pytest_cache/ +.mypy_cache/ +.ruff_cache/ +.coverage +htmlcov/ +.venv/ +.uv-cache/ +.env +*.log diff --git a/.python-version b/.python-version new file mode 100644 index 0000000..e0aad9e --- /dev/null +++ b/.python-version @@ -0,0 +1,2 @@ +3.12 + diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..bfa7e00 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,54 @@ +FROM python:3.12-slim AS base + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + UV_LINK_MODE=copy \ + UV_PROJECT_ENVIRONMENT=/opt/venv \ + PATH="/opt/venv/bin:$PATH" + +WORKDIR /app + +RUN apt-get update \ + && apt-get install -y --no-install-recommends build-essential curl \ + && rm -rf /var/lib/apt/lists/* + +COPY --from=ghcr.io/astral-sh/uv:0.6.5 /uv /uvx /bin/ + +COPY pyproject.toml uv.lock README.md ./ +COPY app ./app +COPY alembic.ini ./alembic.ini +COPY alembic ./alembic +COPY scripts ./scripts + +RUN uv sync --frozen --extra dev + +FROM base AS dev +ENTRYPOINT ["/bin/sh", "/app/scripts/entrypoint.sh"] + +FROM python:3.12-slim AS prod + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + UV_LINK_MODE=copy \ + UV_PROJECT_ENVIRONMENT=/opt/venv \ + PATH="/opt/venv/bin:$PATH" \ + APP_RELOAD=0 \ + UVICORN_WORKERS=1 + +WORKDIR /app + +RUN apt-get update \ + && apt-get install -y --no-install-recommends curl \ + && rm -rf /var/lib/apt/lists/* + +COPY --from=ghcr.io/astral-sh/uv:0.6.5 /uv /uvx /bin/ + +COPY pyproject.toml uv.lock README.md ./ +COPY app ./app +COPY alembic.ini ./alembic.ini +COPY alembic ./alembic + +RUN uv sync --frozen + +COPY scripts ./scripts +ENTRYPOINT ["/bin/sh", "/app/scripts/entrypoint.sh"] diff --git a/README.md b/README.md new file mode 100644 index 0000000..21c0da2 --- /dev/null +++ b/README.md @@ -0,0 +1,206 @@ +# scholarr + +Container-first, self-hosted scholar tracking in the spirit of the `*arr` ecosystem, with strict multi-user isolation and an intentionally small operational footprint. + +## Current Scope + +- Admin-managed users only (no public signup) +- Session auth with CSRF protection and login rate limiting +- Per-user scholar tracking list (add, enable/disable, delete) +- Per-user automation settings +- Manual per-user ingestion runs from dashboard +- Per-user run history and "new since last run" publication stream +- Publications library view with `new` and `all` modes plus scholar filtering +- Dedicated run diagnostics pages (`/runs`, `/runs/{id}`) with failed-only filtering +- Continuation queue visibility + controls on runs page (`retry`, `drop`, `clear`) +- Baseline-aware discovery tracking (`new` = discovered in latest completed run) +- Read state is user-controlled (`Mark All Read`), including baseline items +- Multi-page profile ingestion (`cstart`) to capture full publication lists +- Dashboard rendered via a presentation-layer view model and section template registry +- Structured application logging with request IDs and field redaction +- Ingestion overlap guard via Postgres advisory locks +- Network-error retry support for ingestion attempts +- Automatic continuation queue for resumable/limit-hit scholar runs +- Background scheduler for per-user automatic runs +- User self-service password changes +- Admin user management (create users, activate/deactivate, reset passwords) +- PostgreSQL + Alembic migrations + containerized test workflow +- Versioned backend API (`/api/v1`) for frontend decoupling (same-origin cookie session) + +## Tech Stack + +- Python 3.12+ +- FastAPI +- PostgreSQL 15+ +- SQLAlchemy 2 + Alembic +- Jinja2 templates + modular theme tokens +- `uv` for dependency and runtime workflow +- Docker / Docker Compose for development and CI parity + +## Quick Start + +1. Copy environment defaults: + +```bash +cp .env.example .env +``` + +2. Bootstrap the first admin on startup (recommended for first run): + +```bash +export BOOTSTRAP_ADMIN_ON_START=1 +export BOOTSTRAP_ADMIN_EMAIL=admin@example.com +export BOOTSTRAP_ADMIN_PASSWORD=change-me-now +``` + +3. Start the stack: + +```bash +docker compose up --build +``` + +4. Open: + +- Login: `http://localhost:8000/login` +- Dashboard: `http://localhost:8000/` +- Healthcheck: `http://localhost:8000/healthz` + +5. After first successful admin login, disable auto-bootstrap: + +```bash +export BOOTSTRAP_ADMIN_ON_START=0 +``` + +## Admin Bootstrap (Manual) + +You can create/update an admin account without restarting the app: + +```bash +docker compose run --rm app uv run python scripts/bootstrap_admin.py \ + --email admin@example.com \ + --password change-me-now \ + --force-password +``` + +Notes: +- If the user does not exist, this creates an active admin account. +- If the user exists, it ensures admin + active flags. +- `--force-password` resets the password for existing users. + +## Test Workflow + +- Integration/smoke tests run against `TEST_DATABASE_URL`. +- If `TEST_DATABASE_URL` is empty, test runs derive an isolated database from `DATABASE_URL` by appending `_test`. +- This keeps app data in `DATABASE_URL` untouched during test runs. +- Parser regression tests are pinned to captured real-world fixture pages in `tests/fixtures/scholar/regression`. + +- Unit tests: + +```bash +docker compose run --rm app uv run pytest tests/unit +``` + +- Integration tests: + +```bash +docker compose run --rm app uv run pytest -m integration +``` + +- Smoke checks (container + DB + migration sanity): + +```bash +./scripts/smoke_compose.sh +``` + +CI runs migration, unit, and integration checks on push/PR via `.github/workflows/ci.yml`. + +## Scholar Ingestion Notes + +- Current ingestion targets public profile pages: `/citations?user=`. +- Runs are intentionally conservative and prioritize reliability over aggressive scraping. +- Ingestion follows pagination (`cstart`) to collect all reachable profile pages. +- If pagination cannot be completed (max pages hit, cursor stall, or page-state failure), the scholar result is marked partial with explicit debug context. +- Resumable partial/failure states are automatically queued and retried by the scheduler with bounded backoff. +- Queue items keep explicit status (`queued`, `retrying`, `dropped`) and last-error context for UI diagnostics. +- Failed scholar attempts persist structured debug context in `crawl_runs.error_log` (state reason, fetch metadata, marker evidence, and compact response excerpt). +- Only resumable states are retried automatically (`network_error` and continuation-eligible pagination truncations); blocked/layout states are recorded immediately as failures. + +## API v1 (Frontend Prep) + +- Base path: `/api/v1` +- Auth model: same-origin cookie session (no token auth added) +- CSRF model: required for unsafe methods (`POST`, `PUT`, `PATCH`, `DELETE`) via `X-CSRF-Token` header. +- Bootstrap CSRF via `GET /api/v1/auth/csrf` (works for anonymous and authenticated sessions). +- You can also fetch `csrf_token` from `GET /api/v1/auth/me` after login. +- Manual run idempotency is supported via `Idempotency-Key` header on `POST /api/v1/runs/manual`. +- Response envelopes: + - Success: `{"data": ..., "meta": {"request_id": "..."}}` + - Error: `{"error": {"code": "...", "message": "...", "details": ...}, "meta": {"request_id": "..."}}` +- Initial API domains exposed: + - `auth`: `/auth/csrf`, `/auth/login`, `/auth/me`, `/auth/change-password`, `/auth/logout` + - `admin users`: `/admin/users` (list/create), `/admin/users/{id}/active`, `/admin/users/{id}/reset-password` + - `scholars`: list/create/toggle/delete + - `settings`: get/update + - `runs`: list/detail/manual trigger + queue list/retry/drop/clear + - `publications`: list + mark-all-read +- Queue action semantics: + - `retry` is valid only for dropped items; retrying/queued states return `409`. + - `drop` returns the updated queue item (status `dropped`); dropping an already dropped item returns `409`. + - `clear` only works for dropped items and returns `{queue_item_id, previous_status, status="cleared"}`. + +## Logging + +- Default output is concise console logs to stdout. +- Timestamp format is compact UTC: `YYYY-MM-DD HH:MM:SSZ`. +- Every request gets an `X-Request-ID` (propagated if caller provides one). +- Sensitive fields are redacted before log emission (`password`, `csrf_token`, `cookie`, etc.). +- Configure via env: + - `LOG_LEVEL` (default `INFO`) + - `LOG_FORMAT` (`console` or `json`, default `console`) + - `LOG_REQUESTS` (`1`/`0`, default `1`) + - `LOG_UVICORN_ACCESS` (`1`/`0`, default `0`) + - `LOG_REQUEST_SKIP_PATHS` (comma-separated prefixes, default `/healthz,/static/`) + - `LOG_REDACT_FIELDS` (comma-separated additional keys) + - `SCHEDULER_ENABLED` (`1`/`0`, default `1`) + - `SCHEDULER_TICK_SECONDS` (default `60`) + - `INGESTION_NETWORK_ERROR_RETRIES` (default `1`) + - `INGESTION_RETRY_BACKOFF_SECONDS` (default `1.0`) + - `INGESTION_MAX_PAGES_PER_SCHOLAR` (default `30`) + - `INGESTION_PAGE_SIZE` (default `100`) + - `INGESTION_CONTINUATION_QUEUE_ENABLED` (`1`/`0`, default `1`) + - `INGESTION_CONTINUATION_BASE_DELAY_SECONDS` (default `120`) + - `INGESTION_CONTINUATION_MAX_DELAY_SECONDS` (default `3600`) + - `INGESTION_CONTINUATION_MAX_ATTEMPTS` (default `6`) + - `SCHEDULER_QUEUE_BATCH_SIZE` (default `10`) + +## Optional Local `uv` Workflow + +```bash +uv sync --extra dev +uv run pytest +uv run uvicorn app.main:app --reload +``` + +Update the lockfile after dependency changes: + +```bash +uv lock +``` + +## Theming + +- Theme registry: `app/theme.py` +- Semantic tokens: `app/static/theme.css` +- App styling: `app/static/app.css` +- Theme persistence: `app/static/theme.js` +- Dashboard presentation contract: `app/presentation/dashboard.py` + +## Project Layout + +```text +app/ FastAPI app, auth/security modules, templates, services, DB wiring, presentation view-models +app/web/ Router modules, shared web helpers, and request middleware +alembic/ Migration environment and versions +scripts/ Entrypoint, DB wait, bootstrap, smoke automation +tests/ Unit, integration, and smoke suites +``` diff --git a/alembic.ini b/alembic.ini new file mode 100644 index 0000000..f5ccb1d --- /dev/null +++ b/alembic.ini @@ -0,0 +1,37 @@ +[alembic] +script_location = alembic +prepend_sys_path = . +path_separator = os +sqlalchemy.url = postgresql+asyncpg://scholar:scholar@db:5432/scholar + +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s diff --git a/alembic/env.py b/alembic/env.py new file mode 100644 index 0000000..17575e4 --- /dev/null +++ b/alembic/env.py @@ -0,0 +1,66 @@ +from logging.config import fileConfig +import os + +from alembic import context +from sqlalchemy import pool +from sqlalchemy.engine import Connection +from sqlalchemy.ext.asyncio import async_engine_from_config + +from app.db.base import metadata +from app.db import models # noqa: F401 + +config = context.config + +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +database_url = os.getenv("DATABASE_URL") +if database_url: + config.set_main_option("sqlalchemy.url", database_url) + +target_metadata = metadata + + +def run_migrations_offline() -> None: + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + ) + + with context.begin_transaction(): + context.run_migrations() + + +def do_run_migrations(connection: Connection) -> None: + context.configure(connection=connection, target_metadata=target_metadata) + + with context.begin_transaction(): + context.run_migrations() + + +async def run_async_migrations() -> None: + connectable = async_engine_from_config( + config.get_section(config.config_ini_section, {}), + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + + async with connectable.connect() as connection: + await connection.run_sync(do_run_migrations) + + await connectable.dispose() + + +def run_migrations_online() -> None: + import asyncio + + asyncio.run(run_async_migrations()) + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/alembic/script.py.mako b/alembic/script.py.mako new file mode 100644 index 0000000..8460fee --- /dev/null +++ b/alembic/script.py.mako @@ -0,0 +1,27 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from collections.abc import Sequence + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision: str = ${repr(up_revision)} +down_revision: str | Sequence[str] | None = ${repr(down_revision)} +branch_labels: str | Sequence[str] | None = ${repr(branch_labels)} +depends_on: str | Sequence[str] | None = ${repr(depends_on)} + + +def upgrade() -> None: + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + ${downgrades if downgrades else "pass"} + diff --git a/alembic/versions/.gitkeep b/alembic/versions/.gitkeep new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/alembic/versions/.gitkeep @@ -0,0 +1 @@ + diff --git a/alembic/versions/20260216_0001_multi_user_schema.py b/alembic/versions/20260216_0001_multi_user_schema.py new file mode 100644 index 0000000..221bbf7 --- /dev/null +++ b/alembic/versions/20260216_0001_multi_user_schema.py @@ -0,0 +1,353 @@ +"""Create multi-user core schema + +Revision ID: 20260216_0001 +Revises: +Create Date: 2026-02-16 17:10:00.000000 + +""" + +from collections.abc import Sequence + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision: str = "20260216_0001" +down_revision: str | Sequence[str] | None = None +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +run_status_enum = sa.Enum( + "running", + "success", + "partial_failure", + "failed", + name="run_status", +) +run_trigger_type_enum = sa.Enum( + "manual", + "scheduled", + name="run_trigger_type", +) +run_status_ref = postgresql.ENUM( + "running", + "success", + "partial_failure", + "failed", + name="run_status", + create_type=False, +) +run_trigger_type_ref = postgresql.ENUM( + "manual", + "scheduled", + name="run_trigger_type", + create_type=False, +) + + +def upgrade() -> None: + bind = op.get_bind() + run_status_enum.create(bind, checkfirst=True) + run_trigger_type_enum.create(bind, checkfirst=True) + + op.create_table( + "users", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("email", sa.String(length=255), nullable=False), + sa.Column("password_hash", sa.String(length=255), nullable=False), + sa.Column( + "is_active", + sa.Boolean(), + nullable=False, + server_default=sa.text("true"), + ), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.now(), + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.now(), + ), + sa.PrimaryKeyConstraint("id", name="pk_users"), + sa.UniqueConstraint("email", name="uq_users_email"), + ) + + op.create_table( + "user_settings", + sa.Column("user_id", sa.Integer(), nullable=False), + sa.Column( + "auto_run_enabled", + sa.Boolean(), + nullable=False, + server_default=sa.text("false"), + ), + sa.Column( + "run_interval_minutes", + sa.Integer(), + nullable=False, + server_default=sa.text("1440"), + ), + sa.Column( + "request_delay_seconds", + sa.Integer(), + nullable=False, + server_default=sa.text("10"), + ), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.now(), + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.now(), + ), + sa.CheckConstraint( + "run_interval_minutes >= 15", + name="ck_user_settings_run_interval_minutes_min_15", + ), + sa.CheckConstraint( + "request_delay_seconds >= 1", + name="ck_user_settings_request_delay_seconds_min_1", + ), + sa.ForeignKeyConstraint( + ["user_id"], + ["users.id"], + name="fk_user_settings_user_id_users", + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("user_id", name="pk_user_settings"), + ) + + op.create_table( + "publications", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("cluster_id", sa.String(length=64), nullable=True), + sa.Column("fingerprint_sha256", sa.String(length=64), nullable=False), + sa.Column("title_raw", sa.Text(), nullable=False), + sa.Column("title_normalized", sa.Text(), nullable=False), + sa.Column("year", sa.Integer(), nullable=True), + sa.Column( + "citation_count", + sa.Integer(), + nullable=False, + server_default=sa.text("0"), + ), + sa.Column("author_text", sa.Text(), nullable=True), + sa.Column("venue_text", sa.Text(), nullable=True), + sa.Column("pub_url", sa.Text(), nullable=True), + sa.Column("pdf_url", sa.Text(), nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.now(), + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.now(), + ), + sa.PrimaryKeyConstraint("id", name="pk_publications"), + sa.UniqueConstraint( + "fingerprint_sha256", + name="uq_publications_fingerprint", + ), + ) + op.create_index( + "uq_publications_cluster_id_not_null", + "publications", + ["cluster_id"], + unique=True, + postgresql_where=sa.text("cluster_id IS NOT NULL"), + ) + + op.create_table( + "scholar_profiles", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("user_id", sa.Integer(), nullable=False), + sa.Column("scholar_id", sa.String(length=64), nullable=False), + sa.Column("display_name", sa.String(length=255), nullable=True), + sa.Column( + "is_enabled", + sa.Boolean(), + nullable=False, + server_default=sa.text("true"), + ), + sa.Column( + "baseline_completed", + sa.Boolean(), + nullable=False, + server_default=sa.text("false"), + ), + sa.Column("last_run_dt", sa.DateTime(timezone=True), nullable=True), + sa.Column("last_run_status", run_status_ref, nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.now(), + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.now(), + ), + sa.ForeignKeyConstraint( + ["user_id"], + ["users.id"], + name="fk_scholar_profiles_user_id_users", + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id", name="pk_scholar_profiles"), + sa.UniqueConstraint( + "user_id", + "scholar_id", + name="uq_scholar_profiles_user_scholar", + ), + ) + op.create_index( + "ix_scholar_profiles_user_enabled", + "scholar_profiles", + ["user_id", "is_enabled"], + unique=False, + ) + + op.create_table( + "crawl_runs", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("user_id", sa.Integer(), nullable=False), + sa.Column("trigger_type", run_trigger_type_ref, nullable=False), + sa.Column("status", run_status_ref, nullable=False), + sa.Column( + "start_dt", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.now(), + ), + sa.Column("end_dt", sa.DateTime(timezone=True), nullable=True), + sa.Column( + "scholar_count", + sa.Integer(), + nullable=False, + server_default=sa.text("0"), + ), + sa.Column( + "new_pub_count", + sa.Integer(), + nullable=False, + server_default=sa.text("0"), + ), + sa.Column( + "error_log", + postgresql.JSONB(astext_type=sa.Text()), + nullable=False, + server_default=sa.text("'{}'::jsonb"), + ), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.now(), + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.now(), + ), + sa.ForeignKeyConstraint( + ["user_id"], + ["users.id"], + name="fk_crawl_runs_user_id_users", + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id", name="pk_crawl_runs"), + ) + op.create_index( + "ix_crawl_runs_user_start", + "crawl_runs", + ["user_id", "start_dt"], + unique=False, + ) + + op.create_table( + "scholar_publications", + sa.Column("scholar_profile_id", sa.Integer(), nullable=False), + sa.Column("publication_id", sa.Integer(), nullable=False), + sa.Column( + "is_read", + sa.Boolean(), + nullable=False, + server_default=sa.text("false"), + ), + sa.Column("first_seen_run_id", sa.Integer(), nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.now(), + ), + sa.ForeignKeyConstraint( + ["first_seen_run_id"], + ["crawl_runs.id"], + name="fk_scholar_publications_first_seen_run_id_crawl_runs", + ondelete="SET NULL", + ), + sa.ForeignKeyConstraint( + ["publication_id"], + ["publications.id"], + name="fk_scholar_publications_publication_id_publications", + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["scholar_profile_id"], + ["scholar_profiles.id"], + name="fk_scholar_publications_scholar_profile_id_scholar_profiles", + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint( + "scholar_profile_id", + "publication_id", + name="pk_scholar_publications", + ), + ) + op.create_index( + "ix_scholar_publications_is_read", + "scholar_publications", + ["is_read"], + unique=False, + ) + + +def downgrade() -> None: + op.drop_index("ix_scholar_publications_is_read", table_name="scholar_publications") + op.drop_table("scholar_publications") + + op.drop_index("ix_crawl_runs_user_start", table_name="crawl_runs") + op.drop_table("crawl_runs") + + op.drop_index("ix_scholar_profiles_user_enabled", table_name="scholar_profiles") + op.drop_table("scholar_profiles") + + op.drop_index("uq_publications_cluster_id_not_null", table_name="publications") + op.drop_table("publications") + + op.drop_table("user_settings") + op.drop_table("users") + + bind = op.get_bind() + run_trigger_type_enum.drop(bind, checkfirst=True) + run_status_enum.drop(bind, checkfirst=True) diff --git a/alembic/versions/20260216_0002_add_user_admin_flag.py b/alembic/versions/20260216_0002_add_user_admin_flag.py new file mode 100644 index 0000000..e60de0f --- /dev/null +++ b/alembic/versions/20260216_0002_add_user_admin_flag.py @@ -0,0 +1,34 @@ +"""Add admin flag to users + +Revision ID: 20260216_0002 +Revises: 20260216_0001 +Create Date: 2026-02-16 17:40:00.000000 +""" + +from collections.abc import Sequence + +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision: str = "20260216_0002" +down_revision: str | Sequence[str] | None = "20260216_0001" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + op.add_column( + "users", + sa.Column( + "is_admin", + sa.Boolean(), + nullable=False, + server_default=sa.text("false"), + ), + ) + + +def downgrade() -> None: + op.drop_column("users", "is_admin") + diff --git a/alembic/versions/20260216_0003_add_ingestion_queue.py b/alembic/versions/20260216_0003_add_ingestion_queue.py new file mode 100644 index 0000000..cc03867 --- /dev/null +++ b/alembic/versions/20260216_0003_add_ingestion_queue.py @@ -0,0 +1,94 @@ +"""Add ingestion continuation queue table + +Revision ID: 20260216_0003 +Revises: 20260216_0002 +Create Date: 2026-02-16 23:45:00.000000 +""" + +from collections.abc import Sequence + +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision: str = "20260216_0003" +down_revision: str | Sequence[str] | None = "20260216_0002" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + op.create_table( + "ingestion_queue_items", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("user_id", sa.Integer(), nullable=False), + sa.Column("scholar_profile_id", sa.Integer(), nullable=False), + sa.Column( + "resume_cstart", + sa.Integer(), + nullable=False, + server_default=sa.text("0"), + ), + sa.Column("reason", sa.String(length=64), nullable=False), + sa.Column( + "attempt_count", + sa.Integer(), + nullable=False, + server_default=sa.text("0"), + ), + sa.Column( + "next_attempt_dt", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.now(), + ), + sa.Column("last_run_id", sa.Integer(), nullable=True), + sa.Column("last_error", sa.Text(), nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.now(), + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.now(), + ), + sa.ForeignKeyConstraint( + ["last_run_id"], + ["crawl_runs.id"], + name="fk_ingestion_queue_items_last_run_id_crawl_runs", + ondelete="SET NULL", + ), + sa.ForeignKeyConstraint( + ["scholar_profile_id"], + ["scholar_profiles.id"], + name="fk_ingestion_queue_items_scholar_profile_id_scholar_profiles", + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["user_id"], + ["users.id"], + name="fk_ingestion_queue_items_user_id_users", + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id", name="pk_ingestion_queue_items"), + sa.UniqueConstraint( + "user_id", + "scholar_profile_id", + name="uq_ingestion_queue_user_scholar", + ), + ) + op.create_index( + "ix_ingestion_queue_next_attempt", + "ingestion_queue_items", + ["next_attempt_dt"], + unique=False, + ) + + +def downgrade() -> None: + op.drop_index("ix_ingestion_queue_next_attempt", table_name="ingestion_queue_items") + op.drop_table("ingestion_queue_items") diff --git a/alembic/versions/20260217_0004_queue_status_fields.py b/alembic/versions/20260217_0004_queue_status_fields.py new file mode 100644 index 0000000..68da9dd --- /dev/null +++ b/alembic/versions/20260217_0004_queue_status_fields.py @@ -0,0 +1,96 @@ +"""Add queue status and dropped diagnostics fields + +Revision ID: 20260217_0004 +Revises: 20260216_0003 +Create Date: 2026-02-17 10:15:00.000000 +""" + +from collections.abc import Sequence + +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision: str = "20260217_0004" +down_revision: str | Sequence[str] | None = "20260216_0003" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + bind = op.get_bind() + inspector = sa.inspect(bind) + columns = {column["name"] for column in inspector.get_columns("ingestion_queue_items")} + indexes = {index["name"] for index in inspector.get_indexes("ingestion_queue_items")} + checks = { + constraint["name"] for constraint in inspector.get_check_constraints("ingestion_queue_items") + } + + if "status" not in columns: + op.add_column( + "ingestion_queue_items", + sa.Column( + "status", + sa.String(length=16), + nullable=False, + server_default=sa.text("'queued'"), + ), + ) + columns.add("status") + + if "dropped_reason" not in columns: + op.add_column( + "ingestion_queue_items", + sa.Column("dropped_reason", sa.String(length=128), nullable=True), + ) + + if "dropped_at" not in columns: + op.add_column( + "ingestion_queue_items", + sa.Column("dropped_at", sa.DateTime(timezone=True), nullable=True), + ) + + if "status" in columns: + op.execute( + """ + UPDATE ingestion_queue_items + SET status = 'queued' + WHERE status IS NULL OR status = '' + """ + ) + + if "ingestion_queue_status_valid" not in checks: + op.create_check_constraint( + "ingestion_queue_status_valid", + "ingestion_queue_items", + "status IN ('queued', 'retrying', 'dropped')", + ) + + if "ix_ingestion_queue_status_next_attempt" not in indexes: + op.create_index( + "ix_ingestion_queue_status_next_attempt", + "ingestion_queue_items", + ["status", "next_attempt_dt"], + unique=False, + ) + + +def downgrade() -> None: + bind = op.get_bind() + inspector = sa.inspect(bind) + columns = {column["name"] for column in inspector.get_columns("ingestion_queue_items")} + indexes = {index["name"] for index in inspector.get_indexes("ingestion_queue_items")} + checks = { + constraint["name"] for constraint in inspector.get_check_constraints("ingestion_queue_items") + } + + if "ix_ingestion_queue_status_next_attempt" in indexes: + op.drop_index("ix_ingestion_queue_status_next_attempt", table_name="ingestion_queue_items") + if "ingestion_queue_status_valid" in checks: + op.drop_constraint("ingestion_queue_status_valid", "ingestion_queue_items", type_="check") + if "dropped_at" in columns: + op.drop_column("ingestion_queue_items", "dropped_at") + if "dropped_reason" in columns: + op.drop_column("ingestion_queue_items", "dropped_reason") + if "status" in columns: + op.drop_column("ingestion_queue_items", "status") diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/app/__init__.py @@ -0,0 +1 @@ + diff --git a/app/api/__init__.py b/app/api/__init__.py new file mode 100644 index 0000000..3ce55a6 --- /dev/null +++ b/app/api/__init__.py @@ -0,0 +1,2 @@ +from __future__ import annotations + diff --git a/app/api/deps.py b/app/api/deps.py new file mode 100644 index 0000000..9dd3a68 --- /dev/null +++ b/app/api/deps.py @@ -0,0 +1,36 @@ +from __future__ import annotations + +from fastapi import Depends, Request +from sqlalchemy.ext.asyncio import AsyncSession + +from app.api.errors import ApiException +from app.db.models import User +from app.db.session import get_db_session +from app.web import common as web_common + + +async def get_api_current_user( + request: Request, + db_session: AsyncSession = Depends(get_db_session), +) -> User: + current_user = await web_common.get_authenticated_user(request, db_session) + if current_user is None: + raise ApiException( + status_code=401, + code="auth_required", + message="Authentication required.", + ) + return current_user + + +async def get_api_admin_user( + current_user: User = Depends(get_api_current_user), +) -> User: + if not current_user.is_admin: + raise ApiException( + status_code=403, + code="forbidden", + message="Admin access required.", + ) + return current_user + diff --git a/app/api/errors.py b/app/api/errors.py new file mode 100644 index 0000000..976ccb8 --- /dev/null +++ b/app/api/errors.py @@ -0,0 +1,85 @@ +from __future__ import annotations + +from typing import Any + +from fastapi import FastAPI, HTTPException, Request +from fastapi.exception_handlers import ( + http_exception_handler as fastapi_http_exception_handler, +) +from fastapi.exception_handlers import ( + request_validation_exception_handler as fastapi_validation_exception_handler, +) +from fastapi.exceptions import RequestValidationError + +from app.api.responses import error_response + + +def _is_api_path(path: str) -> bool: + return path.startswith("/api/") + + +def _status_code_to_error_code(status_code: int) -> str: + mapping = { + 400: "bad_request", + 401: "auth_required", + 403: "forbidden", + 404: "not_found", + 409: "conflict", + 422: "validation_error", + 429: "rate_limited", + 500: "internal_error", + } + return mapping.get(status_code, "error") + + +class ApiException(Exception): + def __init__( + self, + *, + status_code: int, + code: str, + message: str, + details: Any | None = None, + ) -> None: + super().__init__(message) + self.status_code = status_code + self.code = code + self.message = message + self.details = details + + +def register_api_exception_handlers(app: FastAPI) -> None: + @app.exception_handler(ApiException) + async def _handle_api_exception(request: Request, exc: ApiException): + return error_response( + request, + status_code=exc.status_code, + code=exc.code, + message=exc.message, + details=exc.details, + ) + + @app.exception_handler(HTTPException) + async def _handle_http_exception(request: Request, exc: HTTPException): + if not _is_api_path(request.url.path): + return await fastapi_http_exception_handler(request, exc) + return error_response( + request, + status_code=exc.status_code, + code=_status_code_to_error_code(exc.status_code), + message=str(exc.detail) if exc.detail is not None else "Request failed.", + details=exc.detail if isinstance(exc.detail, (dict, list)) else None, + ) + + @app.exception_handler(RequestValidationError) + async def _handle_validation_exception(request: Request, exc: RequestValidationError): + if not _is_api_path(request.url.path): + return await fastapi_validation_exception_handler(request, exc) + return error_response( + request, + status_code=422, + code="validation_error", + message="Request validation failed.", + details=exc.errors(), + ) + diff --git a/app/api/responses.py b/app/api/responses.py new file mode 100644 index 0000000..9976aba --- /dev/null +++ b/app/api/responses.py @@ -0,0 +1,64 @@ +from __future__ import annotations + +from typing import Any + +from fastapi.encoders import jsonable_encoder +from starlette.requests import Request +from starlette.responses import JSONResponse + + +def _request_id(request: Request) -> str | None: + request_state = getattr(request, "state", None) + if request_state is None: + return None + return getattr(request_state, "request_id", None) + + +def success_payload( + request: Request, + *, + data: Any, +) -> dict[str, Any]: + return { + "data": data, + "meta": { + "request_id": _request_id(request), + }, + } + + +def success_response( + request: Request, + *, + data: Any, + status_code: int = 200, +) -> JSONResponse: + payload = success_payload(request, data=data) + return JSONResponse( + status_code=status_code, + content=jsonable_encoder(payload), + ) + + +def error_response( + request: Request, + *, + status_code: int, + code: str, + message: str, + details: Any | None = None, +) -> JSONResponse: + payload = { + "error": { + "code": code, + "message": message, + "details": details, + }, + "meta": { + "request_id": _request_id(request), + }, + } + return JSONResponse( + status_code=status_code, + content=jsonable_encoder(payload), + ) diff --git a/app/api/router.py b/app/api/router.py new file mode 100644 index 0000000..09d49b2 --- /dev/null +++ b/app/api/router.py @@ -0,0 +1,13 @@ +from __future__ import annotations + +from fastapi import APIRouter + +from app.api.routers import admin, auth, publications, runs, scholars, settings + +router = APIRouter(prefix="/api/v1") +router.include_router(auth.router) +router.include_router(admin.router) +router.include_router(scholars.router) +router.include_router(settings.router) +router.include_router(runs.router) +router.include_router(publications.router) diff --git a/app/api/routers/__init__.py b/app/api/routers/__init__.py new file mode 100644 index 0000000..3ce55a6 --- /dev/null +++ b/app/api/routers/__init__.py @@ -0,0 +1,2 @@ +from __future__ import annotations + diff --git a/app/api/routers/admin.py b/app/api/routers/admin.py new file mode 100644 index 0000000..dbf00bf --- /dev/null +++ b/app/api/routers/admin.py @@ -0,0 +1,202 @@ +from __future__ import annotations + +import logging + +from fastapi import APIRouter, Depends, Request +from sqlalchemy.ext.asyncio import AsyncSession + +from app.api.deps import get_api_admin_user +from app.api.errors import ApiException +from app.api.responses import success_payload +from app.api.schemas import ( + AdminResetPasswordRequest, + AdminUserActiveUpdateRequest, + AdminUserCreateRequest, + AdminUserEnvelope, + AdminUsersListEnvelope, + MessageEnvelope, +) +from app.auth.deps import get_auth_service +from app.auth.service import AuthService +from app.db.models import User +from app.db.session import get_db_session +from app.services import users as user_service + +logger = logging.getLogger(__name__) + +router = APIRouter(prefix="/admin/users", tags=["api-admin-users"]) + + +def _serialize_user(user: User) -> dict[str, object]: + return { + "id": int(user.id), + "email": user.email, + "is_active": bool(user.is_active), + "is_admin": bool(user.is_admin), + "created_at": user.created_at, + "updated_at": user.updated_at, + } + + +@router.get( + "", + response_model=AdminUsersListEnvelope, +) +async def list_users( + request: Request, + db_session: AsyncSession = Depends(get_db_session), + admin_user: User = Depends(get_api_admin_user), +): + users = await user_service.list_users(db_session) + logger.info( + "api.admin.users_listed", + extra={ + "event": "api.admin.users_listed", + "admin_user_id": int(admin_user.id), + "user_count": len(users), + }, + ) + return success_payload( + request, + data={ + "users": [_serialize_user(user) for user in users], + }, + ) + + +@router.post( + "", + response_model=AdminUserEnvelope, + status_code=201, +) +async def create_user( + payload: AdminUserCreateRequest, + request: Request, + db_session: AsyncSession = Depends(get_db_session), + auth_service: AuthService = Depends(get_auth_service), + admin_user: User = Depends(get_api_admin_user), +): + try: + validated_email = user_service.validate_email(payload.email) + validated_password = user_service.validate_password(payload.password) + created_user = await user_service.create_user( + db_session, + email=validated_email, + password_hash=auth_service.hash_password(validated_password), + is_admin=bool(payload.is_admin), + ) + except user_service.UserServiceError as exc: + raise ApiException( + status_code=400, + code="invalid_user_input", + message=str(exc), + ) from exc + + logger.info( + "api.admin.user_created", + extra={ + "event": "api.admin.user_created", + "admin_user_id": int(admin_user.id), + "target_user_id": int(created_user.id), + "target_is_admin": bool(created_user.is_admin), + }, + ) + return success_payload( + request, + data=_serialize_user(created_user), + ) + + +@router.patch( + "/{user_id}/active", + response_model=AdminUserEnvelope, +) +async def set_user_active( + user_id: int, + payload: AdminUserActiveUpdateRequest, + request: Request, + db_session: AsyncSession = Depends(get_db_session), + admin_user: User = Depends(get_api_admin_user), +): + target_user = await user_service.get_user_by_id(db_session, user_id) + if target_user is None: + raise ApiException( + status_code=404, + code="user_not_found", + message="User not found.", + ) + if ( + int(target_user.id) == int(admin_user.id) + and bool(target_user.is_active) + and not bool(payload.is_active) + ): + raise ApiException( + status_code=400, + code="cannot_deactivate_self", + message="You cannot deactivate your own account.", + ) + updated_user = await user_service.set_user_active( + db_session, + user=target_user, + is_active=bool(payload.is_active), + ) + logger.info( + "api.admin.user_active_updated", + extra={ + "event": "api.admin.user_active_updated", + "admin_user_id": int(admin_user.id), + "target_user_id": int(updated_user.id), + "is_active": bool(updated_user.is_active), + }, + ) + return success_payload( + request, + data=_serialize_user(updated_user), + ) + + +@router.post( + "/{user_id}/reset-password", + response_model=MessageEnvelope, +) +async def reset_user_password( + user_id: int, + payload: AdminResetPasswordRequest, + request: Request, + db_session: AsyncSession = Depends(get_db_session), + auth_service: AuthService = Depends(get_auth_service), + admin_user: User = Depends(get_api_admin_user), +): + target_user = await user_service.get_user_by_id(db_session, user_id) + if target_user is None: + raise ApiException( + status_code=404, + code="user_not_found", + message="User not found.", + ) + try: + validated_password = user_service.validate_password(payload.new_password) + except user_service.UserServiceError as exc: + raise ApiException( + status_code=400, + code="invalid_password", + message=str(exc), + ) from exc + + await user_service.set_user_password_hash( + db_session, + user=target_user, + password_hash=auth_service.hash_password(validated_password), + ) + logger.info( + "api.admin.user_password_reset", + extra={ + "event": "api.admin.user_password_reset", + "admin_user_id": int(admin_user.id), + "target_user_id": int(target_user.id), + }, + ) + return success_payload( + request, + data={"message": f"Password reset: {target_user.email}"}, + ) diff --git a/app/api/routers/auth.py b/app/api/routers/auth.py new file mode 100644 index 0000000..3b53dc8 --- /dev/null +++ b/app/api/routers/auth.py @@ -0,0 +1,230 @@ +from __future__ import annotations + +import logging + +from fastapi import APIRouter, Depends, Request +from sqlalchemy.ext.asyncio import AsyncSession + +from app.api.errors import ApiException +from app.api.deps import get_api_current_user +from app.api.responses import success_payload +from app.api.schemas import ( + AuthMeEnvelope, + ChangePasswordRequest, + CsrfBootstrapEnvelope, + LoginEnvelope, + LoginRequest, + MessageEnvelope, +) +from app.auth.deps import get_auth_service, get_login_rate_limiter +from app.auth.rate_limit import SlidingWindowRateLimiter +from app.auth.service import AuthService +from app.auth.session import set_session_user +from app.db.models import User +from app.db.session import get_db_session +from app.security.csrf import ensure_csrf_token +from app.services import users as user_service +from app.web import common as web_common + +logger = logging.getLogger(__name__) + +router = APIRouter(prefix="/auth", tags=["api-auth"]) + + +@router.post( + "/login", + response_model=LoginEnvelope, +) +async def login( + payload: LoginRequest, + request: Request, + db_session: AsyncSession = Depends(get_db_session), + auth_service: AuthService = Depends(get_auth_service), + rate_limiter: SlidingWindowRateLimiter = Depends(get_login_rate_limiter), +): + limiter_key = web_common.login_rate_limit_key(request, payload.email) + decision = rate_limiter.check(limiter_key) + normalized_email = payload.email.strip().lower() + if not decision.allowed: + logger.warning( + "api.auth.login_rate_limited", + extra={ + "event": "api.auth.login_rate_limited", + "email": normalized_email, + "retry_after_seconds": decision.retry_after_seconds, + }, + ) + raise ApiException( + status_code=429, + code="rate_limited", + message="Too many login attempts. Please try again later.", + details={"retry_after_seconds": decision.retry_after_seconds}, + ) + + user = await auth_service.authenticate_user( + db_session, + email=payload.email, + password=payload.password, + ) + if user is None: + rate_limiter.record_failure(limiter_key) + logger.info( + "api.auth.login_failed", + extra={ + "event": "api.auth.login_failed", + "email": normalized_email, + }, + ) + raise ApiException( + status_code=401, + code="invalid_credentials", + message="Invalid email or password.", + ) + + rate_limiter.reset(limiter_key) + set_session_user( + request, + user_id=user.id, + email=user.email, + is_admin=user.is_admin, + ) + logger.info( + "api.auth.login_succeeded", + extra={ + "event": "api.auth.login_succeeded", + "user_id": user.id, + "is_admin": user.is_admin, + }, + ) + return success_payload( + request, + data={ + "authenticated": True, + "csrf_token": ensure_csrf_token(request), + "user": { + "id": int(user.id), + "email": user.email, + "is_admin": bool(user.is_admin), + "is_active": bool(user.is_active), + }, + }, + ) + + +@router.get( + "/me", + response_model=AuthMeEnvelope, +) +async def get_current_session( + request: Request, + current_user: User = Depends(get_api_current_user), +): + return success_payload( + request, + data={ + "authenticated": True, + "csrf_token": ensure_csrf_token(request), + "user": { + "id": int(current_user.id), + "email": current_user.email, + "is_admin": bool(current_user.is_admin), + "is_active": bool(current_user.is_active), + }, + }, + ) + + +@router.get( + "/csrf", + response_model=CsrfBootstrapEnvelope, +) +async def get_csrf_bootstrap( + request: Request, + db_session: AsyncSession = Depends(get_db_session), +): + current_user = await web_common.get_authenticated_user(request, db_session) + return success_payload( + request, + data={ + "csrf_token": ensure_csrf_token(request), + "authenticated": current_user is not None, + }, + ) + + +@router.post( + "/change-password", + response_model=MessageEnvelope, +) +async def change_password( + payload: ChangePasswordRequest, + request: Request, + current_user: User = Depends(get_api_current_user), + db_session: AsyncSession = Depends(get_db_session), + auth_service: AuthService = Depends(get_auth_service), +): + if not auth_service.verify_password( + password_hash=current_user.password_hash, + password=payload.current_password, + ): + raise ApiException( + status_code=400, + code="invalid_current_password", + message="Current password is incorrect.", + ) + if payload.new_password != payload.confirm_password: + raise ApiException( + status_code=400, + code="password_confirmation_mismatch", + message="New password and confirmation do not match.", + ) + try: + validated_password = user_service.validate_password(payload.new_password) + except user_service.UserServiceError as exc: + raise ApiException( + status_code=400, + code="invalid_password", + message=str(exc), + ) from exc + + await user_service.set_user_password_hash( + db_session, + user=current_user, + password_hash=auth_service.hash_password(validated_password), + ) + logger.info( + "api.auth.password_changed", + extra={ + "event": "api.auth.password_changed", + "user_id": int(current_user.id), + }, + ) + return success_payload( + request, + data={"message": "Password updated successfully."}, + ) + + +@router.post( + "/logout", + response_model=MessageEnvelope, +) +async def logout( + request: Request, + db_session: AsyncSession = Depends(get_db_session), +): + current_user = await web_common.get_authenticated_user(request, db_session) + web_common.invalidate_session(request) + logger.info( + "api.auth.logout", + extra={ + "event": "api.auth.logout", + "user_id": int(current_user.id) if current_user is not None else None, + }, + ) + return success_payload( + request, + data={ + "message": "Logged out.", + }, + ) diff --git a/app/api/routers/publications.py b/app/api/routers/publications.py new file mode 100644 index 0000000..a783147 --- /dev/null +++ b/app/api/routers/publications.py @@ -0,0 +1,123 @@ +from __future__ import annotations + +import logging +from typing import Literal + +from fastapi import APIRouter, Depends, Query, Request +from sqlalchemy.ext.asyncio import AsyncSession + +from app.api.deps import get_api_current_user +from app.api.errors import ApiException +from app.api.responses import success_payload +from app.api.schemas import MarkAllReadEnvelope, PublicationsListEnvelope +from app.db.models import User +from app.db.session import get_db_session +from app.services import publications as publication_service +from app.services import scholars as scholar_service + +logger = logging.getLogger(__name__) + +router = APIRouter(prefix="/publications", tags=["api-publications"]) + + +@router.get( + "", + response_model=PublicationsListEnvelope, +) +async def list_publications( + request: Request, + mode: Literal["all", "new"] | None = Query(default=None), + scholar_profile_id: int | None = Query(default=None, ge=1), + limit: int = Query(default=300, ge=1, le=1000), + db_session: AsyncSession = Depends(get_db_session), + current_user: User = Depends(get_api_current_user), +): + resolved_mode = publication_service.resolve_mode(mode) + selected_scholar_id = scholar_profile_id + if selected_scholar_id is not None: + selected_profile = await scholar_service.get_user_scholar_by_id( + db_session, + user_id=current_user.id, + scholar_profile_id=selected_scholar_id, + ) + if selected_profile is None: + raise ApiException( + status_code=404, + code="scholar_not_found", + message="Scholar filter not found.", + ) + + publications = await publication_service.list_for_user( + db_session, + user_id=current_user.id, + mode=resolved_mode, + scholar_profile_id=selected_scholar_id, + limit=limit, + ) + new_count = await publication_service.count_for_user( + db_session, + user_id=current_user.id, + mode=publication_service.MODE_NEW, + scholar_profile_id=selected_scholar_id, + ) + total_count = await publication_service.count_for_user( + db_session, + user_id=current_user.id, + mode=publication_service.MODE_ALL, + scholar_profile_id=selected_scholar_id, + ) + return success_payload( + request, + data={ + "mode": resolved_mode, + "selected_scholar_profile_id": selected_scholar_id, + "new_count": new_count, + "total_count": total_count, + "publications": [ + { + "publication_id": item.publication_id, + "scholar_profile_id": item.scholar_profile_id, + "scholar_label": item.scholar_label, + "title": item.title, + "year": item.year, + "citation_count": item.citation_count, + "venue_text": item.venue_text, + "pub_url": item.pub_url, + "is_read": item.is_read, + "first_seen_at": item.first_seen_at, + "is_new_in_latest_run": item.is_new_in_latest_run, + } + for item in publications + ], + }, + ) + + +@router.post( + "/mark-all-read", + response_model=MarkAllReadEnvelope, +) +async def mark_all_publications_read( + request: Request, + db_session: AsyncSession = Depends(get_db_session), + current_user: User = Depends(get_api_current_user), +): + updated_count = await publication_service.mark_all_unread_as_read_for_user( + db_session, + user_id=current_user.id, + ) + logger.info( + "api.publications.mark_all_read", + extra={ + "event": "api.publications.mark_all_read", + "user_id": current_user.id, + "updated_count": updated_count, + }, + ) + return success_payload( + request, + data={ + "message": "Marked all unread publications as read.", + "updated_count": updated_count, + }, + ) diff --git a/app/api/routers/runs.py b/app/api/routers/runs.py new file mode 100644 index 0000000..00f18d1 --- /dev/null +++ b/app/api/routers/runs.py @@ -0,0 +1,560 @@ +from __future__ import annotations + +import logging +import re +from typing import Any + +from fastapi import APIRouter, Depends, Query, Request +from sqlalchemy.ext.asyncio import AsyncSession + +from app.api.deps import get_api_current_user +from app.api.errors import ApiException +from app.api.responses import success_payload +from app.api.schemas import ( + ManualRunEnvelope, + QueueClearEnvelope, + QueueItemEnvelope, + QueueListEnvelope, + RunDetailEnvelope, + RunsListEnvelope, +) +from app.db.models import RunStatus, RunTriggerType, User +from app.db.session import get_db_session +from app.services import ingestion as ingestion_service +from app.services import runs as run_service +from app.services import user_settings as user_settings_service +from app.settings import settings +from app.web.deps import get_ingestion_service + +logger = logging.getLogger(__name__) + +router = APIRouter(prefix="/runs", tags=["api-runs"]) + +IDEMPOTENCY_HEADER = "Idempotency-Key" +IDEMPOTENCY_MAX_LENGTH = 128 +IDEMPOTENCY_PATTERN = re.compile(r"^[A-Za-z0-9._:-]{8,128}$") + + +def _int_value(value: Any, default: int = 0) -> int: + try: + return int(value) + except (TypeError, ValueError): + return default + + +def _str_value(value: Any) -> str | None: + if value is None: + return None + text = str(value).strip() + return text if text else None + + +def _bool_value(value: Any, default: bool = False) -> bool: + if isinstance(value, bool): + return value + if isinstance(value, str): + lowered = value.strip().lower() + if lowered in {"1", "true", "yes", "on"}: + return True + if lowered in {"0", "false", "no", "off"}: + return False + return default + + +def _normalize_idempotency_key(raw_value: str | None) -> str | None: + if raw_value is None: + return None + candidate = raw_value.strip() + if not candidate: + return None + if len(candidate) > IDEMPOTENCY_MAX_LENGTH or not IDEMPOTENCY_PATTERN.match(candidate): + raise ApiException( + status_code=400, + code="invalid_idempotency_key", + message=( + "Invalid Idempotency-Key. Use 8-128 characters from: " + "A-Z a-z 0-9 . _ : -" + ), + ) + return candidate + + +def _serialize_run(run) -> dict[str, Any]: + summary = run_service.extract_run_summary(run.error_log) + return { + "id": int(run.id), + "trigger_type": run.trigger_type.value, + "status": run.status.value, + "start_dt": run.start_dt, + "end_dt": run.end_dt, + "scholar_count": int(run.scholar_count or 0), + "new_publication_count": int(run.new_pub_count or 0), + "failed_count": int(summary["failed_count"]), + "partial_count": int(summary["partial_count"]), + } + + +def _serialize_queue_item(item) -> dict[str, Any]: + return { + "id": int(item.id), + "scholar_profile_id": int(item.scholar_profile_id), + "scholar_label": item.scholar_label, + "status": item.status, + "reason": item.reason, + "dropped_reason": item.dropped_reason, + "attempt_count": int(item.attempt_count), + "resume_cstart": int(item.resume_cstart), + "next_attempt_dt": item.next_attempt_dt, + "updated_at": item.updated_at, + "last_error": item.last_error, + "last_run_id": item.last_run_id, + } + + +def _normalize_attempt_log(value: Any) -> list[dict[str, Any]]: + if not isinstance(value, list): + return [] + normalized: list[dict[str, Any]] = [] + for item in value: + if not isinstance(item, dict): + continue + normalized.append( + { + "attempt": _int_value(item.get("attempt"), 0), + "cstart": _int_value(item.get("cstart"), 0), + "state": _str_value(item.get("state")), + "state_reason": _str_value(item.get("state_reason")), + "status_code": ( + _int_value(item.get("status_code")) + if item.get("status_code") is not None + else None + ), + "fetch_error": _str_value(item.get("fetch_error")), + } + ) + return normalized + + +def _normalize_page_logs(value: Any) -> list[dict[str, Any]]: + if not isinstance(value, list): + return [] + normalized: list[dict[str, Any]] = [] + for item in value: + if not isinstance(item, dict): + continue + warning_codes = item.get("warning_codes") + normalized.append( + { + "page": _int_value(item.get("page"), 0), + "cstart": _int_value(item.get("cstart"), 0), + "state": _str_value(item.get("state")) or "unknown", + "state_reason": _str_value(item.get("state_reason")), + "status_code": ( + _int_value(item.get("status_code")) + if item.get("status_code") is not None + else None + ), + "publication_count": _int_value(item.get("publication_count"), 0), + "attempt_count": _int_value(item.get("attempt_count"), 0), + "has_show_more_button": _bool_value(item.get("has_show_more_button"), False), + "articles_range": _str_value(item.get("articles_range")), + "warning_codes": [ + str(code) + for code in (warning_codes if isinstance(warning_codes, list) else []) + ], + } + ) + return normalized + + +def _normalize_debug(value: Any) -> dict[str, Any] | None: + if not isinstance(value, dict): + return None + marker_counts = value.get("marker_counts_nonzero") + warning_codes = value.get("warning_codes") + return { + "status_code": ( + _int_value(value.get("status_code")) + if value.get("status_code") is not None + else None + ), + "final_url": _str_value(value.get("final_url")), + "fetch_error": _str_value(value.get("fetch_error")), + "body_sha256": _str_value(value.get("body_sha256")), + "body_length": ( + _int_value(value.get("body_length")) + if value.get("body_length") is not None + else None + ), + "has_show_more_button": ( + _bool_value(value.get("has_show_more_button"), False) + if value.get("has_show_more_button") is not None + else None + ), + "articles_range": _str_value(value.get("articles_range")), + "state_reason": _str_value(value.get("state_reason")), + "warning_codes": [ + str(code) + for code in (warning_codes if isinstance(warning_codes, list) else []) + ], + "marker_counts_nonzero": { + str(key): _int_value(count, 0) + for key, count in (marker_counts.items() if isinstance(marker_counts, dict) else []) + }, + "page_logs": _normalize_page_logs(value.get("page_logs")), + "attempt_log": _normalize_attempt_log(value.get("attempt_log")), + } + + +def _normalize_scholar_result(value: Any) -> dict[str, Any]: + if not isinstance(value, dict): + return { + "scholar_profile_id": 0, + "scholar_id": "unknown", + "state": "unknown", + "state_reason": None, + "outcome": "failed", + "attempt_count": 0, + "publication_count": 0, + "start_cstart": 0, + "continuation_cstart": None, + "continuation_enqueued": False, + "continuation_cleared": False, + "warnings": [], + "error": None, + "debug": None, + } + warnings = value.get("warnings") + return { + "scholar_profile_id": _int_value(value.get("scholar_profile_id"), 0), + "scholar_id": _str_value(value.get("scholar_id")) or "unknown", + "state": _str_value(value.get("state")) or "unknown", + "state_reason": _str_value(value.get("state_reason")), + "outcome": _str_value(value.get("outcome")) or "failed", + "attempt_count": _int_value(value.get("attempt_count"), 0), + "publication_count": _int_value(value.get("publication_count"), 0), + "start_cstart": _int_value(value.get("start_cstart"), 0), + "continuation_cstart": ( + _int_value(value.get("continuation_cstart")) + if value.get("continuation_cstart") is not None + else None + ), + "continuation_enqueued": _bool_value(value.get("continuation_enqueued"), False), + "continuation_cleared": _bool_value(value.get("continuation_cleared"), False), + "warnings": [str(item) for item in (warnings if isinstance(warnings, list) else [])], + "error": _str_value(value.get("error")), + "debug": _normalize_debug(value.get("debug")), + } + + +def _manual_run_payload_from_run( + run, + *, + idempotency_key: str | None, + reused_existing_run: bool, +) -> dict[str, Any]: + summary = run_service.extract_run_summary(run.error_log) + return { + "run_id": int(run.id), + "status": run.status.value, + "scholar_count": int(run.scholar_count or 0), + "succeeded_count": int(summary["succeeded_count"]), + "failed_count": int(summary["failed_count"]), + "partial_count": int(summary["partial_count"]), + "new_publication_count": int(run.new_pub_count or 0), + "reused_existing_run": reused_existing_run, + "idempotency_key": idempotency_key, + } + + +@router.get( + "", + response_model=RunsListEnvelope, +) +async def list_runs( + request: Request, + failed_only: bool = Query(default=False), + limit: int = Query(default=100, ge=1, le=500), + db_session: AsyncSession = Depends(get_db_session), + current_user: User = Depends(get_api_current_user), +): + runs = await run_service.list_runs_for_user( + db_session, + user_id=current_user.id, + limit=limit, + failed_only=failed_only, + ) + return success_payload( + request, + data={ + "runs": [_serialize_run(run) for run in runs], + }, + ) + + +@router.get( + "/{run_id}", + response_model=RunDetailEnvelope, +) +async def get_run( + run_id: int, + request: Request, + db_session: AsyncSession = Depends(get_db_session), + current_user: User = Depends(get_api_current_user), +): + run = await run_service.get_run_for_user( + db_session, + user_id=current_user.id, + run_id=run_id, + ) + if run is None: + raise ApiException( + status_code=404, + code="run_not_found", + message="Run not found.", + ) + error_log = run.error_log if isinstance(run.error_log, dict) else {} + scholar_results = error_log.get("scholar_results") + if not isinstance(scholar_results, list): + scholar_results = [] + return success_payload( + request, + data={ + "run": _serialize_run(run), + "summary": run_service.extract_run_summary(error_log), + "scholar_results": [_normalize_scholar_result(item) for item in scholar_results], + }, + ) + + +@router.post( + "/manual", + response_model=ManualRunEnvelope, +) +async def run_manual( + request: Request, + db_session: AsyncSession = Depends(get_db_session), + current_user: User = Depends(get_api_current_user), + ingest_service: ingestion_service.ScholarIngestionService = Depends(get_ingestion_service), +): + idempotency_key = _normalize_idempotency_key(request.headers.get(IDEMPOTENCY_HEADER)) + if idempotency_key is not None: + previous_run = await run_service.get_manual_run_by_idempotency_key( + db_session, + user_id=current_user.id, + idempotency_key=idempotency_key, + ) + if previous_run is not None: + if previous_run.status == RunStatus.RUNNING: + raise ApiException( + status_code=409, + code="run_in_progress", + message="A run with this idempotency key is still in progress.", + details={ + "run_id": int(previous_run.id), + "idempotency_key": idempotency_key, + }, + ) + return success_payload( + request, + data=_manual_run_payload_from_run( + previous_run, + idempotency_key=idempotency_key, + reused_existing_run=True, + ), + ) + + user_settings = await user_settings_service.get_or_create_settings( + db_session, + user_id=current_user.id, + ) + try: + run_summary = await ingest_service.run_for_user( + db_session, + user_id=current_user.id, + trigger_type=RunTriggerType.MANUAL, + request_delay_seconds=user_settings.request_delay_seconds, + network_error_retries=settings.ingestion_network_error_retries, + retry_backoff_seconds=settings.ingestion_retry_backoff_seconds, + max_pages_per_scholar=settings.ingestion_max_pages_per_scholar, + page_size=settings.ingestion_page_size, + auto_queue_continuations=settings.ingestion_continuation_queue_enabled, + queue_delay_seconds=settings.ingestion_continuation_base_delay_seconds, + ) + except ingestion_service.RunAlreadyInProgressError as exc: + await db_session.rollback() + raise ApiException( + status_code=409, + code="run_in_progress", + message="A run is already in progress for this account.", + ) from exc + except Exception as exc: + await db_session.rollback() + logger.exception( + "api.runs.manual_failed", + extra={ + "event": "api.runs.manual_failed", + "user_id": current_user.id, + }, + ) + raise ApiException( + status_code=500, + code="manual_run_failed", + message="Manual run failed.", + ) from exc + + if idempotency_key is not None: + await run_service.set_manual_run_idempotency_key( + db_session, + user_id=current_user.id, + run_id=run_summary.crawl_run_id, + idempotency_key=idempotency_key, + ) + + return success_payload( + request, + data={ + "run_id": run_summary.crawl_run_id, + "status": run_summary.status.value, + "scholar_count": run_summary.scholar_count, + "succeeded_count": run_summary.succeeded_count, + "failed_count": run_summary.failed_count, + "partial_count": run_summary.partial_count, + "new_publication_count": run_summary.new_publication_count, + "reused_existing_run": False, + "idempotency_key": idempotency_key, + }, + ) + + +@router.get( + "/queue/items", + response_model=QueueListEnvelope, +) +async def list_queue_items( + request: Request, + limit: int = Query(default=200, ge=1, le=500), + db_session: AsyncSession = Depends(get_db_session), + current_user: User = Depends(get_api_current_user), +): + items = await run_service.list_queue_items_for_user( + db_session, + user_id=current_user.id, + limit=limit, + ) + return success_payload( + request, + data={ + "queue_items": [_serialize_queue_item(item) for item in items], + }, + ) + + +@router.post( + "/queue/{queue_item_id}/retry", + response_model=QueueItemEnvelope, +) +async def retry_queue_item( + queue_item_id: int, + request: Request, + db_session: AsyncSession = Depends(get_db_session), + current_user: User = Depends(get_api_current_user), +): + try: + queue_item = await run_service.retry_queue_item_for_user( + db_session, + user_id=current_user.id, + queue_item_id=queue_item_id, + ) + except run_service.QueueTransitionError as exc: + raise ApiException( + status_code=409, + code=exc.code, + message=exc.message, + details={"current_status": exc.current_status}, + ) from exc + if queue_item is None: + raise ApiException( + status_code=404, + code="queue_item_not_found", + message="Queue item not found.", + ) + return success_payload( + request, + data=_serialize_queue_item(queue_item), + ) + + +@router.post( + "/queue/{queue_item_id}/drop", + response_model=QueueItemEnvelope, +) +async def drop_queue_item( + queue_item_id: int, + request: Request, + db_session: AsyncSession = Depends(get_db_session), + current_user: User = Depends(get_api_current_user), +): + try: + dropped = await run_service.drop_queue_item_for_user( + db_session, + user_id=current_user.id, + queue_item_id=queue_item_id, + ) + except run_service.QueueTransitionError as exc: + raise ApiException( + status_code=409, + code=exc.code, + message=exc.message, + details={"current_status": exc.current_status}, + ) from exc + if dropped is None: + raise ApiException( + status_code=404, + code="queue_item_not_found", + message="Queue item not found.", + ) + return success_payload( + request, + data=_serialize_queue_item(dropped), + ) + + +@router.delete( + "/queue/{queue_item_id}", + response_model=QueueClearEnvelope, +) +async def clear_queue_item( + queue_item_id: int, + request: Request, + db_session: AsyncSession = Depends(get_db_session), + current_user: User = Depends(get_api_current_user), +): + try: + deleted = await run_service.clear_queue_item_for_user( + db_session, + user_id=current_user.id, + queue_item_id=queue_item_id, + ) + except run_service.QueueTransitionError as exc: + raise ApiException( + status_code=409, + code=exc.code, + message=exc.message, + details={"current_status": exc.current_status}, + ) from exc + if deleted is None: + raise ApiException( + status_code=404, + code="queue_item_not_found", + message="Queue item not found.", + ) + return success_payload( + request, + data={ + "queue_item_id": deleted.queue_item_id, + "previous_status": deleted.previous_status, + "status": "cleared", + "message": "Queue item cleared.", + }, + ) diff --git a/app/api/routers/scholars.py b/app/api/routers/scholars.py new file mode 100644 index 0000000..2f109b5 --- /dev/null +++ b/app/api/routers/scholars.py @@ -0,0 +1,169 @@ +from __future__ import annotations + +import logging + +from fastapi import APIRouter, Depends, Request +from sqlalchemy.ext.asyncio import AsyncSession + +from app.api.deps import get_api_current_user +from app.api.errors import ApiException +from app.api.responses import success_payload +from app.api.schemas import ( + MessageEnvelope, + ScholarCreateRequest, + ScholarEnvelope, + ScholarsListEnvelope, +) +from app.db.models import User +from app.db.session import get_db_session +from app.services import scholars as scholar_service + +logger = logging.getLogger(__name__) + +router = APIRouter(prefix="/scholars", tags=["api-scholars"]) + + +def _serialize_scholar(profile) -> dict[str, object]: + return { + "id": int(profile.id), + "scholar_id": profile.scholar_id, + "display_name": profile.display_name, + "is_enabled": bool(profile.is_enabled), + "baseline_completed": bool(profile.baseline_completed), + "last_run_dt": profile.last_run_dt, + "last_run_status": ( + profile.last_run_status.value if profile.last_run_status is not None else None + ), + } + + +@router.get( + "", + response_model=ScholarsListEnvelope, +) +async def list_scholars( + request: Request, + db_session: AsyncSession = Depends(get_db_session), + current_user: User = Depends(get_api_current_user), +): + scholars = await scholar_service.list_scholars_for_user( + db_session, + user_id=current_user.id, + ) + return success_payload( + request, + data={ + "scholars": [_serialize_scholar(profile) for profile in scholars], + }, + ) + + +@router.post( + "", + response_model=ScholarEnvelope, + status_code=201, +) +async def create_scholar( + payload: ScholarCreateRequest, + request: Request, + db_session: AsyncSession = Depends(get_db_session), + current_user: User = Depends(get_api_current_user), +): + try: + created = await scholar_service.create_scholar_for_user( + db_session, + user_id=current_user.id, + scholar_id=payload.scholar_id, + display_name=payload.display_name or "", + ) + except scholar_service.ScholarServiceError as exc: + raise ApiException( + status_code=400, + code="invalid_scholar", + message=str(exc), + ) from exc + logger.info( + "api.scholars.created", + extra={ + "event": "api.scholars.created", + "user_id": current_user.id, + "scholar_profile_id": created.id, + }, + ) + return success_payload( + request, + data=_serialize_scholar(created), + ) + + +@router.patch( + "/{scholar_profile_id}/toggle", + response_model=ScholarEnvelope, +) +async def toggle_scholar( + scholar_profile_id: int, + request: Request, + db_session: AsyncSession = Depends(get_db_session), + current_user: User = Depends(get_api_current_user), +): + profile = await scholar_service.get_user_scholar_by_id( + db_session, + user_id=current_user.id, + scholar_profile_id=scholar_profile_id, + ) + if profile is None: + raise ApiException( + status_code=404, + code="scholar_not_found", + message="Scholar not found.", + ) + updated = await scholar_service.toggle_scholar_enabled(db_session, profile=profile) + logger.info( + "api.scholars.toggled", + extra={ + "event": "api.scholars.toggled", + "user_id": current_user.id, + "scholar_profile_id": updated.id, + "is_enabled": updated.is_enabled, + }, + ) + return success_payload( + request, + data=_serialize_scholar(updated), + ) + + +@router.delete( + "/{scholar_profile_id}", + response_model=MessageEnvelope, +) +async def delete_scholar( + scholar_profile_id: int, + request: Request, + db_session: AsyncSession = Depends(get_db_session), + current_user: User = Depends(get_api_current_user), +): + profile = await scholar_service.get_user_scholar_by_id( + db_session, + user_id=current_user.id, + scholar_profile_id=scholar_profile_id, + ) + if profile is None: + raise ApiException( + status_code=404, + code="scholar_not_found", + message="Scholar not found.", + ) + await scholar_service.delete_scholar(db_session, profile=profile) + logger.info( + "api.scholars.deleted", + extra={ + "event": "api.scholars.deleted", + "user_id": current_user.id, + "scholar_profile_id": scholar_profile_id, + }, + ) + return success_payload( + request, + data={"message": "Scholar deleted."}, + ) diff --git a/app/api/routers/settings.py b/app/api/routers/settings.py new file mode 100644 index 0000000..7659591 --- /dev/null +++ b/app/api/routers/settings.py @@ -0,0 +1,96 @@ +from __future__ import annotations + +import logging + +from fastapi import APIRouter, Depends, Request +from sqlalchemy.ext.asyncio import AsyncSession + +from app.api.deps import get_api_current_user +from app.api.errors import ApiException +from app.api.responses import success_payload +from app.api.schemas import SettingsEnvelope, SettingsUpdateRequest +from app.db.models import User +from app.db.session import get_db_session +from app.services import user_settings as user_settings_service + +logger = logging.getLogger(__name__) + +router = APIRouter(prefix="/settings", tags=["api-settings"]) + + +def _serialize_settings(settings) -> dict[str, object]: + return { + "auto_run_enabled": bool(settings.auto_run_enabled), + "run_interval_minutes": int(settings.run_interval_minutes), + "request_delay_seconds": int(settings.request_delay_seconds), + } + + +@router.get( + "", + response_model=SettingsEnvelope, +) +async def get_settings( + request: Request, + db_session: AsyncSession = Depends(get_db_session), + current_user: User = Depends(get_api_current_user), +): + settings = await user_settings_service.get_or_create_settings( + db_session, + user_id=current_user.id, + ) + return success_payload( + request, + data=_serialize_settings(settings), + ) + + +@router.put( + "", + response_model=SettingsEnvelope, +) +async def update_settings( + payload: SettingsUpdateRequest, + request: Request, + db_session: AsyncSession = Depends(get_db_session), + current_user: User = Depends(get_api_current_user), +): + try: + parsed_interval = user_settings_service.parse_run_interval_minutes( + str(payload.run_interval_minutes) + ) + parsed_delay = user_settings_service.parse_request_delay_seconds( + str(payload.request_delay_seconds) + ) + except user_settings_service.UserSettingsServiceError as exc: + raise ApiException( + status_code=400, + code="invalid_settings", + message=str(exc), + ) from exc + + settings = await user_settings_service.get_or_create_settings( + db_session, + user_id=current_user.id, + ) + updated = await user_settings_service.update_settings( + db_session, + settings=settings, + auto_run_enabled=bool(payload.auto_run_enabled), + run_interval_minutes=parsed_interval, + request_delay_seconds=parsed_delay, + ) + logger.info( + "api.settings.updated", + extra={ + "event": "api.settings.updated", + "user_id": current_user.id, + "auto_run_enabled": updated.auto_run_enabled, + "run_interval_minutes": updated.run_interval_minutes, + "request_delay_seconds": updated.request_delay_seconds, + }, + ) + return success_payload( + request, + data=_serialize_settings(updated), + ) diff --git a/app/api/schemas.py b/app/api/schemas.py new file mode 100644 index 0000000..ab5934b --- /dev/null +++ b/app/api/schemas.py @@ -0,0 +1,456 @@ +from __future__ import annotations + +from datetime import datetime +from typing import Any + +from pydantic import BaseModel, ConfigDict, Field + + +class ApiMeta(BaseModel): + request_id: str | None = None + + model_config = ConfigDict(extra="forbid") + + +class ApiErrorData(BaseModel): + code: str + message: str + details: Any | None = None + + model_config = ConfigDict(extra="forbid") + + +class ApiErrorEnvelope(BaseModel): + error: ApiErrorData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") + + +class MessageData(BaseModel): + message: str + + model_config = ConfigDict(extra="forbid") + + +class MessageEnvelope(BaseModel): + data: MessageData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") + + +class SessionUserData(BaseModel): + id: int + email: str + is_admin: bool + is_active: bool + + model_config = ConfigDict(extra="forbid") + + +class AuthMeData(BaseModel): + authenticated: bool + csrf_token: str + user: SessionUserData + + model_config = ConfigDict(extra="forbid") + + +class AuthMeEnvelope(BaseModel): + data: AuthMeData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") + + +class CsrfBootstrapData(BaseModel): + csrf_token: str + authenticated: bool + + model_config = ConfigDict(extra="forbid") + + +class CsrfBootstrapEnvelope(BaseModel): + data: CsrfBootstrapData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") + + +class LoginRequest(BaseModel): + email: str + password: str + + model_config = ConfigDict(extra="forbid") + + +class LoginData(BaseModel): + authenticated: bool + csrf_token: str + user: SessionUserData + + model_config = ConfigDict(extra="forbid") + + +class LoginEnvelope(BaseModel): + data: LoginData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") + + +class ChangePasswordRequest(BaseModel): + current_password: str + new_password: str + confirm_password: str + + model_config = ConfigDict(extra="forbid") + + +class ScholarItemData(BaseModel): + id: int + scholar_id: str + display_name: str | None + is_enabled: bool + baseline_completed: bool + last_run_dt: datetime | None + last_run_status: str | None + + model_config = ConfigDict(extra="forbid") + + +class ScholarsListData(BaseModel): + scholars: list[ScholarItemData] + + model_config = ConfigDict(extra="forbid") + + +class ScholarsListEnvelope(BaseModel): + data: ScholarsListData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") + + +class ScholarCreateRequest(BaseModel): + scholar_id: str + display_name: str | None = None + + model_config = ConfigDict(extra="forbid") + + +class ScholarEnvelope(BaseModel): + data: ScholarItemData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") + + +class RunListItemData(BaseModel): + id: int + trigger_type: str + status: str + start_dt: datetime + end_dt: datetime | None + scholar_count: int + new_publication_count: int + failed_count: int + partial_count: int + + model_config = ConfigDict(extra="forbid") + + +class RunsListData(BaseModel): + runs: list[RunListItemData] + + model_config = ConfigDict(extra="forbid") + + +class RunsListEnvelope(BaseModel): + data: RunsListData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") + + +class RunSummaryData(BaseModel): + succeeded_count: int + failed_count: int + partial_count: int + failed_state_counts: dict[str, int] = Field(default_factory=dict) + failed_reason_counts: dict[str, int] = Field(default_factory=dict) + + model_config = ConfigDict(extra="forbid") + + +class RunAttemptLogData(BaseModel): + attempt: int + cstart: int + state: str | None = None + state_reason: str | None = None + status_code: int | None = None + fetch_error: str | None = None + + model_config = ConfigDict(extra="forbid") + + +class RunPageLogData(BaseModel): + page: int + cstart: int + state: str + state_reason: str | None = None + status_code: int | None = None + publication_count: int = 0 + attempt_count: int = 0 + has_show_more_button: bool = False + articles_range: str | None = None + warning_codes: list[str] = Field(default_factory=list) + + model_config = ConfigDict(extra="forbid") + + +class RunDebugData(BaseModel): + status_code: int | None = None + final_url: str | None = None + fetch_error: str | None = None + body_sha256: str | None = None + body_length: int | None = None + has_show_more_button: bool | None = None + articles_range: str | None = None + state_reason: str | None = None + warning_codes: list[str] = Field(default_factory=list) + marker_counts_nonzero: dict[str, int] = Field(default_factory=dict) + page_logs: list[RunPageLogData] = Field(default_factory=list) + attempt_log: list[RunAttemptLogData] = Field(default_factory=list) + + model_config = ConfigDict(extra="forbid") + + +class RunScholarResultData(BaseModel): + scholar_profile_id: int + scholar_id: str + state: str + state_reason: str | None = None + outcome: str + attempt_count: int = 0 + publication_count: int = 0 + start_cstart: int = 0 + continuation_cstart: int | None = None + continuation_enqueued: bool = False + continuation_cleared: bool = False + warnings: list[str] = Field(default_factory=list) + error: str | None = None + debug: RunDebugData | None = None + + model_config = ConfigDict(extra="forbid") + + +class RunDetailData(BaseModel): + run: RunListItemData + summary: RunSummaryData + scholar_results: list[RunScholarResultData] + + model_config = ConfigDict(extra="forbid") + + +class RunDetailEnvelope(BaseModel): + data: RunDetailData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") + + +class ManualRunData(BaseModel): + run_id: int + status: str + scholar_count: int + succeeded_count: int + failed_count: int + partial_count: int + new_publication_count: int + reused_existing_run: bool + idempotency_key: str | None = None + + model_config = ConfigDict(extra="forbid") + + +class ManualRunEnvelope(BaseModel): + data: ManualRunData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") + + +class QueueItemData(BaseModel): + id: int + scholar_profile_id: int + scholar_label: str + status: str + reason: str + dropped_reason: str | None + attempt_count: int + resume_cstart: int + next_attempt_dt: datetime | None + updated_at: datetime + last_error: str | None + last_run_id: int | None + + model_config = ConfigDict(extra="forbid") + + +class QueueListData(BaseModel): + queue_items: list[QueueItemData] + + model_config = ConfigDict(extra="forbid") + + +class QueueListEnvelope(BaseModel): + data: QueueListData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") + + +class QueueItemEnvelope(BaseModel): + data: QueueItemData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") + + +class QueueClearData(BaseModel): + queue_item_id: int + previous_status: str + status: str + message: str + + model_config = ConfigDict(extra="forbid") + + +class QueueClearEnvelope(BaseModel): + data: QueueClearData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") + + +class AdminUserData(BaseModel): + id: int + email: str + is_active: bool + is_admin: bool + created_at: datetime + updated_at: datetime + + model_config = ConfigDict(extra="forbid") + + +class AdminUsersListData(BaseModel): + users: list[AdminUserData] + + model_config = ConfigDict(extra="forbid") + + +class AdminUsersListEnvelope(BaseModel): + data: AdminUsersListData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") + + +class AdminUserCreateRequest(BaseModel): + email: str + password: str + is_admin: bool = False + + model_config = ConfigDict(extra="forbid") + + +class AdminUserActiveUpdateRequest(BaseModel): + is_active: bool + + model_config = ConfigDict(extra="forbid") + + +class AdminUserEnvelope(BaseModel): + data: AdminUserData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") + + +class AdminResetPasswordRequest(BaseModel): + new_password: str + + model_config = ConfigDict(extra="forbid") + + +class SettingsData(BaseModel): + auto_run_enabled: bool + run_interval_minutes: int + request_delay_seconds: int + + model_config = ConfigDict(extra="forbid") + + +class SettingsEnvelope(BaseModel): + data: SettingsData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") + + +class SettingsUpdateRequest(BaseModel): + auto_run_enabled: bool + run_interval_minutes: int + request_delay_seconds: int + + model_config = ConfigDict(extra="forbid") + + +class PublicationItemData(BaseModel): + publication_id: int + scholar_profile_id: int + scholar_label: str + title: str + year: int | None + citation_count: int + venue_text: str | None + pub_url: str | None + is_read: bool + first_seen_at: datetime + is_new_in_latest_run: bool + + model_config = ConfigDict(extra="forbid") + + +class PublicationsListData(BaseModel): + mode: str + selected_scholar_profile_id: int | None + new_count: int + total_count: int + publications: list[PublicationItemData] + + model_config = ConfigDict(extra="forbid") + + +class PublicationsListEnvelope(BaseModel): + data: PublicationsListData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") + + +class MarkAllReadData(BaseModel): + message: str + updated_count: int + + model_config = ConfigDict(extra="forbid") + + +class MarkAllReadEnvelope(BaseModel): + data: MarkAllReadData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") diff --git a/app/auth/__init__.py b/app/auth/__init__.py new file mode 100644 index 0000000..79644a4 --- /dev/null +++ b/app/auth/__init__.py @@ -0,0 +1,2 @@ +"""Authentication package for scholarr.""" + diff --git a/app/auth/deps.py b/app/auth/deps.py new file mode 100644 index 0000000..50424ae --- /dev/null +++ b/app/auth/deps.py @@ -0,0 +1,27 @@ +from __future__ import annotations + +from functools import lru_cache + +from app.auth.rate_limit import SlidingWindowRateLimiter +from app.auth.security import PasswordService +from app.auth.service import AuthService +from app.settings import settings + + +@lru_cache +def get_password_service() -> PasswordService: + return PasswordService() + + +@lru_cache +def get_auth_service() -> AuthService: + return AuthService(password_service=get_password_service()) + + +@lru_cache +def get_login_rate_limiter() -> SlidingWindowRateLimiter: + return SlidingWindowRateLimiter( + max_attempts=settings.login_rate_limit_attempts, + window_seconds=settings.login_rate_limit_window_seconds, + ) + diff --git a/app/auth/rate_limit.py b/app/auth/rate_limit.py new file mode 100644 index 0000000..a397265 --- /dev/null +++ b/app/auth/rate_limit.py @@ -0,0 +1,62 @@ +from __future__ import annotations + +from collections import defaultdict, deque +from collections.abc import Callable +from dataclasses import dataclass +from math import ceil +from threading import Lock +from time import monotonic + + +@dataclass(frozen=True) +class RateLimitDecision: + allowed: bool + retry_after_seconds: int = 0 + + +class SlidingWindowRateLimiter: + def __init__( + self, + *, + max_attempts: int, + window_seconds: int, + now: Callable[[], float] = monotonic, + ) -> None: + self._max_attempts = max_attempts + self._window_seconds = window_seconds + self._now = now + self._attempts: dict[str, deque[float]] = defaultdict(deque) + self._lock = Lock() + + def check(self, key: str) -> RateLimitDecision: + with self._lock: + now_value = self._now() + attempts = self._attempts[key] + self._trim_expired(attempts, now_value) + if len(attempts) >= self._max_attempts: + retry_after = self._window_seconds - (now_value - attempts[0]) + return RateLimitDecision( + allowed=False, + retry_after_seconds=max(1, ceil(retry_after)), + ) + return RateLimitDecision(allowed=True) + + def record_failure(self, key: str) -> None: + with self._lock: + now_value = self._now() + attempts = self._attempts[key] + self._trim_expired(attempts, now_value) + attempts.append(now_value) + + def reset(self, key: str) -> None: + with self._lock: + self._attempts.pop(key, None) + + def clear_all(self) -> None: + with self._lock: + self._attempts.clear() + + def _trim_expired(self, attempts: deque[float], now_value: float) -> None: + while attempts and now_value - attempts[0] >= self._window_seconds: + attempts.popleft() + diff --git a/app/auth/security.py b/app/auth/security.py new file mode 100644 index 0000000..f89b122 --- /dev/null +++ b/app/auth/security.py @@ -0,0 +1,19 @@ +from __future__ import annotations + +from argon2 import PasswordHasher +from argon2.exceptions import InvalidHashError, VerificationError + + +class PasswordService: + def __init__(self, hasher: PasswordHasher | None = None) -> None: + self._hasher = hasher or PasswordHasher() + + def hash_password(self, password: str) -> str: + return self._hasher.hash(password) + + def verify_password(self, password_hash: str, password: str) -> bool: + try: + return bool(self._hasher.verify(password_hash, password)) + except (InvalidHashError, VerificationError): + return False + diff --git a/app/auth/service.py b/app/auth/service.py new file mode 100644 index 0000000..9772358 --- /dev/null +++ b/app/auth/service.py @@ -0,0 +1,38 @@ +from __future__ import annotations + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.auth.security import PasswordService +from app.db.models import User + + +class AuthService: + def __init__(self, password_service: PasswordService) -> None: + self._password_service = password_service + + async def authenticate_user( + self, + db_session: AsyncSession, + *, + email: str, + password: str, + ) -> User | None: + normalized_email = email.strip().lower() + if not normalized_email or not password: + return None + result = await db_session.execute( + select(User).where(User.email == normalized_email) + ) + user = result.scalar_one_or_none() + if user is None or not user.is_active: + return None + if not self._password_service.verify_password(user.password_hash, password): + return None + return user + + def hash_password(self, password: str) -> str: + return self._password_service.hash_password(password) + + def verify_password(self, *, password_hash: str, password: str) -> bool: + return self._password_service.verify_password(password_hash, password) diff --git a/app/auth/session.py b/app/auth/session.py new file mode 100644 index 0000000..8c83197 --- /dev/null +++ b/app/auth/session.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +from dataclasses import dataclass + +from starlette.requests import Request + + +SESSION_USER_ID_KEY = "auth_user_id" +SESSION_USER_EMAIL_KEY = "auth_user_email" +SESSION_USER_IS_ADMIN_KEY = "auth_user_is_admin" + + +@dataclass(frozen=True) +class SessionUser: + id: int + email: str + is_admin: bool + + +def get_session_user(request: Request) -> SessionUser | None: + user_id = request.session.get(SESSION_USER_ID_KEY) + email = request.session.get(SESSION_USER_EMAIL_KEY) + is_admin = request.session.get(SESSION_USER_IS_ADMIN_KEY) + if user_id is None or email is None or is_admin is None: + return None + try: + parsed_user_id = int(user_id) + except (TypeError, ValueError): + return None + if not isinstance(email, str): + return None + return SessionUser(id=parsed_user_id, email=email, is_admin=bool(is_admin)) + + +def set_session_user( + request: Request, + *, + user_id: int, + email: str, + is_admin: bool, +) -> None: + request.session[SESSION_USER_ID_KEY] = int(user_id) + request.session[SESSION_USER_EMAIL_KEY] = email + request.session[SESSION_USER_IS_ADMIN_KEY] = bool(is_admin) + + +def clear_session_user(request: Request) -> None: + request.session.pop(SESSION_USER_ID_KEY, None) + request.session.pop(SESSION_USER_EMAIL_KEY, None) + request.session.pop(SESSION_USER_IS_ADMIN_KEY, None) diff --git a/app/db/__init__.py b/app/db/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/app/db/__init__.py @@ -0,0 +1 @@ + diff --git a/app/db/base.py b/app/db/base.py new file mode 100644 index 0000000..29adb07 --- /dev/null +++ b/app/db/base.py @@ -0,0 +1,18 @@ +from sqlalchemy import MetaData +from sqlalchemy.orm import DeclarativeBase + + +NAMING_CONVENTION = { + "ix": "ix_%(column_0_label)s", + "uq": "uq_%(table_name)s_%(column_0_name)s", + "ck": "ck_%(table_name)s_%(constraint_name)s", + "fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s", + "pk": "pk_%(table_name)s", +} + + +class Base(DeclarativeBase): + metadata = MetaData(naming_convention=NAMING_CONVENTION) + + +metadata = Base.metadata diff --git a/app/db/models.py b/app/db/models.py new file mode 100644 index 0000000..428c654 --- /dev/null +++ b/app/db/models.py @@ -0,0 +1,292 @@ +from __future__ import annotations + +from datetime import datetime +from enum import StrEnum + +from sqlalchemy import ( + Boolean, + CheckConstraint, + DateTime, + Enum, + ForeignKey, + Index, + Integer, + String, + Text, + UniqueConstraint, + func, + text, +) +from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.orm import Mapped, mapped_column + +from app.db.base import Base + + +class RunTriggerType(StrEnum): + MANUAL = "manual" + SCHEDULED = "scheduled" + + +class RunStatus(StrEnum): + RUNNING = "running" + SUCCESS = "success" + PARTIAL_FAILURE = "partial_failure" + FAILED = "failed" + + +class QueueItemStatus(StrEnum): + QUEUED = "queued" + RETRYING = "retrying" + DROPPED = "dropped" + + +RUN_STATUS_DB_ENUM = Enum( + RunStatus, + name="run_status", + values_callable=lambda members: [member.value for member in members], +) +RUN_TRIGGER_TYPE_DB_ENUM = Enum( + RunTriggerType, + name="run_trigger_type", + values_callable=lambda members: [member.value for member in members], +) + + +class User(Base): + __tablename__ = "users" + + id: Mapped[int] = mapped_column(primary_key=True) + email: Mapped[str] = mapped_column(String(255), nullable=False, unique=True) + password_hash: Mapped[str] = mapped_column(String(255), nullable=False) + is_active: Mapped[bool] = mapped_column( + Boolean, nullable=False, server_default=text("true") + ) + is_admin: Mapped[bool] = mapped_column( + Boolean, nullable=False, server_default=text("false") + ) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) + + +class UserSetting(Base): + __tablename__ = "user_settings" + __table_args__ = ( + CheckConstraint( + "run_interval_minutes >= 15", + name="run_interval_minutes_min_15", + ), + CheckConstraint( + "request_delay_seconds >= 1", + name="request_delay_seconds_min_1", + ), + ) + + user_id: Mapped[int] = mapped_column( + ForeignKey("users.id", ondelete="CASCADE"), primary_key=True + ) + auto_run_enabled: Mapped[bool] = mapped_column( + Boolean, nullable=False, server_default=text("false") + ) + run_interval_minutes: Mapped[int] = mapped_column( + Integer, nullable=False, server_default=text("1440") + ) + request_delay_seconds: Mapped[int] = mapped_column( + Integer, nullable=False, server_default=text("10") + ) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) + + +class ScholarProfile(Base): + __tablename__ = "scholar_profiles" + __table_args__ = ( + UniqueConstraint("user_id", "scholar_id", name="uq_scholar_profiles_user_scholar"), + Index("ix_scholar_profiles_user_enabled", "user_id", "is_enabled"), + ) + + id: Mapped[int] = mapped_column(primary_key=True) + user_id: Mapped[int] = mapped_column( + ForeignKey("users.id", ondelete="CASCADE"), nullable=False + ) + scholar_id: Mapped[str] = mapped_column(String(64), nullable=False) + display_name: Mapped[str | None] = mapped_column(String(255)) + is_enabled: Mapped[bool] = mapped_column( + Boolean, nullable=False, server_default=text("true") + ) + baseline_completed: Mapped[bool] = mapped_column( + Boolean, nullable=False, server_default=text("false") + ) + last_run_dt: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + last_run_status: Mapped[RunStatus | None] = mapped_column( + RUN_STATUS_DB_ENUM, + ) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) + + +class CrawlRun(Base): + __tablename__ = "crawl_runs" + __table_args__ = ( + Index("ix_crawl_runs_user_start", "user_id", "start_dt"), + ) + + id: Mapped[int] = mapped_column(primary_key=True) + user_id: Mapped[int] = mapped_column( + ForeignKey("users.id", ondelete="CASCADE"), nullable=False + ) + trigger_type: Mapped[RunTriggerType] = mapped_column( + RUN_TRIGGER_TYPE_DB_ENUM, nullable=False + ) + status: Mapped[RunStatus] = mapped_column( + RUN_STATUS_DB_ENUM, nullable=False + ) + start_dt: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) + end_dt: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + scholar_count: Mapped[int] = mapped_column( + Integer, nullable=False, server_default=text("0") + ) + new_pub_count: Mapped[int] = mapped_column( + Integer, nullable=False, server_default=text("0") + ) + error_log: Mapped[dict] = mapped_column( + JSONB, nullable=False, server_default=text("'{}'::jsonb") + ) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) + + +class Publication(Base): + __tablename__ = "publications" + __table_args__ = ( + UniqueConstraint("fingerprint_sha256", name="uq_publications_fingerprint"), + Index( + "uq_publications_cluster_id_not_null", + "cluster_id", + unique=True, + postgresql_where=text("cluster_id IS NOT NULL"), + ), + ) + + id: Mapped[int] = mapped_column(primary_key=True) + cluster_id: Mapped[str | None] = mapped_column(String(64)) + fingerprint_sha256: Mapped[str] = mapped_column(String(64), nullable=False) + title_raw: Mapped[str] = mapped_column(Text, nullable=False) + title_normalized: Mapped[str] = mapped_column(Text, nullable=False) + year: Mapped[int | None] = mapped_column(Integer) + citation_count: Mapped[int] = mapped_column( + Integer, nullable=False, server_default=text("0") + ) + author_text: Mapped[str | None] = mapped_column(Text) + venue_text: Mapped[str | None] = mapped_column(Text) + pub_url: Mapped[str | None] = mapped_column(Text) + pdf_url: Mapped[str | None] = mapped_column(Text) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) + + +class ScholarPublication(Base): + __tablename__ = "scholar_publications" + __table_args__ = ( + Index("ix_scholar_publications_is_read", "is_read"), + ) + + scholar_profile_id: Mapped[int] = mapped_column( + ForeignKey("scholar_profiles.id", ondelete="CASCADE"), + primary_key=True, + ) + publication_id: Mapped[int] = mapped_column( + ForeignKey("publications.id", ondelete="CASCADE"), + primary_key=True, + ) + is_read: Mapped[bool] = mapped_column( + Boolean, nullable=False, server_default=text("false") + ) + first_seen_run_id: Mapped[int | None] = mapped_column( + ForeignKey("crawl_runs.id", ondelete="SET NULL") + ) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) + + +class IngestionQueueItem(Base): + __tablename__ = "ingestion_queue_items" + __table_args__ = ( + UniqueConstraint( + "user_id", + "scholar_profile_id", + name="uq_ingestion_queue_user_scholar", + ), + CheckConstraint( + "status IN ('queued', 'retrying', 'dropped')", + name="ingestion_queue_status_valid", + ), + Index("ix_ingestion_queue_next_attempt", "next_attempt_dt"), + Index("ix_ingestion_queue_status_next_attempt", "status", "next_attempt_dt"), + ) + + id: Mapped[int] = mapped_column(primary_key=True) + user_id: Mapped[int] = mapped_column( + ForeignKey("users.id", ondelete="CASCADE"), + nullable=False, + ) + scholar_profile_id: Mapped[int] = mapped_column( + ForeignKey("scholar_profiles.id", ondelete="CASCADE"), + nullable=False, + ) + resume_cstart: Mapped[int] = mapped_column( + Integer, + nullable=False, + server_default=text("0"), + ) + reason: Mapped[str] = mapped_column(String(64), nullable=False) + status: Mapped[str] = mapped_column( + String(16), + nullable=False, + server_default=text("'queued'"), + ) + attempt_count: Mapped[int] = mapped_column( + Integer, + nullable=False, + server_default=text("0"), + ) + next_attempt_dt: Mapped[datetime] = mapped_column( + DateTime(timezone=True), + nullable=False, + server_default=func.now(), + ) + last_run_id: Mapped[int | None] = mapped_column( + ForeignKey("crawl_runs.id", ondelete="SET NULL"), + ) + last_error: Mapped[str | None] = mapped_column(Text) + dropped_reason: Mapped[str | None] = mapped_column(String(128)) + dropped_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) diff --git a/app/db/session.py b/app/db/session.py new file mode 100644 index 0000000..d1ccc04 --- /dev/null +++ b/app/db/session.py @@ -0,0 +1,64 @@ +from collections.abc import AsyncIterator +import logging + +from sqlalchemy import text +from sqlalchemy.ext.asyncio import ( + AsyncEngine, + AsyncSession, + async_sessionmaker, + create_async_engine, +) +from sqlalchemy.pool import NullPool + +from app.settings import settings + +logger = logging.getLogger(__name__) + +_engine: AsyncEngine | None = None +_session_factory: async_sessionmaker[AsyncSession] | None = None + + +def get_engine() -> AsyncEngine: + global _engine + if _engine is None: + # NullPool avoids cross-event-loop connection reuse in tests and dev tools. + _engine = create_async_engine( + settings.database_url, + pool_pre_ping=True, + poolclass=NullPool, + ) + logger.info("db.engine_initialized", extra={"event": "db.engine_initialized"}) + return _engine + + +def get_session_factory() -> async_sessionmaker[AsyncSession]: + global _session_factory + if _session_factory is None: + _session_factory = async_sessionmaker(get_engine(), expire_on_commit=False) + return _session_factory + + +async def get_db_session() -> AsyncIterator[AsyncSession]: + session_factory = get_session_factory() + async with session_factory() as session: + yield session + + +async def check_database() -> bool: + engine = get_engine() + try: + async with engine.connect() as conn: + result = await conn.execute(text("SELECT 1")) + return result.scalar_one() == 1 + except Exception: + logger.exception("db.healthcheck_failed", extra={"event": "db.healthcheck_failed"}) + return False + + +async def close_engine() -> None: + global _engine, _session_factory + if _engine is not None: + await _engine.dispose() + logger.info("db.engine_disposed", extra={"event": "db.engine_disposed"}) + _engine = None + _session_factory = None diff --git a/app/logging_config.py b/app/logging_config.py new file mode 100644 index 0000000..22907be --- /dev/null +++ b/app/logging_config.py @@ -0,0 +1,183 @@ +from __future__ import annotations + +from datetime import datetime, timezone +import json +import logging +import sys +from typing import Any + +from app.logging_context import get_request_id + +DEFAULT_REDACT_FIELDS = { + "authorization", + "cookie", + "csrf_token", + "new_password", + "password", + "password_hash", + "session", + "session_secret_key", +} + +_BASE_RECORD = logging.makeLogRecord({}) +_STANDARD_RECORD_FIELDS = set(_BASE_RECORD.__dict__.keys()) | {"message", "asctime"} +_NOISY_RECORD_FIELDS = {"color_message"} + + +def parse_redact_fields(raw: str | None) -> set[str]: + fields = {field.strip().lower() for field in (raw or "").split(",") if field.strip()} + return DEFAULT_REDACT_FIELDS | fields + + +def configure_logging( + *, + level: str, + log_format: str, + redact_fields: set[str], + include_uvicorn_access: bool, +) -> None: + normalized_level = _normalize_level(level) + root_logger = logging.getLogger() + root_logger.handlers.clear() + root_logger.setLevel(normalized_level) + + handler = logging.StreamHandler(stream=sys.stdout) + handler.setLevel(normalized_level) + handler.addFilter(RequestContextFilter()) + + normalized_format = log_format.strip().lower() + if normalized_format == "json": + handler.setFormatter(JsonLogFormatter(redact_fields=redact_fields)) + else: + handler.setFormatter(ConsoleLogFormatter(redact_fields=redact_fields)) + + root_logger.addHandler(handler) + + # Route server/framework logs through our single root handler. + for logger_name in ("uvicorn", "uvicorn.error"): + framework_logger = logging.getLogger(logger_name) + framework_logger.handlers.clear() + framework_logger.propagate = True + framework_logger.setLevel(normalized_level) + + access_logger = logging.getLogger("uvicorn.access") + access_logger.handlers.clear() + access_logger.propagate = True + access_logger.setLevel(normalized_level if include_uvicorn_access else logging.WARNING) + + +class RequestContextFilter(logging.Filter): + def filter(self, record: logging.LogRecord) -> bool: + if not getattr(record, "request_id", None): + request_id = get_request_id() + if request_id: + record.request_id = request_id + return True + + +class JsonLogFormatter(logging.Formatter): + def __init__(self, *, redact_fields: set[str]) -> None: + super().__init__() + self._redact_fields = {field.lower() for field in redact_fields} + + def format(self, record: logging.LogRecord) -> str: + payload: dict[str, Any] = { + "timestamp": _format_timestamp(record.created), + "level": record.levelname.lower(), + "logger": record.name, + "event": getattr(record, "event", record.getMessage()), + } + request_id = getattr(record, "request_id", None) + if request_id: + payload["request_id"] = request_id + payload.update(self._redact_mapping(_extra_fields(record))) + if record.exc_info: + payload["exception"] = self.formatException(record.exc_info) + return json.dumps(payload, ensure_ascii=True, default=str) + + def _redact_mapping(self, value: dict[str, Any]) -> dict[str, Any]: + return {key: self._redact_value(key, item) for key, item in value.items()} + + def _redact_value(self, key: str, value: Any) -> Any: + if key.lower() in self._redact_fields: + return "[REDACTED]" + if isinstance(value, dict): + return {nested_key: self._redact_value(nested_key, nested_value) for nested_key, nested_value in value.items()} + if isinstance(value, (list, tuple)): + return [self._redact_value(key, item) for item in value] + return value + + +class ConsoleLogFormatter(logging.Formatter): + def __init__(self, *, redact_fields: set[str]) -> None: + super().__init__() + self._json_formatter = JsonLogFormatter(redact_fields=redact_fields) + + def format(self, record: logging.LogRecord) -> str: + payload = json.loads(self._json_formatter.format(record)) + timestamp = payload.get("timestamp", "") + level = _short_level(payload.get("level", "info")) + logger_name = str(payload.get("logger", "app")) + event = str(payload.get("event", "")) + + parts = [timestamp, level, logger_name, event] + + request_id = payload.pop("request_id", None) + method = payload.pop("method", None) + path = payload.pop("path", None) + status_code = payload.pop("status_code", None) + duration_ms = payload.pop("duration_ms", None) + + if request_id: + parts.append(f"rid={request_id}") + if method and path: + parts.append(f"{method} {path}") + if status_code is not None: + parts.append(str(status_code)) + if duration_ms is not None: + parts.append(f"{duration_ms}ms") + + for key in sorted(payload.keys()): + if key in {"timestamp", "level", "logger", "event", "exception"}: + continue + parts.append(f"{key}={payload[key]}") + + if "exception" in payload: + parts.append(f"exception={payload['exception']}") + + return " | ".join(str(part) for part in parts if part) + + +def _extra_fields(record: logging.LogRecord) -> dict[str, Any]: + extras: dict[str, Any] = {} + for key, value in record.__dict__.items(): + if key in _STANDARD_RECORD_FIELDS or key.startswith("_"): + continue + if key in _NOISY_RECORD_FIELDS: + continue + extras[key] = value + return extras + + +def _normalize_level(level: str) -> int: + normalized = level.strip().upper() + mapping = logging.getLevelNamesMapping() + if normalized not in mapping: + return logging.INFO + return mapping[normalized] + + +def _format_timestamp(created_ts: float) -> str: + dt = datetime.fromtimestamp(created_ts, tz=timezone.utc) + return dt.strftime("%Y-%m-%d %H:%M:%SZ") + + +def _short_level(level: str) -> str: + mapping = { + "debug": "DBG", + "info": "INF", + "warning": "WRN", + "error": "ERR", + "critical": "CRT", + } + return mapping.get(level.lower(), level[:3].upper()) diff --git a/app/logging_context.py b/app/logging_context.py new file mode 100644 index 0000000..ee578fb --- /dev/null +++ b/app/logging_context.py @@ -0,0 +1,15 @@ +from __future__ import annotations + +from contextvars import ContextVar + + +_request_id_ctx: ContextVar[str | None] = ContextVar("request_id", default=None) + + +def get_request_id() -> str | None: + return _request_id_ctx.get() + + +def set_request_id(value: str | None) -> None: + _request_id_ctx.set(value) + diff --git a/app/main.py b/app/main.py new file mode 100644 index 0000000..75ee94d --- /dev/null +++ b/app/main.py @@ -0,0 +1,87 @@ +from __future__ import annotations + +from contextlib import asynccontextmanager + +from fastapi import FastAPI +from fastapi.staticfiles import StaticFiles +from starlette.middleware.sessions import SessionMiddleware + +from app.api.errors import register_api_exception_handlers +from app.api.router import router as api_router +from app.db.session import close_engine +from app.logging_config import configure_logging, parse_redact_fields +from app.security.csrf import CSRFMiddleware +from app.services.scheduler import SchedulerService +from app.settings import settings +from app.web import common as web_common +from app.web.deps import get_ingestion_service, get_scholar_source +from app.web.middleware import RequestLoggingMiddleware, parse_skip_paths +from app.web.routers import ( + admin, + auth, + dashboard, + health, + publications, + runs, + scholars, + settings as settings_router, +) + +configure_logging( + level=settings.log_level, + log_format=settings.log_format, + redact_fields=parse_redact_fields(settings.log_redact_fields), + include_uvicorn_access=settings.log_uvicorn_access, +) + +scheduler_service = SchedulerService( + enabled=settings.scheduler_enabled, + tick_seconds=settings.scheduler_tick_seconds, + network_error_retries=settings.ingestion_network_error_retries, + retry_backoff_seconds=settings.ingestion_retry_backoff_seconds, + max_pages_per_scholar=settings.ingestion_max_pages_per_scholar, + page_size=settings.ingestion_page_size, + continuation_queue_enabled=settings.ingestion_continuation_queue_enabled, + continuation_base_delay_seconds=settings.ingestion_continuation_base_delay_seconds, + continuation_max_delay_seconds=settings.ingestion_continuation_max_delay_seconds, + continuation_max_attempts=settings.ingestion_continuation_max_attempts, + queue_batch_size=settings.scheduler_queue_batch_size, +) + + +@asynccontextmanager +async def lifespan(_: FastAPI): + await scheduler_service.start() + yield + await scheduler_service.stop() + await close_engine() + + +app = FastAPI(title=settings.app_name, lifespan=lifespan) +register_api_exception_handlers(app) +app.add_middleware(CSRFMiddleware) +app.add_middleware( + SessionMiddleware, + secret_key=settings.session_secret_key, + same_site="lax", + https_only=settings.session_cookie_secure, +) +app.add_middleware( + RequestLoggingMiddleware, + log_requests=settings.log_requests, + skip_paths=parse_skip_paths(settings.log_request_skip_paths), +) +app.mount("/static", StaticFiles(directory="app/static"), name="static") + +app.include_router(api_router) +app.include_router(auth.router) +app.include_router(admin.router) +app.include_router(scholars.router) +app.include_router(settings_router.router) +app.include_router(runs.router) +app.include_router(publications.router) +app.include_router(dashboard.router) +app.include_router(health.router) + +# Backward-compatible export kept for tests and any existing local scripts. +_get_authenticated_user = web_common.get_authenticated_user diff --git a/app/presentation/__init__.py b/app/presentation/__init__.py new file mode 100644 index 0000000..d3d8cc8 --- /dev/null +++ b/app/presentation/__init__.py @@ -0,0 +1,2 @@ +"""Presentation-layer view models used by template routes.""" + diff --git a/app/presentation/dashboard.py b/app/presentation/dashboard.py new file mode 100644 index 0000000..2f5d609 --- /dev/null +++ b/app/presentation/dashboard.py @@ -0,0 +1,108 @@ +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime, timezone +from typing import Sequence + +from app.db.models import CrawlRun +from app.services.publications import UnreadPublicationItem + +SECTION_TEMPLATES: tuple[str, ...] = ( + "dashboard/_run_controls.html", + "dashboard/_unread_publications.html", + "dashboard/_run_history.html", +) + + +@dataclass(frozen=True) +class RunControlsViewModel: + request_delay_seconds: int + queue_queued_count: int + queue_retrying_count: int + queue_dropped_count: int + run_manual_action: str = "/runs/manual" + mark_all_read_action: str = "/publications/mark-all-read" + + +@dataclass(frozen=True) +class UnreadPublicationViewModel: + title: str + scholar_label: str + year_display: str + citation_count: int + venue_text: str | None + pub_url: str | None + + +@dataclass(frozen=True) +class RunHistoryItemViewModel: + run_id: int | None + detail_url: str | None + started_at_display: str + status_label: str + status_badge: str + scholar_count: int + new_publication_count: int + + +@dataclass(frozen=True) +class DashboardViewModel: + section_templates: tuple[str, ...] + run_controls: RunControlsViewModel + unread_publications: list[UnreadPublicationViewModel] + run_history: list[RunHistoryItemViewModel] + + +def build_dashboard_view_model( + *, + unread_publications: Sequence[UnreadPublicationItem], + recent_runs: Sequence[CrawlRun], + request_delay_seconds: int, + queue_counts: dict[str, int] | None = None, +) -> DashboardViewModel: + queue_counts = queue_counts or {} + return DashboardViewModel( + section_templates=SECTION_TEMPLATES, + run_controls=RunControlsViewModel( + request_delay_seconds=request_delay_seconds, + queue_queued_count=int(queue_counts.get("queued", 0)), + queue_retrying_count=int(queue_counts.get("retrying", 0)), + queue_dropped_count=int(queue_counts.get("dropped", 0)), + ), + unread_publications=[ + UnreadPublicationViewModel( + title=item.title, + scholar_label=item.scholar_label, + year_display=str(item.year) if item.year is not None else "-", + citation_count=item.citation_count, + venue_text=item.venue_text, + pub_url=item.pub_url, + ) + for item in unread_publications + ], + run_history=[ + RunHistoryItemViewModel( + run_id=run.id, + detail_url=f"/runs/{run.id}" if run.id is not None else None, + started_at_display=_format_started_at(run.start_dt), + status_label=run.status.value, + status_badge=_status_badge(run.status.value), + scholar_count=run.scholar_count, + new_publication_count=run.new_pub_count, + ) + for run in recent_runs + ], + ) + + +def _status_badge(status: str) -> str: + if status == "success": + return "ok" + if status == "failed": + return "danger" + return "warn" + + +def _format_started_at(dt: datetime) -> str: + local_aware = dt.astimezone(timezone.utc) + return local_aware.strftime("%Y-%m-%d %H:%M UTC") diff --git a/app/security/__init__.py b/app/security/__init__.py new file mode 100644 index 0000000..4033692 --- /dev/null +++ b/app/security/__init__.py @@ -0,0 +1,2 @@ +"""Security middleware and helpers for scholarr.""" + diff --git a/app/security/csrf.py b/app/security/csrf.py new file mode 100644 index 0000000..1f5c1a1 --- /dev/null +++ b/app/security/csrf.py @@ -0,0 +1,135 @@ +from __future__ import annotations + +import logging +from secrets import compare_digest, token_urlsafe +from urllib.parse import parse_qs + +from starlette.middleware.base import BaseHTTPMiddleware +from starlette.requests import Request +from starlette.responses import JSONResponse, PlainTextResponse, Response +from starlette.types import Message + + +CSRF_SESSION_KEY = "csrf_token" +CSRF_FORM_FIELD = "csrf_token" +CSRF_HEADER_NAME = "x-csrf-token" +SAFE_METHODS = {"GET", "HEAD", "OPTIONS", "TRACE"} +logger = logging.getLogger(__name__) + + +def ensure_csrf_token(request: Request) -> str: + token = request.session.get(CSRF_SESSION_KEY) + if token is None: + token = token_urlsafe(32) + request.session[CSRF_SESSION_KEY] = token + return str(token) + + +class CSRFMiddleware(BaseHTTPMiddleware): + def __init__(self, app, *, exempt_paths: set[str] | None = None) -> None: + super().__init__(app) + self._exempt_paths = exempt_paths or set() + + async def dispatch(self, request: Request, call_next) -> Response: + if self._should_skip(request): + return await call_next(request) + + session_token = request.session.get(CSRF_SESSION_KEY) + if not session_token: + logger.warning( + "csrf.missing_session_token", + extra={ + "event": "csrf.missing_session_token", + "method": request.method, + "path": request.url.path, + }, + ) + return self._csrf_error_response( + request, + code="csrf_missing", + message="CSRF token missing.", + ) + + request_token = request.headers.get(CSRF_HEADER_NAME) + if request_token is None and self._is_form_payload(request): + body = await request.body() + request_token = self._token_from_form_body(request, body) + request._receive = self._build_receive(body) # type: ignore[attr-defined] + + if not request_token or not compare_digest(str(session_token), str(request_token)): + logger.warning( + "csrf.invalid_token", + extra={ + "event": "csrf.invalid_token", + "method": request.method, + "path": request.url.path, + }, + ) + return self._csrf_error_response( + request, + code="csrf_invalid", + message="CSRF token invalid.", + ) + + return await call_next(request) + + def _should_skip(self, request: Request) -> bool: + if request.method in SAFE_METHODS: + return True + path = request.url.path + if path.startswith("/static/"): + return True + return path in self._exempt_paths + + def _is_form_payload(self, request: Request) -> bool: + content_type = request.headers.get("content-type", "") + return content_type.startswith("application/x-www-form-urlencoded") or ( + content_type.startswith("multipart/form-data") + ) + + def _token_from_form_body(self, request: Request, body: bytes) -> str | None: + content_type = request.headers.get("content-type", "") + if not content_type.startswith("application/x-www-form-urlencoded"): + return None + parsed = parse_qs(body.decode("utf-8"), keep_blank_values=True) + values = parsed.get(CSRF_FORM_FIELD) + if not values: + return None + return values[0] + + def _build_receive(self, body: bytes): + consumed = False + + async def receive() -> Message: + nonlocal consumed + if consumed: + return {"type": "http.request", "body": b"", "more_body": False} + consumed = True + return {"type": "http.request", "body": body, "more_body": False} + + return receive + + def _csrf_error_response( + self, + request: Request, + *, + code: str, + message: str, + ) -> Response: + if request.url.path.startswith("/api/"): + request_state = getattr(request, "state", None) + request_id = getattr(request_state, "request_id", None) if request_state else None + return JSONResponse( + { + "error": { + "code": code, + "message": message, + "details": None, + }, + "meta": { + "request_id": request_id, + }, + }, + status_code=403, + ) + return PlainTextResponse(message, status_code=403) diff --git a/app/services/__init__.py b/app/services/__init__.py new file mode 100644 index 0000000..167f869 --- /dev/null +++ b/app/services/__init__.py @@ -0,0 +1,2 @@ +"""Service layer for scholarr application workflows.""" + diff --git a/app/services/continuation_queue.py b/app/services/continuation_queue.py new file mode 100644 index 0000000..cb5b175 --- /dev/null +++ b/app/services/continuation_queue.py @@ -0,0 +1,278 @@ +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db.models import IngestionQueueItem, QueueItemStatus + + +@dataclass(frozen=True) +class ContinuationQueueJob: + id: int + user_id: int + scholar_profile_id: int + resume_cstart: int + reason: str + status: str + attempt_count: int + next_attempt_dt: datetime + + +ACTIVE_QUEUE_STATUSES: tuple[str, ...] = ( + QueueItemStatus.QUEUED.value, + QueueItemStatus.RETRYING.value, +) + + +def normalize_cstart(value: int | None) -> int: + if value is None: + return 0 + return max(0, int(value)) + + +def compute_backoff_seconds(*, base_seconds: int, attempt_count: int, max_seconds: int) -> int: + base = max(1, int(base_seconds)) + attempts = max(1, int(attempt_count)) + maximum = max(base, int(max_seconds)) + seconds = base * (2 ** max(0, attempts - 1)) + return min(seconds, maximum) + + +async def upsert_job( + db_session: AsyncSession, + *, + user_id: int, + scholar_profile_id: int, + resume_cstart: int, + reason: str, + run_id: int | None, + delay_seconds: int, +) -> IngestionQueueItem: + now = datetime.now(timezone.utc) + next_attempt_dt = now + timedelta(seconds=max(0, int(delay_seconds))) + result = await db_session.execute( + select(IngestionQueueItem).where( + IngestionQueueItem.user_id == user_id, + IngestionQueueItem.scholar_profile_id == scholar_profile_id, + ) + ) + item = result.scalar_one_or_none() + normalized_cstart = normalize_cstart(resume_cstart) + if item is None: + item = IngestionQueueItem( + user_id=user_id, + scholar_profile_id=scholar_profile_id, + resume_cstart=normalized_cstart, + reason=reason, + status=QueueItemStatus.QUEUED.value, + attempt_count=0, + next_attempt_dt=next_attempt_dt, + last_run_id=run_id, + last_error=None, + dropped_reason=None, + dropped_at=None, + created_at=now, + updated_at=now, + ) + db_session.add(item) + return item + + item.resume_cstart = normalized_cstart + item.reason = reason + if item.status == QueueItemStatus.DROPPED.value: + item.attempt_count = 0 + item.status = QueueItemStatus.QUEUED.value + item.next_attempt_dt = next_attempt_dt + item.last_run_id = run_id + item.last_error = None + item.dropped_reason = None + item.dropped_at = None + item.updated_at = now + return item + + +async def clear_job_for_scholar( + db_session: AsyncSession, + *, + user_id: int, + scholar_profile_id: int, +) -> bool: + result = await db_session.execute( + select(IngestionQueueItem).where( + IngestionQueueItem.user_id == user_id, + IngestionQueueItem.scholar_profile_id == scholar_profile_id, + ) + ) + item = result.scalar_one_or_none() + if item is None: + return False + await db_session.delete(item) + return True + + +async def delete_job_by_id( + db_session: AsyncSession, + *, + job_id: int, +) -> bool: + result = await db_session.execute( + select(IngestionQueueItem).where(IngestionQueueItem.id == job_id) + ) + item = result.scalar_one_or_none() + if item is None: + return False + await db_session.delete(item) + return True + + +async def list_due_jobs( + db_session: AsyncSession, + *, + now: datetime, + limit: int, +) -> list[ContinuationQueueJob]: + result = await db_session.execute( + select(IngestionQueueItem) + .where( + IngestionQueueItem.next_attempt_dt <= now, + IngestionQueueItem.status.in_(ACTIVE_QUEUE_STATUSES), + ) + .order_by( + IngestionQueueItem.next_attempt_dt.asc(), + IngestionQueueItem.id.asc(), + ) + .limit(limit) + ) + rows = list(result.scalars().all()) + jobs: list[ContinuationQueueJob] = [] + for row in rows: + jobs.append( + ContinuationQueueJob( + id=int(row.id), + user_id=int(row.user_id), + scholar_profile_id=int(row.scholar_profile_id), + resume_cstart=normalize_cstart(row.resume_cstart), + reason=row.reason, + status=row.status, + attempt_count=int(row.attempt_count), + next_attempt_dt=row.next_attempt_dt, + ) + ) + return jobs + + +async def increment_attempt_count( + db_session: AsyncSession, + *, + job_id: int, +) -> IngestionQueueItem | None: + now = datetime.now(timezone.utc) + result = await db_session.execute( + select(IngestionQueueItem).where(IngestionQueueItem.id == job_id) + ) + item = result.scalar_one_or_none() + if item is None: + return None + item.attempt_count = int(item.attempt_count or 0) + 1 + item.updated_at = now + return item + + +async def reschedule_job( + db_session: AsyncSession, + *, + job_id: int, + delay_seconds: int, + reason: str, + error: str | None = None, +) -> IngestionQueueItem | None: + now = datetime.now(timezone.utc) + result = await db_session.execute( + select(IngestionQueueItem).where(IngestionQueueItem.id == job_id) + ) + item = result.scalar_one_or_none() + if item is None: + return None + item.next_attempt_dt = now + timedelta(seconds=max(1, int(delay_seconds))) + item.status = QueueItemStatus.QUEUED.value + item.reason = reason + item.last_error = error + item.dropped_reason = None + item.dropped_at = None + item.updated_at = now + return item + + +async def mark_retrying( + db_session: AsyncSession, + *, + job_id: int, + reason: str | None = None, +) -> IngestionQueueItem | None: + now = datetime.now(timezone.utc) + result = await db_session.execute( + select(IngestionQueueItem).where(IngestionQueueItem.id == job_id) + ) + item = result.scalar_one_or_none() + if item is None: + return None + if item.status == QueueItemStatus.DROPPED.value: + return item + item.status = QueueItemStatus.RETRYING.value + if reason: + item.reason = reason + item.updated_at = now + return item + + +async def mark_dropped( + db_session: AsyncSession, + *, + job_id: int, + reason: str, + error: str | None = None, +) -> IngestionQueueItem | None: + now = datetime.now(timezone.utc) + result = await db_session.execute( + select(IngestionQueueItem).where(IngestionQueueItem.id == job_id) + ) + item = result.scalar_one_or_none() + if item is None: + return None + item.status = QueueItemStatus.DROPPED.value + item.reason = "dropped" + item.dropped_reason = reason + item.dropped_at = now + if error is not None: + item.last_error = error + item.updated_at = now + return item + + +async def mark_queued_now( + db_session: AsyncSession, + *, + job_id: int, + reason: str, + reset_attempt_count: bool = False, +) -> IngestionQueueItem | None: + now = datetime.now(timezone.utc) + result = await db_session.execute( + select(IngestionQueueItem).where(IngestionQueueItem.id == job_id) + ) + item = result.scalar_one_or_none() + if item is None: + return None + item.status = QueueItemStatus.QUEUED.value + item.reason = reason + item.next_attempt_dt = now + if reset_attempt_count: + item.attempt_count = 0 + item.last_error = None + item.dropped_reason = None + item.dropped_at = None + item.updated_at = now + return item diff --git a/app/services/ingestion.py b/app/services/ingestion.py new file mode 100644 index 0000000..61025bb --- /dev/null +++ b/app/services/ingestion.py @@ -0,0 +1,967 @@ +from __future__ import annotations + +import asyncio +from dataclasses import dataclass +from datetime import datetime, timezone +import hashlib +import logging +import re +from typing import Any +from urllib.parse import urljoin + +from sqlalchemy import select, text +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db.models import ( + CrawlRun, + Publication, + RunStatus, + RunTriggerType, + ScholarProfile, + ScholarPublication, +) +from app.services import continuation_queue as queue_service +from app.services.scholar_parser import ( + ParseState, + ParsedProfilePage, + PublicationCandidate, + parse_profile_page, +) +from app.services.scholar_source import FetchResult, ScholarSource + +TITLE_ALNUM_RE = re.compile(r"[^a-z0-9]+") +WORD_RE = re.compile(r"[a-z0-9]+") +HTML_TAG_RE = re.compile(r"<[^>]+>", re.S) +SPACE_RE = re.compile(r"\s+") +FAILED_STATES = { + ParseState.BLOCKED_OR_CAPTCHA.value, + ParseState.LAYOUT_CHANGED.value, + ParseState.NETWORK_ERROR.value, + "ingestion_error", +} +RUN_LOCK_NAMESPACE = 8217 +RESUMABLE_PARTIAL_REASONS = { + "max_pages_reached", + "pagination_cursor_stalled", +} +RESUMABLE_PARTIAL_REASON_PREFIXES = ("page_state_network_error",) +logger = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class RunExecutionSummary: + crawl_run_id: int + status: RunStatus + scholar_count: int + succeeded_count: int + failed_count: int + partial_count: int + new_publication_count: int + + +@dataclass(frozen=True) +class PagedParseResult: + fetch_result: FetchResult + parsed_page: ParsedProfilePage + publications: list[PublicationCandidate] + attempt_log: list[dict[str, Any]] + page_logs: list[dict[str, Any]] + pages_fetched: int + pages_attempted: int + has_more_remaining: bool + pagination_truncated_reason: str | None + continuation_cstart: int | None + + +class RunAlreadyInProgressError(RuntimeError): + """Raised when a run lock for a user is already held by another process.""" + + +class ScholarIngestionService: + def __init__(self, *, source: ScholarSource) -> None: + self._source = source + + async def run_for_user( + self, + db_session: AsyncSession, + *, + user_id: int, + trigger_type: RunTriggerType, + request_delay_seconds: int, + network_error_retries: int = 1, + retry_backoff_seconds: float = 1.0, + max_pages_per_scholar: int = 30, + page_size: int = 100, + scholar_profile_ids: set[int] | None = None, + start_cstart_by_scholar_id: dict[int, int] | None = None, + auto_queue_continuations: bool = True, + queue_delay_seconds: int = 60, + ) -> RunExecutionSummary: + lock_acquired = await self._try_acquire_user_lock( + db_session, + user_id=user_id, + ) + if not lock_acquired: + raise RunAlreadyInProgressError(f"Run already in progress for user_id={user_id}.") + + filtered_scholar_ids = ( + {int(value) for value in scholar_profile_ids} + if scholar_profile_ids is not None + else None + ) + start_cstart_map = { + int(key): max(0, int(value)) + for key, value in (start_cstart_by_scholar_id or {}).items() + } + + scholars_stmt = ( + select(ScholarProfile) + .where( + ScholarProfile.user_id == user_id, + ScholarProfile.is_enabled.is_(True), + ) + .order_by(ScholarProfile.created_at.asc(), ScholarProfile.id.asc()) + ) + if filtered_scholar_ids is not None: + scholars_stmt = scholars_stmt.where( + ScholarProfile.id.in_(filtered_scholar_ids) + ) + + scholars_result = await db_session.execute( + scholars_stmt + ) + scholars = list(scholars_result.scalars().all()) + if filtered_scholar_ids is not None: + found_ids = {int(scholar.id) for scholar in scholars} + missing_ids = filtered_scholar_ids - found_ids + for scholar_profile_id in missing_ids: + await queue_service.clear_job_for_scholar( + db_session, + user_id=user_id, + scholar_profile_id=scholar_profile_id, + ) + logger.info( + "ingestion.run_started", + extra={ + "event": "ingestion.run_started", + "user_id": user_id, + "trigger_type": trigger_type.value, + "scholar_count": len(scholars), + "is_filtered_run": filtered_scholar_ids is not None, + "request_delay_seconds": request_delay_seconds, + "network_error_retries": network_error_retries, + "retry_backoff_seconds": retry_backoff_seconds, + "max_pages_per_scholar": max_pages_per_scholar, + "page_size": page_size, + }, + ) + + run = CrawlRun( + user_id=user_id, + trigger_type=trigger_type, + status=RunStatus.RUNNING, + scholar_count=len(scholars), + new_pub_count=0, + error_log={}, + ) + db_session.add(run) + await db_session.flush() + + succeeded_count = 0 + failed_count = 0 + partial_count = 0 + scholar_results: list[dict[str, Any]] = [] + + for index, scholar in enumerate(scholars): + if index > 0 and request_delay_seconds > 0: + await asyncio.sleep(float(request_delay_seconds)) + + run_dt = datetime.now(timezone.utc) + start_cstart = int(start_cstart_map.get(int(scholar.id), 0)) + + paged_parse_result = await self._fetch_and_parse_all_pages_with_retry( + scholar_id=scholar.scholar_id, + start_cstart=start_cstart, + request_delay_seconds=request_delay_seconds, + network_error_retries=network_error_retries, + retry_backoff_seconds=retry_backoff_seconds, + max_pages=max_pages_per_scholar, + page_size=page_size, + ) + fetch_result = paged_parse_result.fetch_result + parsed_page = paged_parse_result.parsed_page + publications = paged_parse_result.publications + attempt_log = paged_parse_result.attempt_log + page_logs = paged_parse_result.page_logs + + logger.info( + "ingestion.scholar_parsed", + extra={ + "event": "ingestion.scholar_parsed", + "user_id": user_id, + "crawl_run_id": run.id, + "scholar_profile_id": scholar.id, + "scholar_id": scholar.scholar_id, + "state": parsed_page.state.value, + "publication_count": len(publications), + "has_show_more_button": parsed_page.has_show_more_button, + "pages_fetched": paged_parse_result.pages_fetched, + "pages_attempted": paged_parse_result.pages_attempted, + "has_more_remaining": paged_parse_result.has_more_remaining, + "pagination_truncated_reason": paged_parse_result.pagination_truncated_reason, + "warning_count": len(parsed_page.warnings), + }, + ) + + result_entry = { + "scholar_profile_id": scholar.id, + "scholar_id": scholar.scholar_id, + "state": parsed_page.state.value, + "state_reason": parsed_page.state_reason, + "outcome": "failed", + "attempt_count": len(attempt_log), + "publication_count": len(publications), + "start_cstart": start_cstart, + "articles_range": parsed_page.articles_range, + "warnings": parsed_page.warnings, + "has_show_more_button": parsed_page.has_show_more_button, + "pages_fetched": paged_parse_result.pages_fetched, + "pages_attempted": paged_parse_result.pages_attempted, + "has_more_remaining": paged_parse_result.has_more_remaining, + "pagination_truncated_reason": paged_parse_result.pagination_truncated_reason, + "continuation_cstart": paged_parse_result.continuation_cstart, + } + had_page_failure = parsed_page.state not in {ParseState.OK, ParseState.NO_RESULTS} + has_partial_publication_set = len(publications) > 0 and had_page_failure + is_partial_due_to_pagination = ( + paged_parse_result.has_more_remaining + or paged_parse_result.pagination_truncated_reason is not None + ) + + if (not had_page_failure) or has_partial_publication_set: + try: + discovered_publication_count = await self._upsert_profile_publications( + db_session, + run=run, + scholar=scholar, + publications=publications, + ) + run.new_pub_count = int(run.new_pub_count or 0) + discovered_publication_count + + is_partial_scholar = is_partial_due_to_pagination or has_partial_publication_set + scholar.last_run_status = ( + RunStatus.PARTIAL_FAILURE if is_partial_scholar else RunStatus.SUCCESS + ) + scholar.last_run_dt = run_dt + succeeded_count += 1 + result_entry["outcome"] = "partial" if is_partial_scholar else "success" + + if is_partial_scholar: + partial_count += 1 + result_entry["debug"] = self._build_failure_debug_context( + fetch_result=fetch_result, + parsed_page=parsed_page, + attempt_log=attempt_log, + page_logs=page_logs, + ) + except Exception as exc: + failed_count += 1 + scholar.last_run_status = RunStatus.FAILED + scholar.last_run_dt = run_dt + result_entry["state"] = "ingestion_error" + result_entry["state_reason"] = "publication_upsert_exception" + result_entry["outcome"] = "failed" + result_entry["error"] = str(exc) + result_entry["debug"] = self._build_failure_debug_context( + fetch_result=fetch_result, + parsed_page=parsed_page, + attempt_log=attempt_log, + page_logs=page_logs, + exception=exc, + ) + logger.exception( + "ingestion.scholar_failed", + extra={ + "event": "ingestion.scholar_failed", + "user_id": user_id, + "crawl_run_id": run.id, + "scholar_profile_id": scholar.id, + "scholar_id": scholar.scholar_id, + }, + ) + else: + failed_count += 1 + scholar.last_run_status = RunStatus.FAILED + scholar.last_run_dt = run_dt + result_entry["debug"] = self._build_failure_debug_context( + fetch_result=fetch_result, + parsed_page=parsed_page, + attempt_log=attempt_log, + page_logs=page_logs, + ) + logger.warning( + "ingestion.scholar_parse_failed", + extra={ + "event": "ingestion.scholar_parse_failed", + "user_id": user_id, + "crawl_run_id": run.id, + "scholar_profile_id": scholar.id, + "scholar_id": scholar.scholar_id, + "state": parsed_page.state.value, + "state_reason": parsed_page.state_reason, + "status_code": fetch_result.status_code, + }, + ) + + queue_reason, queue_cstart = self._resolve_continuation_queue_target( + outcome=str(result_entry.get("outcome", "")), + state=str(result_entry.get("state", "")), + pagination_truncated_reason=paged_parse_result.pagination_truncated_reason, + continuation_cstart=paged_parse_result.continuation_cstart, + fallback_cstart=start_cstart, + ) + if auto_queue_continuations and queue_reason is not None: + await queue_service.upsert_job( + db_session, + user_id=user_id, + scholar_profile_id=scholar.id, + resume_cstart=queue_cstart, + reason=queue_reason, + run_id=run.id, + delay_seconds=queue_delay_seconds, + ) + result_entry["continuation_enqueued"] = True + result_entry["continuation_reason"] = queue_reason + result_entry["continuation_cstart"] = queue_cstart + else: + cleared = await queue_service.clear_job_for_scholar( + db_session, + user_id=user_id, + scholar_profile_id=scholar.id, + ) + if cleared: + result_entry["continuation_cleared"] = True + + scholar_results.append(result_entry) + + failed_state_counts: dict[str, int] = {} + failed_reason_counts: dict[str, int] = {} + for entry in scholar_results: + if str(entry.get("outcome", "")) != "failed": + continue + state = str(entry.get("state", "")) + if state not in FAILED_STATES: + continue + failed_state_counts[state] = failed_state_counts.get(state, 0) + 1 + reason = str(entry.get("state_reason", "")).strip() + if reason: + failed_reason_counts[reason] = failed_reason_counts.get(reason, 0) + 1 + + run.end_dt = datetime.now(timezone.utc) + run.status = self._resolve_run_status( + scholar_count=len(scholars), + succeeded_count=succeeded_count, + failed_count=failed_count, + partial_count=partial_count, + ) + run.error_log = { + "scholar_results": scholar_results, + "summary": { + "succeeded_count": succeeded_count, + "failed_count": failed_count, + "partial_count": partial_count, + "failed_state_counts": failed_state_counts, + "failed_reason_counts": failed_reason_counts, + }, + } + + await db_session.commit() + logger.info( + "ingestion.run_completed", + extra={ + "event": "ingestion.run_completed", + "user_id": user_id, + "crawl_run_id": run.id, + "status": run.status.value, + "scholar_count": len(scholars), + "succeeded_count": succeeded_count, + "failed_count": failed_count, + "partial_count": partial_count, + "new_publication_count": run.new_pub_count, + }, + ) + + return RunExecutionSummary( + crawl_run_id=run.id, + status=run.status, + scholar_count=len(scholars), + succeeded_count=succeeded_count, + failed_count=failed_count, + partial_count=partial_count, + new_publication_count=run.new_pub_count, + ) + + async def _fetch_profile_page( + self, + *, + scholar_id: str, + cstart: int, + page_size: int, + ) -> FetchResult: + try: + page_fetcher = getattr(self._source, "fetch_profile_page_html", None) + if callable(page_fetcher): + return await page_fetcher( + scholar_id, + cstart=cstart, + pagesize=page_size, + ) + if cstart <= 0: + return await self._source.fetch_profile_html(scholar_id) + return FetchResult( + requested_url=( + "https://scholar.google.com/citations" + f"?hl=en&user={scholar_id}&cstart={cstart}&pagesize={page_size}" + ), + status_code=None, + final_url=None, + body="", + error="source_does_not_support_pagination", + ) + except Exception as exc: + logger.exception( + "ingestion.fetch_unexpected_error", + extra={ + "event": "ingestion.fetch_unexpected_error", + "scholar_id": scholar_id, + "cstart": cstart, + "page_size": page_size, + }, + ) + return FetchResult( + requested_url=( + "https://scholar.google.com/citations" + f"?hl=en&user={scholar_id}&cstart={cstart}&pagesize={page_size}" + ), + status_code=None, + final_url=None, + body="", + error=str(exc), + ) + + async def _fetch_and_parse_page_with_retry( + self, + *, + scholar_id: str, + cstart: int, + page_size: int, + network_error_retries: int, + retry_backoff_seconds: float, + ) -> tuple[FetchResult, ParsedProfilePage, list[dict[str, Any]]]: + max_attempts = max(1, int(network_error_retries) + 1) + backoff = max(float(retry_backoff_seconds), 0.0) + attempt_log: list[dict[str, Any]] = [] + fetch_result: FetchResult | None = None + parsed_page: ParsedProfilePage | None = None + + for attempt_index in range(max_attempts): + fetch_result = await self._fetch_profile_page( + scholar_id=scholar_id, + cstart=cstart, + page_size=page_size, + ) + parsed_page = parse_profile_page(fetch_result) + attempt_log.append( + { + "attempt": attempt_index + 1, + "cstart": cstart, + "state": parsed_page.state.value, + "state_reason": parsed_page.state_reason, + "status_code": fetch_result.status_code, + "fetch_error": fetch_result.error, + } + ) + + should_retry = ( + parsed_page.state == ParseState.NETWORK_ERROR + and attempt_index < max_attempts - 1 + ) + if not should_retry: + break + + sleep_seconds = backoff * (2**attempt_index) + logger.warning( + "ingestion.scholar_retry_scheduled", + extra={ + "event": "ingestion.scholar_retry_scheduled", + "scholar_id": scholar_id, + "cstart": cstart, + "attempt": attempt_index + 1, + "next_attempt": attempt_index + 2, + "sleep_seconds": sleep_seconds, + "state_reason": parsed_page.state_reason, + }, + ) + if sleep_seconds > 0: + await asyncio.sleep(sleep_seconds) + + if fetch_result is None or parsed_page is None: + raise RuntimeError("Fetch-and-parse retry loop produced no result.") + return fetch_result, parsed_page, attempt_log + + async def _fetch_and_parse_all_pages_with_retry( + self, + *, + scholar_id: str, + start_cstart: int, + request_delay_seconds: int, + network_error_retries: int, + retry_backoff_seconds: float, + max_pages: int, + page_size: int, + ) -> PagedParseResult: + bounded_max_pages = max(1, int(max_pages)) + bounded_page_size = max(1, int(page_size)) + + ( + fetch_result, + parsed_page, + first_attempt_log, + ) = await self._fetch_and_parse_page_with_retry( + scholar_id=scholar_id, + cstart=start_cstart, + page_size=bounded_page_size, + network_error_retries=network_error_retries, + retry_backoff_seconds=retry_backoff_seconds, + ) + + attempt_log: list[dict[str, Any]] = list(first_attempt_log) + page_logs: list[dict[str, Any]] = [] + page_logs.append( + { + "page": 1, + "cstart": start_cstart, + "state": parsed_page.state.value, + "state_reason": parsed_page.state_reason, + "status_code": fetch_result.status_code, + "publication_count": len(parsed_page.publications), + "articles_range": parsed_page.articles_range, + "has_show_more_button": parsed_page.has_show_more_button, + "warning_codes": parsed_page.warnings, + "attempt_count": len(first_attempt_log), + } + ) + pages_attempted = 1 + + # Immediate hard failure: nothing to salvage. + if parsed_page.state not in {ParseState.OK, ParseState.NO_RESULTS}: + return PagedParseResult( + fetch_result=fetch_result, + parsed_page=parsed_page, + publications=[], + attempt_log=attempt_log, + page_logs=page_logs, + pages_fetched=0, + pages_attempted=pages_attempted, + has_more_remaining=False, + pagination_truncated_reason=None, + continuation_cstart=( + start_cstart if parsed_page.state == ParseState.NETWORK_ERROR else None + ), + ) + + publications = list(parsed_page.publications) + pages_fetched = 1 + has_more_remaining = False + pagination_truncated_reason: str | None = None + continuation_cstart: int | None = None + current_cstart = start_cstart + next_cstart = _next_cstart_value( + articles_range=parsed_page.articles_range, + fallback=current_cstart + len(parsed_page.publications), + ) + + while parsed_page.has_show_more_button: + if pages_fetched >= bounded_max_pages: + has_more_remaining = True + pagination_truncated_reason = "max_pages_reached" + continuation_cstart = next_cstart if next_cstart > current_cstart else current_cstart + break + if next_cstart <= current_cstart: + has_more_remaining = True + pagination_truncated_reason = "pagination_cursor_stalled" + continuation_cstart = current_cstart + break + if request_delay_seconds > 0: + await asyncio.sleep(float(request_delay_seconds)) + + current_cstart = next_cstart + ( + page_fetch_result, + page_parsed, + page_attempt_log, + ) = await self._fetch_and_parse_page_with_retry( + scholar_id=scholar_id, + cstart=current_cstart, + page_size=bounded_page_size, + network_error_retries=network_error_retries, + retry_backoff_seconds=retry_backoff_seconds, + ) + + pages_attempted += 1 + attempt_log.extend(page_attempt_log) + page_logs.append( + { + "page": pages_attempted, + "cstart": current_cstart, + "state": page_parsed.state.value, + "state_reason": page_parsed.state_reason, + "status_code": page_fetch_result.status_code, + "publication_count": len(page_parsed.publications), + "articles_range": page_parsed.articles_range, + "has_show_more_button": page_parsed.has_show_more_button, + "warning_codes": page_parsed.warnings, + "attempt_count": len(page_attempt_log), + } + ) + + fetch_result = page_fetch_result + parsed_page = page_parsed + if parsed_page.state not in {ParseState.OK, ParseState.NO_RESULTS}: + has_more_remaining = True + pagination_truncated_reason = f"page_state_{parsed_page.state.value}" + continuation_cstart = current_cstart + break + + # Google may keep a stale/disabled "show more" marker while returning an empty tail page. + # Treat this as a terminal page to avoid false cursor-stalled partial runs. + if parsed_page.state == ParseState.NO_RESULTS and len(parsed_page.publications) == 0: + pages_fetched += 1 + break + + pages_fetched += 1 + publications.extend(parsed_page.publications) + next_cstart = _next_cstart_value( + articles_range=parsed_page.articles_range, + fallback=current_cstart + len(parsed_page.publications), + ) + + return PagedParseResult( + fetch_result=fetch_result, + parsed_page=parsed_page, + publications=_dedupe_publication_candidates(publications), + attempt_log=attempt_log, + page_logs=page_logs, + pages_fetched=pages_fetched, + pages_attempted=pages_attempted, + has_more_remaining=has_more_remaining, + pagination_truncated_reason=pagination_truncated_reason, + continuation_cstart=continuation_cstart, + ) + + async def _upsert_profile_publications( + self, + db_session: AsyncSession, + *, + run: CrawlRun, + scholar: ScholarProfile, + publications: list[PublicationCandidate], + ) -> int: + seen_publication_ids: set[int] = set() + discovered_count = 0 + + for candidate in publications: + publication = await self._resolve_publication(db_session, candidate) + if publication.id in seen_publication_ids: + continue + seen_publication_ids.add(publication.id) + + link_result = await db_session.execute( + select(ScholarPublication).where( + ScholarPublication.scholar_profile_id == scholar.id, + ScholarPublication.publication_id == publication.id, + ) + ) + link = link_result.scalar_one_or_none() + if link is not None: + continue + + link = ScholarPublication( + scholar_profile_id=scholar.id, + publication_id=publication.id, + is_read=False, + first_seen_run_id=run.id, + ) + db_session.add(link) + discovered_count += 1 + + logger.debug( + "ingestion.publication_discovered", + extra={ + "event": "ingestion.publication_discovered", + "scholar_profile_id": scholar.id, + "publication_id": publication.id, + "crawl_run_id": run.id, + }, + ) + + if not scholar.baseline_completed: + scholar.baseline_completed = True + + return discovered_count + + async def _resolve_publication( + self, + db_session: AsyncSession, + candidate: PublicationCandidate, + ) -> Publication: + fingerprint = build_publication_fingerprint(candidate) + + publication: Publication | None = None + cluster_publication: Publication | None = None + + if candidate.cluster_id: + cluster_result = await db_session.execute( + select(Publication).where(Publication.cluster_id == candidate.cluster_id) + ) + cluster_publication = cluster_result.scalar_one_or_none() + publication = cluster_publication + + if publication is None: + fingerprint_result = await db_session.execute( + select(Publication).where(Publication.fingerprint_sha256 == fingerprint) + ) + publication = fingerprint_result.scalar_one_or_none() + + if publication is not None and cluster_publication is not None and publication.id != cluster_publication.id: + publication = cluster_publication + + if publication is None: + publication = Publication( + cluster_id=candidate.cluster_id, + fingerprint_sha256=fingerprint, + title_raw=candidate.title, + title_normalized=normalize_title(candidate.title), + year=candidate.year, + citation_count=int(candidate.citation_count or 0), + author_text=candidate.authors_text, + venue_text=candidate.venue_text, + pub_url=build_publication_url(candidate.title_url), + pdf_url=None, + ) + db_session.add(publication) + await db_session.flush() + logger.debug( + "ingestion.publication_created", + extra={ + "event": "ingestion.publication_created", + "publication_id": publication.id, + "cluster_id": publication.cluster_id, + }, + ) + return publication + + if candidate.cluster_id and publication.cluster_id is None: + publication.cluster_id = candidate.cluster_id + publication.title_raw = candidate.title + publication.title_normalized = normalize_title(candidate.title) + if candidate.year is not None: + publication.year = candidate.year + if candidate.citation_count is not None: + publication.citation_count = int(candidate.citation_count) + if candidate.authors_text: + publication.author_text = candidate.authors_text + if candidate.venue_text: + publication.venue_text = candidate.venue_text + if candidate.title_url: + publication.pub_url = build_publication_url(candidate.title_url) + + return publication + + def _resolve_run_status( + self, + *, + scholar_count: int, + succeeded_count: int, + failed_count: int, + partial_count: int, + ) -> RunStatus: + if scholar_count == 0: + return RunStatus.SUCCESS + if failed_count == scholar_count: + return RunStatus.FAILED + if failed_count > 0 or partial_count > 0: + return RunStatus.PARTIAL_FAILURE + if succeeded_count > 0: + return RunStatus.SUCCESS + return RunStatus.FAILED + + def _resolve_continuation_queue_target( + self, + *, + outcome: str, + state: str, + pagination_truncated_reason: str | None, + continuation_cstart: int | None, + fallback_cstart: int, + ) -> tuple[str | None, int]: + if outcome == "partial": + reason = (pagination_truncated_reason or "").strip() + if reason in RESUMABLE_PARTIAL_REASONS or reason.startswith( + RESUMABLE_PARTIAL_REASON_PREFIXES + ): + return reason, queue_service.normalize_cstart( + continuation_cstart if continuation_cstart is not None else fallback_cstart + ) + return None, queue_service.normalize_cstart(fallback_cstart) + + if outcome == "failed" and state == ParseState.NETWORK_ERROR.value: + return "network_error_retry", queue_service.normalize_cstart( + continuation_cstart if continuation_cstart is not None else fallback_cstart + ) + + return None, queue_service.normalize_cstart(fallback_cstart) + + def _build_failure_debug_context( + self, + *, + fetch_result: FetchResult, + parsed_page: ParsedProfilePage, + attempt_log: list[dict[str, Any]], + page_logs: list[dict[str, Any]] | None = None, + exception: Exception | None = None, + ) -> dict[str, Any]: + context: dict[str, Any] = { + "requested_url": fetch_result.requested_url, + "final_url": fetch_result.final_url, + "status_code": fetch_result.status_code, + "fetch_error": fetch_result.error, + "state_reason": parsed_page.state_reason, + "profile_name": parsed_page.profile_name, + "articles_range": parsed_page.articles_range, + "has_show_more_button": parsed_page.has_show_more_button, + "has_operation_error_banner": parsed_page.has_operation_error_banner, + "warning_codes": parsed_page.warnings, + "marker_counts_nonzero": { + key: value for key, value in parsed_page.marker_counts.items() if value > 0 + }, + "body_length": len(fetch_result.body), + "body_sha256": hashlib.sha256(fetch_result.body.encode("utf-8")).hexdigest() + if fetch_result.body + else None, + "body_excerpt": _build_body_excerpt(fetch_result.body), + "attempt_log": attempt_log, + } + if page_logs: + context["page_logs"] = page_logs + if exception is not None: + context["exception_type"] = type(exception).__name__ + context["exception_message"] = str(exception) + return context + + async def _try_acquire_user_lock( + self, + db_session: AsyncSession, + *, + user_id: int, + ) -> bool: + result = await db_session.execute( + text( + "SELECT pg_try_advisory_xact_lock(:namespace, :user_key)" + ), + { + "namespace": RUN_LOCK_NAMESPACE, + "user_key": int(user_id), + }, + ) + return bool(result.scalar_one()) + + +def normalize_title(value: str) -> str: + lowered = value.lower() + cleaned = TITLE_ALNUM_RE.sub("", lowered) + return cleaned + + +def _first_author_last_name(authors_text: str | None) -> str: + if not authors_text: + return "" + first_author = authors_text.split(",", maxsplit=1)[0].strip().lower() + words = WORD_RE.findall(first_author) + if not words: + return "" + return words[-1] + + +def _first_venue_word(venue_text: str | None) -> str: + if not venue_text: + return "" + words = WORD_RE.findall(venue_text.lower()) + if not words: + return "" + return words[0] + + +def build_publication_fingerprint(candidate: PublicationCandidate) -> str: + canonical = "|".join( + [ + normalize_title(candidate.title), + str(candidate.year) if candidate.year is not None else "", + _first_author_last_name(candidate.authors_text), + _first_venue_word(candidate.venue_text), + ] + ) + return hashlib.sha256(canonical.encode("utf-8")).hexdigest() + + +def build_publication_url(path_or_url: str | None) -> str | None: + if not path_or_url: + return None + return urljoin("https://scholar.google.com", path_or_url) + + +def _next_cstart_value(*, articles_range: str | None, fallback: int) -> int: + if articles_range: + numbers = re.findall(r"\d+", articles_range) + if len(numbers) >= 2: + try: + return int(numbers[1]) + except ValueError: + pass + return int(fallback) + + +def _dedupe_publication_candidates( + publications: list[PublicationCandidate], +) -> list[PublicationCandidate]: + deduped: list[PublicationCandidate] = [] + seen: set[str] = set() + for publication in publications: + if publication.cluster_id: + identity = f"cluster:{publication.cluster_id}" + else: + identity = "|".join( + [ + "fallback", + normalize_title(publication.title), + str(publication.year) if publication.year is not None else "", + publication.authors_text or "", + publication.venue_text or "", + ] + ) + if identity in seen: + continue + seen.add(identity) + deduped.append(publication) + return deduped + + +def _build_body_excerpt(body: str, *, max_chars: int = 220) -> str | None: + if not body: + return None + flattened = SPACE_RE.sub(" ", HTML_TAG_RE.sub(" ", body)).strip() + if not flattened: + return None + if len(flattened) <= max_chars: + return flattened + return f"{flattened[:max_chars - 1]}..." diff --git a/app/services/publications.py b/app/services/publications.py new file mode 100644 index 0000000..5856b74 --- /dev/null +++ b/app/services/publications.py @@ -0,0 +1,297 @@ +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime + +from sqlalchemy import Select, func, select, update +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db.models import ( + CrawlRun, + Publication, + RunStatus, + ScholarProfile, + ScholarPublication, +) + +MODE_NEW = "new" +MODE_ALL = "all" + + +@dataclass(frozen=True) +class PublicationListItem: + publication_id: int + scholar_profile_id: int + scholar_label: str + title: str + year: int | None + citation_count: int + venue_text: str | None + pub_url: str | None + is_read: bool + first_seen_at: datetime + is_new_in_latest_run: bool + + +@dataclass(frozen=True) +class UnreadPublicationItem: + publication_id: int + scholar_profile_id: int + scholar_label: str + title: str + year: int | None + citation_count: int + venue_text: str | None + pub_url: str | None + + +def resolve_mode(value: str | None) -> str: + if value == MODE_ALL: + return MODE_ALL + return MODE_NEW + + +async def get_latest_completed_run_id_for_user( + db_session: AsyncSession, + *, + user_id: int, +) -> int | None: + result = await db_session.execute( + select(func.max(CrawlRun.id)).where( + CrawlRun.user_id == user_id, + CrawlRun.status != RunStatus.RUNNING, + ) + ) + latest_run_id = result.scalar_one_or_none() + return int(latest_run_id) if latest_run_id is not None else None + + +def publications_query( + *, + user_id: int, + mode: str, + latest_run_id: int | None, + scholar_profile_id: int | None, + limit: int, +) -> Select[tuple]: + scholar_label = ScholarProfile.display_name + + stmt = ( + select( + Publication.id, + ScholarProfile.id, + scholar_label, + ScholarProfile.scholar_id, + Publication.title_raw, + Publication.year, + Publication.citation_count, + Publication.venue_text, + Publication.pub_url, + ScholarPublication.is_read, + ScholarPublication.first_seen_run_id, + ScholarPublication.created_at, + ) + .join(ScholarPublication, ScholarPublication.publication_id == Publication.id) + .join(ScholarProfile, ScholarProfile.id == ScholarPublication.scholar_profile_id) + .where(ScholarProfile.user_id == user_id) + .order_by(ScholarPublication.created_at.desc(), Publication.id.desc()) + .limit(limit) + ) + if scholar_profile_id is not None: + stmt = stmt.where(ScholarProfile.id == scholar_profile_id) + if mode == MODE_NEW: + # "New" means discovered in the latest completed run. + if latest_run_id is None: + stmt = stmt.where(False) + else: + stmt = stmt.where(ScholarPublication.first_seen_run_id == latest_run_id) + return stmt + + +async def list_for_user( + db_session: AsyncSession, + *, + user_id: int, + mode: str = MODE_ALL, + scholar_profile_id: int | None = None, + limit: int = 300, +) -> list[PublicationListItem]: + resolved_mode = resolve_mode(mode) + latest_run_id = await get_latest_completed_run_id_for_user( + db_session, + user_id=user_id, + ) + result = await db_session.execute( + publications_query( + user_id=user_id, + mode=resolved_mode, + latest_run_id=latest_run_id, + scholar_profile_id=scholar_profile_id, + limit=limit, + ) + ) + + rows = result.all() + items: list[PublicationListItem] = [] + for row in rows: + ( + publication_id, + scholar_profile_id, + display_name, + scholar_id, + title_raw, + year, + citation_count, + venue_text, + pub_url, + is_read, + first_seen_run_id, + created_at, + ) = row + items.append( + PublicationListItem( + publication_id=int(publication_id), + scholar_profile_id=int(scholar_profile_id), + scholar_label=(display_name or scholar_id), + title=title_raw, + year=year, + citation_count=int(citation_count or 0), + venue_text=venue_text, + pub_url=pub_url, + is_read=bool(is_read), + first_seen_at=created_at, + is_new_in_latest_run=( + latest_run_id is not None and int(first_seen_run_id or 0) == latest_run_id + ), + ) + ) + return items + + +async def list_unread_for_user( + db_session: AsyncSession, + *, + user_id: int, + limit: int = 100, +) -> list[UnreadPublicationItem]: + result = await db_session.execute( + publications_query( + user_id=user_id, + mode=MODE_ALL, + latest_run_id=None, + scholar_profile_id=None, + limit=limit, + ).where(ScholarPublication.is_read.is_(False)) + ) + rows = result.all() + items: list[UnreadPublicationItem] = [] + for row in rows: + ( + publication_id, + scholar_profile_id, + display_name, + scholar_id, + title_raw, + year, + citation_count, + venue_text, + pub_url, + _is_read, + _first_seen_run_id, + _created_at, + ) = row + items.append( + UnreadPublicationItem( + publication_id=int(publication_id), + scholar_profile_id=int(scholar_profile_id), + scholar_label=(display_name or scholar_id), + title=title_raw, + year=year, + citation_count=int(citation_count or 0), + venue_text=venue_text, + pub_url=pub_url, + ) + ) + return items + + +async def list_new_for_latest_run_for_user( + db_session: AsyncSession, + *, + user_id: int, + limit: int = 100, +) -> list[UnreadPublicationItem]: + rows = await list_for_user( + db_session, + user_id=user_id, + mode=MODE_NEW, + scholar_profile_id=None, + limit=limit, + ) + return [ + UnreadPublicationItem( + publication_id=row.publication_id, + scholar_profile_id=row.scholar_profile_id, + scholar_label=row.scholar_label, + title=row.title, + year=row.year, + citation_count=row.citation_count, + venue_text=row.venue_text, + pub_url=row.pub_url, + ) + for row in rows + ] + + +async def count_for_user( + db_session: AsyncSession, + *, + user_id: int, + mode: str = MODE_ALL, + scholar_profile_id: int | None = None, +) -> int: + resolved_mode = resolve_mode(mode) + latest_run_id = await get_latest_completed_run_id_for_user( + db_session, + user_id=user_id, + ) + stmt = ( + select(func.count()) + .select_from(ScholarPublication) + .join(ScholarProfile, ScholarProfile.id == ScholarPublication.scholar_profile_id) + .where(ScholarProfile.user_id == user_id) + ) + if scholar_profile_id is not None: + stmt = stmt.where(ScholarProfile.id == scholar_profile_id) + if resolved_mode == MODE_NEW: + if latest_run_id is None: + return 0 + stmt = stmt.where(ScholarPublication.first_seen_run_id == latest_run_id) + result = await db_session.execute(stmt) + return int(result.scalar_one() or 0) + + +async def mark_all_unread_as_read_for_user( + db_session: AsyncSession, + *, + user_id: int, +) -> int: + scholar_ids = ( + select(ScholarProfile.id) + .where(ScholarProfile.user_id == user_id) + .scalar_subquery() + ) + + stmt = ( + update(ScholarPublication) + .where( + ScholarPublication.scholar_profile_id.in_(scholar_ids), + ScholarPublication.is_read.is_(False), + ) + .values(is_read=True) + ) + result = await db_session.execute(stmt) + await db_session.commit() + + rowcount = result.rowcount + return int(rowcount or 0) diff --git a/app/services/runs.py b/app/services/runs.py new file mode 100644 index 0000000..e313a9d --- /dev/null +++ b/app/services/runs.py @@ -0,0 +1,464 @@ +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime +from typing import Any + +from sqlalchemy import and_, case, func, select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db.models import ( + CrawlRun, + IngestionQueueItem, + RunStatus, + RunTriggerType, + ScholarProfile, +) +from app.services import continuation_queue as queue_service + +QUEUE_STATUS_QUEUED = "queued" +QUEUE_STATUS_RETRYING = "retrying" +QUEUE_STATUS_DROPPED = "dropped" + + +@dataclass(frozen=True) +class QueueListItem: + id: int + scholar_profile_id: int + scholar_label: str + status: str + reason: str + dropped_reason: str | None + attempt_count: int + resume_cstart: int + next_attempt_dt: datetime | None + updated_at: datetime + last_error: str | None + last_run_id: int | None + + +@dataclass(frozen=True) +class QueueClearResult: + queue_item_id: int + previous_status: str + + +class QueueTransitionError(RuntimeError): + def __init__(self, *, code: str, message: str, current_status: str) -> None: + super().__init__(message) + self.code = code + self.message = message + self.current_status = current_status + + +def _safe_int(value: object, default: int = 0) -> int: + try: + return int(value) + except (TypeError, ValueError): + return default + + +def _summary_dict(error_log: object) -> dict[str, Any]: + if not isinstance(error_log, dict): + return {} + summary = error_log.get("summary") + if not isinstance(summary, dict): + return {} + return summary + + +def extract_run_summary(error_log: object) -> dict[str, Any]: + summary = _summary_dict(error_log) + return { + "succeeded_count": _safe_int(summary.get("succeeded_count", 0)), + "failed_count": _safe_int(summary.get("failed_count", 0)), + "partial_count": _safe_int(summary.get("partial_count", 0)), + "failed_state_counts": { + str(key): _safe_int(value, 0) + for key, value in summary.get("failed_state_counts", {}).items() + if isinstance(key, str) + } + if isinstance(summary.get("failed_state_counts"), dict) + else {}, + "failed_reason_counts": { + str(key): _safe_int(value, 0) + for key, value in summary.get("failed_reason_counts", {}).items() + if isinstance(key, str) + } + if isinstance(summary.get("failed_reason_counts"), dict) + else {}, + } + + +async def list_recent_runs_for_user( + db_session: AsyncSession, + *, + user_id: int, + limit: int = 20, +) -> list[CrawlRun]: + result = await db_session.execute( + select(CrawlRun) + .where(CrawlRun.user_id == user_id) + .order_by(CrawlRun.start_dt.desc(), CrawlRun.id.desc()) + .limit(limit) + ) + return list(result.scalars().all()) + + +async def list_runs_for_user( + db_session: AsyncSession, + *, + user_id: int, + limit: int = 100, + failed_only: bool = False, +) -> list[CrawlRun]: + stmt = ( + select(CrawlRun) + .where(CrawlRun.user_id == user_id) + .order_by(CrawlRun.start_dt.desc(), CrawlRun.id.desc()) + .limit(limit) + ) + if failed_only: + stmt = stmt.where( + CrawlRun.status.in_([RunStatus.FAILED, RunStatus.PARTIAL_FAILURE]) + ) + result = await db_session.execute(stmt) + return list(result.scalars().all()) + + +async def get_run_for_user( + db_session: AsyncSession, + *, + user_id: int, + run_id: int, +) -> CrawlRun | None: + result = await db_session.execute( + select(CrawlRun).where( + CrawlRun.user_id == user_id, + CrawlRun.id == run_id, + ) + ) + return result.scalar_one_or_none() + + +async def get_manual_run_by_idempotency_key( + db_session: AsyncSession, + *, + user_id: int, + idempotency_key: str, +) -> CrawlRun | None: + result = await db_session.execute( + select(CrawlRun) + .where( + CrawlRun.user_id == user_id, + CrawlRun.trigger_type == RunTriggerType.MANUAL, + CrawlRun.error_log["meta"]["idempotency_key"].astext == idempotency_key, + ) + .order_by(CrawlRun.start_dt.desc(), CrawlRun.id.desc()) + .limit(1) + ) + return result.scalar_one_or_none() + + +async def set_manual_run_idempotency_key( + db_session: AsyncSession, + *, + user_id: int, + run_id: int, + idempotency_key: str, +) -> bool: + run = await get_run_for_user( + db_session, + user_id=user_id, + run_id=run_id, + ) + if run is None: + return False + if run.trigger_type != RunTriggerType.MANUAL: + return False + + payload = dict(run.error_log) if isinstance(run.error_log, dict) else {} + meta = payload.get("meta") + meta_dict = dict(meta) if isinstance(meta, dict) else {} + meta_dict["idempotency_key"] = idempotency_key + payload["meta"] = meta_dict + run.error_log = payload + await db_session.commit() + return True + + +async def list_queue_items_for_user( + db_session: AsyncSession, + *, + user_id: int, + limit: int = 200, +) -> list[QueueListItem]: + result = await db_session.execute( + select( + IngestionQueueItem.id, + IngestionQueueItem.scholar_profile_id, + ScholarProfile.display_name, + ScholarProfile.scholar_id, + IngestionQueueItem.status, + IngestionQueueItem.reason, + IngestionQueueItem.dropped_reason, + IngestionQueueItem.attempt_count, + IngestionQueueItem.resume_cstart, + IngestionQueueItem.next_attempt_dt, + IngestionQueueItem.updated_at, + IngestionQueueItem.last_error, + IngestionQueueItem.last_run_id, + ) + .join( + ScholarProfile, + and_( + ScholarProfile.id == IngestionQueueItem.scholar_profile_id, + ScholarProfile.user_id == IngestionQueueItem.user_id, + ), + ) + .where(IngestionQueueItem.user_id == user_id) + .order_by( + case((IngestionQueueItem.status == "dropped", 1), else_=0).asc(), + IngestionQueueItem.next_attempt_dt.asc(), + IngestionQueueItem.id.asc(), + ) + .limit(limit) + ) + + items: list[QueueListItem] = [] + for row in result.all(): + ( + item_id, + scholar_profile_id, + display_name, + scholar_id, + status, + reason, + dropped_reason, + attempt_count, + resume_cstart, + next_attempt_dt, + updated_at, + last_error, + last_run_id, + ) = row + items.append( + QueueListItem( + id=int(item_id), + scholar_profile_id=int(scholar_profile_id), + scholar_label=(display_name or scholar_id), + status=str(status), + reason=str(reason), + dropped_reason=dropped_reason, + attempt_count=int(attempt_count or 0), + resume_cstart=int(resume_cstart or 0), + next_attempt_dt=next_attempt_dt, + updated_at=updated_at, + last_error=last_error, + last_run_id=int(last_run_id) if last_run_id is not None else None, + ) + ) + return items + + +async def get_queue_item_for_user( + db_session: AsyncSession, + *, + user_id: int, + queue_item_id: int, +) -> QueueListItem | None: + result = await db_session.execute( + select( + IngestionQueueItem.id, + IngestionQueueItem.scholar_profile_id, + ScholarProfile.display_name, + ScholarProfile.scholar_id, + IngestionQueueItem.status, + IngestionQueueItem.reason, + IngestionQueueItem.dropped_reason, + IngestionQueueItem.attempt_count, + IngestionQueueItem.resume_cstart, + IngestionQueueItem.next_attempt_dt, + IngestionQueueItem.updated_at, + IngestionQueueItem.last_error, + IngestionQueueItem.last_run_id, + ) + .join( + ScholarProfile, + and_( + ScholarProfile.id == IngestionQueueItem.scholar_profile_id, + ScholarProfile.user_id == IngestionQueueItem.user_id, + ), + ) + .where( + IngestionQueueItem.user_id == user_id, + IngestionQueueItem.id == queue_item_id, + ) + .limit(1) + ) + row = result.one_or_none() + if row is None: + return None + ( + item_id, + scholar_profile_id, + display_name, + scholar_id, + status, + reason, + dropped_reason, + attempt_count, + resume_cstart, + next_attempt_dt, + updated_at, + last_error, + last_run_id, + ) = row + return QueueListItem( + id=int(item_id), + scholar_profile_id=int(scholar_profile_id), + scholar_label=(display_name or scholar_id), + status=str(status), + reason=str(reason), + dropped_reason=dropped_reason, + attempt_count=int(attempt_count or 0), + resume_cstart=int(resume_cstart or 0), + next_attempt_dt=next_attempt_dt, + updated_at=updated_at, + last_error=last_error, + last_run_id=int(last_run_id) if last_run_id is not None else None, + ) + + +async def retry_queue_item_for_user( + db_session: AsyncSession, + *, + user_id: int, + queue_item_id: int, +) -> QueueListItem | None: + result = await db_session.execute( + select(IngestionQueueItem).where( + IngestionQueueItem.id == queue_item_id, + IngestionQueueItem.user_id == user_id, + ) + ) + item = result.scalar_one_or_none() + if item is None: + return None + if item.status == QUEUE_STATUS_QUEUED: + raise QueueTransitionError( + code="queue_item_already_queued", + message="Queue item is already queued.", + current_status=item.status, + ) + if item.status == QUEUE_STATUS_RETRYING: + raise QueueTransitionError( + code="queue_item_retrying", + message="Queue item is currently retrying.", + current_status=item.status, + ) + + await queue_service.mark_queued_now( + db_session, + job_id=item.id, + reason="manual_retry", + reset_attempt_count=(item.status == QUEUE_STATUS_DROPPED), + ) + await db_session.commit() + + return await get_queue_item_for_user( + db_session, + user_id=user_id, + queue_item_id=queue_item_id, + ) + + +async def drop_queue_item_for_user( + db_session: AsyncSession, + *, + user_id: int, + queue_item_id: int, +) -> QueueListItem | None: + result = await db_session.execute( + select(IngestionQueueItem).where( + IngestionQueueItem.id == queue_item_id, + IngestionQueueItem.user_id == user_id, + ) + ) + item = result.scalar_one_or_none() + if item is None: + return None + if item.status == QUEUE_STATUS_DROPPED: + raise QueueTransitionError( + code="queue_item_already_dropped", + message="Queue item is already dropped.", + current_status=item.status, + ) + + await queue_service.mark_dropped( + db_session, + job_id=int(item.id), + reason="manual_drop", + ) + await db_session.commit() + return await get_queue_item_for_user( + db_session, + user_id=user_id, + queue_item_id=queue_item_id, + ) + + +async def clear_queue_item_for_user( + db_session: AsyncSession, + *, + user_id: int, + queue_item_id: int, +) -> QueueClearResult | None: + result = await db_session.execute( + select(IngestionQueueItem).where( + IngestionQueueItem.id == queue_item_id, + IngestionQueueItem.user_id == user_id, + ) + ) + item = result.scalar_one_or_none() + if item is None: + return None + if item.status != QUEUE_STATUS_DROPPED: + raise QueueTransitionError( + code="queue_item_not_dropped", + message="Queue item can only be cleared after it is dropped.", + current_status=item.status, + ) + + item_id = int(item.id) + previous_status = str(item.status) + deleted = await queue_service.delete_job_by_id( + db_session, + job_id=item_id, + ) + await db_session.commit() + if not deleted: + return None + return QueueClearResult( + queue_item_id=item_id, + previous_status=previous_status, + ) + + +async def queue_status_counts_for_user( + db_session: AsyncSession, + *, + user_id: int, +) -> dict[str, int]: + result = await db_session.execute( + select( + IngestionQueueItem.status, + func.count(IngestionQueueItem.id), + ) + .where(IngestionQueueItem.user_id == user_id) + .group_by(IngestionQueueItem.status) + ) + counts: dict[str, int] = {"queued": 0, "retrying": 0, "dropped": 0} + for status, count in result.all(): + counts[str(status)] = int(count or 0) + return counts diff --git a/app/services/scheduler.py b/app/services/scheduler.py new file mode 100644 index 0000000..80ca1cb --- /dev/null +++ b/app/services/scheduler.py @@ -0,0 +1,478 @@ +from __future__ import annotations + +import asyncio +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +import logging + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db.models import ( + CrawlRun, + QueueItemStatus, + RunTriggerType, + ScholarProfile, + User, + UserSetting, +) +from app.db.session import get_session_factory +from app.services import continuation_queue as queue_service +from app.services.ingestion import RunAlreadyInProgressError, ScholarIngestionService +from app.services.scholar_source import LiveScholarSource + +logger = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class _AutoRunCandidate: + user_id: int + run_interval_minutes: int + request_delay_seconds: int + + +class SchedulerService: + def __init__( + self, + *, + enabled: bool, + tick_seconds: int, + network_error_retries: int, + retry_backoff_seconds: float, + max_pages_per_scholar: int, + page_size: int, + continuation_queue_enabled: bool, + continuation_base_delay_seconds: int, + continuation_max_delay_seconds: int, + continuation_max_attempts: int, + queue_batch_size: int, + ) -> None: + self._enabled = enabled + self._tick_seconds = max(5, int(tick_seconds)) + self._network_error_retries = max(0, int(network_error_retries)) + self._retry_backoff_seconds = max(0.0, float(retry_backoff_seconds)) + self._max_pages_per_scholar = max(1, int(max_pages_per_scholar)) + self._page_size = max(1, int(page_size)) + self._continuation_queue_enabled = bool(continuation_queue_enabled) + self._continuation_base_delay_seconds = max(1, int(continuation_base_delay_seconds)) + self._continuation_max_delay_seconds = max( + self._continuation_base_delay_seconds, + int(continuation_max_delay_seconds), + ) + self._continuation_max_attempts = max(1, int(continuation_max_attempts)) + self._queue_batch_size = max(1, int(queue_batch_size)) + self._task: asyncio.Task[None] | None = None + self._source = LiveScholarSource() + + async def start(self) -> None: + if not self._enabled: + logger.info( + "scheduler.disabled", + extra={ + "event": "scheduler.disabled", + }, + ) + return + if self._task is not None: + return + self._task = asyncio.create_task(self._run_loop(), name="scholarr-scheduler") + logger.info( + "scheduler.started", + extra={ + "event": "scheduler.started", + "tick_seconds": self._tick_seconds, + "network_error_retries": self._network_error_retries, + "retry_backoff_seconds": self._retry_backoff_seconds, + "max_pages_per_scholar": self._max_pages_per_scholar, + "page_size": self._page_size, + "continuation_queue_enabled": self._continuation_queue_enabled, + "continuation_base_delay_seconds": self._continuation_base_delay_seconds, + "continuation_max_delay_seconds": self._continuation_max_delay_seconds, + "continuation_max_attempts": self._continuation_max_attempts, + "queue_batch_size": self._queue_batch_size, + }, + ) + + async def stop(self) -> None: + if self._task is None: + return + self._task.cancel() + try: + await self._task + except asyncio.CancelledError: + pass + finally: + self._task = None + logger.info("scheduler.stopped", extra={"event": "scheduler.stopped"}) + + async def _run_loop(self) -> None: + while True: + try: + await self._tick_once() + except asyncio.CancelledError: + raise + except Exception: + logger.exception( + "scheduler.tick_failed", + extra={ + "event": "scheduler.tick_failed", + }, + ) + await asyncio.sleep(float(self._tick_seconds)) + + async def _tick_once(self) -> None: + if self._continuation_queue_enabled: + await self._drain_continuation_queue() + candidates = await self._load_candidates() + if not candidates: + return + now = datetime.now(timezone.utc) + for candidate in candidates: + if not await self._is_due(candidate, now=now): + continue + await self._run_candidate(candidate) + + async def _load_candidates(self) -> list[_AutoRunCandidate]: + session_factory = get_session_factory() + async with session_factory() as session: + result = await session.execute( + select( + UserSetting.user_id, + UserSetting.run_interval_minutes, + UserSetting.request_delay_seconds, + ) + .join(User, User.id == UserSetting.user_id) + .where( + User.is_active.is_(True), + UserSetting.auto_run_enabled.is_(True), + ) + .order_by(UserSetting.user_id.asc()) + ) + rows = result.all() + return [ + _AutoRunCandidate( + user_id=int(user_id), + run_interval_minutes=int(run_interval_minutes), + request_delay_seconds=int(request_delay_seconds), + ) + for user_id, run_interval_minutes, request_delay_seconds in rows + ] + + async def _is_due(self, candidate: _AutoRunCandidate, *, now: datetime) -> bool: + session_factory = get_session_factory() + async with session_factory() as session: + result = await session.execute( + select(CrawlRun.start_dt) + .where( + CrawlRun.user_id == candidate.user_id, + ) + .order_by(CrawlRun.start_dt.desc(), CrawlRun.id.desc()) + .limit(1) + ) + last_run = result.scalar_one_or_none() + + if last_run is None: + return True + + next_due_dt = last_run + timedelta( + minutes=candidate.run_interval_minutes + ) + return now >= next_due_dt + + async def _run_candidate(self, candidate: _AutoRunCandidate) -> None: + session_factory = get_session_factory() + async with session_factory() as session: + ingestion = ScholarIngestionService(source=self._source) + try: + run_summary = await ingestion.run_for_user( + session, + user_id=candidate.user_id, + trigger_type=RunTriggerType.SCHEDULED, + request_delay_seconds=candidate.request_delay_seconds, + network_error_retries=self._network_error_retries, + retry_backoff_seconds=self._retry_backoff_seconds, + max_pages_per_scholar=self._max_pages_per_scholar, + page_size=self._page_size, + auto_queue_continuations=self._continuation_queue_enabled, + queue_delay_seconds=self._continuation_base_delay_seconds, + ) + except RunAlreadyInProgressError: + await session.rollback() + logger.info( + "scheduler.run_skipped_locked", + extra={ + "event": "scheduler.run_skipped_locked", + "user_id": candidate.user_id, + }, + ) + return + except Exception: + await session.rollback() + logger.exception( + "scheduler.run_failed", + extra={ + "event": "scheduler.run_failed", + "user_id": candidate.user_id, + }, + ) + return + + logger.info( + "scheduler.run_completed", + extra={ + "event": "scheduler.run_completed", + "user_id": candidate.user_id, + "run_id": run_summary.crawl_run_id, + "status": run_summary.status.value, + "scholar_count": run_summary.scholar_count, + "new_publication_count": run_summary.new_publication_count, + }, + ) + + async def _drain_continuation_queue(self) -> None: + now = datetime.now(timezone.utc) + session_factory = get_session_factory() + async with session_factory() as session: + jobs = await queue_service.list_due_jobs( + session, + now=now, + limit=self._queue_batch_size, + ) + for job in jobs: + await self._run_queue_job(job) + + async def _run_queue_job(self, job: queue_service.ContinuationQueueJob) -> None: + if job.attempt_count >= self._continuation_max_attempts: + session_factory = get_session_factory() + async with session_factory() as session: + dropped = await queue_service.mark_dropped( + session, + job_id=job.id, + reason="max_attempts_reached", + ) + await session.commit() + if dropped is not None: + logger.warning( + "scheduler.queue_item_dropped_max_attempts", + extra={ + "event": "scheduler.queue_item_dropped_max_attempts", + "queue_item_id": job.id, + "user_id": job.user_id, + "scholar_profile_id": job.scholar_profile_id, + "attempt_count": job.attempt_count, + "max_attempts": self._continuation_max_attempts, + }, + ) + return + + session_factory = get_session_factory() + async with session_factory() as session: + queue_item = await queue_service.mark_retrying(session, job_id=job.id) + if queue_item is None: + await session.commit() + return + if queue_item.status == QueueItemStatus.DROPPED.value: + await session.commit() + return + await session.commit() + + async with session_factory() as session: + scholar_result = await session.execute( + select(ScholarProfile.id).where( + ScholarProfile.user_id == job.user_id, + ScholarProfile.id == job.scholar_profile_id, + ScholarProfile.is_enabled.is_(True), + ) + ) + scholar_id = scholar_result.scalar_one_or_none() + if scholar_id is None: + dropped = await queue_service.mark_dropped( + session, + job_id=job.id, + reason="scholar_unavailable", + ) + await session.commit() + if dropped is not None: + logger.info( + "scheduler.queue_item_dropped_scholar_unavailable", + extra={ + "event": "scheduler.queue_item_dropped_scholar_unavailable", + "queue_item_id": job.id, + "user_id": job.user_id, + "scholar_profile_id": job.scholar_profile_id, + }, + ) + return + + async with session_factory() as session: + request_delay_seconds = await self._load_request_delay_for_user( + session, + user_id=job.user_id, + ) + ingestion = ScholarIngestionService(source=self._source) + try: + run_summary = await ingestion.run_for_user( + session, + user_id=job.user_id, + trigger_type=RunTriggerType.SCHEDULED, + request_delay_seconds=request_delay_seconds, + network_error_retries=self._network_error_retries, + retry_backoff_seconds=self._retry_backoff_seconds, + max_pages_per_scholar=self._max_pages_per_scholar, + page_size=self._page_size, + scholar_profile_ids={job.scholar_profile_id}, + start_cstart_by_scholar_id={ + job.scholar_profile_id: job.resume_cstart, + }, + auto_queue_continuations=self._continuation_queue_enabled, + queue_delay_seconds=self._continuation_base_delay_seconds, + ) + except RunAlreadyInProgressError: + await session.rollback() + async with session_factory() as recovery_session: + await queue_service.reschedule_job( + recovery_session, + job_id=job.id, + delay_seconds=max(self._tick_seconds, 15), + reason="user_run_lock_active", + error="run_already_in_progress", + ) + await recovery_session.commit() + logger.info( + "scheduler.queue_item_deferred_lock", + extra={ + "event": "scheduler.queue_item_deferred_lock", + "queue_item_id": job.id, + "user_id": job.user_id, + }, + ) + return + except Exception as exc: + await session.rollback() + async with session_factory() as recovery_session: + queue_item = await queue_service.increment_attempt_count( + recovery_session, + job_id=job.id, + ) + if queue_item is None: + await recovery_session.commit() + return + if int(queue_item.attempt_count) >= self._continuation_max_attempts: + await queue_service.mark_dropped( + recovery_session, + job_id=job.id, + reason="scheduler_exception_max_attempts", + error=str(exc), + ) + await recovery_session.commit() + logger.warning( + "scheduler.queue_item_dropped_after_exception", + extra={ + "event": "scheduler.queue_item_dropped_after_exception", + "queue_item_id": job.id, + "user_id": job.user_id, + "attempt_count": queue_item.attempt_count, + }, + ) + return + delay_seconds = queue_service.compute_backoff_seconds( + base_seconds=self._continuation_base_delay_seconds, + attempt_count=int(queue_item.attempt_count), + max_seconds=self._continuation_max_delay_seconds, + ) + await queue_service.reschedule_job( + recovery_session, + job_id=job.id, + delay_seconds=delay_seconds, + reason="scheduler_exception", + error=str(exc), + ) + await recovery_session.commit() + logger.exception( + "scheduler.queue_item_run_failed", + extra={ + "event": "scheduler.queue_item_run_failed", + "queue_item_id": job.id, + "user_id": job.user_id, + }, + ) + return + + async with session_factory() as session: + queue_item = await queue_service.increment_attempt_count( + session, + job_id=job.id, + ) + if queue_item is None: + await session.commit() + logger.info( + "scheduler.queue_item_resolved", + extra={ + "event": "scheduler.queue_item_resolved", + "queue_item_id": job.id, + "user_id": job.user_id, + "run_id": run_summary.crawl_run_id, + "status": run_summary.status.value, + }, + ) + return + + if int(queue_item.attempt_count) >= self._continuation_max_attempts: + await queue_service.mark_dropped( + session, + job_id=job.id, + reason="max_attempts_after_run", + ) + await session.commit() + logger.warning( + "scheduler.queue_item_dropped_max_attempts_after_run", + extra={ + "event": "scheduler.queue_item_dropped_max_attempts_after_run", + "queue_item_id": job.id, + "user_id": job.user_id, + "attempt_count": queue_item.attempt_count, + "run_id": run_summary.crawl_run_id, + "status": run_summary.status.value, + }, + ) + return + + delay_seconds = queue_service.compute_backoff_seconds( + base_seconds=self._continuation_base_delay_seconds, + attempt_count=int(queue_item.attempt_count), + max_seconds=self._continuation_max_delay_seconds, + ) + await queue_service.reschedule_job( + session, + job_id=job.id, + delay_seconds=delay_seconds, + reason=queue_item.reason, + error=queue_item.last_error, + ) + await session.commit() + logger.info( + "scheduler.queue_item_rescheduled", + extra={ + "event": "scheduler.queue_item_rescheduled", + "queue_item_id": job.id, + "user_id": job.user_id, + "attempt_count": queue_item.attempt_count, + "delay_seconds": delay_seconds, + "run_id": run_summary.crawl_run_id, + "status": run_summary.status.value, + }, + ) + + async def _load_request_delay_for_user( + self, + db_session: AsyncSession, + *, + user_id: int, + ) -> int: + result = await db_session.execute( + select(UserSetting.request_delay_seconds).where(UserSetting.user_id == user_id) + ) + delay = result.scalar_one_or_none() + if delay is None: + return 10 + return max(1, int(delay)) diff --git a/app/services/scholar_parser.py b/app/services/scholar_parser.py new file mode 100644 index 0000000..4cb9ed3 --- /dev/null +++ b/app/services/scholar_parser.py @@ -0,0 +1,370 @@ +from __future__ import annotations + +import re +from dataclasses import dataclass +from enum import StrEnum +from html import unescape +from html.parser import HTMLParser +from typing import Any +from urllib.parse import parse_qs, urlparse + +from app.services.scholar_source import FetchResult + +BLOCKED_KEYWORDS = [ + "unusual traffic", + "sorry/index", + "not a robot", + "our systems have detected", + "automated queries", +] + +NO_RESULTS_KEYWORDS = [ + "didn't match any articles", + "did not match any articles", + "no articles", + "no documents", +] + +MARKER_KEYS = [ + "gsc_a_tr", + "gsc_a_at", + "gsc_a_ac", + "gsc_a_h", + "gsc_a_y", + "gs_gray", + "gsc_prf_in", + "gsc_rsb_st", +] + +TAG_RE = re.compile(r"<[^>]+>", re.S) +SCRIPT_STYLE_RE = re.compile(r"<(script|style)\b[^>]*>.*?", re.I | re.S) +SHOW_MORE_BUTTON_RE = re.compile( + r"]*\bid\s*=\s*['\"]gsc_bpf_more['\"][^>]*>", + re.I | re.S, +) + + +class ParseState(StrEnum): + OK = "ok" + NO_RESULTS = "no_results" + BLOCKED_OR_CAPTCHA = "blocked_or_captcha" + LAYOUT_CHANGED = "layout_changed" + NETWORK_ERROR = "network_error" + + +@dataclass(frozen=True) +class PublicationCandidate: + title: str + title_url: str | None + cluster_id: str | None + year: int | None + citation_count: int | None + authors_text: str | None + venue_text: str | None + + +@dataclass(frozen=True) +class ParsedProfilePage: + state: ParseState + state_reason: str + profile_name: str | None + publications: list[PublicationCandidate] + marker_counts: dict[str, int] + warnings: list[str] + has_show_more_button: bool + has_operation_error_banner: bool + articles_range: str | None + + +def normalize_space(value: str) -> str: + return " ".join(unescape(value).split()) + + +def strip_tags(value: str) -> str: + return normalize_space(TAG_RE.sub(" ", value)) + + +def attr_class(attrs: list[tuple[str, str | None]]) -> str: + for name, raw_value in attrs: + if name.lower() == "class": + return raw_value or "" + return "" + + +def attr_href(attrs: list[tuple[str, str | None]]) -> str | None: + for name, raw_value in attrs: + if name.lower() == "href": + return raw_value + return None + + +class ScholarRowParser(HTMLParser): + def __init__(self) -> None: + super().__init__(convert_charrefs=True) + self.title_href: str | None = None + self.title_parts: list[str] = [] + self.citation_parts: list[str] = [] + self.year_parts: list[str] = [] + self.gray_texts: list[str] = [] + + self._title_depth = 0 + self._citation_depth = 0 + self._year_depth = 0 + self._gray_stack: list[dict[str, Any]] = [] + + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + if self._title_depth > 0: + self._title_depth += 1 + if self._citation_depth > 0: + self._citation_depth += 1 + if self._year_depth > 0: + self._year_depth += 1 + if self._gray_stack: + self._gray_stack[-1]["depth"] += 1 + + classes = attr_class(attrs) + + if tag == "a" and "gsc_a_at" in classes: + self._title_depth = 1 + self.title_href = attr_href(attrs) + return + + if tag == "a" and "gsc_a_ac" in classes: + self._citation_depth = 1 + return + + if tag in {"span", "a"} and ("gsc_a_h" in classes or "gsc_a_y" in classes): + self._year_depth = 1 + return + + if tag == "div" and "gs_gray" in classes: + self._gray_stack.append({"depth": 1, "parts": []}) + return + + def handle_data(self, data: str) -> None: + if self._title_depth > 0: + self.title_parts.append(data) + if self._citation_depth > 0: + self.citation_parts.append(data) + if self._year_depth > 0: + self.year_parts.append(data) + if self._gray_stack: + self._gray_stack[-1]["parts"].append(data) + + def handle_endtag(self, _tag: str) -> None: + if self._title_depth > 0: + self._title_depth -= 1 + if self._citation_depth > 0: + self._citation_depth -= 1 + if self._year_depth > 0: + self._year_depth -= 1 + if self._gray_stack: + self._gray_stack[-1]["depth"] -= 1 + if self._gray_stack[-1]["depth"] == 0: + text = normalize_space("".join(self._gray_stack[-1]["parts"])) + if text: + self.gray_texts.append(text) + self._gray_stack.pop() + + +def extract_rows(html: str) -> list[str]: + pattern = re.compile( + r"]*\bclass\s*=\s*['\"][^'\"]*\bgsc_a_tr\b[^'\"]*['\"])[^>]*>(.*?)", + re.I | re.S, + ) + return [match.group(1) for match in pattern.finditer(html)] + + +def parse_cluster_id_from_href(href: str | None) -> str | None: + if not href: + return None + parsed = urlparse(href) + query = parse_qs(parsed.query) + + citation_for_view = query.get("citation_for_view") + if citation_for_view: + token = citation_for_view[0].strip() + if token: + if ":" in token: + return token.rsplit(":", 1)[-1] or None + return token + + cluster = query.get("cluster") + if cluster: + token = cluster[0].strip() + if token: + return token + return None + + +def parse_year(parts: list[str]) -> int | None: + text = normalize_space(" ".join(parts)) + match = re.search(r"\b(19|20)\d{2}\b", text) + if not match: + return None + try: + return int(match.group(0)) + except ValueError: + return None + + +def parse_citation_count(parts: list[str]) -> int | None: + text = normalize_space(" ".join(parts)) + if not text: + return 0 + match = re.search(r"\d+", text) + if not match: + return None + return int(match.group(0)) + + +def parse_publications(html: str) -> tuple[list[PublicationCandidate], list[str]]: + rows = extract_rows(html) + warnings: list[str] = [] + publications: list[PublicationCandidate] = [] + + for row_html in rows: + parser = ScholarRowParser() + parser.feed(row_html) + + title = normalize_space("".join(parser.title_parts)) + if not title: + warnings.append("row_missing_title") + continue + + authors_text = parser.gray_texts[0] if len(parser.gray_texts) > 0 else None + venue_text = parser.gray_texts[1] if len(parser.gray_texts) > 1 else None + + publications.append( + PublicationCandidate( + title=title, + title_url=parser.title_href, + cluster_id=parse_cluster_id_from_href(parser.title_href), + year=parse_year(parser.year_parts), + citation_count=parse_citation_count(parser.citation_parts), + authors_text=authors_text, + venue_text=venue_text, + ) + ) + + if not rows: + warnings.append("no_rows_detected") + + return publications, sorted(set(warnings)) + + +def extract_profile_name(html: str) -> str | None: + pattern = re.compile( + r"<[^>]*\bid\s*=\s*['\"]gsc_prf_in['\"][^>]*>(.*?)]+>", + re.I | re.S, + ) + match = pattern.search(html) + if not match: + return None + value = strip_tags(match.group(1)) + return value or None + + +def extract_articles_range(html: str) -> str | None: + pattern = re.compile( + r"<[^>]*\bid\s*=\s*['\"]gsc_a_nn['\"][^>]*>(.*?)]+>", + re.I | re.S, + ) + match = pattern.search(html) + if not match: + return None + value = strip_tags(match.group(1)) + return value or None + + +def has_show_more_button(html: str) -> bool: + match = SHOW_MORE_BUTTON_RE.search(html) + if match is None: + return False + + button_tag = match.group(0).lower() + if "disabled" in button_tag: + return False + if 'aria-disabled="true"' in button_tag or "aria-disabled='true'" in button_tag: + return False + if "gs_dis" in button_tag: + return False + return True + + +def has_operation_error_banner(html: str) -> bool: + lowered = html.lower() + if "id=\"gsc_a_err\"" not in lowered and "id='gsc_a_err'" not in lowered: + return False + return "can't perform the operation now" in lowered or "cannot perform the operation now" in lowered + + +def count_markers(html: str) -> dict[str, int]: + lowered = html.lower() + return {key: lowered.count(key.lower()) for key in MARKER_KEYS} + + +def detect_state( + fetch_result: FetchResult, + publications: list[PublicationCandidate], + marker_counts: dict[str, int], + *, + visible_text: str, +) -> tuple[ParseState, str]: + if fetch_result.status_code is None: + return ParseState.NETWORK_ERROR, "network_error_missing_status_code" + + lowered = fetch_result.body.lower() + final = (fetch_result.final_url or "").lower() + + if "accounts.google.com" in final and ("signin" in final or "servicelogin" in final): + return ParseState.BLOCKED_OR_CAPTCHA, "blocked_accounts_redirect" + if any(keyword in lowered for keyword in BLOCKED_KEYWORDS) or "sorry/index" in final: + return ParseState.BLOCKED_OR_CAPTCHA, "blocked_keyword_detected" + + if not publications and any(keyword in visible_text for keyword in NO_RESULTS_KEYWORDS): + return ParseState.NO_RESULTS, "no_results_keyword_detected" + + if not publications: + has_profile_markers = marker_counts.get("gsc_prf_in", 0) > 0 + has_table_markers = marker_counts.get("gsc_a_tr", 0) > 0 or marker_counts.get("gsc_a_at", 0) > 0 + if not has_profile_markers and not has_table_markers: + return ParseState.LAYOUT_CHANGED, "layout_markers_missing" + return ParseState.OK, "no_rows_with_known_markers" + + return ParseState.OK, "publications_extracted" + + +def parse_profile_page(fetch_result: FetchResult) -> ParsedProfilePage: + publications, warnings = parse_publications(fetch_result.body) + marker_counts = count_markers(fetch_result.body) + visible_text = strip_tags(SCRIPT_STYLE_RE.sub(" ", fetch_result.body)).lower() + + show_more = has_show_more_button(fetch_result.body) + operation_error_banner = has_operation_error_banner(fetch_result.body) + + if show_more: + warnings.append("possible_partial_page_show_more_present") + if operation_error_banner: + warnings.append("operation_error_banner_present") + + warnings = sorted(set(warnings)) + + state, state_reason = detect_state( + fetch_result, + publications, + marker_counts, + visible_text=visible_text, + ) + + return ParsedProfilePage( + state=state, + state_reason=state_reason, + profile_name=extract_profile_name(fetch_result.body), + publications=publications, + marker_counts=marker_counts, + warnings=warnings, + has_show_more_button=show_more, + has_operation_error_banner=operation_error_banner, + articles_range=extract_articles_range(fetch_result.body), + ) diff --git a/app/services/scholar_source.py b/app/services/scholar_source.py new file mode 100644 index 0000000..75e260a --- /dev/null +++ b/app/services/scholar_source.py @@ -0,0 +1,171 @@ +from __future__ import annotations + +import asyncio +import logging +import random +from dataclasses import dataclass +from typing import Protocol +from urllib.error import HTTPError, URLError +from urllib.parse import urlencode +from urllib.request import Request, urlopen + +SCHOLAR_PROFILE_URL = "https://scholar.google.com/citations" +DEFAULT_PAGE_SIZE = 100 + +DEFAULT_USER_AGENTS = [ + ( + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36" + ), + ( + "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_7) AppleWebKit/605.1.15 " + "(KHTML, like Gecko) Version/18.1 Safari/605.1.15" + ), + ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:131.0) " + "Gecko/20100101 Firefox/131.0" + ), +] + +logger = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class FetchResult: + requested_url: str + status_code: int | None + final_url: str | None + body: str + error: str | None + + +class ScholarSource(Protocol): + async def fetch_profile_html(self, scholar_id: str) -> FetchResult: + ... + + async def fetch_profile_page_html( + self, + scholar_id: str, + *, + cstart: int, + pagesize: int, + ) -> FetchResult: + ... + + +class LiveScholarSource: + def __init__( + self, + *, + timeout_seconds: float = 25.0, + user_agents: list[str] | None = None, + ) -> None: + self._timeout_seconds = timeout_seconds + self._user_agents = user_agents or DEFAULT_USER_AGENTS + + async def fetch_profile_html(self, scholar_id: str) -> FetchResult: + return await self.fetch_profile_page_html( + scholar_id, + cstart=0, + pagesize=DEFAULT_PAGE_SIZE, + ) + + async def fetch_profile_page_html( + self, + scholar_id: str, + *, + cstart: int, + pagesize: int = DEFAULT_PAGE_SIZE, + ) -> FetchResult: + requested_url = _build_profile_url( + scholar_id=scholar_id, + cstart=cstart, + pagesize=pagesize, + ) + logger.debug( + "scholar_source.fetch_started", + extra={ + "event": "scholar_source.fetch_started", + "scholar_id": scholar_id, + "requested_url": requested_url, + "cstart": cstart, + "pagesize": pagesize, + }, + ) + return await asyncio.to_thread(self._fetch_sync, requested_url) + + def _fetch_sync(self, requested_url: str) -> FetchResult: + request = Request( + requested_url, + headers={ + "User-Agent": random.choice(self._user_agents), + "Accept": "text/html,application/xhtml+xml", + "Accept-Language": "en-US,en;q=0.9", + "Connection": "close", + }, + ) + + try: + with urlopen(request, timeout=self._timeout_seconds) as response: + body = response.read().decode("utf-8", errors="replace") + status_code = getattr(response, "status", 200) + logger.debug( + "scholar_source.fetch_succeeded", + extra={ + "event": "scholar_source.fetch_succeeded", + "requested_url": requested_url, + "status_code": status_code, + }, + ) + return FetchResult( + requested_url=requested_url, + status_code=status_code, + final_url=response.geturl(), + body=body, + error=None, + ) + except HTTPError as exc: + body = "" + try: + body = exc.read().decode("utf-8", errors="replace") + except Exception: + body = "" + logger.warning( + "scholar_source.fetch_http_error", + extra={ + "event": "scholar_source.fetch_http_error", + "requested_url": requested_url, + "status_code": exc.code, + }, + ) + return FetchResult( + requested_url=requested_url, + status_code=exc.code, + final_url=exc.geturl(), + body=body, + error=str(exc), + ) + except URLError as exc: + logger.warning( + "scholar_source.fetch_network_error", + extra={ + "event": "scholar_source.fetch_network_error", + "requested_url": requested_url, + }, + ) + return FetchResult( + requested_url=requested_url, + status_code=None, + final_url=None, + body="", + error=str(exc), + ) + + +def _build_profile_url(*, scholar_id: str, cstart: int, pagesize: int) -> str: + query: dict[str, int | str] = {"hl": "en", "user": scholar_id} + if cstart > 0: + query["cstart"] = int(cstart) + if pagesize > 0 and cstart > 0: + query["pagesize"] = int(pagesize) + return f"{SCHOLAR_PROFILE_URL}?{urlencode(query)}" diff --git a/app/services/scholars.py b/app/services/scholars.py new file mode 100644 index 0000000..aacf210 --- /dev/null +++ b/app/services/scholars.py @@ -0,0 +1,98 @@ +from __future__ import annotations + +import re + +from sqlalchemy import select +from sqlalchemy.exc import IntegrityError +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db.models import ScholarProfile + +SCHOLAR_ID_PATTERN = re.compile(r"^[a-zA-Z0-9_-]{12}$") + + +class ScholarServiceError(ValueError): + """Raised for expected scholar-management validation failures.""" + + +def validate_scholar_id(value: str) -> str: + scholar_id = value.strip() + if not SCHOLAR_ID_PATTERN.fullmatch(scholar_id): + raise ScholarServiceError("Scholar ID must match [a-zA-Z0-9_-]{12}.") + return scholar_id + + +def normalize_display_name(value: str) -> str | None: + normalized = value.strip() + return normalized if normalized else None + + +async def list_scholars_for_user( + db_session: AsyncSession, + *, + user_id: int, +) -> list[ScholarProfile]: + result = await db_session.execute( + select(ScholarProfile) + .where(ScholarProfile.user_id == user_id) + .order_by(ScholarProfile.created_at.desc(), ScholarProfile.id.desc()) + ) + return list(result.scalars().all()) + + +async def create_scholar_for_user( + db_session: AsyncSession, + *, + user_id: int, + scholar_id: str, + display_name: str, +) -> ScholarProfile: + profile = ScholarProfile( + user_id=user_id, + scholar_id=validate_scholar_id(scholar_id), + display_name=normalize_display_name(display_name), + ) + db_session.add(profile) + try: + await db_session.commit() + except IntegrityError as exc: + await db_session.rollback() + raise ScholarServiceError("That scholar is already tracked for this account.") from exc + await db_session.refresh(profile) + return profile + + +async def get_user_scholar_by_id( + db_session: AsyncSession, + *, + user_id: int, + scholar_profile_id: int, +) -> ScholarProfile | None: + result = await db_session.execute( + select(ScholarProfile).where( + ScholarProfile.id == scholar_profile_id, + ScholarProfile.user_id == user_id, + ) + ) + return result.scalar_one_or_none() + + +async def toggle_scholar_enabled( + db_session: AsyncSession, + *, + profile: ScholarProfile, +) -> ScholarProfile: + profile.is_enabled = not profile.is_enabled + await db_session.commit() + await db_session.refresh(profile) + return profile + + +async def delete_scholar( + db_session: AsyncSession, + *, + profile: ScholarProfile, +) -> None: + await db_session.delete(profile) + await db_session.commit() + diff --git a/app/services/user_settings.py b/app/services/user_settings.py new file mode 100644 index 0000000..92aff32 --- /dev/null +++ b/app/services/user_settings.py @@ -0,0 +1,66 @@ +from __future__ import annotations + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db.models import UserSetting + + +class UserSettingsServiceError(ValueError): + """Raised for expected settings-validation failures.""" + + +def parse_run_interval_minutes(value: str) -> int: + try: + parsed = int(value) + except ValueError as exc: + raise UserSettingsServiceError("Run interval must be a whole number.") from exc + if parsed < 15: + raise UserSettingsServiceError("Run interval must be at least 15 minutes.") + return parsed + + +def parse_request_delay_seconds(value: str) -> int: + try: + parsed = int(value) + except ValueError as exc: + raise UserSettingsServiceError("Request delay must be a whole number.") from exc + if parsed < 1: + raise UserSettingsServiceError("Request delay must be at least 1 second.") + return parsed + + +async def get_or_create_settings( + db_session: AsyncSession, + *, + user_id: int, +) -> UserSetting: + result = await db_session.execute( + select(UserSetting).where(UserSetting.user_id == user_id) + ) + settings = result.scalar_one_or_none() + if settings is not None: + return settings + + settings = UserSetting(user_id=user_id) + db_session.add(settings) + await db_session.commit() + await db_session.refresh(settings) + return settings + + +async def update_settings( + db_session: AsyncSession, + *, + settings: UserSetting, + auto_run_enabled: bool, + run_interval_minutes: int, + request_delay_seconds: int, +) -> UserSetting: + settings.auto_run_enabled = auto_run_enabled + settings.run_interval_minutes = run_interval_minutes + settings.request_delay_seconds = request_delay_seconds + await db_session.commit() + await db_session.refresh(settings) + return settings + diff --git a/app/services/users.py b/app/services/users.py new file mode 100644 index 0000000..7f019c4 --- /dev/null +++ b/app/services/users.py @@ -0,0 +1,98 @@ +from __future__ import annotations + +import re + +from sqlalchemy import select +from sqlalchemy.exc import IntegrityError +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db.models import User + +EMAIL_PATTERN = re.compile(r"^[^@\s]+@[^@\s]+\.[^@\s]+$") + + +class UserServiceError(ValueError): + """Raised for expected user-management validation failures.""" + + +def normalize_email(value: str) -> str: + return value.strip().lower() + + +def validate_email(value: str) -> str: + email = normalize_email(value) + if not EMAIL_PATTERN.fullmatch(email): + raise UserServiceError("Enter a valid email address.") + return email + + +def validate_password(value: str) -> str: + password = value.strip() + if len(password) < 8: + raise UserServiceError("Password must be at least 8 characters.") + return password + + +async def get_user_by_id(db_session: AsyncSession, user_id: int) -> User | None: + result = await db_session.execute(select(User).where(User.id == user_id)) + return result.scalar_one_or_none() + + +async def get_user_by_email(db_session: AsyncSession, email: str) -> User | None: + result = await db_session.execute( + select(User).where(User.email == normalize_email(email)) + ) + return result.scalar_one_or_none() + + +async def list_users(db_session: AsyncSession) -> list[User]: + result = await db_session.execute(select(User).order_by(User.email.asc())) + return list(result.scalars().all()) + + +async def create_user( + db_session: AsyncSession, + *, + email: str, + password_hash: str, + is_admin: bool, +) -> User: + user = User( + email=validate_email(email), + password_hash=password_hash, + is_admin=is_admin, + is_active=True, + ) + db_session.add(user) + try: + await db_session.commit() + except IntegrityError as exc: + await db_session.rollback() + raise UserServiceError("A user with that email already exists.") from exc + await db_session.refresh(user) + return user + + +async def set_user_active( + db_session: AsyncSession, + *, + user: User, + is_active: bool, +) -> User: + user.is_active = is_active + await db_session.commit() + await db_session.refresh(user) + return user + + +async def set_user_password_hash( + db_session: AsyncSession, + *, + user: User, + password_hash: str, +) -> User: + user.password_hash = password_hash + await db_session.commit() + await db_session.refresh(user) + return user + diff --git a/app/settings.py b/app/settings.py new file mode 100644 index 0000000..826fa13 --- /dev/null +++ b/app/settings.py @@ -0,0 +1,84 @@ +from dataclasses import dataclass +import os + + +def _env_bool(name: str, default: bool) -> bool: + value = os.getenv(name) + if value is None: + return default + return value.strip().lower() in {"1", "true", "yes", "on"} + + +def _env_int(name: str, default: int) -> int: + value = os.getenv(name) + if value is None: + return default + return int(value) + + +def _env_float(name: str, default: float) -> float: + value = os.getenv(name) + if value is None: + return default + return float(value) + + +def _env_str(name: str, default: str) -> str: + value = os.getenv(name) + if value is None: + return default + return value.strip() or default + + +@dataclass(frozen=True) +class Settings: + app_name: str = os.getenv("APP_NAME", "scholarr") + database_url: str = os.getenv( + "DATABASE_URL", + "postgresql+asyncpg://scholar:scholar@db:5432/scholar", + ) + session_secret_key: str = os.getenv("SESSION_SECRET_KEY", "dev-insecure-session-key") + session_cookie_secure: bool = _env_bool("SESSION_COOKIE_SECURE", False) + login_rate_limit_attempts: int = _env_int("LOGIN_RATE_LIMIT_ATTEMPTS", 5) + login_rate_limit_window_seconds: int = _env_int( + "LOGIN_RATE_LIMIT_WINDOW_SECONDS", + 60, + ) + log_level: str = _env_str("LOG_LEVEL", "INFO") + log_format: str = _env_str("LOG_FORMAT", "console") + log_requests: bool = _env_bool("LOG_REQUESTS", True) + log_uvicorn_access: bool = _env_bool("LOG_UVICORN_ACCESS", False) + log_request_skip_paths: str = _env_str("LOG_REQUEST_SKIP_PATHS", "/healthz,/static/") + log_redact_fields: str = os.getenv("LOG_REDACT_FIELDS", "") + scheduler_enabled: bool = _env_bool("SCHEDULER_ENABLED", True) + scheduler_tick_seconds: int = _env_int("SCHEDULER_TICK_SECONDS", 60) + ingestion_network_error_retries: int = _env_int("INGESTION_NETWORK_ERROR_RETRIES", 1) + ingestion_retry_backoff_seconds: float = _env_float( + "INGESTION_RETRY_BACKOFF_SECONDS", + 1.0, + ) + ingestion_max_pages_per_scholar: int = _env_int( + "INGESTION_MAX_PAGES_PER_SCHOLAR", + 30, + ) + ingestion_page_size: int = _env_int("INGESTION_PAGE_SIZE", 100) + ingestion_continuation_queue_enabled: bool = _env_bool( + "INGESTION_CONTINUATION_QUEUE_ENABLED", + True, + ) + ingestion_continuation_base_delay_seconds: int = _env_int( + "INGESTION_CONTINUATION_BASE_DELAY_SECONDS", + 120, + ) + ingestion_continuation_max_delay_seconds: int = _env_int( + "INGESTION_CONTINUATION_MAX_DELAY_SECONDS", + 3600, + ) + ingestion_continuation_max_attempts: int = _env_int( + "INGESTION_CONTINUATION_MAX_ATTEMPTS", + 6, + ) + scheduler_queue_batch_size: int = _env_int("SCHEDULER_QUEUE_BATCH_SIZE", 10) + + +settings = Settings() diff --git a/app/static/app.css b/app/static/app.css new file mode 100644 index 0000000..3b332c5 --- /dev/null +++ b/app/static/app.css @@ -0,0 +1,501 @@ +* { + box-sizing: border-box; +} + +body { + margin: 0; + min-height: 100vh; + color: var(--md-sys-color-on-surface); + font-family: var(--font-body); + background: + radial-gradient(circle at 10% 5%, var(--md-sys-color-primary-container) 0%, transparent 30%), + radial-gradient(circle at 90% 0%, var(--md-sys-color-secondary-container) 0%, transparent 28%), + var(--md-sys-color-surface); +} + +.loading-indicator { + position: fixed; + top: 0; + left: 0; + width: 100%; + height: 3px; + z-index: 40; + transform: scaleX(0); + transform-origin: left; + background: linear-gradient(90deg, var(--md-sys-color-primary), var(--md-sys-color-tertiary)); + opacity: 0; +} + +body.is-page-loading .loading-indicator { + opacity: 1; + animation: loading-bar 1.2s ease-in-out infinite; +} + +.app-backdrop { + position: fixed; + inset: 0; + pointer-events: none; + background: + linear-gradient(130deg, var(--md-sys-color-primary-tint), transparent 40%), + linear-gradient(230deg, var(--md-sys-color-surface-tint), transparent 45%); + opacity: 0.28; +} + +.top-app-bar { + position: sticky; + top: 0; + z-index: 10; + display: flex; + align-items: center; + gap: 0.8rem; + flex-wrap: wrap; + padding: 0.8rem 1.1rem; + border-bottom: 1px solid var(--md-sys-color-outline-variant); + backdrop-filter: blur(9px); + background: color-mix(in srgb, var(--md-sys-color-surface) 88%, white 12%); +} + +.brand-wrap { + display: flex; + align-items: center; + gap: 0.65rem; +} + +.brand { + color: var(--md-sys-color-primary); + text-decoration: none; + font-family: var(--font-heading); + font-weight: 700; + font-size: 1.2rem; + letter-spacing: 0.02em; +} + +.session-user { + margin: 0; + padding: 0.24rem 0.58rem; + border-radius: 999px; + border: 1px solid var(--md-sys-color-outline-variant); + background: var(--md-sys-color-surface-container-high); + color: var(--md-sys-color-on-surface-variant); + font-size: 0.78rem; +} + +.site-nav { + display: flex; + align-items: center; + gap: 0.35rem; + flex-wrap: wrap; +} + +.nav-link { + color: var(--md-sys-color-on-surface-variant); + text-decoration: none; + padding: 0.42rem 0.78rem; + border-radius: 999px; + border: 1px solid transparent; + font-size: 0.92rem; +} + +.nav-link:hover { + background: var(--md-sys-color-surface-container); + border-color: var(--md-sys-color-outline-variant); +} + +.nav-link.is-active { + color: var(--md-sys-color-on-secondary-container); + background: var(--md-sys-color-secondary-container); + border-color: transparent; +} + +.header-actions { + margin-left: auto; + display: flex; + align-items: center; + gap: 0.7rem; +} + +.theme-control-wrap { + display: flex; + align-items: center; + gap: 0.45rem; +} + +.theme-control-wrap label { + color: var(--md-sys-color-on-surface-variant); + font-size: 0.82rem; +} + +.theme-control { + border: 1px solid var(--md-sys-color-outline); + border-radius: 999px; + background: var(--md-sys-color-surface-container-low); + color: var(--md-sys-color-on-surface); + padding: 0.32rem 0.64rem; + font: inherit; +} + +.logout-form { + margin: 0; +} + +.page-shell { + position: relative; + display: grid; + gap: 0.9rem; + width: min(1100px, calc(100vw - 1.4rem)); + margin: 0 auto; + padding: 1.1rem 0 1.6rem; +} + +.flash { + margin: 0; + padding: 0.72rem 0.82rem; + border-radius: 14px; + border: 1px solid var(--md-sys-color-outline-variant); + background: var(--md-sys-color-surface-container); +} + +.flash-notice { + color: var(--md-sys-color-on-tertiary-container); + border-color: var(--md-sys-color-tertiary-container); + background: color-mix(in srgb, var(--md-sys-color-tertiary-container) 30%, white 70%); +} + +.flash-error { + color: var(--md-sys-color-on-error-container); + border-color: var(--md-sys-color-error-container); + background: color-mix(in srgb, var(--md-sys-color-error-container) 25%, white 75%); +} + +.hero, +.sandbox { + border-radius: 24px; + border: 1px solid var(--md-sys-color-outline-variant); + background: var(--md-sys-color-surface-container-low); + box-shadow: var(--elevation-card); +} + +.hero { + padding: 1.3rem; +} + +.sandbox { + padding: 1.05rem; +} + +.eyebrow { + margin: 0; + font-size: 0.78rem; + color: var(--md-sys-color-primary); + text-transform: uppercase; + letter-spacing: 0.11em; + font-weight: 600; +} + +h1 { + margin: 0.28rem 0 0.5rem; + color: var(--md-sys-color-on-surface); + font-family: var(--font-heading); + font-size: clamp(1.45rem, 3vw, 2.05rem); + line-height: 1.15; +} + +h2 { + margin: 0; + color: var(--md-sys-color-on-surface); + font-family: var(--font-heading); + font-size: 1.08rem; +} + +.lede { + margin: 0; + color: var(--md-sys-color-on-surface-variant); + max-width: 68ch; +} + +.section-heading { + display: flex; + align-items: center; + justify-content: space-between; + gap: 0.55rem; + margin-bottom: 0.7rem; + flex-wrap: wrap; +} + +.stat-strip { + margin-top: 0.95rem; + display: grid; + gap: 0.75rem; + grid-template-columns: repeat(auto-fit, minmax(170px, 1fr)); +} + +.stat-item { + margin: 0; + padding: 0.72rem 0.78rem; + border-radius: 16px; + border: 1px solid var(--md-sys-color-outline-variant); + background: var(--md-sys-color-surface-container); +} + +.stat-label { + margin: 0; + color: var(--md-sys-color-on-surface-variant); + font-size: 0.8rem; +} + +.stat-value { + margin: 0.2rem 0 0; + color: var(--md-sys-color-on-surface); + font-size: 1.36rem; + font-weight: 700; +} + +form { + display: grid; + gap: 0.45rem; +} + +.stack-form { + max-width: 560px; +} + +label { + font-size: 0.9rem; + color: var(--md-sys-color-on-surface-variant); +} + +input, +select { + width: 100%; + border: 1px solid var(--md-sys-color-outline); + border-radius: 12px; + padding: 0.56rem 0.64rem; + font: inherit; + color: var(--md-sys-color-on-surface); + background: var(--md-sys-color-surface); +} + +input:focus, +select:focus { + outline: 2px solid color-mix(in srgb, var(--md-sys-color-primary) 35%, transparent); + border-color: var(--md-sys-color-primary); +} + +.checkbox-row { + display: flex; + align-items: center; + gap: 0.54rem; +} + +.checkbox-row input { + width: auto; +} + +button, +.button { + border: 0; + border-radius: 999px; + padding: 0.54rem 0.86rem; + font: inherit; + font-weight: 600; + color: var(--md-sys-color-on-primary); + background: var(--md-sys-color-primary); + cursor: pointer; +} + +button:hover, +.button:hover { + filter: brightness(0.97); +} + +button:disabled, +.button:disabled { + cursor: not-allowed; + opacity: 0.5; +} + +.button-text { + color: var(--md-sys-color-primary); + background: transparent; + border: 1px solid var(--md-sys-color-outline-variant); +} + +.danger-button { + color: var(--md-sys-color-on-error); + background: var(--md-sys-color-error); +} + +.inline-actions { + display: flex; + align-items: center; + flex-wrap: wrap; + gap: 0.45rem; +} + +.inline-actions form { + display: inline-flex; + gap: 0.42rem; + align-items: center; +} + +.inline-reset-form input { + min-width: 160px; +} + +form.is-loading button[type="submit"], +form.is-loading input[type="submit"] { + opacity: 0.75; +} + +table { + width: 100%; + border-collapse: collapse; + border-radius: 12px; + overflow: hidden; + background: var(--md-sys-color-surface); +} + +th, +td { + padding: 0.54rem; + text-align: left; + border-bottom: 1px solid var(--md-sys-color-outline-variant); + vertical-align: top; +} + +thead th { + color: var(--md-sys-color-on-surface-variant); + font-size: 0.84rem; + font-weight: 600; + background: var(--md-sys-color-surface-container-high); +} + +tbody tr:last-child td { + border-bottom: 0; +} + +a { + color: var(--md-sys-color-primary); +} + +.link-button { + display: inline-flex; + align-items: center; + padding: 0.45rem 0.78rem; + border: 1px solid var(--md-sys-color-outline-variant); + border-radius: 999px; + color: var(--md-sys-color-primary); + text-decoration: none; + background: var(--md-sys-color-surface-container-low); +} + +.link-button.is-active { + color: var(--md-sys-color-on-secondary-container); + background: var(--md-sys-color-secondary-container); + border-color: transparent; +} + +.badge { + display: inline-block; + border-radius: 999px; + padding: 0.16rem 0.48rem; + font-size: 0.76rem; + text-transform: lowercase; + border: 1px solid var(--md-sys-color-outline-variant); +} + +.badge.ok { + color: var(--md-sys-color-on-tertiary-container); + background: var(--md-sys-color-tertiary-container); + border-color: transparent; +} + +.badge.warn, +.badge.danger { + color: var(--md-sys-color-on-error-container); + background: var(--md-sys-color-error-container); + border-color: transparent; +} + +.helper-text { + margin: 0.24rem 0 0; + color: var(--md-sys-color-on-surface-variant); + font-size: 0.82rem; +} + +.auth-card { + max-width: 520px; +} + +.auth-form { + max-width: 360px; +} + +.form-error { + margin: 0.5rem 0 0; + color: var(--md-sys-color-error); + font-weight: 600; +} + +.form-note { + margin: 0.35rem 0 0; + color: var(--md-sys-color-on-surface-variant); +} + +code { + font-family: var(--font-code); + background: var(--md-sys-color-surface-container-high); + border-radius: 6px; + padding: 0.1rem 0.25rem; +} + +pre { + margin: 0.6rem 0 0; + border-radius: 12px; + border: 1px solid var(--md-sys-color-outline-variant); + background: var(--md-sys-color-surface-container-high); + color: var(--md-sys-color-on-surface); + padding: 0.75rem; + white-space: pre-wrap; + word-break: break-word; +} + +@media (max-width: 880px) { + .top-app-bar { + padding: 0.7rem 0.8rem; + } + + .header-actions { + width: 100%; + justify-content: space-between; + margin-left: 0; + } + + .page-shell { + width: min(100vw, calc(100vw - 0.9rem)); + padding-top: 0.8rem; + } + + .hero, + .sandbox { + border-radius: 18px; + padding: 0.9rem; + } + + .inline-actions form { + width: 100%; + justify-content: flex-start; + } +} + +@keyframes loading-bar { + 0% { + transform: scaleX(0.1); + } + 50% { + transform: scaleX(0.7); + } + 100% { + transform: scaleX(1); + } +} diff --git a/app/static/app.js b/app/static/app.js new file mode 100644 index 0000000..a2d6d80 --- /dev/null +++ b/app/static/app.js @@ -0,0 +1,76 @@ +(() => { + const body = document.body; + if (!body) { + return; + } + + const defaultLoadingText = "Working..."; + + const shouldHandleAnchor = (anchor) => { + if (!anchor || !anchor.getAttribute) { + return false; + } + const href = anchor.getAttribute("href"); + if (!href) { + return false; + } + if (href.startsWith("#") || href.startsWith("mailto:") || href.startsWith("javascript:")) { + return false; + } + if (anchor.hasAttribute("download")) { + return false; + } + if ((anchor.getAttribute("target") || "").toLowerCase() === "_blank") { + return false; + } + return true; + }; + + document.addEventListener("click", (event) => { + const target = event.target; + if (!(target instanceof Element)) { + return; + } + const anchor = target.closest("a"); + if (!shouldHandleAnchor(anchor)) { + return; + } + body.classList.add("is-page-loading"); + }); + + document.addEventListener("submit", (event) => { + const form = event.target; + if (!(form instanceof HTMLFormElement)) { + return; + } + if (form.dataset.noLoading === "1") { + return; + } + + body.classList.add("is-page-loading"); + form.classList.add("is-loading"); + + const submitElements = form.querySelectorAll("button[type='submit'], input[type='submit']"); + submitElements.forEach((element) => { + if (element.hasAttribute("disabled")) { + return; + } + element.setAttribute("disabled", "disabled"); + if (element instanceof HTMLInputElement) { + const loadingText = element.dataset.loadingText || defaultLoadingText; + element.dataset.originalValue = element.value; + element.value = loadingText; + return; + } + if (element instanceof HTMLButtonElement) { + const loadingText = element.dataset.loadingText || defaultLoadingText; + element.dataset.originalText = element.textContent || ""; + element.textContent = loadingText; + } + }); + }); + + window.addEventListener("pageshow", () => { + body.classList.remove("is-page-loading"); + }); +})(); diff --git a/app/static/theme.css b/app/static/theme.css new file mode 100644 index 0000000..8506a29 --- /dev/null +++ b/app/static/theme.css @@ -0,0 +1,98 @@ +:root { + --font-body: "Plus Jakarta Sans", "Segoe UI", sans-serif; + --font-heading: "DM Serif Text", "Georgia", serif; + --font-code: "JetBrains Mono", "Fira Code", monospace; + + --md-sys-color-primary: #73594d; + --md-sys-color-on-primary: #ffffff; + --md-sys-color-primary-container: #fbded0; + --md-sys-color-on-primary-container: #2b160f; + --md-sys-color-primary-tint: rgba(115, 89, 77, 0.18); + + --md-sys-color-secondary: #6f5b52; + --md-sys-color-on-secondary: #ffffff; + --md-sys-color-secondary-container: #f8ddd2; + --md-sys-color-on-secondary-container: #281913; + + --md-sys-color-tertiary: #5b6550; + --md-sys-color-on-tertiary: #ffffff; + --md-sys-color-tertiary-container: #ddebcf; + --md-sys-color-on-tertiary-container: #17200f; + + --md-sys-color-error: #b3261e; + --md-sys-color-on-error: #ffffff; + --md-sys-color-error-container: #f9dedc; + --md-sys-color-on-error-container: #410e0b; + + --md-sys-color-surface: #fff8f5; + --md-sys-color-surface-tint: rgba(111, 87, 74, 0.12); + --md-sys-color-surface-container-low: #fffaf8; + --md-sys-color-surface-container: #fdf2ec; + --md-sys-color-surface-container-high: #f8e9e1; + --md-sys-color-on-surface: #221a17; + --md-sys-color-on-surface-variant: #55423a; + --md-sys-color-outline: #85736b; + --md-sys-color-outline-variant: #d8c2b8; + + --elevation-card: 0 10px 30px rgba(55, 39, 28, 0.08); +} + +:root[data-theme="fjord"] { + --md-sys-color-primary: #2f6678; + --md-sys-color-on-primary: #ffffff; + --md-sys-color-primary-container: #cde9f4; + --md-sys-color-on-primary-container: #051f28; + --md-sys-color-primary-tint: rgba(47, 102, 120, 0.2); + + --md-sys-color-secondary: #4d626b; + --md-sys-color-on-secondary: #ffffff; + --md-sys-color-secondary-container: #d0e6f2; + --md-sys-color-on-secondary-container: #091f27; + + --md-sys-color-tertiary: #456179; + --md-sys-color-on-tertiary: #ffffff; + --md-sys-color-tertiary-container: #d2e5ff; + --md-sys-color-on-tertiary-container: #031d33; + + --md-sys-color-surface: #f5fbff; + --md-sys-color-surface-tint: rgba(47, 102, 120, 0.12); + --md-sys-color-surface-container-low: #fcfeff; + --md-sys-color-surface-container: #ecf5fa; + --md-sys-color-surface-container-high: #dfeef6; + --md-sys-color-on-surface: #172126; + --md-sys-color-on-surface-variant: #3f4f57; + --md-sys-color-outline: #6f7f88; + --md-sys-color-outline-variant: #becfd8; + + --elevation-card: 0 10px 30px rgba(22, 43, 53, 0.09); +} + +:root[data-theme="spruce"] { + --md-sys-color-primary: #39684c; + --md-sys-color-on-primary: #ffffff; + --md-sys-color-primary-container: #bcefc9; + --md-sys-color-on-primary-container: #062111; + --md-sys-color-primary-tint: rgba(57, 104, 76, 0.18); + + --md-sys-color-secondary: #4f6354; + --md-sys-color-on-secondary: #ffffff; + --md-sys-color-secondary-container: #d2e8d4; + --md-sys-color-on-secondary-container: #0d1f12; + + --md-sys-color-tertiary: #3f655f; + --md-sys-color-on-tertiary: #ffffff; + --md-sys-color-tertiary-container: #c2ece4; + --md-sys-color-on-tertiary-container: #021f1a; + + --md-sys-color-surface: #f5fbf5; + --md-sys-color-surface-tint: rgba(57, 104, 76, 0.1); + --md-sys-color-surface-container-low: #fcfffb; + --md-sys-color-surface-container: #ebf4ea; + --md-sys-color-surface-container-high: #deecdd; + --md-sys-color-on-surface: #172119; + --md-sys-color-on-surface-variant: #415246; + --md-sys-color-outline: #708174; + --md-sys-color-outline-variant: #c1d3c2; + + --elevation-card: 0 10px 30px rgba(24, 48, 32, 0.08); +} diff --git a/app/static/theme.js b/app/static/theme.js new file mode 100644 index 0000000..47b95a1 --- /dev/null +++ b/app/static/theme.js @@ -0,0 +1,56 @@ +(() => { + const STORAGE_KEY = "scholarr_theme"; + const LEGACY_STORAGE_KEY = "scholar_tracker_theme"; + const root = document.documentElement; + const themeControl = document.querySelector("[data-theme-control]"); + + if (!root) { + return; + } + + const defaultTheme = root.dataset.defaultTheme || "terracotta"; + const supportedThemes = themeControl + ? Array.from(themeControl.options).map((option) => option.value) + : []; + + const isSupported = (theme) => + supportedThemes.length === 0 || supportedThemes.includes(theme); + + const applyTheme = (theme) => { + if (!isSupported(theme)) { + return; + } + root.dataset.theme = theme; + }; + + let activeTheme = defaultTheme; + try { + const savedTheme = window.localStorage.getItem(STORAGE_KEY); + const legacyTheme = window.localStorage.getItem(LEGACY_STORAGE_KEY); + if (savedTheme && isSupported(savedTheme)) { + activeTheme = savedTheme; + } else if (legacyTheme && isSupported(legacyTheme)) { + activeTheme = legacyTheme; + window.localStorage.setItem(STORAGE_KEY, legacyTheme); + } + } catch { + activeTheme = defaultTheme; + } + + applyTheme(activeTheme); + + if (!themeControl) { + return; + } + + themeControl.value = activeTheme; + themeControl.addEventListener("change", (event) => { + const selectedTheme = event.target.value; + applyTheme(selectedTheme); + try { + window.localStorage.setItem(STORAGE_KEY, selectedTheme); + } catch { + // Ignore storage errors in locked-down browsers. + } + }); +})(); diff --git a/app/templates/account_password.html b/app/templates/account_password.html new file mode 100644 index 0000000..687ec98 --- /dev/null +++ b/app/templates/account_password.html @@ -0,0 +1,23 @@ +{% extends "base.html" %} + +{% block content %} +
+

Account

+

Change Password

+

Use a strong password. This updates only your own account.

+
+ +
+

Password Update

+
+ + + + + + + + +
+
+{% endblock %} diff --git a/app/templates/base.html b/app/templates/base.html new file mode 100644 index 0000000..fcdd653 --- /dev/null +++ b/app/templates/base.html @@ -0,0 +1,66 @@ + + + + + + {{ page_title }} | scholarr + + + + + + +
+
+ scholarr + {% if session_user %} +

{{ session_user.email }}

+ {% endif %} +
+ + + +
+
+ + +
+ {% if session_user %} +
+ + +
+ {% endif %} +
+
+
+ {% if notice %} +

{{ notice }}

+ {% endif %} + {% if page_error %} +

{{ page_error }}

+ {% endif %} + {% block content %}{% endblock %} +
+ + + + diff --git a/app/templates/dashboard/_run_controls.html b/app/templates/dashboard/_run_controls.html new file mode 100644 index 0000000..24779cd --- /dev/null +++ b/app/templates/dashboard/_run_controls.html @@ -0,0 +1,23 @@ +
+
+

Run Tracking

+
+ New Publications + All Publications + Run Diagnostics + Automation Settings + Manage Scholars +
+
+

Manual runs use your request delay of {{ dashboard.run_controls.request_delay_seconds }} second(s).

+

+ Queue state: + {{ dashboard.run_controls.queue_queued_count }} queued, + {{ dashboard.run_controls.queue_retrying_count }} retrying, + {{ dashboard.run_controls.queue_dropped_count }} dropped. +

+
+ + +
+
diff --git a/app/templates/dashboard/_run_history.html b/app/templates/dashboard/_run_history.html new file mode 100644 index 0000000..aa6f9f1 --- /dev/null +++ b/app/templates/dashboard/_run_history.html @@ -0,0 +1,38 @@ +
+
+

Run History

+
+ View All Runs +
+
+ {% if dashboard.run_history %} + + + + + + + + + + + {% for run in dashboard.run_history %} + + + + + + + {% endfor %} + +
StartedStatusScholarsNew Publications
+ {% if run.detail_url %} + {{ run.started_at_display }} + {% else %} + {{ run.started_at_display }} + {% endif %} + {{ run.status_label }}{{ run.scholar_count }}{{ run.new_publication_count }}
+ {% else %} +

No runs yet.

+ {% endif %} +
diff --git a/app/templates/dashboard/_unread_publications.html b/app/templates/dashboard/_unread_publications.html new file mode 100644 index 0000000..1354215 --- /dev/null +++ b/app/templates/dashboard/_unread_publications.html @@ -0,0 +1,47 @@ +
+
+

New Since Last Run

+
+ View All +
+ + + +
+
+
+ + {% if dashboard.unread_publications %} + + + + + + + + + + + {% for item in dashboard.unread_publications %} + + + + + + + {% endfor %} + +
TitleScholarYearCitations
+ {% if item.pub_url %} + {{ item.title }} + {% else %} + {{ item.title }} + {% endif %} + {% if item.venue_text %} +

{{ item.venue_text }}

+ {% endif %} + 		{{ item.scholar_label }}{{ item.year_display }}{{ item.citation_count }}
+ {% else %} +

No new publications in the latest run.

+ {% endif %} +
diff --git a/app/templates/index.html b/app/templates/index.html new file mode 100644 index 0000000..f5fd22a --- /dev/null +++ b/app/templates/index.html @@ -0,0 +1,27 @@ +{% extends "base.html" %} + +{% block content %} +
+

Dashboard

+

Keep up with your tracked scholars

+

Run ingestion, review fresh publications, and inspect recent outcomes from one screen.

+
+
+

New Since Last Run

+

{{ dashboard.unread_publications | length }}

+
+
+

Recent Runs

+

{{ dashboard.run_history | length }}

+
+
+

Automation Delay

+

{{ dashboard.run_controls.request_delay_seconds }}s

+
+
+
+ +{% for section_template in dashboard.section_templates %} +{% include section_template %} +{% endfor %} +{% endblock %} diff --git a/app/templates/login.html b/app/templates/login.html new file mode 100644 index 0000000..8b0f383 --- /dev/null +++ b/app/templates/login.html @@ -0,0 +1,23 @@ +{% extends "base.html" %} + +{% block content %} +
+

Secure Access

+

Sign in

+

Use an admin-created account to access your tracked scholars and settings.

+ {% if error_message %} + + {% endif %} + {% if retry_after_seconds %} +

Retry after {{ retry_after_seconds }} seconds.

+ {% endif %} +
+ + + + + + +
+
+{% endblock %} diff --git a/app/templates/publications.html b/app/templates/publications.html new file mode 100644 index 0000000..89054a2 --- /dev/null +++ b/app/templates/publications.html @@ -0,0 +1,104 @@ +{% extends "base.html" %} + +{% block content %} +
+

Publications

+

Publications

+

+ Browse publications discovered in the latest run or your complete tracked library. + {% if selected_scholar %} + Current scholar filter: {{ selected_scholar.display_name or selected_scholar.scholar_id }}. + {% endif %} +

+
+ New Since Last Run ({{ new_count }}) + All ({{ total_count }}) + Manage Scholars + Run Diagnostics + {% if mode == 'new' and new_count > 0 %} +
+ + + +
+ {% endif %} +
+
+ +
+
+

Filter

+
+
+ + + + + + +
+ +
+

Results

+
+ {% if publications %} + + + + + + + + + + + + + {% for item in publications %} + + + + + + + + + {% endfor %} + +
TitleScholarYearCitationsNew This RunStatus
+ {% if item.pub_url %} + {{ item.title }} + {% else %} + {{ item.title }} + {% endif %} + {% if item.venue_text %} +

{{ item.venue_text }}

+ {% endif %} + 		{{ item.scholar_label }}{{ item.year if item.year is not none else "-" }}{{ item.citation_count }} + {% if item.is_new_in_latest_run %} + new + {% else %} + older + {% endif %} + + {% if item.is_read %} + read + {% else %} + new + {% endif %} +
+ {% else %} +

No publications match this mode yet.

+ {% endif %} +
+{% endblock %} diff --git a/app/templates/run_detail.html b/app/templates/run_detail.html new file mode 100644 index 0000000..06920ee --- /dev/null +++ b/app/templates/run_detail.html @@ -0,0 +1,98 @@ +{% extends "base.html" %} + +{% block content %} +
+

Diagnostics

+

Run #{{ run.id }}

+

Detailed state and failure context for this execution.

+
+ +
+

Summary

+ + + + + + + + + + + + +
Started{{ run.started_at }}
Finished{{ run.finished_at }}
Status{{ run.status }}
Trigger{{ run.trigger_type }}
Scholars{{ run.scholar_count }}
New Publications{{ run.new_publication_count }}
Succeeded{{ run_summary.succeeded_count or 0 }}
Failed{{ run_summary.failed_count or 0 }}
Partial{{ run_summary.partial_count or 0 }}
+ {% if run_summary.failed_state_counts or run_summary.failed_reason_counts %} +
+ Failure Breakdown + 
{{ {"failed_state_counts": run_summary.failed_state_counts, "failed_reason_counts": run_summary.failed_reason_counts} | tojson(indent=2) }}
+
+ {% endif %} +
+ +
+

Scholar Results

+ {% if scholar_results %} + + + + + + + + + + + + + + + + {% for item in scholar_results %} + + + + + + + + + + + + {% if item.debug %} + + + + {% endif %} + {% endfor %} + +
ScholarOutcomeStateReasonPublicationsPagesAttemptsContinuationWarnings
{{ item.scholar_id }}{{ item.outcome or "-" }}{{ item.state }}{{ item.state_reason or "-" }}{{ item.publication_count }}{{ item.pages_fetched or 0 }} / {{ item.pages_attempted or 0 }}{{ item.attempt_count or 1 }} + {% if item.continuation_enqueued %} + queued +

+ reason: {{ item.continuation_reason or "-" }}, + cstart: {{ item.continuation_cstart or "-" }} +

+ {% elif item.continuation_cleared %} + cleared + {% else %} + - + {% endif %} + 		+ {% if item.warnings %} + {{ item.warnings | join(", ") }} + {% else %} + - + {% endif %} +
+
+ Debug Context + 
{{ item.debug | tojson(indent=2) }}
+
+
+ {% else %} +

No scholar result details were captured.

+ {% endif %} +
+{% endblock %} diff --git a/app/templates/runs.html b/app/templates/runs.html new file mode 100644 index 0000000..6a153c1 --- /dev/null +++ b/app/templates/runs.html @@ -0,0 +1,129 @@ +{% extends "base.html" %} + +{% block content %} +
+

Runs

+

Run History

+

Review execution outcomes and drill into run diagnostics when needed.

+
+ +
+

Filters

+
+ + + {% if failed_only %} + Clear + {% endif %} +
+
+ +
+

Recent Runs

+ {% if runs %} + + + + + + + + + + + + + + {% for run in runs %} + + + + + + + + + + {% endfor %} + +
IDStartedStatusTriggerScholarsNew PublicationsFailures
#{{ run.id }}{{ run.started_at }}{{ run.status }}{{ run.trigger_type }}{{ run.scholar_count }}{{ run.new_publication_count }}{{ run.failed_count }} failed / {{ run.partial_count }} partial
+ {% else %} +

No runs match the current filter.

+ {% endif %} +
+ +
+
+

Continuation Queue

+ Focus Failed Runs +
+ {% if queue_items %} + + + + + + + + + + + + + + {% for item in queue_items %} + + + + + + + + + + {% endfor %} + +
ScholarStatusReasonAttemptsNext AttemptLast ErrorActions
+ {{ item.scholar_label }} +

resume cstart: {{ item.resume_cstart }}

+ 		{{ item.status }} + {% if item.status == "dropped" and item.dropped_reason %} + {{ item.dropped_reason }} + {% else %} + {{ item.reason }} + {% endif %} + {{ item.attempt_count }}{{ item.next_attempt_at }} + {% if item.last_error %} +
+ show + 
{{ item.last_error }}
+
+ {% else %} + - + {% endif %} + 		+
+
+ + +
+ {% if item.status != "dropped" %} +
+ + +
+ {% endif %} +
+ + +
+
+
+ {% else %} +

No queued continuation items.

+ {% endif %} +
+{% endblock %} diff --git a/app/templates/scholars.html b/app/templates/scholars.html new file mode 100644 index 0000000..0cc7821 --- /dev/null +++ b/app/templates/scholars.html @@ -0,0 +1,99 @@ +{% extends "base.html" %} + +{% block content %} +
+

Scholars

+

Your Scholars

+

Add, disable, or remove profiles scoped to your account.

+
+ +
+

Add Scholar

+
+ + + + + + +
+
+ +
+

Tracked Scholars

+ {% if scholars %} + + + + + + + + + + + + + {% for scholar in scholars %} + + + + + + + + + {% endfor %} + +
NameScholar IDStatusLast RunPublicationsActions
{{ scholar.display_name }}{{ scholar.scholar_id }} + {% if scholar.is_enabled %} + enabled + {% else %} + disabled + {% endif %} + {% if scholar.baseline_completed %} +

baseline complete

+ {% else %} +

awaiting first run

+ {% endif %} + 		+ + {{ scholar.last_run_status }} + +

{{ scholar.last_run_dt }}

+ 		+
+ New + All +
+ 		+
+
+ + +
+
+ + +
+
+
+ {% else %} +

No scholars tracked yet.

+ {% endif %} +
+{% endblock %} diff --git a/app/templates/settings.html b/app/templates/settings.html new file mode 100644 index 0000000..c07d064 --- /dev/null +++ b/app/templates/settings.html @@ -0,0 +1,44 @@ +{% extends "base.html" %} + +{% block content %} +
+

Settings

+

Your Settings

+

Control automation cadence and request pacing for your own account.

+
+ +
+

Automation Settings

+
+ + + + + + + + + + +
+
+{% endblock %} diff --git a/app/templates/users.html b/app/templates/users.html new file mode 100644 index 0000000..34b4b7d --- /dev/null +++ b/app/templates/users.html @@ -0,0 +1,75 @@ +{% extends "base.html" %} + +{% block content %} +
+

Admin

+

User Management

+

Admin-only controls for account creation, activation, and password resets.

+
+ +
+

Create User

+
+ + + + + + + +
+
+ +
+

Users

+ + + + + + + + + + + {% for user in users %} + + + + + + + {% endfor %} + +
EmailRoleStatusActions
{{ user.email }} + {% if user.is_admin %} + admin + {% else %} + user + {% endif %} + + {% if user.is_active %} + active + {% else %} + inactive + {% endif %} + +
+
+ + +
+
+ + + +
+
+
+
+{% endblock %} diff --git a/app/theme.py b/app/theme.py new file mode 100644 index 0000000..78e152b --- /dev/null +++ b/app/theme.py @@ -0,0 +1,24 @@ +from dataclasses import dataclass +from typing import Final + + +@dataclass(frozen=True) +class ThemeDefinition: + slug: str + label: str + + +THEMES: Final[tuple[ThemeDefinition, ...]] = ( + ThemeDefinition(slug="terracotta", label="Terracotta"), + ThemeDefinition(slug="fjord", label="Fjord"), + ThemeDefinition(slug="spruce", label="Spruce"), +) +_THEME_SLUGS: Final[frozenset[str]] = frozenset(theme.slug for theme in THEMES) +DEFAULT_THEME: Final[str] = THEMES[0].slug + + +def resolve_theme(candidate: str | None) -> str: + if candidate and candidate in _THEME_SLUGS: + return candidate + return DEFAULT_THEME + diff --git a/app/web/__init__.py b/app/web/__init__.py new file mode 100644 index 0000000..4403b00 --- /dev/null +++ b/app/web/__init__.py @@ -0,0 +1,2 @@ +"""Web-layer routers, middleware, and shared helpers.""" + diff --git a/app/web/common.py b/app/web/common.py new file mode 100644 index 0000000..8c03915 --- /dev/null +++ b/app/web/common.py @@ -0,0 +1,140 @@ +from __future__ import annotations + +import logging +from urllib.parse import urlencode + +from fastapi import Request, status +from fastapi.responses import HTMLResponse, RedirectResponse +from fastapi.templating import Jinja2Templates +from sqlalchemy.ext.asyncio import AsyncSession + +from app.auth.session import ( + SessionUser, + clear_session_user, + get_session_user, + set_session_user, +) +from app.db.models import User +from app.security.csrf import CSRF_SESSION_KEY, ensure_csrf_token +from app.services import users as user_service +from app.theme import THEMES + +logger = logging.getLogger(__name__) + +templates = Jinja2Templates(directory="app/templates") + + +def to_session_user(user: User | None) -> SessionUser | None: + if user is None: + return None + return SessionUser(id=user.id, email=user.email, is_admin=user.is_admin) + + +def build_template_context( + request: Request, + *, + page_title: str, + active_nav: str, + theme_name: str, + session_user: SessionUser | None, + notice: str | None = None, + page_error: str | None = None, +) -> dict[str, object]: + return { + "active_nav": active_nav, + "page_title": page_title, + "theme_name": theme_name, + "themes": THEMES, + "session_user": session_user, + "csrf_token": ensure_csrf_token(request), + "notice": notice, + "page_error": page_error, + } + + +def redirect_with_message( + path: str, + *, + notice: str | None = None, + error: str | None = None, +) -> RedirectResponse: + params: dict[str, str] = {} + if notice: + params["notice"] = notice + if error: + params["error"] = error + if params: + path = f"{path}?{urlencode(params)}" + return RedirectResponse(path, status_code=status.HTTP_303_SEE_OTHER) + + +def redirect_to_login() -> RedirectResponse: + return RedirectResponse("/login", status_code=status.HTTP_303_SEE_OTHER) + + +def invalidate_session(request: Request) -> None: + clear_session_user(request) + request.session.pop(CSRF_SESSION_KEY, None) + + +async def get_authenticated_user( + request: Request, + db_session: AsyncSession, +) -> User | None: + session_user = get_session_user(request) + if session_user is None: + return None + + user = await user_service.get_user_by_id(db_session, session_user.id) + if user is None or not user.is_active: + logger.info( + "auth.session_invalidated", + extra={ + "event": "auth.session_invalidated", + "session_user_id": session_user.id, + }, + ) + invalidate_session(request) + return None + + if user.email != session_user.email or user.is_admin != session_user.is_admin: + set_session_user( + request, + user_id=user.id, + email=user.email, + is_admin=user.is_admin, + ) + + return user + + +def login_rate_limit_key(request: Request, email: str) -> str: + client_host = request.client.host if request.client is not None else "unknown" + normalized_email = email.strip().lower() + return f"{client_host}:{normalized_email or ''}" + + +def render_login_page( + request: Request, + *, + theme_name: str, + error_message: str | None = None, + status_code: int = status.HTTP_200_OK, + retry_after_seconds: int | None = None, +) -> HTMLResponse: + context = build_template_context( + request, + page_title="Login", + active_nav="login", + theme_name=theme_name, + session_user=None, + ) + context["error_message"] = error_message + context["retry_after_seconds"] = retry_after_seconds + return templates.TemplateResponse( + request=request, + name="login.html", + context=context, + status_code=status_code, + ) + diff --git a/app/web/deps.py b/app/web/deps.py new file mode 100644 index 0000000..001e2b8 --- /dev/null +++ b/app/web/deps.py @@ -0,0 +1,17 @@ +from __future__ import annotations + +from fastapi import Depends + +from app.services import ingestion as ingestion_service +from app.services import scholar_source as scholar_source_service + + +def get_scholar_source() -> scholar_source_service.ScholarSource: + return scholar_source_service.LiveScholarSource() + + +def get_ingestion_service( + source: scholar_source_service.ScholarSource = Depends(get_scholar_source), +) -> ingestion_service.ScholarIngestionService: + return ingestion_service.ScholarIngestionService(source=source) + diff --git a/app/web/middleware.py b/app/web/middleware.py new file mode 100644 index 0000000..9de69a6 --- /dev/null +++ b/app/web/middleware.py @@ -0,0 +1,85 @@ +from __future__ import annotations + +from secrets import token_urlsafe +import time +import logging + +from starlette.middleware.base import BaseHTTPMiddleware +from starlette.requests import Request +from starlette.responses import Response + +from app.logging_context import set_request_id + +REQUEST_ID_HEADER = "X-Request-ID" + +logger = logging.getLogger(__name__) + + +class RequestLoggingMiddleware(BaseHTTPMiddleware): + def __init__( + self, + app, + *, + log_requests: bool = True, + skip_paths: tuple[str, ...] = (), + ) -> None: + super().__init__(app) + self._log_requests = log_requests + self._skip_paths = tuple(path for path in skip_paths if path) + + async def dispatch(self, request: Request, call_next) -> Response: + request_id = request.headers.get(REQUEST_ID_HEADER) or token_urlsafe(12) + request.state.request_id = request_id + set_request_id(request_id) + + start = time.perf_counter() + should_log = self._log_requests and not self._is_skipped_path(request.url.path) + if should_log: + logger.info( + "request.started", + extra={ + "event": "request.started", + "method": request.method, + "path": request.url.path, + }, + ) + + try: + response = await call_next(request) + except Exception: + duration_ms = int((time.perf_counter() - start) * 1000) + logger.exception( + "request.failed", + extra={ + "event": "request.failed", + "method": request.method, + "path": request.url.path, + "duration_ms": duration_ms, + }, + ) + raise + else: + duration_ms = int((time.perf_counter() - start) * 1000) + response.headers[REQUEST_ID_HEADER] = request_id + if should_log: + logger.info( + "request.completed", + extra={ + "event": "request.completed", + "method": request.method, + "path": request.url.path, + "status_code": response.status_code, + "duration_ms": duration_ms, + }, + ) + return response + finally: + set_request_id(None) + + def _is_skipped_path(self, path: str) -> bool: + return any(path.startswith(prefix) for prefix in self._skip_paths) + + +def parse_skip_paths(raw_value: str) -> tuple[str, ...]: + parts = [part.strip() for part in raw_value.split(",")] + return tuple(part for part in parts if part) diff --git a/app/web/routers/__init__.py b/app/web/routers/__init__.py new file mode 100644 index 0000000..cbf15d9 --- /dev/null +++ b/app/web/routers/__init__.py @@ -0,0 +1,2 @@ +"""Application routers grouped by concern.""" + diff --git a/app/web/routers/admin.py b/app/web/routers/admin.py new file mode 100644 index 0000000..fda4f45 --- /dev/null +++ b/app/web/routers/admin.py @@ -0,0 +1,209 @@ +from __future__ import annotations + +import logging +from typing import Annotated + +from fastapi import APIRouter, Depends, Form, HTTPException, Request, status +from fastapi.responses import HTMLResponse +from sqlalchemy.ext.asyncio import AsyncSession + +from app.auth.deps import get_auth_service +from app.auth.service import AuthService +from app.db.session import get_db_session +from app.services import users as user_service +from app.theme import resolve_theme +from app.web import common + +logger = logging.getLogger(__name__) + +router = APIRouter() + + +async def _render_users_page( + request: Request, + *, + db_session: AsyncSession, + current_user, + theme_name: str, + notice: str | None = None, + page_error: str | None = None, + status_code: int = status.HTTP_200_OK, + form_email: str = "", + form_is_admin: bool = False, +) -> HTMLResponse: + users = await user_service.list_users(db_session) + context = common.build_template_context( + request, + page_title="Users", + active_nav="users", + theme_name=theme_name, + session_user=common.to_session_user(current_user), + notice=notice, + page_error=page_error, + ) + context["users"] = users + context["current_user_id"] = current_user.id + context["form_email"] = form_email + context["form_is_admin"] = form_is_admin + return common.templates.TemplateResponse( + request=request, + name="users.html", + context=context, + status_code=status_code, + ) + + +@router.get("/users", response_class=HTMLResponse) +async def users_page( + request: Request, + theme: str | None = None, + db_session: AsyncSession = Depends(get_db_session), +) -> HTMLResponse: + current_user = await common.get_authenticated_user(request, db_session) + if current_user is None: + return common.redirect_to_login() + if not current_user.is_admin: + raise HTTPException(status_code=403, detail="Admin access required.") + return await _render_users_page( + request, + db_session=db_session, + current_user=current_user, + theme_name=resolve_theme(theme), + notice=request.query_params.get("notice"), + page_error=request.query_params.get("error"), + ) + + +@router.post("/users") +async def create_user( + request: Request, + email: Annotated[str, Form()], + password: Annotated[str, Form()], + is_admin: Annotated[str | None, Form()] = None, + db_session: AsyncSession = Depends(get_db_session), + auth_service: AuthService = Depends(get_auth_service), +) -> HTMLResponse: + current_user = await common.get_authenticated_user(request, db_session) + if current_user is None: + return common.redirect_to_login() + if not current_user.is_admin: + raise HTTPException(status_code=403, detail="Admin access required.") + + active_theme = resolve_theme(None) + try: + validated_email = user_service.validate_email(email) + validated_password = user_service.validate_password(password) + created_user = await user_service.create_user( + db_session, + email=validated_email, + password_hash=auth_service.hash_password(validated_password), + is_admin=is_admin == "on", + ) + except user_service.UserServiceError as exc: + logger.info( + "admin.user_create_failed", + extra={ + "event": "admin.user_create_failed", + "admin_user_id": current_user.id, + "reason": "validation_or_conflict", + }, + ) + return await _render_users_page( + request, + db_session=db_session, + current_user=current_user, + theme_name=active_theme, + page_error=str(exc), + status_code=status.HTTP_400_BAD_REQUEST, + form_email=email, + form_is_admin=is_admin == "on", + ) + + logger.info( + "admin.user_created", + extra={ + "event": "admin.user_created", + "admin_user_id": current_user.id, + "target_user_id": created_user.id, + "target_is_admin": created_user.is_admin, + }, + ) + return common.redirect_with_message( + "/users", + notice=f"User created: {created_user.email}", + ) + + +@router.post("/users/{user_id}/toggle-active") +async def toggle_user_active( + request: Request, + user_id: int, + db_session: AsyncSession = Depends(get_db_session), +): + current_user = await common.get_authenticated_user(request, db_session) + if current_user is None: + return common.redirect_to_login() + if not current_user.is_admin: + raise HTTPException(status_code=403, detail="Admin access required.") + + target_user = await user_service.get_user_by_id(db_session, user_id) + if target_user is None: + return common.redirect_with_message("/users", error="User not found.") + if target_user.id == current_user.id and target_user.is_active: + return common.redirect_with_message("/users", error="You cannot deactivate your own account.") + + updated_user = await user_service.set_user_active( + db_session, + user=target_user, + is_active=not target_user.is_active, + ) + status_label = "activated" if updated_user.is_active else "deactivated" + logger.info( + "admin.user_active_toggled", + extra={ + "event": "admin.user_active_toggled", + "admin_user_id": current_user.id, + "target_user_id": updated_user.id, + "is_active": updated_user.is_active, + }, + ) + return common.redirect_with_message("/users", notice=f"User {status_label}: {updated_user.email}") + + +@router.post("/users/{user_id}/reset-password") +async def reset_user_password( + request: Request, + user_id: int, + new_password: Annotated[str, Form()], + db_session: AsyncSession = Depends(get_db_session), + auth_service: AuthService = Depends(get_auth_service), +): + current_user = await common.get_authenticated_user(request, db_session) + if current_user is None: + return common.redirect_to_login() + if not current_user.is_admin: + raise HTTPException(status_code=403, detail="Admin access required.") + + target_user = await user_service.get_user_by_id(db_session, user_id) + if target_user is None: + return common.redirect_with_message("/users", error="User not found.") + try: + validated_password = user_service.validate_password(new_password) + except user_service.UserServiceError as exc: + return common.redirect_with_message("/users", error=str(exc)) + + await user_service.set_user_password_hash( + db_session, + user=target_user, + password_hash=auth_service.hash_password(validated_password), + ) + logger.info( + "admin.user_password_reset", + extra={ + "event": "admin.user_password_reset", + "admin_user_id": current_user.id, + "target_user_id": target_user.id, + }, + ) + return common.redirect_with_message("/users", notice=f"Password reset: {target_user.email}") + diff --git a/app/web/routers/auth.py b/app/web/routers/auth.py new file mode 100644 index 0000000..8c1c819 --- /dev/null +++ b/app/web/routers/auth.py @@ -0,0 +1,214 @@ +from __future__ import annotations + +import logging +from typing import Annotated + +from fastapi import APIRouter, Depends, Form, Request, status +from fastapi.responses import HTMLResponse, RedirectResponse +from sqlalchemy.ext.asyncio import AsyncSession + +from app.auth.deps import get_auth_service, get_login_rate_limiter +from app.auth.rate_limit import SlidingWindowRateLimiter +from app.auth.service import AuthService +from app.auth.session import get_session_user, set_session_user +from app.db.session import get_db_session +from app.security.csrf import ensure_csrf_token +from app.services import users as user_service +from app.theme import resolve_theme +from app.web import common + +logger = logging.getLogger(__name__) + +router = APIRouter() + + +@router.get("/login", response_class=HTMLResponse) +async def login_page(request: Request, theme: str | None = None) -> HTMLResponse: + active_theme = resolve_theme(theme) + ensure_csrf_token(request) + if get_session_user(request) is not None: + return RedirectResponse("/", status_code=status.HTTP_303_SEE_OTHER) + return common.render_login_page(request, theme_name=active_theme) + + +@router.post("/login", response_class=HTMLResponse) +async def login( + request: Request, + email: Annotated[str, Form()], + password: Annotated[str, Form()], + db_session: AsyncSession = Depends(get_db_session), + auth_service: AuthService = Depends(get_auth_service), + rate_limiter: SlidingWindowRateLimiter = Depends(get_login_rate_limiter), +) -> HTMLResponse: + active_theme = resolve_theme(None) + limiter_key = common.login_rate_limit_key(request, email) + decision = rate_limiter.check(limiter_key) + normalized_email = email.strip().lower() + if not decision.allowed: + logger.warning( + "auth.login_rate_limited", + extra={ + "event": "auth.login_rate_limited", + "email": normalized_email, + "retry_after_seconds": decision.retry_after_seconds, + }, + ) + response = common.render_login_page( + request, + theme_name=active_theme, + error_message="Too many login attempts. Please try again later.", + status_code=status.HTTP_429_TOO_MANY_REQUESTS, + retry_after_seconds=decision.retry_after_seconds, + ) + response.headers["Retry-After"] = str(decision.retry_after_seconds) + return response + + user = await auth_service.authenticate_user( + db_session, + email=email, + password=password, + ) + if user is None: + rate_limiter.record_failure(limiter_key) + logger.info( + "auth.login_failed", + extra={ + "event": "auth.login_failed", + "email": normalized_email, + }, + ) + return common.render_login_page( + request, + theme_name=active_theme, + error_message="Invalid email or password.", + status_code=status.HTTP_401_UNAUTHORIZED, + ) + + rate_limiter.reset(limiter_key) + set_session_user( + request, + user_id=user.id, + email=user.email, + is_admin=user.is_admin, + ) + ensure_csrf_token(request) + logger.info( + "auth.login_succeeded", + extra={ + "event": "auth.login_succeeded", + "user_id": user.id, + "is_admin": user.is_admin, + }, + ) + return RedirectResponse("/", status_code=status.HTTP_303_SEE_OTHER) + + +@router.post("/logout") +async def logout(request: Request) -> RedirectResponse: + session_user = get_session_user(request) + common.invalidate_session(request) + logger.info( + "auth.logout", + extra={ + "event": "auth.logout", + "user_id": session_user.id if session_user else None, + }, + ) + return RedirectResponse("/login", status_code=status.HTTP_303_SEE_OTHER) + + +@router.get("/account/password", response_class=HTMLResponse) +async def account_password_page( + request: Request, + theme: str | None = None, + db_session: AsyncSession = Depends(get_db_session), +) -> HTMLResponse: + current_user = await common.get_authenticated_user(request, db_session) + if current_user is None: + return common.redirect_to_login() + return common.templates.TemplateResponse( + request=request, + name="account_password.html", + context=common.build_template_context( + request, + page_title="Change Password", + active_nav="account_password", + theme_name=resolve_theme(theme), + session_user=common.to_session_user(current_user), + notice=request.query_params.get("notice"), + page_error=request.query_params.get("error"), + ), + ) + + +@router.post("/account/password") +async def update_account_password( + request: Request, + current_password: Annotated[str, Form()], + new_password: Annotated[str, Form()], + confirm_password: Annotated[str, Form()], + db_session: AsyncSession = Depends(get_db_session), + auth_service: AuthService = Depends(get_auth_service), +) -> RedirectResponse: + current_user = await common.get_authenticated_user(request, db_session) + if current_user is None: + return common.redirect_to_login() + if not auth_service.verify_password( + password_hash=current_user.password_hash, + password=current_password, + ): + logger.info( + "account.password_change_failed", + extra={ + "event": "account.password_change_failed", + "user_id": current_user.id, + "reason": "invalid_current_password", + }, + ) + return common.redirect_with_message( + "/account/password", + error="Current password is incorrect.", + ) + if new_password != confirm_password: + logger.info( + "account.password_change_failed", + extra={ + "event": "account.password_change_failed", + "user_id": current_user.id, + "reason": "confirmation_mismatch", + }, + ) + return common.redirect_with_message( + "/account/password", + error="New password and confirmation do not match.", + ) + try: + validated_password = user_service.validate_password(new_password) + except user_service.UserServiceError as exc: + logger.info( + "account.password_change_failed", + extra={ + "event": "account.password_change_failed", + "user_id": current_user.id, + "reason": "validation_error", + }, + ) + return common.redirect_with_message("/account/password", error=str(exc)) + + await user_service.set_user_password_hash( + db_session, + user=current_user, + password_hash=auth_service.hash_password(validated_password), + ) + logger.info( + "account.password_changed", + extra={ + "event": "account.password_changed", + "user_id": current_user.id, + }, + ) + return common.redirect_with_message( + "/account/password", + notice="Password updated successfully.", + ) + diff --git a/app/web/routers/dashboard.py b/app/web/routers/dashboard.py new file mode 100644 index 0000000..efbff30 --- /dev/null +++ b/app/web/routers/dashboard.py @@ -0,0 +1,167 @@ +from __future__ import annotations + +import logging + +from fastapi import APIRouter, Depends, Request +from fastapi.responses import HTMLResponse +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db.models import RunTriggerType +from app.db.session import get_db_session +from app.presentation import dashboard as dashboard_presenter +from app.services import ingestion as ingestion_service +from app.services import publications as publication_service +from app.services import runs as run_service +from app.services import user_settings as user_settings_service +from app.settings import settings +from app.theme import resolve_theme +from app.web import common +from app.web.deps import get_ingestion_service + +logger = logging.getLogger(__name__) + +router = APIRouter() + + +async def _render_dashboard_page( + request: Request, + *, + db_session: AsyncSession, + current_user, + theme_name: str, + notice: str | None = None, + page_error: str | None = None, +) -> HTMLResponse: + unread_publications = await publication_service.list_new_for_latest_run_for_user( + db_session, + user_id=current_user.id, + limit=50, + ) + recent_runs = await run_service.list_recent_runs_for_user( + db_session, + user_id=current_user.id, + limit=20, + ) + user_settings = await user_settings_service.get_or_create_settings( + db_session, + user_id=current_user.id, + ) + queue_counts = await run_service.queue_status_counts_for_user( + db_session, + user_id=current_user.id, + ) + context = common.build_template_context( + request, + page_title="Dashboard", + active_nav="home", + theme_name=theme_name, + session_user=common.to_session_user(current_user), + notice=notice, + page_error=page_error, + ) + context["dashboard"] = dashboard_presenter.build_dashboard_view_model( + unread_publications=unread_publications, + recent_runs=recent_runs, + request_delay_seconds=user_settings.request_delay_seconds, + queue_counts=queue_counts, + ) + return common.templates.TemplateResponse( + request=request, + name="index.html", + context=context, + ) + + +@router.get("/", response_class=HTMLResponse) +async def home( + request: Request, + theme: str | None = None, + db_session: AsyncSession = Depends(get_db_session), +) -> HTMLResponse: + current_user = await common.get_authenticated_user(request, db_session) + if current_user is None: + return common.redirect_to_login() + return await _render_dashboard_page( + request, + db_session=db_session, + current_user=current_user, + theme_name=resolve_theme(theme), + notice=request.query_params.get("notice"), + page_error=request.query_params.get("error"), + ) + + +@router.post("/runs/manual") +async def run_manual_ingestion( + request: Request, + db_session: AsyncSession = Depends(get_db_session), + ingest_service: ingestion_service.ScholarIngestionService = Depends(get_ingestion_service), +): + current_user = await common.get_authenticated_user(request, db_session) + if current_user is None: + return common.redirect_to_login() + + user_settings = await user_settings_service.get_or_create_settings( + db_session, + user_id=current_user.id, + ) + logger.info( + "runs.manual_started", + extra={ + "event": "runs.manual_started", + "user_id": current_user.id, + "request_delay_seconds": user_settings.request_delay_seconds, + "network_error_retries": settings.ingestion_network_error_retries, + "max_pages_per_scholar": settings.ingestion_max_pages_per_scholar, + "page_size": settings.ingestion_page_size, + }, + ) + try: + run_summary = await ingest_service.run_for_user( + db_session, + user_id=current_user.id, + trigger_type=RunTriggerType.MANUAL, + request_delay_seconds=user_settings.request_delay_seconds, + network_error_retries=settings.ingestion_network_error_retries, + retry_backoff_seconds=settings.ingestion_retry_backoff_seconds, + max_pages_per_scholar=settings.ingestion_max_pages_per_scholar, + page_size=settings.ingestion_page_size, + auto_queue_continuations=settings.ingestion_continuation_queue_enabled, + queue_delay_seconds=settings.ingestion_continuation_base_delay_seconds, + ) + except ingestion_service.RunAlreadyInProgressError: + await db_session.rollback() + return common.redirect_with_message( + "/", + error="A run is already in progress for this account.", + ) + except Exception: + await db_session.rollback() + logger.exception( + "runs.manual_failed", + extra={ + "event": "runs.manual_failed", + "user_id": current_user.id, + }, + ) + return common.redirect_with_message("/", error="Manual run failed. Check logs for details.") + + logger.info( + "runs.manual_completed", + extra={ + "event": "runs.manual_completed", + "user_id": current_user.id, + "run_id": run_summary.crawl_run_id, + "status": run_summary.status.value, + "scholar_count": run_summary.scholar_count, + "new_publication_count": run_summary.new_publication_count, + }, + ) + return common.redirect_with_message( + "/", + notice=( + f"Run #{run_summary.crawl_run_id} complete ({run_summary.status.value}). " + f"Scholars: {run_summary.scholar_count}, " + f"new publications: {run_summary.new_publication_count}." + ), + ) diff --git a/app/web/routers/health.py b/app/web/routers/health.py new file mode 100644 index 0000000..3c7c860 --- /dev/null +++ b/app/web/routers/health.py @@ -0,0 +1,15 @@ +from __future__ import annotations + +from fastapi import APIRouter, HTTPException + +from app.db.session import check_database + +router = APIRouter() + + +@router.get("/healthz") +async def healthz() -> dict[str, str]: + if await check_database(): + return {"status": "ok"} + raise HTTPException(status_code=500, detail="database unavailable") + diff --git a/app/web/routers/publications.py b/app/web/routers/publications.py new file mode 100644 index 0000000..c4ce7a2 --- /dev/null +++ b/app/web/routers/publications.py @@ -0,0 +1,165 @@ +from __future__ import annotations + +import logging +from urllib.parse import urlencode, urlparse + +from fastapi import APIRouter, Depends, Form, Request +from fastapi.responses import HTMLResponse +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db.session import get_db_session +from app.services import publications as publication_service +from app.services import scholars as scholar_service +from app.theme import resolve_theme +from app.web import common + +logger = logging.getLogger(__name__) + +router = APIRouter() + +MODE_NEW = "new" +MODE_ALL = "all" + + +def _resolve_mode(raw_mode: str | None) -> str: + return publication_service.resolve_mode(raw_mode) + + +def _parse_scholar_profile_id(value: str | None) -> int | None: + if not value: + return None + try: + parsed = int(value) + except ValueError: + return None + return parsed if parsed > 0 else None + + +def _build_publications_url(*, mode: str, scholar_profile_id: int | None) -> str: + params: dict[str, str] = {"mode": mode} + if scholar_profile_id is not None: + params["scholar_profile_id"] = str(scholar_profile_id) + return f"/publications?{urlencode(params)}" + + +def _is_safe_return_to(value: str | None) -> bool: + if not value: + return False + parsed = urlparse(value) + if parsed.scheme or parsed.netloc: + return False + if not value.startswith("/"): + return False + return value == "/" or value.startswith("/publications") + + +@router.get("/publications", response_class=HTMLResponse) +async def publications_page( + request: Request, + mode: str | None = None, + scholar_profile_id: str | None = None, + theme: str | None = None, + db_session: AsyncSession = Depends(get_db_session), +) -> HTMLResponse: + current_user = await common.get_authenticated_user(request, db_session) + if current_user is None: + return common.redirect_to_login() + + resolved_mode = _resolve_mode(mode) + scholar_id_filter = _parse_scholar_profile_id(scholar_profile_id) + scholars = await scholar_service.list_scholars_for_user( + db_session, + user_id=current_user.id, + ) + scholar_lookup = {scholar.id: scholar for scholar in scholars} + if scholar_id_filter not in scholar_lookup: + scholar_id_filter = None + + publications = await publication_service.list_for_user( + db_session, + user_id=current_user.id, + mode=resolved_mode, + scholar_profile_id=scholar_id_filter, + limit=500, + ) + new_count = await publication_service.count_for_user( + db_session, + user_id=current_user.id, + mode=MODE_NEW, + scholar_profile_id=scholar_id_filter, + ) + total_count = await publication_service.count_for_user( + db_session, + user_id=current_user.id, + mode=MODE_ALL, + scholar_profile_id=scholar_id_filter, + ) + mode_new_url = _build_publications_url( + mode=MODE_NEW, + scholar_profile_id=scholar_id_filter, + ) + mode_all_url = _build_publications_url( + mode=MODE_ALL, + scholar_profile_id=scholar_id_filter, + ) + selected_scholar = scholar_lookup.get(scholar_id_filter) if scholar_id_filter else None + current_publications_url = _build_publications_url( + mode=resolved_mode, + scholar_profile_id=scholar_id_filter, + ) + + context = common.build_template_context( + request, + page_title="Publications", + active_nav="publications", + theme_name=resolve_theme(theme), + session_user=common.to_session_user(current_user), + notice=request.query_params.get("notice"), + page_error=request.query_params.get("error"), + ) + context["mode"] = resolved_mode + context["publications"] = publications + context["new_count"] = new_count + context["total_count"] = total_count + context["mode_new_url"] = mode_new_url + context["mode_all_url"] = mode_all_url + context["scholars"] = scholars + context["selected_scholar_id"] = scholar_id_filter + context["selected_scholar"] = selected_scholar + context["current_publications_url"] = current_publications_url + return common.templates.TemplateResponse( + request=request, + name="publications.html", + context=context, + ) + + +@router.post("/publications/mark-all-read") +async def mark_all_publications_read( + request: Request, + return_to: str | None = Form(default=None), + db_session: AsyncSession = Depends(get_db_session), +): + current_user = await common.get_authenticated_user(request, db_session) + if current_user is None: + return common.redirect_to_login() + + updated_count = await publication_service.mark_all_unread_as_read_for_user( + db_session, + user_id=current_user.id, + ) + logger.info( + "publications.mark_all_read", + extra={ + "event": "publications.mark_all_read", + "user_id": current_user.id, + "updated_count": updated_count, + }, + ) + redirect_target = "/publications?mode=new" + if _is_safe_return_to(return_to): + redirect_target = return_to + return common.redirect_with_message( + redirect_target, + notice=f"Marked {updated_count} publication(s) as read.", + ) diff --git a/app/web/routers/runs.py b/app/web/routers/runs.py new file mode 100644 index 0000000..c6e395f --- /dev/null +++ b/app/web/routers/runs.py @@ -0,0 +1,274 @@ +from __future__ import annotations + +from datetime import datetime, timezone + +from fastapi import APIRouter, Depends, HTTPException, Request +from fastapi.responses import HTMLResponse +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db.session import get_db_session +from app.services import runs as run_service +from app.theme import resolve_theme +from app.web import common + +router = APIRouter() + + +def _as_bool(value: str | None) -> bool: + if value is None: + return False + return value.strip().lower() in {"1", "true", "yes", "on"} + + +def _status_badge(status_value: str) -> str: + if status_value == "success": + return "ok" + if status_value == "failed": + return "danger" + return "warn" + + +def _queue_status_badge(status_value: str) -> str: + if status_value == "dropped": + return "danger" + if status_value == "retrying": + return "ok" + return "warn" + + +def _format_dt(value: datetime | None) -> str: + if value is None: + return "-" + return value.astimezone(timezone.utc).strftime("%Y-%m-%d %H:%M UTC") + + +def _summary_dict(error_log: object) -> dict[str, object]: + if not isinstance(error_log, dict): + return {} + summary = error_log.get("summary") + if not isinstance(summary, dict): + return {} + return summary + + +@router.get("/runs", response_class=HTMLResponse) +async def runs_page( + request: Request, + theme: str | None = None, + failed_only: str | None = None, + db_session: AsyncSession = Depends(get_db_session), +) -> HTMLResponse: + current_user = await common.get_authenticated_user(request, db_session) + if current_user is None: + return common.redirect_to_login() + + failed_only_enabled = _as_bool(failed_only) + runs = await run_service.list_runs_for_user( + db_session, + user_id=current_user.id, + limit=200, + failed_only=failed_only_enabled, + ) + + run_items = [] + for run in runs: + summary = _summary_dict(run.error_log) + failed_count = summary.get("failed_count", 0) + partial_count = summary.get("partial_count", 0) + run_items.append( + { + "id": run.id, + "started_at": _format_dt(run.start_dt), + "finished_at": _format_dt(run.end_dt), + "status": run.status.value, + "status_badge": _status_badge(run.status.value), + "trigger_type": run.trigger_type.value, + "scholar_count": run.scholar_count, + "new_publication_count": run.new_pub_count, + "failed_count": failed_count, + "partial_count": partial_count, + } + ) + + queue_entries = await run_service.list_queue_items_for_user( + db_session, + user_id=current_user.id, + limit=200, + ) + queue_items = [] + for item in queue_entries: + queue_items.append( + { + "id": item.id, + "scholar_profile_id": item.scholar_profile_id, + "scholar_label": item.scholar_label, + "status": item.status, + "status_badge": _queue_status_badge(item.status), + "reason": item.reason, + "dropped_reason": item.dropped_reason, + "attempt_count": item.attempt_count, + "resume_cstart": item.resume_cstart, + "next_attempt_at": _format_dt(item.next_attempt_dt), + "updated_at": _format_dt(item.updated_at), + "last_error": item.last_error, + "last_run_id": item.last_run_id, + } + ) + + context = common.build_template_context( + request, + page_title="Runs", + active_nav="runs", + theme_name=resolve_theme(theme), + session_user=common.to_session_user(current_user), + notice=request.query_params.get("notice"), + page_error=request.query_params.get("error"), + ) + context["runs"] = run_items + context["failed_only"] = failed_only_enabled + context["queue_items"] = queue_items + return common.templates.TemplateResponse( + request=request, + name="runs.html", + context=context, + ) + + +@router.get("/runs/{run_id}", response_class=HTMLResponse) +async def run_detail_page( + request: Request, + run_id: int, + theme: str | None = None, + db_session: AsyncSession = Depends(get_db_session), +) -> HTMLResponse: + current_user = await common.get_authenticated_user(request, db_session) + if current_user is None: + return common.redirect_to_login() + + run = await run_service.get_run_for_user( + db_session, + user_id=current_user.id, + run_id=run_id, + ) + if run is None: + raise HTTPException(status_code=404, detail="Run not found.") + + error_log = run.error_log if isinstance(run.error_log, dict) else {} + scholar_results = error_log.get("scholar_results", []) + if not isinstance(scholar_results, list): + scholar_results = [] + + context = common.build_template_context( + request, + page_title=f"Run #{run.id}", + active_nav="runs", + theme_name=resolve_theme(theme), + session_user=common.to_session_user(current_user), + ) + context["run"] = { + "id": run.id, + "started_at": _format_dt(run.start_dt), + "finished_at": _format_dt(run.end_dt), + "status": run.status.value, + "status_badge": _status_badge(run.status.value), + "trigger_type": run.trigger_type.value, + "scholar_count": run.scholar_count, + "new_publication_count": run.new_pub_count, + } + context["run_summary"] = _summary_dict(error_log) + context["scholar_results"] = scholar_results + return common.templates.TemplateResponse( + request=request, + name="run_detail.html", + context=context, + ) + + +@router.post("/runs/queue/{queue_item_id}/drop") +async def drop_queue_item( + request: Request, + queue_item_id: int, + db_session: AsyncSession = Depends(get_db_session), +): + current_user = await common.get_authenticated_user(request, db_session) + if current_user is None: + return common.redirect_to_login() + + try: + dropped = await run_service.drop_queue_item_for_user( + db_session, + user_id=current_user.id, + queue_item_id=queue_item_id, + ) + except run_service.QueueTransitionError as exc: + return common.redirect_with_message( + "/runs", + error=f"Queue item #{queue_item_id}: {exc.message}", + ) + if dropped is None: + raise HTTPException(status_code=404, detail="Queue item not found.") + return common.redirect_with_message( + "/runs", + notice=f"Queue item #{queue_item_id} marked as dropped.", + ) + + +@router.post("/runs/queue/{queue_item_id}/clear") +async def clear_queue_item( + request: Request, + queue_item_id: int, + db_session: AsyncSession = Depends(get_db_session), +): + current_user = await common.get_authenticated_user(request, db_session) + if current_user is None: + return common.redirect_to_login() + + try: + cleared = await run_service.clear_queue_item_for_user( + db_session, + user_id=current_user.id, + queue_item_id=queue_item_id, + ) + except run_service.QueueTransitionError as exc: + return common.redirect_with_message( + "/runs", + error=f"Queue item #{queue_item_id}: {exc.message}", + ) + if cleared is None: + raise HTTPException(status_code=404, detail="Queue item not found.") + return common.redirect_with_message( + "/runs", + notice=f"Queue item #{queue_item_id} cleared.", + ) + + +@router.post("/runs/queue/{queue_item_id}/retry") +async def retry_queue_item( + request: Request, + queue_item_id: int, + db_session: AsyncSession = Depends(get_db_session), +): + current_user = await common.get_authenticated_user(request, db_session) + if current_user is None: + return common.redirect_to_login() + + try: + queue_item = await run_service.retry_queue_item_for_user( + db_session, + user_id=current_user.id, + queue_item_id=queue_item_id, + ) + except run_service.QueueTransitionError as exc: + return common.redirect_with_message( + "/runs", + error=f"Queue item #{queue_item_id}: {exc.message}", + ) + if queue_item is None: + raise HTTPException(status_code=404, detail="Queue item not found.") + return common.redirect_with_message( + "/runs", + notice=( + f"Queue item #{queue_item_id} queued for retry " + f"(scholar: {queue_item.scholar_label})." + ), + ) diff --git a/app/web/routers/scholars.py b/app/web/routers/scholars.py new file mode 100644 index 0000000..a761f20 --- /dev/null +++ b/app/web/routers/scholars.py @@ -0,0 +1,212 @@ +from __future__ import annotations + +from datetime import datetime, timezone +import logging +from typing import Annotated + +from fastapi import APIRouter, Depends, Form, HTTPException, Request, status +from fastapi.responses import HTMLResponse +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db.session import get_db_session +from app.services import scholars as scholar_service +from app.theme import resolve_theme +from app.web import common + +logger = logging.getLogger(__name__) + +router = APIRouter() + + +def _format_dt(value: datetime | None) -> str: + if value is None: + return "-" + return value.astimezone(timezone.utc).strftime("%Y-%m-%d %H:%M UTC") + + +async def _render_scholars_page( + request: Request, + *, + db_session: AsyncSession, + current_user, + theme_name: str, + notice: str | None = None, + page_error: str | None = None, + status_code: int = status.HTTP_200_OK, + form_scholar_id: str = "", + form_display_name: str = "", +) -> HTMLResponse: + scholars = await scholar_service.list_scholars_for_user( + db_session, + user_id=current_user.id, + ) + scholar_items = [] + for scholar in scholars: + scholar_items.append( + { + "id": scholar.id, + "scholar_id": scholar.scholar_id, + "display_name": scholar.display_name or "Unnamed", + "is_enabled": scholar.is_enabled, + "baseline_completed": scholar.baseline_completed, + "last_run_status": ( + scholar.last_run_status.value + if scholar.last_run_status is not None + else "never" + ), + "last_run_dt": _format_dt(scholar.last_run_dt), + } + ) + context = common.build_template_context( + request, + page_title="Scholars", + active_nav="scholars", + theme_name=theme_name, + session_user=common.to_session_user(current_user), + notice=notice, + page_error=page_error, + ) + context["scholars"] = scholar_items + context["form_scholar_id"] = form_scholar_id + context["form_display_name"] = form_display_name + return common.templates.TemplateResponse( + request=request, + name="scholars.html", + context=context, + status_code=status_code, + ) + + +@router.get("/scholars", response_class=HTMLResponse) +async def scholars_page( + request: Request, + theme: str | None = None, + db_session: AsyncSession = Depends(get_db_session), +) -> HTMLResponse: + current_user = await common.get_authenticated_user(request, db_session) + if current_user is None: + return common.redirect_to_login() + return await _render_scholars_page( + request, + db_session=db_session, + current_user=current_user, + theme_name=resolve_theme(theme), + notice=request.query_params.get("notice"), + page_error=request.query_params.get("error"), + ) + + +@router.post("/scholars") +async def create_scholar( + request: Request, + scholar_id: Annotated[str, Form()], + display_name: Annotated[str, Form()] = "", + db_session: AsyncSession = Depends(get_db_session), +) -> HTMLResponse: + current_user = await common.get_authenticated_user(request, db_session) + if current_user is None: + return common.redirect_to_login() + active_theme = resolve_theme(None) + try: + created_profile = await scholar_service.create_scholar_for_user( + db_session, + user_id=current_user.id, + scholar_id=scholar_id, + display_name=display_name, + ) + except scholar_service.ScholarServiceError as exc: + logger.info( + "scholars.create_failed", + extra={ + "event": "scholars.create_failed", + "user_id": current_user.id, + "scholar_id": scholar_id.strip(), + }, + ) + return await _render_scholars_page( + request, + db_session=db_session, + current_user=current_user, + theme_name=active_theme, + page_error=str(exc), + status_code=status.HTTP_400_BAD_REQUEST, + form_scholar_id=scholar_id, + form_display_name=display_name, + ) + label = created_profile.display_name or created_profile.scholar_id + logger.info( + "scholars.created", + extra={ + "event": "scholars.created", + "user_id": current_user.id, + "scholar_profile_id": created_profile.id, + }, + ) + return common.redirect_with_message("/scholars", notice=f"Scholar added: {label}") + + +@router.post("/scholars/{scholar_profile_id}/toggle") +async def toggle_scholar( + request: Request, + scholar_profile_id: int, + db_session: AsyncSession = Depends(get_db_session), +): + current_user = await common.get_authenticated_user(request, db_session) + if current_user is None: + return common.redirect_to_login() + + profile = await scholar_service.get_user_scholar_by_id( + db_session, + user_id=current_user.id, + scholar_profile_id=scholar_profile_id, + ) + if profile is None: + raise HTTPException(status_code=404, detail="Scholar not found.") + updated_profile = await scholar_service.toggle_scholar_enabled( + db_session, + profile=profile, + ) + status_label = "enabled" if updated_profile.is_enabled else "disabled" + logger.info( + "scholars.toggled", + extra={ + "event": "scholars.toggled", + "user_id": current_user.id, + "scholar_profile_id": updated_profile.id, + "is_enabled": updated_profile.is_enabled, + }, + ) + return common.redirect_with_message( + "/scholars", + notice=f"Scholar {status_label}: {updated_profile.scholar_id}", + ) + + +@router.post("/scholars/{scholar_profile_id}/delete") +async def delete_scholar( + request: Request, + scholar_profile_id: int, + db_session: AsyncSession = Depends(get_db_session), +): + current_user = await common.get_authenticated_user(request, db_session) + if current_user is None: + return common.redirect_to_login() + + profile = await scholar_service.get_user_scholar_by_id( + db_session, + user_id=current_user.id, + scholar_profile_id=scholar_profile_id, + ) + if profile is None: + raise HTTPException(status_code=404, detail="Scholar not found.") + deleted_label = profile.display_name or profile.scholar_id + await scholar_service.delete_scholar(db_session, profile=profile) + logger.info( + "scholars.deleted", + extra={ + "event": "scholars.deleted", + "user_id": current_user.id, + "scholar_profile_id": scholar_profile_id, + }, + ) + return common.redirect_with_message("/scholars", notice=f"Scholar removed: {deleted_label}") diff --git a/app/web/routers/settings.py b/app/web/routers/settings.py new file mode 100644 index 0000000..e5b7ba5 --- /dev/null +++ b/app/web/routers/settings.py @@ -0,0 +1,94 @@ +from __future__ import annotations + +import logging +from typing import Annotated + +from fastapi import APIRouter, Depends, Form, Request +from fastapi.responses import HTMLResponse +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db.session import get_db_session +from app.services import user_settings as user_settings_service +from app.theme import resolve_theme +from app.web import common + +logger = logging.getLogger(__name__) + +router = APIRouter() + + +@router.get("/settings", response_class=HTMLResponse) +async def settings_page( + request: Request, + theme: str | None = None, + db_session: AsyncSession = Depends(get_db_session), +) -> HTMLResponse: + current_user = await common.get_authenticated_user(request, db_session) + if current_user is None: + return common.redirect_to_login() + + user_settings = await user_settings_service.get_or_create_settings( + db_session, + user_id=current_user.id, + ) + context = common.build_template_context( + request, + page_title="Settings", + active_nav="settings", + theme_name=resolve_theme(theme), + session_user=common.to_session_user(current_user), + notice=request.query_params.get("notice"), + page_error=request.query_params.get("error"), + ) + context["user_settings"] = user_settings + return common.templates.TemplateResponse( + request=request, + name="settings.html", + context=context, + ) + + +@router.post("/settings") +async def update_settings( + request: Request, + run_interval_minutes: Annotated[str, Form()], + request_delay_seconds: Annotated[str, Form()], + auto_run_enabled: Annotated[str | None, Form()] = None, + db_session: AsyncSession = Depends(get_db_session), +): + current_user = await common.get_authenticated_user(request, db_session) + if current_user is None: + return common.redirect_to_login() + try: + parsed_interval = user_settings_service.parse_run_interval_minutes( + run_interval_minutes + ) + parsed_delay = user_settings_service.parse_request_delay_seconds( + request_delay_seconds + ) + except user_settings_service.UserSettingsServiceError as exc: + return common.redirect_with_message("/settings", error=str(exc)) + + user_settings = await user_settings_service.get_or_create_settings( + db_session, + user_id=current_user.id, + ) + await user_settings_service.update_settings( + db_session, + settings=user_settings, + auto_run_enabled=auto_run_enabled == "on", + run_interval_minutes=parsed_interval, + request_delay_seconds=parsed_delay, + ) + logger.info( + "settings.updated", + extra={ + "event": "settings.updated", + "user_id": current_user.id, + "auto_run_enabled": auto_run_enabled == "on", + "run_interval_minutes": parsed_interval, + "request_delay_seconds": parsed_delay, + }, + ) + return common.redirect_with_message("/settings", notice="Settings updated.") + diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..a6d4cda --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,68 @@ +name: scholarr + +services: + db: + image: postgres:15-alpine + environment: + POSTGRES_DB: ${POSTGRES_DB:-scholar} + POSTGRES_USER: ${POSTGRES_USER:-scholar} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-scholar} + ports: + - "${POSTGRES_PORT:-5432}:5432" + volumes: + - postgres_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-scholar} -d ${POSTGRES_DB:-scholar}"] + interval: 5s + timeout: 5s + retries: 20 + + app: + build: + context: . + target: dev + environment: + DATABASE_URL: ${DATABASE_URL:-postgresql+asyncpg://scholar:scholar@db:5432/scholar} + TEST_DATABASE_URL: ${TEST_DATABASE_URL:-} + MIGRATE_ON_START: ${MIGRATE_ON_START:-1} + APP_NAME: ${APP_NAME:-scholarr} + APP_HOST: ${APP_HOST:-0.0.0.0} + APP_PORT: ${APP_PORT:-8000} + APP_RELOAD: ${APP_RELOAD:-1} + SESSION_SECRET_KEY: ${SESSION_SECRET_KEY:-dev-insecure-session-key} + SESSION_COOKIE_SECURE: ${SESSION_COOKIE_SECURE:-0} + LOGIN_RATE_LIMIT_ATTEMPTS: ${LOGIN_RATE_LIMIT_ATTEMPTS:-5} + LOGIN_RATE_LIMIT_WINDOW_SECONDS: ${LOGIN_RATE_LIMIT_WINDOW_SECONDS:-60} + LOG_LEVEL: ${LOG_LEVEL:-INFO} + LOG_FORMAT: ${LOG_FORMAT:-console} + LOG_REQUESTS: ${LOG_REQUESTS:-1} + LOG_UVICORN_ACCESS: ${LOG_UVICORN_ACCESS:-0} + LOG_REQUEST_SKIP_PATHS: ${LOG_REQUEST_SKIP_PATHS:-/healthz,/static/} + LOG_REDACT_FIELDS: ${LOG_REDACT_FIELDS:-} + SCHEDULER_ENABLED: ${SCHEDULER_ENABLED:-1} + SCHEDULER_TICK_SECONDS: ${SCHEDULER_TICK_SECONDS:-60} + INGESTION_NETWORK_ERROR_RETRIES: ${INGESTION_NETWORK_ERROR_RETRIES:-1} + INGESTION_RETRY_BACKOFF_SECONDS: ${INGESTION_RETRY_BACKOFF_SECONDS:-1.0} + INGESTION_MAX_PAGES_PER_SCHOLAR: ${INGESTION_MAX_PAGES_PER_SCHOLAR:-30} + INGESTION_PAGE_SIZE: ${INGESTION_PAGE_SIZE:-100} + BOOTSTRAP_ADMIN_ON_START: ${BOOTSTRAP_ADMIN_ON_START:-0} + BOOTSTRAP_ADMIN_EMAIL: ${BOOTSTRAP_ADMIN_EMAIL:-} + BOOTSTRAP_ADMIN_PASSWORD: ${BOOTSTRAP_ADMIN_PASSWORD:-} + BOOTSTRAP_ADMIN_FORCE_PASSWORD: ${BOOTSTRAP_ADMIN_FORCE_PASSWORD:-0} + DB_WAIT_TIMEOUT_SECONDS: ${DB_WAIT_TIMEOUT_SECONDS:-60} + DB_WAIT_INTERVAL_SECONDS: ${DB_WAIT_INTERVAL_SECONDS:-2} + ports: + - "${APP_HOST_PORT:-8000}:8000" + volumes: + - ./:/app + depends_on: + db: + condition: service_healthy + healthcheck: + test: ["CMD-SHELL", "curl -fsS http://localhost:8000/healthz >/dev/null || exit 1"] + interval: 10s + timeout: 5s + retries: 12 + +volumes: + postgres_data: diff --git a/planning/mvp_implementation_reminders.md b/planning/mvp_implementation_reminders.md new file mode 100644 index 0000000..87bc41f --- /dev/null +++ b/planning/mvp_implementation_reminders.md @@ -0,0 +1,92 @@ +# MVP Implementation Reminders (From Scholar Probe) + +Last validated probe run: `planning/scholar_probe_tmp/notes/probe_report_run_20260216T182334Z.md`. + +## Decision: Are we ready? + +Yes, for the scoped MVP. + +- We have stable selectors and field coverage for first-page profile parsing. +- We have explicit failure modes for blocked/inaccessible pages. +- We have enough fixtures to start parser + ingestion tests immediately. + +Not guaranteed (and intentionally out of MVP): + +- Full historical completeness from profile pagination. +- Robots currently disallows `citations?*cstart=`; treat deep pagination as out of scope unless policy changes. + +## Non-negotiables + +- Keep app container-first (`docker compose`) and test while developing. +- Keep feature scope small; prefer deterministic behavior over clever scraping. +- Never crash a whole run because one scholar page fails. +- Preserve strict tenant isolation for all run/output/read-state data. + +## Scraping Contract + +Target URL: + +- `https://scholar.google.com/citations?hl=en&user=` + +Primary parse markers: + +- Row: `tr.gsc_a_tr` +- Title/link: `a.gsc_a_at` +- Citation count: `a.gsc_a_ac` +- Year: `span.gsc_a_h` (fallback year regex) +- Metadata lines: first/second `div.gs_gray` => authors/venue +- Cluster id from `citation_for_view=:` + +## Required Parse States + +Use and persist one of: + +- `ok` +- `no_results` +- `blocked_or_captcha` +- `layout_changed` +- `network_error` + +Plus these page-level flags: + +- `has_show_more_button` +- `articles_range` + +## Data Quality Expectations + +Observed from probe sample: + +- `title`: 100% +- `cluster_id`: 100% +- `citation_count`: 100% +- `authors_text`: 100% +- `year`: 98.2% +- `venue_text`: 94.7% + +Implications: + +- `year` and `venue_text` must remain nullable. +- Dedupe must not depend solely on `year`/`venue_text` presence. + +## Run Semantics + +- First successful run per scholar = baseline. +- If `has_show_more_button=true`, label result as partial in run status/UI. +- Invalid/inaccessible scholar IDs are expected and should become structured run errors, not exceptions. + +## Testing Priorities (Do Before Feature Expansion) + +- Fixture parser unit tests using probe HTML snapshots. +- Parse-state classification tests (ok/blocked/layout/no_results). +- Dedupe integration tests (`cluster_id` first, fingerprint fallback). +- Tenant isolation tests for run records and read-state. +- Smoke test for manual run path through Docker. + +## Stop Conditions + +Pause implementation and re-probe if: + +- Selector markers drop out unexpectedly. +- Login/redirect pages become frequent for valid IDs. +- Robots policy for profile endpoints changes. + diff --git a/planning/scholar_probe_tmp/README.md b/planning/scholar_probe_tmp/README.md new file mode 100644 index 0000000..b6c6ad6 --- /dev/null +++ b/planning/scholar_probe_tmp/README.md @@ -0,0 +1,13 @@ +# Scholar Scrape Probe (Temporary) + +Purpose: short-lived workspace for information gathering and planning. + +Rules: +- No production code changes here. +- Use this for HTML samples, parsing experiments, and extraction notes. +- Delete this directory after we lock the scrape contract and parser plan. + +Structure: +- `fixtures/` saved HTML samples for parser validation +- `notes/` extraction decisions, edge cases, and risk log +- `scripts/` one-off probe scripts used only during planning diff --git a/planning/scholar_probe_tmp/fixtures/run_20260216T182029Z/profile_AAAAAAAAAAAA.html b/planning/scholar_probe_tmp/fixtures/run_20260216T182029Z/profile_AAAAAAAAAAAA.html new file mode 100644 index 0000000..3ddd673 --- /dev/null +++ b/planning/scholar_probe_tmp/fixtures/run_20260216T182029Z/profile_AAAAAAAAAAAA.html @@ -0,0 +1,421 @@ +Google Scholar Citations

Sign in

to continue to Google Scholar Citations
CAPTCHA image of text used to distinguish humans from robots
Not your computer? Use Private Browsing windows to sign in. Learn more about using Guest mode
  • Afrikaans
  • azərbaycan
  • bosanski
  • català
  • Čeština
  • Cymraeg
  • Dansk
  • Deutsch
  • eesti
  • English (United Kingdom)
  • English (United States)
  • Español (España)
  • Español (Latinoamérica)
  • euskara
  • Filipino
  • Français (Canada)
  • Français (France)
  • Gaeilge
  • galego
  • Hrvatski
  • Indonesia
  • isiZulu
  • íslenska
  • Italiano
  • Kiswahili
  • latviešu
  • lietuvių
  • magyar
  • Melayu
  • Nederlands
  • norsk
  • o‘zbek
  • polski
  • Português (Brasil)
  • Português (Portugal)
  • română
  • shqip
  • Slovenčina
  • slovenščina
  • srpski (latinica)
  • Suomi
  • Svenska
  • Tiếng Việt
  • Türkçe
  • Ελληνικά
  • беларуская
  • български
  • кыргызча
  • қазақ тілі
  • македонски
  • монгол
  • Русский
  • српски (ћирилица)
  • Українська
  • ქართული
  • հայերեն
  • ‫עברית‬‎
  • ‫اردو‬‎
  • ‫العربية‬‎
  • ‫فارسی‬‎
  • አማርኛ
  • नेपाली
  • मराठी
  • हिन्दी
  • অসমীয়া
  • বাংলা
  • ਪੰਜਾਬੀ
  • ગુજરાતી
  • ଓଡ଼ିଆ
  • தமிழ்
  • తెలుగు
  • ಕನ್ನಡ
  • മലയാളം
  • සිංහල
  • ไทย
  • ລາວ
  • မြန်မာ
  • ខ្មែរ
  • 한국어
  • 中文（香港）
  • 日本語
  • 简体中文
  • 繁體中文
\ No newline at end of file diff --git a/planning/scholar_probe_tmp/fixtures/run_20260216T182029Z/profile_LZ5D_p4AAAAJ.html b/planning/scholar_probe_tmp/fixtures/run_20260216T182029Z/profile_LZ5D_p4AAAAJ.html new file mode 100644 index 0000000..a9fa214 --- /dev/null +++ b/planning/scholar_probe_tmp/fixtures/run_20260216T182029Z/profile_LZ5D_p4AAAAJ.html @@ -0,0 +1,76 @@ +‪Doaa Elmatary‬ - ‪Google Scholar‬
Follow
Doaa Elmatary
Doaa Elmatary
مدرس هندسة الاتصالات في معهد الصفوة العالي للهندسة
Verified email at alsafwa.edu.eg
Title
Cited by
Cited by
Year
Performance comparison of various machine learning approaches to identify the best one in predicting heart disease
EM Abd Allah, DE El-Matary, EM Eid, AST El Dien
Journal of Computer and Communications 10 (2), 1-18, 2022
252022
Energy efficient key management scheme based on elliptic curve signcryption for wireless sensor networks
EAAA Hagras, D El-Saied, HH Aly
2011 28th National Radio Science Conference (NRSC), 1-9, 2011
152011
A new forward secure elliptic curve signcryption key management (fs-ecskm) scheme for heterogeneous wireless sensor networks
EA Hagras, D El-Saied, HH Aly
International Journal of Computer Science and Technology 2 (2), 19-23, 2011
62011
Performance of polar codes for OFDM-based UWB channel
DEE Matary, EA Hagras, HM Abdel-Kader
Journal of Computer and Communications 6 (03), 102-117, 2018
42018
Multi-user Communication Based OFDM-UWB System under Gaussian and Non-Gaussian Noisy Channel
DEE Matary, EAA Hagras, HM Abdel-Kader
American Journal of Electrical and Electronic Engineering 5 (4), 136-143, 2017
32017
Doaa El-Saied, Dr. Hazem H. Aly,“A New Forward Secure Elliptic Curve Signcryption Key Management (FS-ECSKM) Scheme for Heterogeneous Wireless Sensor Networks”
EAAA Hagras
International Journal of Computer Science and Technology 2 (2), 19-23, 2011
22011
Performance Analysis of LDPC-IDMA-UWB Signals in Non-Gaussian Noisy Channel
DE El-Matary, EAA Hagras, HM Abdel-Kader
12013
Intelligent Sign Multi-Language Real-Time Prediction System with Effective Data Preprocessing
DE Elmatary
Journal of Computer and Communications, 120-134, 2023
2023
Smart Healthcare System Based IoT with Frequency Analysis of Heart Rate Data across FFT Algorithmto be used During Covid-19 Pandemic
E M AbdAllah, D E El Matary
Engineering Research Journal (Shoubra) 51 (1), 88-94, 2022
2022
Polar coded Interleave Division Multiple Access Ultra Wide Band (IDMA-UWB) communication system
DE El Matary, EAA Hagras, HM Abdel-Kader
2018 35th National Radio Science Conference (NRSC), 283-292, 2018
2018
Performance Analysis of LDPC-IDMA-UWB Signals in Non-Gaussian Noisy Channel
HMAK Doaa E.El-Matary, Esam A.A. Hagras
International Journal of Scientific & Engineering Research, 2013
2013
Non-Gaussian Noisy Channel Effect on Multi-user IDMA-UWB Communication System
DEE Matary, EAA Hagras, HM Abdel-Kader
The system can't perform the operation now. Try again later.
Articles 1–12
\ No newline at end of file diff --git a/planning/scholar_probe_tmp/fixtures/run_20260216T182029Z/profile_P1RwlvoAAAAJ.html b/planning/scholar_probe_tmp/fixtures/run_20260216T182029Z/profile_P1RwlvoAAAAJ.html new file mode 100644 index 0000000..fac3935 --- /dev/null +++ b/planning/scholar_probe_tmp/fixtures/run_20260216T182029Z/profile_P1RwlvoAAAAJ.html @@ -0,0 +1,76 @@ +‪WENRUI ZUO‬ - ‪Google Scholar‬
Follow
WENRUI ZUO
Title
Cited by
Cited by
Year
Set in stone: Analysis of an immutable web3 social media platform
W Zuo, A Raman, RJ Mondragón, G Tyson
Proceedings of the ACM Web Conference 2023, 1865-1874, 2023
182023
Understanding and improving content moderation in web3 platforms
W Zuo, RJ Mondragon, A Raman, G Tyson
Proceedings of the International AAAI Conference on Web and Social Media 18 …, 2024
102024
A first look at user-controlled moderation on web3 social media: The case of memo. cash
W Zuo, A Raman, RJ MondragÓN, G Tyson
Proceedings of the 3rd International Workshop on Open Challenges in Online …, 2023
72023
Promoting the Culture of Qinhuai River Lantern Shadow Puppetry with a Digital Archive and Immersive Experience
Y Liu, RM Williams, G Xie, Y Wang, W Zuo
arXiv preprint arXiv:2410.03532, 2024
42024
Understanding and Improving User-controlled Content Moderation Systems on Social Media
W Zuo
Queen Mary University of London, 2025
2025
The system can't perform the operation now. Try again later.
Articles 1–5
\ No newline at end of file diff --git a/planning/scholar_probe_tmp/fixtures/run_20260216T182029Z/profile_RxmmtT8AAAAJ.html b/planning/scholar_probe_tmp/fixtures/run_20260216T182029Z/profile_RxmmtT8AAAAJ.html new file mode 100644 index 0000000..1ab0532 --- /dev/null +++ b/planning/scholar_probe_tmp/fixtures/run_20260216T182029Z/profile_RxmmtT8AAAAJ.html @@ -0,0 +1,76 @@ +‪K. Srinivasan‬ - ‪Google Scholar‬
Follow
K. Srinivasan
Title
Cited by
Cited by
Year
Black pepper and its pungent principle-piperine: a review of diverse physiological effects
K Srinivasan
Critical reviews in food science and nutrition 47 (8), 735-748, 2007
11852007
Digestive stimulant action of spices: a myth or reality?
K Platel, K Srinivasan
Indian Journal of Medical Research 119 (5), 167, 2004
7422004
Influence of dietary spices and their active principles on pancreatic digestive enzymes in albino rats
K Platel, K Srinivasan
Food/Nahrung 44 (1), 42-46, 2000
6822000
Fenugreek (Trigonella foenum-graecum): A Review of Health Beneficial Physiological Effects
K Srinivasan
Food reviews international 22 (2), 203-224, 2006
6632006
Biological activities of red pepper (Capsicum annuum) and its pungent principle capsaicin: a review
K Srinivasan
Critical reviews in food science and nutrition 56 (9), 1488-1500, 2016
5742016
Plant foods in the management of diabetes mellitus: spices as beneficial antidiabetic food adjuncts
K Srinivasan
International journal of food sciences and nutrition 56 (6), 399-414, 2005
5552005
Hypolipidemic action of curcumin, the active principle of turmeric (Curcuma longa) in streptozotocin induced diabetic rats
PS Babu, K Srinivasan
Molecular and cellular biochemistry 166 (1), 169-175, 1997
5391997
Spices as influencers of body metabolism: an overview of three decades of research
K Srinivasan
Food Research International 38 (1), 77-86, 2005
5132005
Role of spices beyond food flavoring: Nutraceuticals with multiple health effects
K Srinivasan
Food reviews international 21 (2), 167-188, 2005
4552005
Antioxidant potential of spices and their active constituents
K Srinivasan
Critical Reviews in Food Science and Nutrition 54 (3), 252-272, 2013
4462013
Tissue distribution and elimination of capsaicin, piperine & curcumin following oral intake in rats.
D Suresh, K Srinivasan
Indian Journal of Medical Research 131, 682-691, 2010
3872010
In vitro influence of spices and spice‐active principles on digestive enzymes of rat pancreas and small intestine
R Ramakrishna Rao, K Platel, K Srinivasan
Food/Nahrung 47 (6), 408-412, 2003
3682003
Cumin (Cuminum cyminum) and black cumin (Nigella sativa) seeds: traditional uses, chemical constituents, and nutraceutical effects
K Srinivasan
Food quality and safety 2 (1), 1-16, 2018
3672018
Ginger rhizomes (Zingiber officinale): A spice with multiple health beneficial potentials
K Srinivasan
PharmaNutrition 5 (1), 18-28, 2017
3642017
Influence of dietary spices or their active principles on digestive enzymes of small intestinal mucosa in rats
K Platel, K Srinivasan
International Journal of Food Sciences and Nutrition 47 (1), 55-59, 1996
3641996
Plant foods in the management of diabetes mellitus: vegetables as potential hypoglycaemic agents
K Platel, K Srinivasan
Food/Nahrung 41 (2), 68-74, 1997
2961997
The effect of spices on cholesterol 7α-hydroxylase activity and on serum and hepatic cholesterol levels in the rat.
K Srinivasan, K Sambaiah
2921991
Zinc and iron contents and their bioaccessibility in cereals and pulses consumed in India
S Hemalatha, K Platel, K Srinivasan
Food chemistry 102 (4), 1328-1336, 2007
2692007
Studies on the influence of dietary spices on food transit time in experimental rats
K Platel, K Srinivasan
Nutrition Research 21 (9), 1309-1314, 2001
2582001
Bioavailability of micronutrients from plant foods: an update
K Platel, K Srinivasan
Critical reviews in food science and nutrition 56 (10), 1608-1619, 2016
2402016
The system can't perform the operation now. Try again later.
Articles 1–20
\ No newline at end of file diff --git a/planning/scholar_probe_tmp/fixtures/run_20260216T182029Z/profile_amIMrIEAAAAJ.html b/planning/scholar_probe_tmp/fixtures/run_20260216T182029Z/profile_amIMrIEAAAAJ.html new file mode 100644 index 0000000..b1d4ad4 --- /dev/null +++ b/planning/scholar_probe_tmp/fixtures/run_20260216T182029Z/profile_amIMrIEAAAAJ.html @@ -0,0 +1,76 @@ +‪Bangar Raju Cherukuri‬ - ‪Google Scholar‬
Follow
Bangar Raju Cherukuri
Bangar Raju Cherukuri
Senior .NET Web Developer
Verified email at dc.gov
Title
Cited by
Cited by
Year
Optimization of Data Structures and Trade-Offs with Concurrency Control in Multithread Software Structures Using Artificial Intelligence
BR Cherukuri, V Arulkumar
2024 IEEE International Conference on Computing, Power and Communication …, 2024
1312024
Development of Design Patterns with Adaptive User Interface for Cloud Native Microservice Architecture Using Deep Learning With IoT
BR Cherukuri
2024 IEEE International Conference on Computing, Power and Communication …, 2024
812024
AI-powered personalization: How machine learning is shaping the future of user experience
BR Cherukuri
International Journal of Science and Research Archive (IJSRA) 12 (1), 3111–3126, 2024
332024
Future of cloud computing: Innovations in multi-cloud and hybrid architectures
BR Cherukuri
World Journal of Advanced Research and Reviews (WJARR) 1 (1), 068-081, 2019
302019
Microservices and containerization: Accelerating web development cycles
BR Cherukuri
World Journal of Advanced Research and Reviews (WJARR) 6 (01), 283-296, 2020
272020
Ethical AI in cloud: Mitigating risks in machine learning models
BR Cherukuri
World Journal of Advanced Engineering Technology and Sciences ((WJAETS) 1 …, 2020
182020
Quantum machine learning: Transforming cloud-based AI solutions
BR Cherukuri
International Journal of Science and Research Archive (IJSRA) 1 (01), 110-122, 2020
142020
Enhanced Trimodal Emotion Recognition Using Multibranch Fusion Attention with Epistemic Neural Networks and Fire Hawk Optimization
BR Cherukuri
Journal of Machine and Computing 5 (1), 2025
132025
Containerization in cloud computing: comparing Docker and Kubernetes for scalable web applications
BR Cherukuri
International Journal of Science and Research Archive (IJSRA) 13 (1), 3302–3315, 2024
132024
Serverless computing: How to build and deploy applications without managing infrastructure
BR Cherukuri
World Journal of Advanced Engineering Technology and Sciences ((WJAETS) 11 …, 2024
132024
Edge Computing vs. Cloud Computing: A Comparative Analysis for Real-Time AI Applications
BR Cherukuri
International Journal For Multidisciplinary Research (IJFMR) 6 (5), 1-17, 2024
122024
Serverless revolution: Redefining application scalability and cost efficiency
BR Cherukuri
World Journal of Advanced Research and Reviews (WJARR) 2 (03), 039-053, 2019
92019
Progressive Web Apps (PWAs): Enhancing User Experience through Modern Web Development
BR Cherukuri
International Journal of Science and Research (IJSR) 13 (10), 1550-1560, 2024
82024
Building Scalable Web Applications: Best Practices for Backend Architecture
BR Cherukuri
International Journal of Science and Research (IJSR) 13 (10), 126-139, 2024
72024
Advanced Multi Class Cyber Security Attack Classification in IoT Based Wireless Sensor Networks Using Context Aware Depthwise Separable Convolutional Neural Network
BR Cherukuri
Journal of Machine and Computing 5 (2), 2025
52025
Maintenance of Web Development Standard for Multiple Devices with Serverless Computing through Cross Browser Affinity Using Hybrid Optimization
BR Cherukuri
2024 IEEE International Conference on Computing, Power and Communication …, 2024
32024
Developing Intelligent Chatbots for Real-Time Customer Support in E-Commerce
BR Cherukuri
International Journal of Science and Research (IJSR) 11 (01), 1709-1719, 2022
32022
Enhancing Web Application Performance with AI - Driven Optimization Techniques
BR Cherukuri
International Journal of Science and Research (IJSR) 10 (2), 1779-1788, 2021
32021
Federated Learning: Privacy-Preserving Machine Learning in Cloud Environments
BR Cherukuri
International Journal of Science and Research (IJSR) 13 (10), 1539-1549, 2024
12024
Scalable machine learning model deployment using serverless cloud architectures
BR Cherukuri
World Journal of Advanced Engineering Technology and Sciences ((WJAETS) 5 (1 …, 2022
12022
The system can't perform the operation now. Try again later.
Articles 1–20
\ No newline at end of file diff --git a/planning/scholar_probe_tmp/fixtures/run_20260216T182029Z/robots.txt b/planning/scholar_probe_tmp/fixtures/run_20260216T182029Z/robots.txt new file mode 100644 index 0000000..50e596d --- /dev/null +++ b/planning/scholar_probe_tmp/fixtures/run_20260216T182029Z/robots.txt @@ -0,0 +1,24 @@ +User-agent: * +Disallow: /search +Disallow: /index.html +Disallow: /scholar +Disallow: /citations? +Allow: /citations?user= +Disallow: /citations?*cstart= +Disallow: /citations?user=*%40 +Disallow: /citations?user=*@ +Allow: /citations?view_op=list_classic_articles +Allow: /citations?view_op=mandates_leaderboard +Allow: /citations?view_op=metrics_intro +Allow: /citations?view_op=new_profile +Allow: /citations?view_op=sitemap +Allow: /citations?view_op=top_venues + +User-agent: Twitterbot +Disallow: + +User-agent: facebookexternalhit +Disallow: + +User-agent: PetalBot +Disallow: / diff --git a/planning/scholar_probe_tmp/fixtures/run_20260216T182157Z/profile_AAAAAAAAAAAA.html b/planning/scholar_probe_tmp/fixtures/run_20260216T182157Z/profile_AAAAAAAAAAAA.html new file mode 100644 index 0000000..5eb3896 --- /dev/null +++ b/planning/scholar_probe_tmp/fixtures/run_20260216T182157Z/profile_AAAAAAAAAAAA.html @@ -0,0 +1,421 @@ +Google Scholar Citations

Sign in

to continue to Google Scholar Citations
CAPTCHA image of text used to distinguish humans from robots
Not your computer? Use Private Browsing windows to sign in. Learn more about using Guest mode
  • Afrikaans
  • azərbaycan
  • bosanski
  • català
  • Čeština
  • Cymraeg
  • Dansk
  • Deutsch
  • eesti
  • English (United Kingdom)
  • English (United States)
  • Español (España)
  • Español (Latinoamérica)
  • euskara
  • Filipino
  • Français (Canada)
  • Français (France)
  • Gaeilge
  • galego
  • Hrvatski
  • Indonesia
  • isiZulu
  • íslenska
  • Italiano
  • Kiswahili
  • latviešu
  • lietuvių
  • magyar
  • Melayu
  • Nederlands
  • norsk
  • o‘zbek
  • polski
  • Português (Brasil)
  • Português (Portugal)
  • română
  • shqip
  • Slovenčina
  • slovenščina
  • srpski (latinica)
  • Suomi
  • Svenska
  • Tiếng Việt
  • Türkçe
  • Ελληνικά
  • беларуская
  • български
  • кыргызча
  • қазақ тілі
  • македонски
  • монгол
  • Русский
  • српски (ћирилица)
  • Українська
  • ქართული
  • հայերեն
  • ‫עברית‬‎
  • ‫اردو‬‎
  • ‫العربية‬‎
  • ‫فارسی‬‎
  • አማርኛ
  • नेपाली
  • मराठी
  • हिन्दी
  • অসমীয়া
  • বাংলা
  • ਪੰਜਾਬੀ
  • ગુજરાતી
  • ଓଡ଼ିଆ
  • தமிழ்
  • తెలుగు
  • ಕನ್ನಡ
  • മലയാളം
  • සිංහල
  • ไทย
  • ລາວ
  • မြန်မာ
  • ខ្មែរ
  • 한국어
  • 中文（香港）
  • 日本語
  • 简体中文
  • 繁體中文
\ No newline at end of file diff --git a/planning/scholar_probe_tmp/fixtures/run_20260216T182157Z/profile_LZ5D_p4AAAAJ.html b/planning/scholar_probe_tmp/fixtures/run_20260216T182157Z/profile_LZ5D_p4AAAAJ.html new file mode 100644 index 0000000..fa93dbd --- /dev/null +++ b/planning/scholar_probe_tmp/fixtures/run_20260216T182157Z/profile_LZ5D_p4AAAAJ.html @@ -0,0 +1,76 @@ +‪Doaa Elmatary‬ - ‪Google Scholar‬
Follow
Doaa Elmatary
Doaa Elmatary
مدرس هندسة الاتصالات في معهد الصفوة العالي للهندسة
Verified email at alsafwa.edu.eg
Title
Cited by
Cited by
Year
Performance comparison of various machine learning approaches to identify the best one in predicting heart disease
EM Abd Allah, DE El-Matary, EM Eid, AST El Dien
Journal of Computer and Communications 10 (2), 1-18, 2022
252022
Energy efficient key management scheme based on elliptic curve signcryption for wireless sensor networks
EAAA Hagras, D El-Saied, HH Aly
2011 28th National Radio Science Conference (NRSC), 1-9, 2011
152011
A new forward secure elliptic curve signcryption key management (fs-ecskm) scheme for heterogeneous wireless sensor networks
EA Hagras, D El-Saied, HH Aly
International Journal of Computer Science and Technology 2 (2), 19-23, 2011
62011
Performance of polar codes for OFDM-based UWB channel
DEE Matary, EA Hagras, HM Abdel-Kader
Journal of Computer and Communications 6 (03), 102-117, 2018
42018
Multi-user Communication Based OFDM-UWB System under Gaussian and Non-Gaussian Noisy Channel
DEE Matary, EAA Hagras, HM Abdel-Kader
American Journal of Electrical and Electronic Engineering 5 (4), 136-143, 2017
32017
Doaa El-Saied, Dr. Hazem H. Aly,“A New Forward Secure Elliptic Curve Signcryption Key Management (FS-ECSKM) Scheme for Heterogeneous Wireless Sensor Networks”
EAAA Hagras
International Journal of Computer Science and Technology 2 (2), 19-23, 2011
22011
Performance Analysis of LDPC-IDMA-UWB Signals in Non-Gaussian Noisy Channel
DE El-Matary, EAA Hagras, HM Abdel-Kader
12013
Intelligent Sign Multi-Language Real-Time Prediction System with Effective Data Preprocessing
DE Elmatary
Journal of Computer and Communications, 120-134, 2023
2023
Smart Healthcare System Based IoT with Frequency Analysis of Heart Rate Data across FFT Algorithmto be used During Covid-19 Pandemic
E M AbdAllah, D E El Matary
Engineering Research Journal (Shoubra) 51 (1), 88-94, 2022
2022
Polar coded Interleave Division Multiple Access Ultra Wide Band (IDMA-UWB) communication system
DE El Matary, EAA Hagras, HM Abdel-Kader
2018 35th National Radio Science Conference (NRSC), 283-292, 2018
2018
Performance Analysis of LDPC-IDMA-UWB Signals in Non-Gaussian Noisy Channel
HMAK Doaa E.El-Matary, Esam A.A. Hagras
International Journal of Scientific & Engineering Research, 2013
2013
Non-Gaussian Noisy Channel Effect on Multi-user IDMA-UWB Communication System
DEE Matary, EAA Hagras, HM Abdel-Kader
The system can't perform the operation now. Try again later.
Articles 1–12
\ No newline at end of file diff --git a/planning/scholar_probe_tmp/fixtures/run_20260216T182157Z/profile_P1RwlvoAAAAJ.html b/planning/scholar_probe_tmp/fixtures/run_20260216T182157Z/profile_P1RwlvoAAAAJ.html new file mode 100644 index 0000000..eb61184 --- /dev/null +++ b/planning/scholar_probe_tmp/fixtures/run_20260216T182157Z/profile_P1RwlvoAAAAJ.html @@ -0,0 +1,76 @@ +‪WENRUI ZUO‬ - ‪Google Scholar‬
Follow
WENRUI ZUO
Title
Cited by
Cited by
Year
Set in stone: Analysis of an immutable web3 social media platform
W Zuo, A Raman, RJ Mondragón, G Tyson
Proceedings of the ACM Web Conference 2023, 1865-1874, 2023
182023
Understanding and improving content moderation in web3 platforms
W Zuo, RJ Mondragon, A Raman, G Tyson
Proceedings of the International AAAI Conference on Web and Social Media 18 …, 2024
102024
A first look at user-controlled moderation on web3 social media: The case of memo. cash
W Zuo, A Raman, RJ MondragÓN, G Tyson
Proceedings of the 3rd International Workshop on Open Challenges in Online …, 2023
72023
Promoting the Culture of Qinhuai River Lantern Shadow Puppetry with a Digital Archive and Immersive Experience
Y Liu, RM Williams, G Xie, Y Wang, W Zuo
arXiv preprint arXiv:2410.03532, 2024
42024
Understanding and Improving User-controlled Content Moderation Systems on Social Media
W Zuo
Queen Mary University of London, 2025
2025
The system can't perform the operation now. Try again later.
Articles 1–5
\ No newline at end of file diff --git a/planning/scholar_probe_tmp/fixtures/run_20260216T182157Z/profile_RxmmtT8AAAAJ.html b/planning/scholar_probe_tmp/fixtures/run_20260216T182157Z/profile_RxmmtT8AAAAJ.html new file mode 100644 index 0000000..f4e0a65 --- /dev/null +++ b/planning/scholar_probe_tmp/fixtures/run_20260216T182157Z/profile_RxmmtT8AAAAJ.html @@ -0,0 +1,76 @@ +‪K. Srinivasan‬ - ‪Google Scholar‬
Follow
K. Srinivasan
Title
Cited by
Cited by
Year
Black pepper and its pungent principle-piperine: a review of diverse physiological effects
K Srinivasan
Critical reviews in food science and nutrition 47 (8), 735-748, 2007
11852007
Digestive stimulant action of spices: a myth or reality?
K Platel, K Srinivasan
Indian Journal of Medical Research 119 (5), 167, 2004
7422004
Influence of dietary spices and their active principles on pancreatic digestive enzymes in albino rats
K Platel, K Srinivasan
Food/Nahrung 44 (1), 42-46, 2000
6822000
Fenugreek (Trigonella foenum-graecum): A Review of Health Beneficial Physiological Effects
K Srinivasan
Food reviews international 22 (2), 203-224, 2006
6632006
Biological activities of red pepper (Capsicum annuum) and its pungent principle capsaicin: a review
K Srinivasan
Critical reviews in food science and nutrition 56 (9), 1488-1500, 2016
5742016
Plant foods in the management of diabetes mellitus: spices as beneficial antidiabetic food adjuncts
K Srinivasan
International journal of food sciences and nutrition 56 (6), 399-414, 2005
5552005
Hypolipidemic action of curcumin, the active principle of turmeric (Curcuma longa) in streptozotocin induced diabetic rats
PS Babu, K Srinivasan
Molecular and cellular biochemistry 166 (1), 169-175, 1997
5391997
Spices as influencers of body metabolism: an overview of three decades of research
K Srinivasan
Food Research International 38 (1), 77-86, 2005
5132005
Role of spices beyond food flavoring: Nutraceuticals with multiple health effects
K Srinivasan
Food reviews international 21 (2), 167-188, 2005
4552005
Antioxidant potential of spices and their active constituents
K Srinivasan
Critical Reviews in Food Science and Nutrition 54 (3), 252-272, 2013
4462013
Tissue distribution and elimination of capsaicin, piperine & curcumin following oral intake in rats.
D Suresh, K Srinivasan
Indian Journal of Medical Research 131, 682-691, 2010
3872010
In vitro influence of spices and spice‐active principles on digestive enzymes of rat pancreas and small intestine
R Ramakrishna Rao, K Platel, K Srinivasan
Food/Nahrung 47 (6), 408-412, 2003
3682003
Cumin (Cuminum cyminum) and black cumin (Nigella sativa) seeds: traditional uses, chemical constituents, and nutraceutical effects
K Srinivasan
Food quality and safety 2 (1), 1-16, 2018
3672018
Ginger rhizomes (Zingiber officinale): A spice with multiple health beneficial potentials
K Srinivasan
PharmaNutrition 5 (1), 18-28, 2017
3642017
Influence of dietary spices or their active principles on digestive enzymes of small intestinal mucosa in rats
K Platel, K Srinivasan
International Journal of Food Sciences and Nutrition 47 (1), 55-59, 1996
3641996
Plant foods in the management of diabetes mellitus: vegetables as potential hypoglycaemic agents
K Platel, K Srinivasan
Food/Nahrung 41 (2), 68-74, 1997
2961997
The effect of spices on cholesterol 7α-hydroxylase activity and on serum and hepatic cholesterol levels in the rat.
K Srinivasan, K Sambaiah
2921991
Zinc and iron contents and their bioaccessibility in cereals and pulses consumed in India
S Hemalatha, K Platel, K Srinivasan
Food chemistry 102 (4), 1328-1336, 2007
2692007
Studies on the influence of dietary spices on food transit time in experimental rats
K Platel, K Srinivasan
Nutrition Research 21 (9), 1309-1314, 2001
2582001
Bioavailability of micronutrients from plant foods: an update
K Platel, K Srinivasan
Critical reviews in food science and nutrition 56 (10), 1608-1619, 2016
2402016
The system can't perform the operation now. Try again later.
Articles 1–20
\ No newline at end of file diff --git a/planning/scholar_probe_tmp/fixtures/run_20260216T182157Z/profile_amIMrIEAAAAJ.html b/planning/scholar_probe_tmp/fixtures/run_20260216T182157Z/profile_amIMrIEAAAAJ.html new file mode 100644 index 0000000..f9589a1 --- /dev/null +++ b/planning/scholar_probe_tmp/fixtures/run_20260216T182157Z/profile_amIMrIEAAAAJ.html @@ -0,0 +1,76 @@ +‪Bangar Raju Cherukuri‬ - ‪Google Scholar‬
Follow
Bangar Raju Cherukuri
Bangar Raju Cherukuri
Senior .NET Web Developer
Verified email at dc.gov
Title
Cited by
Cited by
Year
Optimization of Data Structures and Trade-Offs with Concurrency Control in Multithread Software Structures Using Artificial Intelligence
BR Cherukuri, V Arulkumar
2024 IEEE International Conference on Computing, Power and Communication …, 2024
1312024
Development of Design Patterns with Adaptive User Interface for Cloud Native Microservice Architecture Using Deep Learning With IoT
BR Cherukuri
2024 IEEE International Conference on Computing, Power and Communication …, 2024
812024
AI-powered personalization: How machine learning is shaping the future of user experience
BR Cherukuri
International Journal of Science and Research Archive (IJSRA) 12 (1), 3111–3126, 2024
332024
Future of cloud computing: Innovations in multi-cloud and hybrid architectures
BR Cherukuri
World Journal of Advanced Research and Reviews (WJARR) 1 (1), 068-081, 2019
302019
Microservices and containerization: Accelerating web development cycles
BR Cherukuri
World Journal of Advanced Research and Reviews (WJARR) 6 (01), 283-296, 2020
272020
Ethical AI in cloud: Mitigating risks in machine learning models
BR Cherukuri
World Journal of Advanced Engineering Technology and Sciences ((WJAETS) 1 …, 2020
182020
Quantum machine learning: Transforming cloud-based AI solutions
BR Cherukuri
International Journal of Science and Research Archive (IJSRA) 1 (01), 110-122, 2020
142020
Enhanced Trimodal Emotion Recognition Using Multibranch Fusion Attention with Epistemic Neural Networks and Fire Hawk Optimization
BR Cherukuri
Journal of Machine and Computing 5 (1), 2025
132025
Containerization in cloud computing: comparing Docker and Kubernetes for scalable web applications
BR Cherukuri
International Journal of Science and Research Archive (IJSRA) 13 (1), 3302–3315, 2024
132024
Serverless computing: How to build and deploy applications without managing infrastructure
BR Cherukuri
World Journal of Advanced Engineering Technology and Sciences ((WJAETS) 11 …, 2024
132024
Edge Computing vs. Cloud Computing: A Comparative Analysis for Real-Time AI Applications
BR Cherukuri
International Journal For Multidisciplinary Research (IJFMR) 6 (5), 1-17, 2024
122024
Serverless revolution: Redefining application scalability and cost efficiency
BR Cherukuri
World Journal of Advanced Research and Reviews (WJARR) 2 (03), 039-053, 2019
92019
Progressive Web Apps (PWAs): Enhancing User Experience through Modern Web Development
BR Cherukuri
International Journal of Science and Research (IJSR) 13 (10), 1550-1560, 2024
82024
Building Scalable Web Applications: Best Practices for Backend Architecture
BR Cherukuri
International Journal of Science and Research (IJSR) 13 (10), 126-139, 2024
72024
Advanced Multi Class Cyber Security Attack Classification in IoT Based Wireless Sensor Networks Using Context Aware Depthwise Separable Convolutional Neural Network
BR Cherukuri
Journal of Machine and Computing 5 (2), 2025
52025
Maintenance of Web Development Standard for Multiple Devices with Serverless Computing through Cross Browser Affinity Using Hybrid Optimization
BR Cherukuri
2024 IEEE International Conference on Computing, Power and Communication …, 2024
32024
Developing Intelligent Chatbots for Real-Time Customer Support in E-Commerce
BR Cherukuri
International Journal of Science and Research (IJSR) 11 (01), 1709-1719, 2022
32022
Enhancing Web Application Performance with AI - Driven Optimization Techniques
BR Cherukuri
International Journal of Science and Research (IJSR) 10 (2), 1779-1788, 2021
32021
Federated Learning: Privacy-Preserving Machine Learning in Cloud Environments
BR Cherukuri
International Journal of Science and Research (IJSR) 13 (10), 1539-1549, 2024
12024
Scalable machine learning model deployment using serverless cloud architectures
BR Cherukuri
World Journal of Advanced Engineering Technology and Sciences ((WJAETS) 5 (1 …, 2022
12022
The system can't perform the operation now. Try again later.
Articles 1–20
\ No newline at end of file diff --git a/planning/scholar_probe_tmp/fixtures/run_20260216T182157Z/robots.txt b/planning/scholar_probe_tmp/fixtures/run_20260216T182157Z/robots.txt new file mode 100644 index 0000000..50e596d --- /dev/null +++ b/planning/scholar_probe_tmp/fixtures/run_20260216T182157Z/robots.txt @@ -0,0 +1,24 @@ +User-agent: * +Disallow: /search +Disallow: /index.html +Disallow: /scholar +Disallow: /citations? +Allow: /citations?user= +Disallow: /citations?*cstart= +Disallow: /citations?user=*%40 +Disallow: /citations?user=*@ +Allow: /citations?view_op=list_classic_articles +Allow: /citations?view_op=mandates_leaderboard +Allow: /citations?view_op=metrics_intro +Allow: /citations?view_op=new_profile +Allow: /citations?view_op=sitemap +Allow: /citations?view_op=top_venues + +User-agent: Twitterbot +Disallow: + +User-agent: facebookexternalhit +Disallow: + +User-agent: PetalBot +Disallow: / diff --git a/planning/scholar_probe_tmp/fixtures/run_20260216T182334Z/profile_AAAAAAAAAAAA.html b/planning/scholar_probe_tmp/fixtures/run_20260216T182334Z/profile_AAAAAAAAAAAA.html new file mode 100644 index 0000000..b3707e0 --- /dev/null +++ b/planning/scholar_probe_tmp/fixtures/run_20260216T182334Z/profile_AAAAAAAAAAAA.html @@ -0,0 +1,421 @@ +Google Scholar Citations

Sign in

to continue to Google Scholar Citations
CAPTCHA image of text used to distinguish humans from robots
Not your computer? Use Private Browsing windows to sign in. Learn more about using Guest mode
  • Afrikaans
  • azərbaycan
  • bosanski
  • català
  • Čeština
  • Cymraeg
  • Dansk
  • Deutsch
  • eesti
  • English (United Kingdom)
  • English (United States)
  • Español (España)
  • Español (Latinoamérica)
  • euskara
  • Filipino
  • Français (Canada)
  • Français (France)
  • Gaeilge
  • galego
  • Hrvatski
  • Indonesia
  • isiZulu
  • íslenska
  • Italiano
  • Kiswahili
  • latviešu
  • lietuvių
  • magyar
  • Melayu
  • Nederlands
  • norsk
  • o‘zbek
  • polski
  • Português (Brasil)
  • Português (Portugal)
  • română
  • shqip
  • Slovenčina
  • slovenščina
  • srpski (latinica)
  • Suomi
  • Svenska
  • Tiếng Việt
  • Türkçe
  • Ελληνικά
  • беларуская
  • български
  • кыргызча
  • қазақ тілі
  • македонски
  • монгол
  • Русский
  • српски (ћирилица)
  • Українська
  • ქართული
  • հայերեն
  • ‫עברית‬‎
  • ‫اردو‬‎
  • ‫العربية‬‎
  • ‫فارسی‬‎
  • አማርኛ
  • नेपाली
  • मराठी
  • हिन्दी
  • অসমীয়া
  • বাংলা
  • ਪੰਜਾਬੀ
  • ગુજરાતી
  • ଓଡ଼ିଆ
  • தமிழ்
  • తెలుగు
  • ಕನ್ನಡ
  • മലയാളം
  • සිංහල
  • ไทย
  • ລາວ
  • မြန်မာ
  • ខ្មែរ
  • 한국어
  • 中文（香港）
  • 日本語
  • 简体中文
  • 繁體中文
\ No newline at end of file diff --git a/planning/scholar_probe_tmp/fixtures/run_20260216T182334Z/profile_LZ5D_p4AAAAJ.html b/planning/scholar_probe_tmp/fixtures/run_20260216T182334Z/profile_LZ5D_p4AAAAJ.html new file mode 100644 index 0000000..034d74e --- /dev/null +++ b/planning/scholar_probe_tmp/fixtures/run_20260216T182334Z/profile_LZ5D_p4AAAAJ.html @@ -0,0 +1,76 @@ +‪Doaa Elmatary‬ - ‪Google Scholar‬
Follow
Doaa Elmatary
Doaa Elmatary
مدرس هندسة الاتصالات في معهد الصفوة العالي للهندسة
Verified email at alsafwa.edu.eg
Title
Cited by
Cited by
Year
Performance comparison of various machine learning approaches to identify the best one in predicting heart disease
EM Abd Allah, DE El-Matary, EM Eid, AST El Dien
Journal of Computer and Communications 10 (2), 1-18, 2022
252022
Energy efficient key management scheme based on elliptic curve signcryption for wireless sensor networks
EAAA Hagras, D El-Saied, HH Aly
2011 28th National Radio Science Conference (NRSC), 1-9, 2011
152011
A new forward secure elliptic curve signcryption key management (fs-ecskm) scheme for heterogeneous wireless sensor networks
EA Hagras, D El-Saied, HH Aly
International Journal of Computer Science and Technology 2 (2), 19-23, 2011
62011
Performance of polar codes for OFDM-based UWB channel
DEE Matary, EA Hagras, HM Abdel-Kader
Journal of Computer and Communications 6 (03), 102-117, 2018
42018
Multi-user Communication Based OFDM-UWB System under Gaussian and Non-Gaussian Noisy Channel
DEE Matary, EAA Hagras, HM Abdel-Kader
American Journal of Electrical and Electronic Engineering 5 (4), 136-143, 2017
32017
Doaa El-Saied, Dr. Hazem H. Aly,“A New Forward Secure Elliptic Curve Signcryption Key Management (FS-ECSKM) Scheme for Heterogeneous Wireless Sensor Networks”
EAAA Hagras
International Journal of Computer Science and Technology 2 (2), 19-23, 2011
22011
Performance Analysis of LDPC-IDMA-UWB Signals in Non-Gaussian Noisy Channel
DE El-Matary, EAA Hagras, HM Abdel-Kader
12013
Intelligent Sign Multi-Language Real-Time Prediction System with Effective Data Preprocessing
DE Elmatary
Journal of Computer and Communications, 120-134, 2023
2023
Smart Healthcare System Based IoT with Frequency Analysis of Heart Rate Data across FFT Algorithmto be used During Covid-19 Pandemic
E M AbdAllah, D E El Matary
Engineering Research Journal (Shoubra) 51 (1), 88-94, 2022
2022
Polar coded Interleave Division Multiple Access Ultra Wide Band (IDMA-UWB) communication system
DE El Matary, EAA Hagras, HM Abdel-Kader
2018 35th National Radio Science Conference (NRSC), 283-292, 2018
2018
Performance Analysis of LDPC-IDMA-UWB Signals in Non-Gaussian Noisy Channel
HMAK Doaa E.El-Matary, Esam A.A. Hagras
International Journal of Scientific & Engineering Research, 2013
2013
Non-Gaussian Noisy Channel Effect on Multi-user IDMA-UWB Communication System
DEE Matary, EAA Hagras, HM Abdel-Kader
The system can't perform the operation now. Try again later.
Articles 1–12
\ No newline at end of file diff --git a/planning/scholar_probe_tmp/fixtures/run_20260216T182334Z/profile_P1RwlvoAAAAJ.html b/planning/scholar_probe_tmp/fixtures/run_20260216T182334Z/profile_P1RwlvoAAAAJ.html new file mode 100644 index 0000000..47072eb --- /dev/null +++ b/planning/scholar_probe_tmp/fixtures/run_20260216T182334Z/profile_P1RwlvoAAAAJ.html @@ -0,0 +1,76 @@ +‪WENRUI ZUO‬ - ‪Google Scholar‬
Follow
WENRUI ZUO
Title
Cited by
Cited by
Year
Set in stone: Analysis of an immutable web3 social media platform
W Zuo, A Raman, RJ Mondragón, G Tyson
Proceedings of the ACM Web Conference 2023, 1865-1874, 2023
182023
Understanding and improving content moderation in web3 platforms
W Zuo, RJ Mondragon, A Raman, G Tyson
Proceedings of the International AAAI Conference on Web and Social Media 18 …, 2024
102024
A first look at user-controlled moderation on web3 social media: The case of memo. cash
W Zuo, A Raman, RJ MondragÓN, G Tyson
Proceedings of the 3rd International Workshop on Open Challenges in Online …, 2023
72023
Promoting the Culture of Qinhuai River Lantern Shadow Puppetry with a Digital Archive and Immersive Experience
Y Liu, RM Williams, G Xie, Y Wang, W Zuo
arXiv preprint arXiv:2410.03532, 2024
42024
Understanding and Improving User-controlled Content Moderation Systems on Social Media
W Zuo
Queen Mary University of London, 2025
2025
The system can't perform the operation now. Try again later.
Articles 1–5
\ No newline at end of file diff --git a/planning/scholar_probe_tmp/fixtures/run_20260216T182334Z/profile_RxmmtT8AAAAJ.html b/planning/scholar_probe_tmp/fixtures/run_20260216T182334Z/profile_RxmmtT8AAAAJ.html new file mode 100644 index 0000000..7bd6c72 --- /dev/null +++ b/planning/scholar_probe_tmp/fixtures/run_20260216T182334Z/profile_RxmmtT8AAAAJ.html @@ -0,0 +1,76 @@ +‪K. Srinivasan‬ - ‪Google Scholar‬
Follow
K. Srinivasan
Title
Cited by
Cited by
Year
Black pepper and its pungent principle-piperine: a review of diverse physiological effects
K Srinivasan
Critical reviews in food science and nutrition 47 (8), 735-748, 2007
11852007
Digestive stimulant action of spices: a myth or reality?
K Platel, K Srinivasan
Indian Journal of Medical Research 119 (5), 167, 2004
7422004
Influence of dietary spices and their active principles on pancreatic digestive enzymes in albino rats
K Platel, K Srinivasan
Food/Nahrung 44 (1), 42-46, 2000
6822000
Fenugreek (Trigonella foenum-graecum): A Review of Health Beneficial Physiological Effects
K Srinivasan
Food reviews international 22 (2), 203-224, 2006
6632006
Biological activities of red pepper (Capsicum annuum) and its pungent principle capsaicin: a review
K Srinivasan
Critical reviews in food science and nutrition 56 (9), 1488-1500, 2016
5742016
Plant foods in the management of diabetes mellitus: spices as beneficial antidiabetic food adjuncts
K Srinivasan
International journal of food sciences and nutrition 56 (6), 399-414, 2005
5552005
Hypolipidemic action of curcumin, the active principle of turmeric (Curcuma longa) in streptozotocin induced diabetic rats
PS Babu, K Srinivasan
Molecular and cellular biochemistry 166 (1), 169-175, 1997
5391997
Spices as influencers of body metabolism: an overview of three decades of research
K Srinivasan
Food Research International 38 (1), 77-86, 2005
5132005
Role of spices beyond food flavoring: Nutraceuticals with multiple health effects
K Srinivasan
Food reviews international 21 (2), 167-188, 2005
4552005
Antioxidant potential of spices and their active constituents
K Srinivasan
Critical Reviews in Food Science and Nutrition 54 (3), 252-272, 2013
4462013
Tissue distribution and elimination of capsaicin, piperine & curcumin following oral intake in rats.
D Suresh, K Srinivasan
Indian Journal of Medical Research 131, 682-691, 2010
3872010
In vitro influence of spices and spice‐active principles on digestive enzymes of rat pancreas and small intestine
R Ramakrishna Rao, K Platel, K Srinivasan
Food/Nahrung 47 (6), 408-412, 2003
3682003
Cumin (Cuminum cyminum) and black cumin (Nigella sativa) seeds: traditional uses, chemical constituents, and nutraceutical effects
K Srinivasan
Food quality and safety 2 (1), 1-16, 2018
3672018
Ginger rhizomes (Zingiber officinale): A spice with multiple health beneficial potentials
K Srinivasan
PharmaNutrition 5 (1), 18-28, 2017
3642017
Influence of dietary spices or their active principles on digestive enzymes of small intestinal mucosa in rats
K Platel, K Srinivasan
International Journal of Food Sciences and Nutrition 47 (1), 55-59, 1996
3641996
Plant foods in the management of diabetes mellitus: vegetables as potential hypoglycaemic agents
K Platel, K Srinivasan
Food/Nahrung 41 (2), 68-74, 1997
2961997
The effect of spices on cholesterol 7α-hydroxylase activity and on serum and hepatic cholesterol levels in the rat.
K Srinivasan, K Sambaiah
2921991
Zinc and iron contents and their bioaccessibility in cereals and pulses consumed in India
S Hemalatha, K Platel, K Srinivasan
Food chemistry 102 (4), 1328-1336, 2007
2692007
Studies on the influence of dietary spices on food transit time in experimental rats
K Platel, K Srinivasan
Nutrition Research 21 (9), 1309-1314, 2001
2582001
Bioavailability of micronutrients from plant foods: an update
K Platel, K Srinivasan
Critical reviews in food science and nutrition 56 (10), 1608-1619, 2016
2402016
The system can't perform the operation now. Try again later.
Articles 1–20
\ No newline at end of file diff --git a/planning/scholar_probe_tmp/fixtures/run_20260216T182334Z/profile_amIMrIEAAAAJ.html b/planning/scholar_probe_tmp/fixtures/run_20260216T182334Z/profile_amIMrIEAAAAJ.html new file mode 100644 index 0000000..dfa4a70 --- /dev/null +++ b/planning/scholar_probe_tmp/fixtures/run_20260216T182334Z/profile_amIMrIEAAAAJ.html @@ -0,0 +1,76 @@ +‪Bangar Raju Cherukuri‬ - ‪Google Scholar‬
Follow
Bangar Raju Cherukuri
Bangar Raju Cherukuri
Senior .NET Web Developer
Verified email at dc.gov
Title
Cited by
Cited by
Year
Optimization of Data Structures and Trade-Offs with Concurrency Control in Multithread Software Structures Using Artificial Intelligence
BR Cherukuri, V Arulkumar
2024 IEEE International Conference on Computing, Power and Communication …, 2024
1312024
Development of Design Patterns with Adaptive User Interface for Cloud Native Microservice Architecture Using Deep Learning With IoT
BR Cherukuri
2024 IEEE International Conference on Computing, Power and Communication …, 2024
812024
AI-powered personalization: How machine learning is shaping the future of user experience
BR Cherukuri
International Journal of Science and Research Archive (IJSRA) 12 (1), 3111–3126, 2024
332024
Future of cloud computing: Innovations in multi-cloud and hybrid architectures
BR Cherukuri
World Journal of Advanced Research and Reviews (WJARR) 1 (1), 068-081, 2019
302019
Microservices and containerization: Accelerating web development cycles
BR Cherukuri
World Journal of Advanced Research and Reviews (WJARR) 6 (01), 283-296, 2020
272020
Ethical AI in cloud: Mitigating risks in machine learning models
BR Cherukuri
World Journal of Advanced Engineering Technology and Sciences ((WJAETS) 1 …, 2020
182020
Quantum machine learning: Transforming cloud-based AI solutions
BR Cherukuri
International Journal of Science and Research Archive (IJSRA) 1 (01), 110-122, 2020
142020
Enhanced Trimodal Emotion Recognition Using Multibranch Fusion Attention with Epistemic Neural Networks and Fire Hawk Optimization
BR Cherukuri
Journal of Machine and Computing 5 (1), 2025
132025
Containerization in cloud computing: comparing Docker and Kubernetes for scalable web applications
BR Cherukuri
International Journal of Science and Research Archive (IJSRA) 13 (1), 3302–3315, 2024
132024
Serverless computing: How to build and deploy applications without managing infrastructure
BR Cherukuri
World Journal of Advanced Engineering Technology and Sciences ((WJAETS) 11 …, 2024
132024
Edge Computing vs. Cloud Computing: A Comparative Analysis for Real-Time AI Applications
BR Cherukuri
International Journal For Multidisciplinary Research (IJFMR) 6 (5), 1-17, 2024
122024
Serverless revolution: Redefining application scalability and cost efficiency
BR Cherukuri
World Journal of Advanced Research and Reviews (WJARR) 2 (03), 039-053, 2019
92019
Progressive Web Apps (PWAs): Enhancing User Experience through Modern Web Development
BR Cherukuri
International Journal of Science and Research (IJSR) 13 (10), 1550-1560, 2024
82024
Building Scalable Web Applications: Best Practices for Backend Architecture
BR Cherukuri
International Journal of Science and Research (IJSR) 13 (10), 126-139, 2024
72024
Advanced Multi Class Cyber Security Attack Classification in IoT Based Wireless Sensor Networks Using Context Aware Depthwise Separable Convolutional Neural Network
BR Cherukuri
Journal of Machine and Computing 5 (2), 2025
52025
Maintenance of Web Development Standard for Multiple Devices with Serverless Computing through Cross Browser Affinity Using Hybrid Optimization
BR Cherukuri
2024 IEEE International Conference on Computing, Power and Communication …, 2024
32024
Developing Intelligent Chatbots for Real-Time Customer Support in E-Commerce
BR Cherukuri
International Journal of Science and Research (IJSR) 11 (01), 1709-1719, 2022
32022
Enhancing Web Application Performance with AI - Driven Optimization Techniques
BR Cherukuri
International Journal of Science and Research (IJSR) 10 (2), 1779-1788, 2021
32021
Federated Learning: Privacy-Preserving Machine Learning in Cloud Environments
BR Cherukuri
International Journal of Science and Research (IJSR) 13 (10), 1539-1549, 2024
12024
Scalable machine learning model deployment using serverless cloud architectures
BR Cherukuri
World Journal of Advanced Engineering Technology and Sciences ((WJAETS) 5 (1 …, 2022
12022
The system can't perform the operation now. Try again later.
Articles 1–20
\ No newline at end of file diff --git a/planning/scholar_probe_tmp/fixtures/run_20260216T182334Z/robots.txt b/planning/scholar_probe_tmp/fixtures/run_20260216T182334Z/robots.txt new file mode 100644 index 0000000..50e596d --- /dev/null +++ b/planning/scholar_probe_tmp/fixtures/run_20260216T182334Z/robots.txt @@ -0,0 +1,24 @@ +User-agent: * +Disallow: /search +Disallow: /index.html +Disallow: /scholar +Disallow: /citations? +Allow: /citations?user= +Disallow: /citations?*cstart= +Disallow: /citations?user=*%40 +Disallow: /citations?user=*@ +Allow: /citations?view_op=list_classic_articles +Allow: /citations?view_op=mandates_leaderboard +Allow: /citations?view_op=metrics_intro +Allow: /citations?view_op=new_profile +Allow: /citations?view_op=sitemap +Allow: /citations?view_op=top_venues + +User-agent: Twitterbot +Disallow: + +User-agent: facebookexternalhit +Disallow: + +User-agent: PetalBot +Disallow: / diff --git a/planning/scholar_probe_tmp/notes/findings_phase1_input.md b/planning/scholar_probe_tmp/notes/findings_phase1_input.md new file mode 100644 index 0000000..4a404e1 --- /dev/null +++ b/planning/scholar_probe_tmp/notes/findings_phase1_input.md @@ -0,0 +1,71 @@ +# Probe Findings -> Phase 1 Input + +Generated from live probe run: `run_20260216T182334Z`. + +## What is stable enough to build on + +- Public profile endpoint works for anonymous access: + - `GET /citations?hl=en&user=` +- Core row structure is present and parseable: + - row: `tr.gsc_a_tr` + - title + detail URL: `a.gsc_a_at` + - citation count: `a.gsc_a_ac` + - year: `span.gsc_a_h` (fallback: `td.gsc_a_y` text regex) + - metadata lines: first/second `div.gs_gray` => authors/venue +- `cluster_id` can be extracted reliably from `citation_for_view=:` in title URLs. + +## What can break / where to degrade gracefully + +- Invalid or inaccessible profile IDs may redirect to Google sign-in page. + - Treat as `blocked_or_captcha` / `inaccessible` state, not parser crash. +- `Show more` is present for tested profiles. + - We currently parse first page only. + - Because robots currently disallows `citations?*cstart=`, deep pagination should not be assumed. +- Some rows are missing year or venue text. + - Keep nullable fields and avoid failing dedupe for these gaps. + +## Observed quality from probe + +- Parsed publication rows: 57 across 4 accessible profiles. +- Field coverage: + - title: 100% + - cluster_id: 100% + - citation_count: 100% + - authors_text: 100% + - year: 98.2% + - venue_text: 94.7% + +## Recommended parser contract for implementation + +- Input -> `PublicationCandidate` + - `title` (required) + - `cluster_id` (required when present in URL; expected high coverage) + - `year` (nullable) + - `citation_count` (nullable/int default fallback 0) + - `authors_text` (nullable) + - `venue_text` (nullable) + - `title_url` (nullable) +- Page-level parse status enum: + - `ok`, `no_results`, `blocked_or_captcha`, `layout_changed`, `network_error` +- Page-level flags: + - `has_show_more_button` + - `articles_range` string (e.g. `Articles 1–20`) + +## Immediate test plan unlocked by this probe + +- Add fixture-driven unit tests using captured HTML from `fixtures/run_20260216T182334Z`. +- Add assertions for: + - row count > 0 on known-good fixtures + - profile name extraction + - cluster_id extraction from title URLs + - nullable handling for year/venue + - blocked/inaccessible page classification + - show-more partial warning classification + +## Phase 1 implementation guardrails + +- Keep requests low-rate with jitter and explicit timeout. +- Persist parser status per scholar run. +- If `has_show_more_button=True`, mark run as partial and show this in UI. +- Never fail entire run because one scholar page is blocked or malformed. + diff --git a/planning/scholar_probe_tmp/notes/probe_report_run_20260216T182029Z.json b/planning/scholar_probe_tmp/notes/probe_report_run_20260216T182029Z.json new file mode 100644 index 0000000..539ca7a --- /dev/null +++ b/planning/scholar_probe_tmp/notes/probe_report_run_20260216T182029Z.json @@ -0,0 +1,210 @@ +{ + "fetch_records": [ + { + "elapsed_seconds": 0.053, + "error": null, + "fetched_at_utc": "2026-02-16T18:20:29.406443+00:00", + "file_name": "robots.txt", + "final_url": "https://scholar.google.com/robots.txt", + "source": "robots", + "status_code": 200, + "url": "https://scholar.google.com/robots.txt" + }, + { + "elapsed_seconds": 0.817, + "error": null, + "fetched_at_utc": "2026-02-16T18:20:30.223671+00:00", + "file_name": "profile_amIMrIEAAAAJ.html", + "final_url": "https://scholar.google.com/citations?hl=en&user=amIMrIEAAAAJ", + "source": "profile_amIMrIEAAAAJ", + "status_code": 200, + "url": "https://scholar.google.com/citations?hl=en&user=amIMrIEAAAAJ" + }, + { + "elapsed_seconds": 0.33, + "error": null, + "fetched_at_utc": "2026-02-16T18:20:35.496043+00:00", + "file_name": "profile_P1RwlvoAAAAJ.html", + "final_url": "https://scholar.google.com/citations?hl=en&user=P1RwlvoAAAAJ", + "source": "profile_P1RwlvoAAAAJ", + "status_code": 200, + "url": "https://scholar.google.com/citations?hl=en&user=P1RwlvoAAAAJ" + }, + { + "elapsed_seconds": 0.816, + "error": null, + "fetched_at_utc": "2026-02-16T18:20:40.845606+00:00", + "file_name": "profile_RxmmtT8AAAAJ.html", + "final_url": "https://scholar.google.com/citations?hl=en&user=RxmmtT8AAAAJ", + "source": "profile_RxmmtT8AAAAJ", + "status_code": 200, + "url": "https://scholar.google.com/citations?hl=en&user=RxmmtT8AAAAJ" + }, + { + "elapsed_seconds": 0.794, + "error": null, + "fetched_at_utc": "2026-02-16T18:20:46.112935+00:00", + "file_name": "profile_LZ5D_p4AAAAJ.html", + "final_url": "https://scholar.google.com/citations?hl=en&user=LZ5D_p4AAAAJ", + "source": "profile_LZ5D_p4AAAAJ", + "status_code": 200, + "url": "https://scholar.google.com/citations?hl=en&user=LZ5D_p4AAAAJ" + }, + { + "elapsed_seconds": 1.07, + "error": null, + "fetched_at_utc": "2026-02-16T18:20:52.026510+00:00", + "file_name": "profile_AAAAAAAAAAAA.html", + "final_url": "https://accounts.google.com/v3/signin/identifier?continue=https%3A%2F%2Fscholar.google.com%2Fcitations%3Fhl%3Den%26user%3DAAAAAAAAAAAA&dsh=S1243570986%3A1771266051695499&hl=en&ifkv=ASfE1-pN_UpkKoR95DaV7cklGidF1KWKiZ1XifD65HzS719DIUlTCnwA_hegw9JWCAi7yLsRYXLCuA&service=citations&flowName=GlifWebSignIn&flowEntry=ServiceLogin", + "source": "profile_AAAAAAAAAAAA", + "status_code": 200, + "url": "https://scholar.google.com/citations?hl=en&user=AAAAAAAAAAAA" + } + ], + "generated_at_utc": "2026-02-16T18:20:52.057911+00:00", + "page_analyses": [ + { + "field_presence": { + "authors_text": 0, + "citation_count": 0, + "cluster_id": 0, + "title": 0, + "venue_text": 0, + "year": 0 + }, + "marker_counts": { + "gs_gray": 0, + "gsc_a_ac": 0, + "gsc_a_at": 0, + "gsc_a_h": 0, + "gsc_a_tr": 0, + "gsc_a_y": 0, + "gsc_prf_in": 0, + "gsc_rsb_st": 0 + }, + "parse_warnings": [ + "no_rows_detected" + ], + "profile_name": null, + "publication_count": 0, + "source": "profile_AAAAAAAAAAAA", + "status": "layout_changed" + }, + { + "field_presence": { + "authors_text": 0, + "citation_count": 0, + "cluster_id": 0, + "title": 0, + "venue_text": 0, + "year": 0 + }, + "marker_counts": { + "gs_gray": 26, + "gsc_a_ac": 18, + "gsc_a_at": 15, + "gsc_a_h": 31, + "gsc_a_tr": 32, + "gsc_a_y": 26, + "gsc_prf_in": 17, + "gsc_rsb_st": 30 + }, + "parse_warnings": [ + "no_rows_detected" + ], + "profile_name": null, + "publication_count": 0, + "source": "profile_LZ5D_p4AAAAJ", + "status": "ok" + }, + { + "field_presence": { + "authors_text": 0, + "citation_count": 0, + "cluster_id": 0, + "title": 0, + "venue_text": 0, + "year": 0 + }, + "marker_counts": { + "gs_gray": 14, + "gsc_a_ac": 11, + "gsc_a_at": 8, + "gsc_a_h": 17, + "gsc_a_tr": 25, + "gsc_a_y": 19, + "gsc_prf_in": 21, + "gsc_rsb_st": 30 + }, + "parse_warnings": [ + "no_rows_detected" + ], + "profile_name": null, + "publication_count": 0, + "source": "profile_P1RwlvoAAAAJ", + "status": "ok" + }, + { + "field_presence": { + "authors_text": 0, + "citation_count": 0, + "cluster_id": 0, + "title": 0, + "venue_text": 0, + "year": 0 + }, + "marker_counts": { + "gs_gray": 42, + "gsc_a_ac": 26, + "gsc_a_at": 23, + "gsc_a_h": 47, + "gsc_a_tr": 40, + "gsc_a_y": 34, + "gsc_prf_in": 20, + "gsc_rsb_st": 30 + }, + "parse_warnings": [ + "no_rows_detected" + ], + "profile_name": null, + "publication_count": 0, + "source": "profile_RxmmtT8AAAAJ", + "status": "ok" + }, + { + "field_presence": { + "authors_text": 0, + "citation_count": 0, + "cluster_id": 0, + "title": 0, + "venue_text": 0, + "year": 0 + }, + "marker_counts": { + "gs_gray": 42, + "gsc_a_ac": 26, + "gsc_a_at": 23, + "gsc_a_h": 47, + "gsc_a_tr": 40, + "gsc_a_y": 34, + "gsc_prf_in": 21, + "gsc_rsb_st": 30 + }, + "parse_warnings": [ + "no_rows_detected" + ], + "profile_name": null, + "publication_count": 0, + "source": "profile_amIMrIEAAAAJ", + "status": "ok" + } + ], + "publications_by_source": { + "profile_AAAAAAAAAAAA": [], + "profile_LZ5D_p4AAAAJ": [], + "profile_P1RwlvoAAAAJ": [], + "profile_RxmmtT8AAAAJ": [], + "profile_amIMrIEAAAAJ": [] + }, + "run_dir": "planning/scholar_probe_tmp/fixtures/run_20260216T182029Z" +} \ No newline at end of file diff --git a/planning/scholar_probe_tmp/notes/probe_report_run_20260216T182029Z.md b/planning/scholar_probe_tmp/notes/probe_report_run_20260216T182029Z.md new file mode 100644 index 0000000..ebd7acb --- /dev/null +++ b/planning/scholar_probe_tmp/notes/probe_report_run_20260216T182029Z.md @@ -0,0 +1,69 @@ +# Scholar Scrape Probe Report + +Generated UTC: `2026-02-16T18:20:52.057911+00:00` +Run fixtures dir: `planning/scholar_probe_tmp/fixtures/run_20260216T182029Z` + +## Robots Snapshot + +```text +User-agent: * +Disallow: /search +Disallow: /index.html +Disallow: /scholar +Disallow: /citations? +Allow: /citations?user= +Disallow: /citations?*cstart= +Disallow: /citations?user=*%40 +Disallow: /citations?user=*@ +Allow: /citations?view_op=list_classic_articles +Allow: /citations?view_op=mandates_leaderboard +Allow: /citations?view_op=metrics_intro +Allow: /citations?view_op=new_profile +Allow: /citations?view_op=sitemap +Allow: /citations?view_op=top_venues + +User-agent: Twitterbot +Disallow: + +User-agent: facebookexternalhit +Disallow: + +User-agent: PetalBot +Disallow: / +``` + +## Fetch Summary + +| Source | Status Code | Status/Error | Final URL | +| --- | --- | --- | --- | +| `robots` | 200 | ok | `https://scholar.google.com/robots.txt` | +| `profile_amIMrIEAAAAJ` | 200 | ok | `https://scholar.google.com/citations?hl=en&user=amIMrIEAAAAJ` | +| `profile_P1RwlvoAAAAJ` | 200 | ok | `https://scholar.google.com/citations?hl=en&user=P1RwlvoAAAAJ` | +| `profile_RxmmtT8AAAAJ` | 200 | ok | `https://scholar.google.com/citations?hl=en&user=RxmmtT8AAAAJ` | +| `profile_LZ5D_p4AAAAJ` | 200 | ok | `https://scholar.google.com/citations?hl=en&user=LZ5D_p4AAAAJ` | +| `profile_AAAAAAAAAAAA` | 200 | ok | `https://accounts.google.com/v3/signin/identifier?continue=https%3A%2F%2Fscholar.google.com%2Fcitations%3Fhl%3Den%26user%3DAAAAAAAAAAAA&dsh=S1243570986%3A1771266051695499&hl=en&ifkv=ASfE1-pN_UpkKoR95DaV7cklGidF1KWKiZ1XifD65HzS719DIUlTCnwA_hegw9JWCAi7yLsRYXLCuA&service=citations&flowName=GlifWebSignIn&flowEntry=ServiceLogin` | + +## Parse Summary + +| Source | Parse Status | Profile | Publications | Warnings | +| --- | --- | --- | --- | --- | +| `profile_AAAAAAAAAAAA` | `layout_changed` | - | 0 | no_rows_detected | +| `profile_LZ5D_p4AAAAJ` | `ok` | - | 0 | no_rows_detected | +| `profile_P1RwlvoAAAAJ` | `ok` | - | 0 | no_rows_detected | +| `profile_RxmmtT8AAAAJ` | `ok` | - | 0 | no_rows_detected | +| `profile_amIMrIEAAAAJ` | `ok` | - | 0 | no_rows_detected | + +## Parser Contract Recommendation + +- Primary row marker: `tr.gsc_a_tr`. +- Title anchor marker: `a.gsc_a_at`; derive `cluster_id` from `citation_for_view` query token. +- Metadata text markers: first/second `div.gs_gray` per row for authors and venue. +- Year marker fallback: classes containing `gsc_a_h` or `gsc_a_y` and 4-digit year regex. +- Failure states to persist: `ok`, `no_results`, `blocked_or_captcha`, `layout_changed`, `network_error`. + +## Future-Proofing Notes + +- Keep raw HTML fixture snapshots and update parser tests on DOM drift. +- Treat blocked pages as retriable with backoff, not parser errors. +- Add marker-count assertions in CI to catch silent layout shifts early. +- Use explicit parse status per run/scholar so automation can degrade gracefully. diff --git a/planning/scholar_probe_tmp/notes/probe_report_run_20260216T182142Z.json b/planning/scholar_probe_tmp/notes/probe_report_run_20260216T182142Z.json new file mode 100644 index 0000000..177602e --- /dev/null +++ b/planning/scholar_probe_tmp/notes/probe_report_run_20260216T182142Z.json @@ -0,0 +1,658 @@ +{ + "fetch_records": [], + "generated_at_utc": "2026-02-16T18:21:42.721714+00:00", + "page_analyses": [ + { + "field_presence": { + "authors_text": 0, + "citation_count": 0, + "cluster_id": 0, + "title": 0, + "venue_text": 0, + "year": 0 + }, + "marker_counts": { + "gs_gray": 0, + "gsc_a_ac": 0, + "gsc_a_at": 0, + "gsc_a_h": 0, + "gsc_a_tr": 0, + "gsc_a_y": 0, + "gsc_prf_in": 0, + "gsc_rsb_st": 0 + }, + "parse_warnings": [ + "no_rows_detected" + ], + "profile_name": null, + "publication_count": 0, + "source": "profile_AAAAAAAAAAAA", + "status": "layout_changed" + }, + { + "field_presence": { + "authors_text": 12, + "citation_count": 12, + "cluster_id": 12, + "title": 12, + "venue_text": 10, + "year": 11 + }, + "marker_counts": { + "gs_gray": 26, + "gsc_a_ac": 18, + "gsc_a_at": 15, + "gsc_a_h": 31, + "gsc_a_tr": 32, + "gsc_a_y": 26, + "gsc_prf_in": 17, + "gsc_rsb_st": 30 + }, + "parse_warnings": [], + "profile_name": "Doaa Elmatary", + "publication_count": 12, + "source": "profile_LZ5D_p4AAAAJ", + "status": "ok" + }, + { + "field_presence": { + "authors_text": 5, + "citation_count": 5, + "cluster_id": 5, + "title": 5, + "venue_text": 5, + "year": 5 + }, + "marker_counts": { + "gs_gray": 14, + "gsc_a_ac": 11, + "gsc_a_at": 8, + "gsc_a_h": 17, + "gsc_a_tr": 25, + "gsc_a_y": 19, + "gsc_prf_in": 21, + "gsc_rsb_st": 30 + }, + "parse_warnings": [], + "profile_name": "WENRUI ZUO", + "publication_count": 5, + "source": "profile_P1RwlvoAAAAJ", + "status": "ok" + }, + { + "field_presence": { + "authors_text": 20, + "citation_count": 20, + "cluster_id": 20, + "title": 20, + "venue_text": 19, + "year": 20 + }, + "marker_counts": { + "gs_gray": 42, + "gsc_a_ac": 26, + "gsc_a_at": 23, + "gsc_a_h": 47, + "gsc_a_tr": 40, + "gsc_a_y": 34, + "gsc_prf_in": 20, + "gsc_rsb_st": 30 + }, + "parse_warnings": [], + "profile_name": "K. Srinivasan", + "publication_count": 20, + "source": "profile_RxmmtT8AAAAJ", + "status": "ok" + }, + { + "field_presence": { + "authors_text": 20, + "citation_count": 20, + "cluster_id": 20, + "title": 20, + "venue_text": 20, + "year": 20 + }, + "marker_counts": { + "gs_gray": 42, + "gsc_a_ac": 26, + "gsc_a_at": 23, + "gsc_a_h": 47, + "gsc_a_tr": 40, + "gsc_a_y": 34, + "gsc_prf_in": 21, + "gsc_rsb_st": 30 + }, + "parse_warnings": [], + "profile_name": "Bangar Raju Cherukuri", + "publication_count": 20, + "source": "profile_amIMrIEAAAAJ", + "status": "ok" + } + ], + "publications_by_source": { + "profile_AAAAAAAAAAAA": [], + "profile_LZ5D_p4AAAAJ": [ + { + "authors_text": "EM Abd Allah, DE El-Matary, EM Eid, AST El Dien", + "citation_count": 25, + "cluster_id": "u-x6o8ySG0sC", + "title": "Performance comparison of various machine learning approaches to identify the best one in predicting heart disease", + "title_url": "/citations?view_op=view_citation&hl=en&user=LZ5D_p4AAAAJ&citation_for_view=LZ5D_p4AAAAJ:u-x6o8ySG0sC", + "venue_text": "Journal of Computer and Communications 10 (2), 1-18, 2022", + "year": 2022 + }, + { + "authors_text": "EAAA Hagras, D El-Saied, HH Aly", + "citation_count": 15, + "cluster_id": "YsMSGLbcyi4C", + "title": "Energy efficient key management scheme based on elliptic curve signcryption for wireless sensor networks", + "title_url": "/citations?view_op=view_citation&hl=en&user=LZ5D_p4AAAAJ&citation_for_view=LZ5D_p4AAAAJ:YsMSGLbcyi4C", + "venue_text": "2011 28th National Radio Science Conference (NRSC), 1-9, 2011", + "year": 2011 + }, + { + "authors_text": "EA Hagras, D El-Saied, HH Aly", + "citation_count": 6, + "cluster_id": "Y0pCki6q_DkC", + "title": "A new forward secure elliptic curve signcryption key management (fs-ecskm) scheme for heterogeneous wireless sensor networks", + "title_url": "/citations?view_op=view_citation&hl=en&user=LZ5D_p4AAAAJ&citation_for_view=LZ5D_p4AAAAJ:Y0pCki6q_DkC", + "venue_text": "International Journal of Computer Science and Technology 2 (2), 19-23, 2011", + "year": 2011 + }, + { + "authors_text": "DEE Matary, EA Hagras, HM Abdel-Kader", + "citation_count": 4, + "cluster_id": "d1gkVwhDpl0C", + "title": "Performance of polar codes for OFDM-based UWB channel", + "title_url": "/citations?view_op=view_citation&hl=en&user=LZ5D_p4AAAAJ&citation_for_view=LZ5D_p4AAAAJ:d1gkVwhDpl0C", + "venue_text": "Journal of Computer and Communications 6 (03), 102-117, 2018", + "year": 2018 + }, + { + "authors_text": "DEE Matary, EAA Hagras, HM Abdel-Kader", + "citation_count": 3, + "cluster_id": "u5HHmVD_uO8C", + "title": "Multi-user Communication Based OFDM-UWB System under Gaussian and Non-Gaussian Noisy Channel", + "title_url": "/citations?view_op=view_citation&hl=en&user=LZ5D_p4AAAAJ&citation_for_view=LZ5D_p4AAAAJ:u5HHmVD_uO8C", + "venue_text": "American Journal of Electrical and Electronic Engineering 5 (4), 136-143, 2017", + "year": 2017 + }, + { + "authors_text": "EAAA Hagras", + "citation_count": 2, + "cluster_id": "W7OEmFMy1HYC", + "title": "Doaa El-Saied, Dr. Hazem H. Aly,\u201cA New Forward Secure Elliptic Curve Signcryption Key Management (FS-ECSKM) Scheme for Heterogeneous Wireless Sensor Networks\u201d", + "title_url": "/citations?view_op=view_citation&hl=en&user=LZ5D_p4AAAAJ&citation_for_view=LZ5D_p4AAAAJ:W7OEmFMy1HYC", + "venue_text": "International Journal of Computer Science and Technology 2 (2), 19-23, 2011", + "year": 2011 + }, + { + "authors_text": "DE El-Matary, EAA Hagras, HM Abdel-Kader", + "citation_count": 1, + "cluster_id": "_FxGoFyzp5QC", + "title": "Performance Analysis of LDPC-IDMA-UWB Signals in Non-Gaussian Noisy Channel", + "title_url": "/citations?view_op=view_citation&hl=en&user=LZ5D_p4AAAAJ&citation_for_view=LZ5D_p4AAAAJ:_FxGoFyzp5QC", + "venue_text": null, + "year": 2013 + }, + { + "authors_text": "DE Elmatary", + "citation_count": 0, + "cluster_id": "ufrVoPGSRksC", + "title": "Intelligent Sign Multi-Language Real-Time Prediction System with Effective Data Preprocessing", + "title_url": "/citations?view_op=view_citation&hl=en&user=LZ5D_p4AAAAJ&citation_for_view=LZ5D_p4AAAAJ:ufrVoPGSRksC", + "venue_text": "Journal of Computer and Communications, 120-134, 2023", + "year": 2023 + }, + { + "authors_text": "E M AbdAllah, D E El Matary", + "citation_count": 0, + "cluster_id": "WF5omc3nYNoC", + "title": "Smart Healthcare System Based IoT with Frequency Analysis of Heart Rate Data across FFT Algorithmto be used During Covid-19 Pandemic", + "title_url": "/citations?view_op=view_citation&hl=en&user=LZ5D_p4AAAAJ&citation_for_view=LZ5D_p4AAAAJ:WF5omc3nYNoC", + "venue_text": "Engineering Research Journal (Shoubra) 51 (1), 88-94, 2022", + "year": 2022 + }, + { + "authors_text": "DE El Matary, EAA Hagras, HM Abdel-Kader", + "citation_count": 0, + "cluster_id": "9yKSN-GCB0IC", + "title": "Polar coded Interleave Division Multiple Access Ultra Wide Band (IDMA-UWB) communication system", + "title_url": "/citations?view_op=view_citation&hl=en&user=LZ5D_p4AAAAJ&citation_for_view=LZ5D_p4AAAAJ:9yKSN-GCB0IC", + "venue_text": "2018 35th National Radio Science Conference (NRSC), 283-292, 2018", + "year": 2018 + }, + { + "authors_text": "HMAK Doaa E.El-Matary, Esam A.A. Hagras", + "citation_count": 0, + "cluster_id": "zYLM7Y9cAGgC", + "title": "Performance Analysis of LDPC-IDMA-UWB Signals in Non-Gaussian Noisy Channel", + "title_url": "/citations?view_op=view_citation&hl=en&user=LZ5D_p4AAAAJ&citation_for_view=LZ5D_p4AAAAJ:zYLM7Y9cAGgC", + "venue_text": "International Journal of Scientific & Engineering Research, 2013", + "year": 2013 + }, + { + "authors_text": "DEE Matary, EAA Hagras, HM Abdel-Kader", + "citation_count": 0, + "cluster_id": "eQOLeE2rZwMC", + "title": "Non-Gaussian Noisy Channel Effect on Multi-user IDMA-UWB Communication System", + "title_url": "/citations?view_op=view_citation&hl=en&user=LZ5D_p4AAAAJ&citation_for_view=LZ5D_p4AAAAJ:eQOLeE2rZwMC", + "venue_text": null, + "year": null + } + ], + "profile_P1RwlvoAAAAJ": [ + { + "authors_text": "W Zuo, A Raman, RJ Mondrag\u00f3n, G Tyson", + "citation_count": 18, + "cluster_id": "u5HHmVD_uO8C", + "title": "Set in stone: Analysis of an immutable web3 social media platform", + "title_url": "/citations?view_op=view_citation&hl=en&user=P1RwlvoAAAAJ&citation_for_view=P1RwlvoAAAAJ:u5HHmVD_uO8C", + "venue_text": "Proceedings of the ACM Web Conference 2023, 1865-1874, 2023", + "year": 2023 + }, + { + "authors_text": "W Zuo, RJ Mondragon, A Raman, G Tyson", + "citation_count": 10, + "cluster_id": "d1gkVwhDpl0C", + "title": "Understanding and improving content moderation in web3 platforms", + "title_url": "/citations?view_op=view_citation&hl=en&user=P1RwlvoAAAAJ&citation_for_view=P1RwlvoAAAAJ:d1gkVwhDpl0C", + "venue_text": "Proceedings of the International AAAI Conference on Web and Social Media 18 \u2026, 2024", + "year": 2024 + }, + { + "authors_text": "W Zuo, A Raman, RJ Mondrag\u00d3N, G Tyson", + "citation_count": 7, + "cluster_id": "u-x6o8ySG0sC", + "title": "A first look at user-controlled moderation on web3 social media: The case of memo. cash", + "title_url": "/citations?view_op=view_citation&hl=en&user=P1RwlvoAAAAJ&citation_for_view=P1RwlvoAAAAJ:u-x6o8ySG0sC", + "venue_text": "Proceedings of the 3rd International Workshop on Open Challenges in Online \u2026, 2023", + "year": 2023 + }, + { + "authors_text": "Y Liu, RM Williams, G Xie, Y Wang, W Zuo", + "citation_count": 4, + "cluster_id": "9yKSN-GCB0IC", + "title": "Promoting the Culture of Qinhuai River Lantern Shadow Puppetry with a Digital Archive and Immersive Experience", + "title_url": "/citations?view_op=view_citation&hl=en&user=P1RwlvoAAAAJ&citation_for_view=P1RwlvoAAAAJ:9yKSN-GCB0IC", + "venue_text": "arXiv preprint arXiv:2410.03532, 2024", + "year": 2024 + }, + { + "authors_text": "W Zuo", + "citation_count": 0, + "cluster_id": "2osOgNQ5qMEC", + "title": "Understanding and Improving User-controlled Content Moderation Systems on Social Media", + "title_url": "/citations?view_op=view_citation&hl=en&user=P1RwlvoAAAAJ&citation_for_view=P1RwlvoAAAAJ:2osOgNQ5qMEC", + "venue_text": "Queen Mary University of London, 2025", + "year": 2025 + } + ], + "profile_RxmmtT8AAAAJ": [ + { + "authors_text": "K Srinivasan", + "citation_count": 1185, + "cluster_id": "qjMakFHDy7sC", + "title": "Black pepper and its pungent principle-piperine: a review of diverse physiological effects", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:qjMakFHDy7sC", + "venue_text": "Critical reviews in food science and nutrition 47 (8), 735-748, 2007", + "year": 2007 + }, + { + "authors_text": "K Platel, K Srinivasan", + "citation_count": 742, + "cluster_id": "W7OEmFMy1HYC", + "title": "Digestive stimulant action of spices: a myth or reality?", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:W7OEmFMy1HYC", + "venue_text": "Indian Journal of Medical Research 119 (5), 167, 2004", + "year": 2004 + }, + { + "authors_text": "K Platel, K Srinivasan", + "citation_count": 682, + "cluster_id": "u-x6o8ySG0sC", + "title": "Influence of dietary spices and their active principles on pancreatic digestive enzymes in albino rats", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:u-x6o8ySG0sC", + "venue_text": "Food/Nahrung 44 (1), 42-46, 2000", + "year": 2000 + }, + { + "authors_text": "K Srinivasan", + "citation_count": 663, + "cluster_id": "LkGwnXOMwfcC", + "title": "Fenugreek (Trigonella foenum-graecum): A Review of Health Beneficial Physiological Effects", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:LkGwnXOMwfcC", + "venue_text": "Food reviews international 22 (2), 203-224, 2006", + "year": 2006 + }, + { + "authors_text": "K Srinivasan", + "citation_count": 574, + "cluster_id": "FPJr55Dyh1AC", + "title": "Biological activities of red pepper (Capsicum annuum) and its pungent principle capsaicin: a review", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:FPJr55Dyh1AC", + "venue_text": "Critical reviews in food science and nutrition 56 (9), 1488-1500, 2016", + "year": 2016 + }, + { + "authors_text": "K Srinivasan", + "citation_count": 555, + "cluster_id": "9yKSN-GCB0IC", + "title": "Plant foods in the management of diabetes mellitus: spices as beneficial antidiabetic food adjuncts", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:9yKSN-GCB0IC", + "venue_text": "International journal of food sciences and nutrition 56 (6), 399-414, 2005", + "year": 2005 + }, + { + "authors_text": "PS Babu, K Srinivasan", + "citation_count": 539, + "cluster_id": "ZfRJV9d4-WMC", + "title": "Hypolipidemic action of curcumin, the active principle of turmeric (Curcuma longa) in streptozotocin induced diabetic rats", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:ZfRJV9d4-WMC", + "venue_text": "Molecular and cellular biochemistry 166 (1), 169-175, 1997", + "year": 1997 + }, + { + "authors_text": "K Srinivasan", + "citation_count": 513, + "cluster_id": "IjCSPb-OGe4C", + "title": "Spices as influencers of body metabolism: an overview of three decades of research", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:IjCSPb-OGe4C", + "venue_text": "Food Research International 38 (1), 77-86, 2005", + "year": 2005 + }, + { + "authors_text": "K Srinivasan", + "citation_count": 455, + "cluster_id": "Tyk-4Ss8FVUC", + "title": "Role of spices beyond food flavoring: Nutraceuticals with multiple health effects", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:Tyk-4Ss8FVUC", + "venue_text": "Food reviews international 21 (2), 167-188, 2005", + "year": 2005 + }, + { + "authors_text": "K Srinivasan", + "citation_count": 446, + "cluster_id": "tkaPQYYpVKoC", + "title": "Antioxidant potential of spices and their active constituents", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:tkaPQYYpVKoC", + "venue_text": "Critical Reviews in Food Science and Nutrition 54 (3), 252-272, 2013", + "year": 2013 + }, + { + "authors_text": "D Suresh, K Srinivasan", + "citation_count": 387, + "cluster_id": "5nxA0vEk-isC", + "title": "Tissue distribution and elimination of capsaicin, piperine & curcumin following oral intake in rats.", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:5nxA0vEk-isC", + "venue_text": "Indian Journal of Medical Research 131, 682-691, 2010", + "year": 2010 + }, + { + "authors_text": "R Ramakrishna Rao, K Platel, K Srinivasan", + "citation_count": 368, + "cluster_id": "YsMSGLbcyi4C", + "title": "In vitro influence of spices and spice\u2010active principles on digestive enzymes of rat pancreas and small intestine", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:YsMSGLbcyi4C", + "venue_text": "Food/Nahrung 47 (6), 408-412, 2003", + "year": 2003 + }, + { + "authors_text": "K Srinivasan", + "citation_count": 367, + "cluster_id": "zCSUwVk65WsC", + "title": "Cumin (Cuminum cyminum) and black cumin (Nigella sativa) seeds: traditional uses, chemical constituents, and nutraceutical effects", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:zCSUwVk65WsC", + "venue_text": "Food quality and safety 2 (1), 1-16, 2018", + "year": 2018 + }, + { + "authors_text": "K Srinivasan", + "citation_count": 364, + "cluster_id": "yqoGN6RLRZoC", + "title": "Ginger rhizomes (Zingiber officinale): A spice with multiple health beneficial potentials", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:yqoGN6RLRZoC", + "venue_text": "PharmaNutrition 5 (1), 18-28, 2017", + "year": 2017 + }, + { + "authors_text": "K Platel, K Srinivasan", + "citation_count": 364, + "cluster_id": "d1gkVwhDpl0C", + "title": "Influence of dietary spices or their active principles on digestive enzymes of small intestinal mucosa in rats", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:d1gkVwhDpl0C", + "venue_text": "International Journal of Food Sciences and Nutrition 47 (1), 55-59, 1996", + "year": 1996 + }, + { + "authors_text": "K Platel, K Srinivasan", + "citation_count": 296, + "cluster_id": "UeHWp8X0CEIC", + "title": "Plant foods in the management of diabetes mellitus: vegetables as potential hypoglycaemic agents", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:UeHWp8X0CEIC", + "venue_text": "Food/Nahrung 41 (2), 68-74, 1997", + "year": 1997 + }, + { + "authors_text": "K Srinivasan, K Sambaiah", + "citation_count": 292, + "cluster_id": "2osOgNQ5qMEC", + "title": "The effect of spices on cholesterol 7\u03b1-hydroxylase activity and on serum and hepatic cholesterol levels in the rat.", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:2osOgNQ5qMEC", + "venue_text": null, + "year": 1991 + }, + { + "authors_text": "S Hemalatha, K Platel, K Srinivasan", + "citation_count": 269, + "cluster_id": "UebtZRa9Y70C", + "title": "Zinc and iron contents and their bioaccessibility in cereals and pulses consumed in India", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:UebtZRa9Y70C", + "venue_text": "Food chemistry 102 (4), 1328-1336, 2007", + "year": 2007 + }, + { + "authors_text": "K Platel, K Srinivasan", + "citation_count": 258, + "cluster_id": "_FxGoFyzp5QC", + "title": "Studies on the influence of dietary spices on food transit time in experimental rats", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:_FxGoFyzp5QC", + "venue_text": "Nutrition Research 21 (9), 1309-1314, 2001", + "year": 2001 + }, + { + "authors_text": "K Platel, K Srinivasan", + "citation_count": 240, + "cluster_id": "hCrLmN-GePgC", + "title": "Bioavailability of micronutrients from plant foods: an update", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:hCrLmN-GePgC", + "venue_text": "Critical reviews in food science and nutrition 56 (10), 1608-1619, 2016", + "year": 2016 + } + ], + "profile_amIMrIEAAAAJ": [ + { + "authors_text": "BR Cherukuri, V Arulkumar", + "citation_count": 131, + "cluster_id": "u5HHmVD_uO8C", + "title": "Optimization of Data Structures and Trade-Offs with Concurrency Control in Multithread Software Structures Using Artificial Intelligence", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:u5HHmVD_uO8C", + "venue_text": "2024 IEEE International Conference on Computing, Power and Communication \u2026, 2024", + "year": 2024 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 81, + "cluster_id": "d1gkVwhDpl0C", + "title": "Development of Design Patterns with Adaptive User Interface for Cloud Native Microservice Architecture Using Deep Learning With IoT", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:d1gkVwhDpl0C", + "venue_text": "2024 IEEE International Conference on Computing, Power and Communication \u2026, 2024", + "year": 2024 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 33, + "cluster_id": "zYLM7Y9cAGgC", + "title": "AI-powered personalization: How machine learning is shaping the future of user experience", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:zYLM7Y9cAGgC", + "venue_text": "International Journal of Science and Research Archive (IJSRA) 12 (1), 3111\u20133126, 2024", + "year": 2024 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 30, + "cluster_id": "roLk4NBRz8UC", + "title": "Future of cloud computing: Innovations in multi-cloud and hybrid architectures", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:roLk4NBRz8UC", + "venue_text": "World Journal of Advanced Research and Reviews (WJARR) 1 (1), 068-081, 2019", + "year": 2019 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 27, + "cluster_id": "UebtZRa9Y70C", + "title": "Microservices and containerization: Accelerating web development cycles", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:UebtZRa9Y70C", + "venue_text": "World Journal of Advanced Research and Reviews (WJARR) 6 (01), 283-296, 2020", + "year": 2020 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 18, + "cluster_id": "hqOjcs7Dif8C", + "title": "Ethical AI in cloud: Mitigating risks in machine learning models", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:hqOjcs7Dif8C", + "venue_text": "World Journal of Advanced Engineering Technology and Sciences ((WJAETS) 1 \u2026, 2020", + "year": 2020 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 14, + "cluster_id": "0EnyYjriUFMC", + "title": "Quantum machine learning: Transforming cloud-based AI solutions", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:0EnyYjriUFMC", + "venue_text": "International Journal of Science and Research Archive (IJSRA) 1 (01), 110-122, 2020", + "year": 2020 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 13, + "cluster_id": "WF5omc3nYNoC", + "title": "Enhanced Trimodal Emotion Recognition Using Multibranch Fusion Attention with Epistemic Neural Networks and Fire Hawk Optimization", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:WF5omc3nYNoC", + "venue_text": "Journal of Machine and Computing 5 (1), 2025", + "year": 2025 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 13, + "cluster_id": "W7OEmFMy1HYC", + "title": "Containerization in cloud computing: comparing Docker and Kubernetes for scalable web applications", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:W7OEmFMy1HYC", + "venue_text": "International Journal of Science and Research Archive (IJSRA) 13 (1), 3302\u20133315, 2024", + "year": 2024 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 13, + "cluster_id": "IjCSPb-OGe4C", + "title": "Serverless computing: How to build and deploy applications without managing infrastructure", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:IjCSPb-OGe4C", + "venue_text": "World Journal of Advanced Engineering Technology and Sciences ((WJAETS) 11 \u2026, 2024", + "year": 2024 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 12, + "cluster_id": "YsMSGLbcyi4C", + "title": "Edge Computing vs. Cloud Computing: A Comparative Analysis for Real-Time AI Applications", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:YsMSGLbcyi4C", + "venue_text": "International Journal For Multidisciplinary Research (IJFMR) 6 (5), 1-17, 2024", + "year": 2024 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 9, + "cluster_id": "Se3iqnhoufwC", + "title": "Serverless revolution: Redefining application scalability and cost efficiency", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:Se3iqnhoufwC", + "venue_text": "World Journal of Advanced Research and Reviews (WJARR) 2 (03), 039-053, 2019", + "year": 2019 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 8, + "cluster_id": "Tyk-4Ss8FVUC", + "title": "Progressive Web Apps (PWAs): Enhancing User Experience through Modern Web Development", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:Tyk-4Ss8FVUC", + "venue_text": "International Journal of Science and Research (IJSR) 13 (10), 1550-1560, 2024", + "year": 2024 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 7, + "cluster_id": "UeHWp8X0CEIC", + "title": "Building Scalable Web Applications: Best Practices for Backend Architecture", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:UeHWp8X0CEIC", + "venue_text": "International Journal of Science and Research (IJSR) 13 (10), 126-139, 2024", + "year": 2024 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 5, + "cluster_id": "5nxA0vEk-isC", + "title": "Advanced Multi Class Cyber Security Attack Classification in IoT Based Wireless Sensor Networks Using Context Aware Depthwise Separable Convolutional Neural Network", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:5nxA0vEk-isC", + "venue_text": "Journal of Machine and Computing 5 (2), 2025", + "year": 2025 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 3, + "cluster_id": "u-x6o8ySG0sC", + "title": "Maintenance of Web Development Standard for Multiple Devices with Serverless Computing through Cross Browser Affinity Using Hybrid Optimization", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:u-x6o8ySG0sC", + "venue_text": "2024 IEEE International Conference on Computing, Power and Communication \u2026, 2024", + "year": 2024 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 3, + "cluster_id": "LkGwnXOMwfcC", + "title": "Developing Intelligent Chatbots for Real-Time Customer Support in E-Commerce", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:LkGwnXOMwfcC", + "venue_text": "International Journal of Science and Research (IJSR) 11 (01), 1709-1719, 2022", + "year": 2022 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 3, + "cluster_id": "_FxGoFyzp5QC", + "title": "Enhancing Web Application Performance with AI - Driven Optimization Techniques", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:_FxGoFyzp5QC", + "venue_text": "International Journal of Science and Research (IJSR) 10 (2), 1779-1788, 2021", + "year": 2021 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 1, + "cluster_id": "Y0pCki6q_DkC", + "title": "Federated Learning: Privacy-Preserving Machine Learning in Cloud Environments", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:Y0pCki6q_DkC", + "venue_text": "International Journal of Science and Research (IJSR) 13 (10), 1539-1549, 2024", + "year": 2024 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 1, + "cluster_id": "ufrVoPGSRksC", + "title": "Scalable machine learning model deployment using serverless cloud architectures", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:ufrVoPGSRksC", + "venue_text": "World Journal of Advanced Engineering Technology and Sciences ((WJAETS) 5 (1 \u2026, 2022", + "year": 2022 + } + ] + }, + "run_dir": "planning/scholar_probe_tmp/fixtures/run_20260216T182142Z" +} \ No newline at end of file diff --git a/planning/scholar_probe_tmp/notes/probe_report_run_20260216T182142Z.md b/planning/scholar_probe_tmp/notes/probe_report_run_20260216T182142Z.md new file mode 100644 index 0000000..1a596f3 --- /dev/null +++ b/planning/scholar_probe_tmp/notes/probe_report_run_20260216T182142Z.md @@ -0,0 +1,76 @@ +# Scholar Scrape Probe Report + +Generated UTC: `2026-02-16T18:21:42.721714+00:00` +Run fixtures dir: `planning/scholar_probe_tmp/fixtures/run_20260216T182142Z` + +## Robots Snapshot + +```text +User-agent: * +Disallow: /search +Disallow: /index.html +Disallow: /scholar +Disallow: /citations? +Allow: /citations?user= +Disallow: /citations?*cstart= +Disallow: /citations?user=*%40 +Disallow: /citations?user=*@ +Allow: /citations?view_op=list_classic_articles +Allow: /citations?view_op=mandates_leaderboard +Allow: /citations?view_op=metrics_intro +Allow: /citations?view_op=new_profile +Allow: /citations?view_op=sitemap +Allow: /citations?view_op=top_venues + +User-agent: Twitterbot +Disallow: + +User-agent: facebookexternalhit +Disallow: + +User-agent: PetalBot +Disallow: / +``` + +## Fetch Summary + +| Source | Status Code | Status/Error | Final URL | +| --- | --- | --- | --- | + +## Parse Summary + +| Source | Parse Status | Profile | Publications | Warnings | +| --- | --- | --- | --- | --- | +| `profile_AAAAAAAAAAAA` | `layout_changed` | - | 0 | no_rows_detected | +| `profile_LZ5D_p4AAAAJ` | `ok` | Doaa Elmatary | 12 | - | +| `profile_P1RwlvoAAAAJ` | `ok` | WENRUI ZUO | 5 | - | +| `profile_RxmmtT8AAAAJ` | `ok` | K. Srinivasan | 20 | - | +| `profile_amIMrIEAAAAJ` | `ok` | Bangar Raju Cherukuri | 20 | - | + +## Field Coverage + +Total parsed publication rows: **57** + +| Field | Present | Coverage | +| --- | --- | --- | +| `title` | 57 | 100.0% | +| `cluster_id` | 57 | 100.0% | +| `year` | 56 | 98.2% | +| `citation_count` | 57 | 100.0% | +| `authors_text` | 57 | 100.0% | +| `venue_text` | 54 | 94.7% | + +## Parser Contract Recommendation + +- Primary row marker: `tr.gsc_a_tr`. +- Title anchor marker: `a.gsc_a_at`; derive `cluster_id` from `citation_for_view` query token. +- Metadata text markers: first/second `div.gs_gray` per row for authors and venue. +- Year marker fallback: classes containing `gsc_a_h` or `gsc_a_y` and 4-digit year regex. +- Failure states to persist: `ok`, `no_results`, `blocked_or_captcha`, `layout_changed`, `network_error`. + +## Future-Proofing Notes + +- Keep raw HTML fixture snapshots and update parser tests on DOM drift. +- Treat blocked pages as retriable with backoff, not parser errors. +- Add marker-count assertions in CI to catch silent layout shifts early. +- Use explicit parse status per run/scholar so automation can degrade gracefully. diff --git a/planning/scholar_probe_tmp/notes/probe_report_run_20260216T182157Z.json b/planning/scholar_probe_tmp/notes/probe_report_run_20260216T182157Z.json new file mode 100644 index 0000000..34b3767 --- /dev/null +++ b/planning/scholar_probe_tmp/notes/probe_report_run_20260216T182157Z.json @@ -0,0 +1,719 @@ +{ + "fetch_records": [ + { + "elapsed_seconds": 0.072, + "error": null, + "fetched_at_utc": "2026-02-16T18:21:57.261516+00:00", + "file_name": "robots.txt", + "final_url": "https://scholar.google.com/robots.txt", + "source": "robots", + "status_code": 200, + "url": "https://scholar.google.com/robots.txt" + }, + { + "elapsed_seconds": 0.889, + "error": null, + "fetched_at_utc": "2026-02-16T18:21:58.150732+00:00", + "file_name": "profile_amIMrIEAAAAJ.html", + "final_url": "https://scholar.google.com/citations?hl=en&user=amIMrIEAAAAJ", + "source": "profile_amIMrIEAAAAJ", + "status_code": 200, + "url": "https://scholar.google.com/citations?hl=en&user=amIMrIEAAAAJ" + }, + { + "elapsed_seconds": 0.797, + "error": null, + "fetched_at_utc": "2026-02-16T18:22:03.779375+00:00", + "file_name": "profile_P1RwlvoAAAAJ.html", + "final_url": "https://scholar.google.com/citations?hl=en&user=P1RwlvoAAAAJ", + "source": "profile_P1RwlvoAAAAJ", + "status_code": 200, + "url": "https://scholar.google.com/citations?hl=en&user=P1RwlvoAAAAJ" + }, + { + "elapsed_seconds": 0.367, + "error": null, + "fetched_at_utc": "2026-02-16T18:22:08.622977+00:00", + "file_name": "profile_RxmmtT8AAAAJ.html", + "final_url": "https://scholar.google.com/citations?hl=en&user=RxmmtT8AAAAJ", + "source": "profile_RxmmtT8AAAAJ", + "status_code": 200, + "url": "https://scholar.google.com/citations?hl=en&user=RxmmtT8AAAAJ" + }, + { + "elapsed_seconds": 0.281, + "error": null, + "fetched_at_utc": "2026-02-16T18:22:13.568478+00:00", + "file_name": "profile_LZ5D_p4AAAAJ.html", + "final_url": "https://scholar.google.com/citations?hl=en&user=LZ5D_p4AAAAJ", + "source": "profile_LZ5D_p4AAAAJ", + "status_code": 200, + "url": "https://scholar.google.com/citations?hl=en&user=LZ5D_p4AAAAJ" + }, + { + "elapsed_seconds": 0.646, + "error": null, + "fetched_at_utc": "2026-02-16T18:22:18.796882+00:00", + "file_name": "profile_AAAAAAAAAAAA.html", + "final_url": "https://accounts.google.com/v3/signin/identifier?continue=https%3A%2F%2Fscholar.google.com%2Fcitations%3Fhl%3Den%26user%3DAAAAAAAAAAAA&dsh=S-568466874%3A1771266138494353&hl=en&ifkv=ASfE1-pK1BK8frcmUfumfUYfZZe1iJXbTGg6TCfyR-Q1ErPAopCpUGy_sEwLdvOICVCsiXsIJbFm1Q&service=citations&flowName=GlifWebSignIn&flowEntry=ServiceLogin", + "source": "profile_AAAAAAAAAAAA", + "status_code": 200, + "url": "https://scholar.google.com/citations?hl=en&user=AAAAAAAAAAAA" + } + ], + "generated_at_utc": "2026-02-16T18:22:18.839668+00:00", + "page_analyses": [ + { + "field_presence": { + "authors_text": 0, + "citation_count": 0, + "cluster_id": 0, + "title": 0, + "venue_text": 0, + "year": 0 + }, + "marker_counts": { + "gs_gray": 0, + "gsc_a_ac": 0, + "gsc_a_at": 0, + "gsc_a_h": 0, + "gsc_a_tr": 0, + "gsc_a_y": 0, + "gsc_prf_in": 0, + "gsc_rsb_st": 0 + }, + "parse_warnings": [ + "no_rows_detected" + ], + "profile_name": null, + "publication_count": 0, + "source": "profile_AAAAAAAAAAAA", + "status": "blocked_or_captcha" + }, + { + "field_presence": { + "authors_text": 12, + "citation_count": 12, + "cluster_id": 12, + "title": 12, + "venue_text": 10, + "year": 11 + }, + "marker_counts": { + "gs_gray": 26, + "gsc_a_ac": 18, + "gsc_a_at": 15, + "gsc_a_h": 31, + "gsc_a_tr": 32, + "gsc_a_y": 26, + "gsc_prf_in": 17, + "gsc_rsb_st": 30 + }, + "parse_warnings": [], + "profile_name": "Doaa Elmatary", + "publication_count": 12, + "source": "profile_LZ5D_p4AAAAJ", + "status": "ok" + }, + { + "field_presence": { + "authors_text": 5, + "citation_count": 5, + "cluster_id": 5, + "title": 5, + "venue_text": 5, + "year": 5 + }, + "marker_counts": { + "gs_gray": 14, + "gsc_a_ac": 11, + "gsc_a_at": 8, + "gsc_a_h": 17, + "gsc_a_tr": 25, + "gsc_a_y": 19, + "gsc_prf_in": 21, + "gsc_rsb_st": 30 + }, + "parse_warnings": [], + "profile_name": "WENRUI ZUO", + "publication_count": 5, + "source": "profile_P1RwlvoAAAAJ", + "status": "ok" + }, + { + "field_presence": { + "authors_text": 20, + "citation_count": 20, + "cluster_id": 20, + "title": 20, + "venue_text": 19, + "year": 20 + }, + "marker_counts": { + "gs_gray": 42, + "gsc_a_ac": 26, + "gsc_a_at": 23, + "gsc_a_h": 47, + "gsc_a_tr": 40, + "gsc_a_y": 34, + "gsc_prf_in": 20, + "gsc_rsb_st": 30 + }, + "parse_warnings": [], + "profile_name": "K. Srinivasan", + "publication_count": 20, + "source": "profile_RxmmtT8AAAAJ", + "status": "ok" + }, + { + "field_presence": { + "authors_text": 20, + "citation_count": 20, + "cluster_id": 20, + "title": 20, + "venue_text": 20, + "year": 20 + }, + "marker_counts": { + "gs_gray": 42, + "gsc_a_ac": 26, + "gsc_a_at": 23, + "gsc_a_h": 47, + "gsc_a_tr": 40, + "gsc_a_y": 34, + "gsc_prf_in": 21, + "gsc_rsb_st": 30 + }, + "parse_warnings": [], + "profile_name": "Bangar Raju Cherukuri", + "publication_count": 20, + "source": "profile_amIMrIEAAAAJ", + "status": "ok" + } + ], + "publications_by_source": { + "profile_AAAAAAAAAAAA": [], + "profile_LZ5D_p4AAAAJ": [ + { + "authors_text": "EM Abd Allah, DE El-Matary, EM Eid, AST El Dien", + "citation_count": 25, + "cluster_id": "u-x6o8ySG0sC", + "title": "Performance comparison of various machine learning approaches to identify the best one in predicting heart disease", + "title_url": "/citations?view_op=view_citation&hl=en&user=LZ5D_p4AAAAJ&citation_for_view=LZ5D_p4AAAAJ:u-x6o8ySG0sC", + "venue_text": "Journal of Computer and Communications 10 (2), 1-18, 2022", + "year": 2022 + }, + { + "authors_text": "EAAA Hagras, D El-Saied, HH Aly", + "citation_count": 15, + "cluster_id": "YsMSGLbcyi4C", + "title": "Energy efficient key management scheme based on elliptic curve signcryption for wireless sensor networks", + "title_url": "/citations?view_op=view_citation&hl=en&user=LZ5D_p4AAAAJ&citation_for_view=LZ5D_p4AAAAJ:YsMSGLbcyi4C", + "venue_text": "2011 28th National Radio Science Conference (NRSC), 1-9, 2011", + "year": 2011 + }, + { + "authors_text": "EA Hagras, D El-Saied, HH Aly", + "citation_count": 6, + "cluster_id": "Y0pCki6q_DkC", + "title": "A new forward secure elliptic curve signcryption key management (fs-ecskm) scheme for heterogeneous wireless sensor networks", + "title_url": "/citations?view_op=view_citation&hl=en&user=LZ5D_p4AAAAJ&citation_for_view=LZ5D_p4AAAAJ:Y0pCki6q_DkC", + "venue_text": "International Journal of Computer Science and Technology 2 (2), 19-23, 2011", + "year": 2011 + }, + { + "authors_text": "DEE Matary, EA Hagras, HM Abdel-Kader", + "citation_count": 4, + "cluster_id": "d1gkVwhDpl0C", + "title": "Performance of polar codes for OFDM-based UWB channel", + "title_url": "/citations?view_op=view_citation&hl=en&user=LZ5D_p4AAAAJ&citation_for_view=LZ5D_p4AAAAJ:d1gkVwhDpl0C", + "venue_text": "Journal of Computer and Communications 6 (03), 102-117, 2018", + "year": 2018 + }, + { + "authors_text": "DEE Matary, EAA Hagras, HM Abdel-Kader", + "citation_count": 3, + "cluster_id": "u5HHmVD_uO8C", + "title": "Multi-user Communication Based OFDM-UWB System under Gaussian and Non-Gaussian Noisy Channel", + "title_url": "/citations?view_op=view_citation&hl=en&user=LZ5D_p4AAAAJ&citation_for_view=LZ5D_p4AAAAJ:u5HHmVD_uO8C", + "venue_text": "American Journal of Electrical and Electronic Engineering 5 (4), 136-143, 2017", + "year": 2017 + }, + { + "authors_text": "EAAA Hagras", + "citation_count": 2, + "cluster_id": "W7OEmFMy1HYC", + "title": "Doaa El-Saied, Dr. Hazem H. Aly,\u201cA New Forward Secure Elliptic Curve Signcryption Key Management (FS-ECSKM) Scheme for Heterogeneous Wireless Sensor Networks\u201d", + "title_url": "/citations?view_op=view_citation&hl=en&user=LZ5D_p4AAAAJ&citation_for_view=LZ5D_p4AAAAJ:W7OEmFMy1HYC", + "venue_text": "International Journal of Computer Science and Technology 2 (2), 19-23, 2011", + "year": 2011 + }, + { + "authors_text": "DE El-Matary, EAA Hagras, HM Abdel-Kader", + "citation_count": 1, + "cluster_id": "_FxGoFyzp5QC", + "title": "Performance Analysis of LDPC-IDMA-UWB Signals in Non-Gaussian Noisy Channel", + "title_url": "/citations?view_op=view_citation&hl=en&user=LZ5D_p4AAAAJ&citation_for_view=LZ5D_p4AAAAJ:_FxGoFyzp5QC", + "venue_text": null, + "year": 2013 + }, + { + "authors_text": "DE Elmatary", + "citation_count": 0, + "cluster_id": "ufrVoPGSRksC", + "title": "Intelligent Sign Multi-Language Real-Time Prediction System with Effective Data Preprocessing", + "title_url": "/citations?view_op=view_citation&hl=en&user=LZ5D_p4AAAAJ&citation_for_view=LZ5D_p4AAAAJ:ufrVoPGSRksC", + "venue_text": "Journal of Computer and Communications, 120-134, 2023", + "year": 2023 + }, + { + "authors_text": "E M AbdAllah, D E El Matary", + "citation_count": 0, + "cluster_id": "WF5omc3nYNoC", + "title": "Smart Healthcare System Based IoT with Frequency Analysis of Heart Rate Data across FFT Algorithmto be used During Covid-19 Pandemic", + "title_url": "/citations?view_op=view_citation&hl=en&user=LZ5D_p4AAAAJ&citation_for_view=LZ5D_p4AAAAJ:WF5omc3nYNoC", + "venue_text": "Engineering Research Journal (Shoubra) 51 (1), 88-94, 2022", + "year": 2022 + }, + { + "authors_text": "DE El Matary, EAA Hagras, HM Abdel-Kader", + "citation_count": 0, + "cluster_id": "9yKSN-GCB0IC", + "title": "Polar coded Interleave Division Multiple Access Ultra Wide Band (IDMA-UWB) communication system", + "title_url": "/citations?view_op=view_citation&hl=en&user=LZ5D_p4AAAAJ&citation_for_view=LZ5D_p4AAAAJ:9yKSN-GCB0IC", + "venue_text": "2018 35th National Radio Science Conference (NRSC), 283-292, 2018", + "year": 2018 + }, + { + "authors_text": "HMAK Doaa E.El-Matary, Esam A.A. Hagras", + "citation_count": 0, + "cluster_id": "zYLM7Y9cAGgC", + "title": "Performance Analysis of LDPC-IDMA-UWB Signals in Non-Gaussian Noisy Channel", + "title_url": "/citations?view_op=view_citation&hl=en&user=LZ5D_p4AAAAJ&citation_for_view=LZ5D_p4AAAAJ:zYLM7Y9cAGgC", + "venue_text": "International Journal of Scientific & Engineering Research, 2013", + "year": 2013 + }, + { + "authors_text": "DEE Matary, EAA Hagras, HM Abdel-Kader", + "citation_count": 0, + "cluster_id": "eQOLeE2rZwMC", + "title": "Non-Gaussian Noisy Channel Effect on Multi-user IDMA-UWB Communication System", + "title_url": "/citations?view_op=view_citation&hl=en&user=LZ5D_p4AAAAJ&citation_for_view=LZ5D_p4AAAAJ:eQOLeE2rZwMC", + "venue_text": null, + "year": null + } + ], + "profile_P1RwlvoAAAAJ": [ + { + "authors_text": "W Zuo, A Raman, RJ Mondrag\u00f3n, G Tyson", + "citation_count": 18, + "cluster_id": "u5HHmVD_uO8C", + "title": "Set in stone: Analysis of an immutable web3 social media platform", + "title_url": "/citations?view_op=view_citation&hl=en&user=P1RwlvoAAAAJ&citation_for_view=P1RwlvoAAAAJ:u5HHmVD_uO8C", + "venue_text": "Proceedings of the ACM Web Conference 2023, 1865-1874, 2023", + "year": 2023 + }, + { + "authors_text": "W Zuo, RJ Mondragon, A Raman, G Tyson", + "citation_count": 10, + "cluster_id": "d1gkVwhDpl0C", + "title": "Understanding and improving content moderation in web3 platforms", + "title_url": "/citations?view_op=view_citation&hl=en&user=P1RwlvoAAAAJ&citation_for_view=P1RwlvoAAAAJ:d1gkVwhDpl0C", + "venue_text": "Proceedings of the International AAAI Conference on Web and Social Media 18 \u2026, 2024", + "year": 2024 + }, + { + "authors_text": "W Zuo, A Raman, RJ Mondrag\u00d3N, G Tyson", + "citation_count": 7, + "cluster_id": "u-x6o8ySG0sC", + "title": "A first look at user-controlled moderation on web3 social media: The case of memo. cash", + "title_url": "/citations?view_op=view_citation&hl=en&user=P1RwlvoAAAAJ&citation_for_view=P1RwlvoAAAAJ:u-x6o8ySG0sC", + "venue_text": "Proceedings of the 3rd International Workshop on Open Challenges in Online \u2026, 2023", + "year": 2023 + }, + { + "authors_text": "Y Liu, RM Williams, G Xie, Y Wang, W Zuo", + "citation_count": 4, + "cluster_id": "9yKSN-GCB0IC", + "title": "Promoting the Culture of Qinhuai River Lantern Shadow Puppetry with a Digital Archive and Immersive Experience", + "title_url": "/citations?view_op=view_citation&hl=en&user=P1RwlvoAAAAJ&citation_for_view=P1RwlvoAAAAJ:9yKSN-GCB0IC", + "venue_text": "arXiv preprint arXiv:2410.03532, 2024", + "year": 2024 + }, + { + "authors_text": "W Zuo", + "citation_count": 0, + "cluster_id": "2osOgNQ5qMEC", + "title": "Understanding and Improving User-controlled Content Moderation Systems on Social Media", + "title_url": "/citations?view_op=view_citation&hl=en&user=P1RwlvoAAAAJ&citation_for_view=P1RwlvoAAAAJ:2osOgNQ5qMEC", + "venue_text": "Queen Mary University of London, 2025", + "year": 2025 + } + ], + "profile_RxmmtT8AAAAJ": [ + { + "authors_text": "K Srinivasan", + "citation_count": 1185, + "cluster_id": "qjMakFHDy7sC", + "title": "Black pepper and its pungent principle-piperine: a review of diverse physiological effects", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:qjMakFHDy7sC", + "venue_text": "Critical reviews in food science and nutrition 47 (8), 735-748, 2007", + "year": 2007 + }, + { + "authors_text": "K Platel, K Srinivasan", + "citation_count": 742, + "cluster_id": "W7OEmFMy1HYC", + "title": "Digestive stimulant action of spices: a myth or reality?", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:W7OEmFMy1HYC", + "venue_text": "Indian Journal of Medical Research 119 (5), 167, 2004", + "year": 2004 + }, + { + "authors_text": "K Platel, K Srinivasan", + "citation_count": 682, + "cluster_id": "u-x6o8ySG0sC", + "title": "Influence of dietary spices and their active principles on pancreatic digestive enzymes in albino rats", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:u-x6o8ySG0sC", + "venue_text": "Food/Nahrung 44 (1), 42-46, 2000", + "year": 2000 + }, + { + "authors_text": "K Srinivasan", + "citation_count": 663, + "cluster_id": "LkGwnXOMwfcC", + "title": "Fenugreek (Trigonella foenum-graecum): A Review of Health Beneficial Physiological Effects", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:LkGwnXOMwfcC", + "venue_text": "Food reviews international 22 (2), 203-224, 2006", + "year": 2006 + }, + { + "authors_text": "K Srinivasan", + "citation_count": 574, + "cluster_id": "FPJr55Dyh1AC", + "title": "Biological activities of red pepper (Capsicum annuum) and its pungent principle capsaicin: a review", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:FPJr55Dyh1AC", + "venue_text": "Critical reviews in food science and nutrition 56 (9), 1488-1500, 2016", + "year": 2016 + }, + { + "authors_text": "K Srinivasan", + "citation_count": 555, + "cluster_id": "9yKSN-GCB0IC", + "title": "Plant foods in the management of diabetes mellitus: spices as beneficial antidiabetic food adjuncts", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:9yKSN-GCB0IC", + "venue_text": "International journal of food sciences and nutrition 56 (6), 399-414, 2005", + "year": 2005 + }, + { + "authors_text": "PS Babu, K Srinivasan", + "citation_count": 539, + "cluster_id": "ZfRJV9d4-WMC", + "title": "Hypolipidemic action of curcumin, the active principle of turmeric (Curcuma longa) in streptozotocin induced diabetic rats", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:ZfRJV9d4-WMC", + "venue_text": "Molecular and cellular biochemistry 166 (1), 169-175, 1997", + "year": 1997 + }, + { + "authors_text": "K Srinivasan", + "citation_count": 513, + "cluster_id": "IjCSPb-OGe4C", + "title": "Spices as influencers of body metabolism: an overview of three decades of research", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:IjCSPb-OGe4C", + "venue_text": "Food Research International 38 (1), 77-86, 2005", + "year": 2005 + }, + { + "authors_text": "K Srinivasan", + "citation_count": 455, + "cluster_id": "Tyk-4Ss8FVUC", + "title": "Role of spices beyond food flavoring: Nutraceuticals with multiple health effects", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:Tyk-4Ss8FVUC", + "venue_text": "Food reviews international 21 (2), 167-188, 2005", + "year": 2005 + }, + { + "authors_text": "K Srinivasan", + "citation_count": 446, + "cluster_id": "tkaPQYYpVKoC", + "title": "Antioxidant potential of spices and their active constituents", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:tkaPQYYpVKoC", + "venue_text": "Critical Reviews in Food Science and Nutrition 54 (3), 252-272, 2013", + "year": 2013 + }, + { + "authors_text": "D Suresh, K Srinivasan", + "citation_count": 387, + "cluster_id": "5nxA0vEk-isC", + "title": "Tissue distribution and elimination of capsaicin, piperine & curcumin following oral intake in rats.", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:5nxA0vEk-isC", + "venue_text": "Indian Journal of Medical Research 131, 682-691, 2010", + "year": 2010 + }, + { + "authors_text": "R Ramakrishna Rao, K Platel, K Srinivasan", + "citation_count": 368, + "cluster_id": "YsMSGLbcyi4C", + "title": "In vitro influence of spices and spice\u2010active principles on digestive enzymes of rat pancreas and small intestine", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:YsMSGLbcyi4C", + "venue_text": "Food/Nahrung 47 (6), 408-412, 2003", + "year": 2003 + }, + { + "authors_text": "K Srinivasan", + "citation_count": 367, + "cluster_id": "zCSUwVk65WsC", + "title": "Cumin (Cuminum cyminum) and black cumin (Nigella sativa) seeds: traditional uses, chemical constituents, and nutraceutical effects", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:zCSUwVk65WsC", + "venue_text": "Food quality and safety 2 (1), 1-16, 2018", + "year": 2018 + }, + { + "authors_text": "K Srinivasan", + "citation_count": 364, + "cluster_id": "yqoGN6RLRZoC", + "title": "Ginger rhizomes (Zingiber officinale): A spice with multiple health beneficial potentials", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:yqoGN6RLRZoC", + "venue_text": "PharmaNutrition 5 (1), 18-28, 2017", + "year": 2017 + }, + { + "authors_text": "K Platel, K Srinivasan", + "citation_count": 364, + "cluster_id": "d1gkVwhDpl0C", + "title": "Influence of dietary spices or their active principles on digestive enzymes of small intestinal mucosa in rats", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:d1gkVwhDpl0C", + "venue_text": "International Journal of Food Sciences and Nutrition 47 (1), 55-59, 1996", + "year": 1996 + }, + { + "authors_text": "K Platel, K Srinivasan", + "citation_count": 296, + "cluster_id": "UeHWp8X0CEIC", + "title": "Plant foods in the management of diabetes mellitus: vegetables as potential hypoglycaemic agents", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:UeHWp8X0CEIC", + "venue_text": "Food/Nahrung 41 (2), 68-74, 1997", + "year": 1997 + }, + { + "authors_text": "K Srinivasan, K Sambaiah", + "citation_count": 292, + "cluster_id": "2osOgNQ5qMEC", + "title": "The effect of spices on cholesterol 7\u03b1-hydroxylase activity and on serum and hepatic cholesterol levels in the rat.", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:2osOgNQ5qMEC", + "venue_text": null, + "year": 1991 + }, + { + "authors_text": "S Hemalatha, K Platel, K Srinivasan", + "citation_count": 269, + "cluster_id": "UebtZRa9Y70C", + "title": "Zinc and iron contents and their bioaccessibility in cereals and pulses consumed in India", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:UebtZRa9Y70C", + "venue_text": "Food chemistry 102 (4), 1328-1336, 2007", + "year": 2007 + }, + { + "authors_text": "K Platel, K Srinivasan", + "citation_count": 258, + "cluster_id": "_FxGoFyzp5QC", + "title": "Studies on the influence of dietary spices on food transit time in experimental rats", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:_FxGoFyzp5QC", + "venue_text": "Nutrition Research 21 (9), 1309-1314, 2001", + "year": 2001 + }, + { + "authors_text": "K Platel, K Srinivasan", + "citation_count": 240, + "cluster_id": "hCrLmN-GePgC", + "title": "Bioavailability of micronutrients from plant foods: an update", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:hCrLmN-GePgC", + "venue_text": "Critical reviews in food science and nutrition 56 (10), 1608-1619, 2016", + "year": 2016 + } + ], + "profile_amIMrIEAAAAJ": [ + { + "authors_text": "BR Cherukuri, V Arulkumar", + "citation_count": 131, + "cluster_id": "u5HHmVD_uO8C", + "title": "Optimization of Data Structures and Trade-Offs with Concurrency Control in Multithread Software Structures Using Artificial Intelligence", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:u5HHmVD_uO8C", + "venue_text": "2024 IEEE International Conference on Computing, Power and Communication \u2026, 2024", + "year": 2024 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 81, + "cluster_id": "d1gkVwhDpl0C", + "title": "Development of Design Patterns with Adaptive User Interface for Cloud Native Microservice Architecture Using Deep Learning With IoT", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:d1gkVwhDpl0C", + "venue_text": "2024 IEEE International Conference on Computing, Power and Communication \u2026, 2024", + "year": 2024 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 33, + "cluster_id": "zYLM7Y9cAGgC", + "title": "AI-powered personalization: How machine learning is shaping the future of user experience", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:zYLM7Y9cAGgC", + "venue_text": "International Journal of Science and Research Archive (IJSRA) 12 (1), 3111\u20133126, 2024", + "year": 2024 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 30, + "cluster_id": "roLk4NBRz8UC", + "title": "Future of cloud computing: Innovations in multi-cloud and hybrid architectures", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:roLk4NBRz8UC", + "venue_text": "World Journal of Advanced Research and Reviews (WJARR) 1 (1), 068-081, 2019", + "year": 2019 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 27, + "cluster_id": "UebtZRa9Y70C", + "title": "Microservices and containerization: Accelerating web development cycles", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:UebtZRa9Y70C", + "venue_text": "World Journal of Advanced Research and Reviews (WJARR) 6 (01), 283-296, 2020", + "year": 2020 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 18, + "cluster_id": "hqOjcs7Dif8C", + "title": "Ethical AI in cloud: Mitigating risks in machine learning models", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:hqOjcs7Dif8C", + "venue_text": "World Journal of Advanced Engineering Technology and Sciences ((WJAETS) 1 \u2026, 2020", + "year": 2020 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 14, + "cluster_id": "0EnyYjriUFMC", + "title": "Quantum machine learning: Transforming cloud-based AI solutions", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:0EnyYjriUFMC", + "venue_text": "International Journal of Science and Research Archive (IJSRA) 1 (01), 110-122, 2020", + "year": 2020 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 13, + "cluster_id": "WF5omc3nYNoC", + "title": "Enhanced Trimodal Emotion Recognition Using Multibranch Fusion Attention with Epistemic Neural Networks and Fire Hawk Optimization", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:WF5omc3nYNoC", + "venue_text": "Journal of Machine and Computing 5 (1), 2025", + "year": 2025 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 13, + "cluster_id": "W7OEmFMy1HYC", + "title": "Containerization in cloud computing: comparing Docker and Kubernetes for scalable web applications", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:W7OEmFMy1HYC", + "venue_text": "International Journal of Science and Research Archive (IJSRA) 13 (1), 3302\u20133315, 2024", + "year": 2024 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 13, + "cluster_id": "IjCSPb-OGe4C", + "title": "Serverless computing: How to build and deploy applications without managing infrastructure", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:IjCSPb-OGe4C", + "venue_text": "World Journal of Advanced Engineering Technology and Sciences ((WJAETS) 11 \u2026, 2024", + "year": 2024 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 12, + "cluster_id": "YsMSGLbcyi4C", + "title": "Edge Computing vs. Cloud Computing: A Comparative Analysis for Real-Time AI Applications", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:YsMSGLbcyi4C", + "venue_text": "International Journal For Multidisciplinary Research (IJFMR) 6 (5), 1-17, 2024", + "year": 2024 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 9, + "cluster_id": "Se3iqnhoufwC", + "title": "Serverless revolution: Redefining application scalability and cost efficiency", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:Se3iqnhoufwC", + "venue_text": "World Journal of Advanced Research and Reviews (WJARR) 2 (03), 039-053, 2019", + "year": 2019 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 8, + "cluster_id": "Tyk-4Ss8FVUC", + "title": "Progressive Web Apps (PWAs): Enhancing User Experience through Modern Web Development", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:Tyk-4Ss8FVUC", + "venue_text": "International Journal of Science and Research (IJSR) 13 (10), 1550-1560, 2024", + "year": 2024 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 7, + "cluster_id": "UeHWp8X0CEIC", + "title": "Building Scalable Web Applications: Best Practices for Backend Architecture", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:UeHWp8X0CEIC", + "venue_text": "International Journal of Science and Research (IJSR) 13 (10), 126-139, 2024", + "year": 2024 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 5, + "cluster_id": "5nxA0vEk-isC", + "title": "Advanced Multi Class Cyber Security Attack Classification in IoT Based Wireless Sensor Networks Using Context Aware Depthwise Separable Convolutional Neural Network", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:5nxA0vEk-isC", + "venue_text": "Journal of Machine and Computing 5 (2), 2025", + "year": 2025 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 3, + "cluster_id": "u-x6o8ySG0sC", + "title": "Maintenance of Web Development Standard for Multiple Devices with Serverless Computing through Cross Browser Affinity Using Hybrid Optimization", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:u-x6o8ySG0sC", + "venue_text": "2024 IEEE International Conference on Computing, Power and Communication \u2026, 2024", + "year": 2024 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 3, + "cluster_id": "LkGwnXOMwfcC", + "title": "Developing Intelligent Chatbots for Real-Time Customer Support in E-Commerce", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:LkGwnXOMwfcC", + "venue_text": "International Journal of Science and Research (IJSR) 11 (01), 1709-1719, 2022", + "year": 2022 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 3, + "cluster_id": "_FxGoFyzp5QC", + "title": "Enhancing Web Application Performance with AI - Driven Optimization Techniques", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:_FxGoFyzp5QC", + "venue_text": "International Journal of Science and Research (IJSR) 10 (2), 1779-1788, 2021", + "year": 2021 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 1, + "cluster_id": "Y0pCki6q_DkC", + "title": "Federated Learning: Privacy-Preserving Machine Learning in Cloud Environments", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:Y0pCki6q_DkC", + "venue_text": "International Journal of Science and Research (IJSR) 13 (10), 1539-1549, 2024", + "year": 2024 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 1, + "cluster_id": "ufrVoPGSRksC", + "title": "Scalable machine learning model deployment using serverless cloud architectures", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:ufrVoPGSRksC", + "venue_text": "World Journal of Advanced Engineering Technology and Sciences ((WJAETS) 5 (1 \u2026, 2022", + "year": 2022 + } + ] + }, + "run_dir": "planning/scholar_probe_tmp/fixtures/run_20260216T182157Z" +} \ No newline at end of file diff --git a/planning/scholar_probe_tmp/notes/probe_report_run_20260216T182157Z.md b/planning/scholar_probe_tmp/notes/probe_report_run_20260216T182157Z.md new file mode 100644 index 0000000..883ce4d --- /dev/null +++ b/planning/scholar_probe_tmp/notes/probe_report_run_20260216T182157Z.md @@ -0,0 +1,82 @@ +# Scholar Scrape Probe Report + +Generated UTC: `2026-02-16T18:22:18.839668+00:00` +Run fixtures dir: `planning/scholar_probe_tmp/fixtures/run_20260216T182157Z` + +## Robots Snapshot + +```text +User-agent: * +Disallow: /search +Disallow: /index.html +Disallow: /scholar +Disallow: /citations? +Allow: /citations?user= +Disallow: /citations?*cstart= +Disallow: /citations?user=*%40 +Disallow: /citations?user=*@ +Allow: /citations?view_op=list_classic_articles +Allow: /citations?view_op=mandates_leaderboard +Allow: /citations?view_op=metrics_intro +Allow: /citations?view_op=new_profile +Allow: /citations?view_op=sitemap +Allow: /citations?view_op=top_venues + +User-agent: Twitterbot +Disallow: + +User-agent: facebookexternalhit +Disallow: + +User-agent: PetalBot +Disallow: / +``` + +## Fetch Summary + +| Source | Status Code | Status/Error | Final URL | +| --- | --- | --- | --- | +| `robots` | 200 | ok | `https://scholar.google.com/robots.txt` | +| `profile_amIMrIEAAAAJ` | 200 | ok | `https://scholar.google.com/citations?hl=en&user=amIMrIEAAAAJ` | +| `profile_P1RwlvoAAAAJ` | 200 | ok | `https://scholar.google.com/citations?hl=en&user=P1RwlvoAAAAJ` | +| `profile_RxmmtT8AAAAJ` | 200 | ok | `https://scholar.google.com/citations?hl=en&user=RxmmtT8AAAAJ` | +| `profile_LZ5D_p4AAAAJ` | 200 | ok | `https://scholar.google.com/citations?hl=en&user=LZ5D_p4AAAAJ` | +| `profile_AAAAAAAAAAAA` | 200 | ok | `https://accounts.google.com/v3/signin/identifier?continue=https%3A%2F%2Fscholar.google.com%2Fcitations%3Fhl%3Den%26user%3DAAAAAAAAAAAA&dsh=S-568466874%3A1771266138494353&hl=en&ifkv=ASfE1-pK1BK8frcmUfumfUYfZZe1iJXbTGg6TCfyR-Q1ErPAopCpUGy_sEwLdvOICVCsiXsIJbFm1Q&service=citations&flowName=GlifWebSignIn&flowEntry=ServiceLogin` | + +## Parse Summary + +| Source | Parse Status | Profile | Publications | Warnings | +| --- | --- | --- | --- | --- | +| `profile_AAAAAAAAAAAA` | `blocked_or_captcha` | - | 0 | no_rows_detected | +| `profile_LZ5D_p4AAAAJ` | `ok` | Doaa Elmatary | 12 | - | +| `profile_P1RwlvoAAAAJ` | `ok` | WENRUI ZUO | 5 | - | +| `profile_RxmmtT8AAAAJ` | `ok` | K. Srinivasan | 20 | - | +| `profile_amIMrIEAAAAJ` | `ok` | Bangar Raju Cherukuri | 20 | - | + +## Field Coverage + +Total parsed publication rows: **57** + +| Field | Present | Coverage | +| --- | --- | --- | +| `title` | 57 | 100.0% | +| `cluster_id` | 57 | 100.0% | +| `year` | 56 | 98.2% | +| `citation_count` | 57 | 100.0% | +| `authors_text` | 57 | 100.0% | +| `venue_text` | 54 | 94.7% | + +## Parser Contract Recommendation + +- Primary row marker: `tr.gsc_a_tr`. +- Title anchor marker: `a.gsc_a_at`; derive `cluster_id` from `citation_for_view` query token. +- Metadata text markers: first/second `div.gs_gray` per row for authors and venue. +- Year marker fallback: classes containing `gsc_a_h` or `gsc_a_y` and 4-digit year regex. +- Failure states to persist: `ok`, `no_results`, `blocked_or_captcha`, `layout_changed`, `network_error`. + +## Future-Proofing Notes + +- Keep raw HTML fixture snapshots and update parser tests on DOM drift. +- Treat blocked pages as retriable with backoff, not parser errors. +- Add marker-count assertions in CI to catch silent layout shifts early. +- Use explicit parse status per run/scholar so automation can degrade gracefully. diff --git a/planning/scholar_probe_tmp/notes/probe_report_run_20260216T182303Z.json b/planning/scholar_probe_tmp/notes/probe_report_run_20260216T182303Z.json new file mode 100644 index 0000000..87ac758 --- /dev/null +++ b/planning/scholar_probe_tmp/notes/probe_report_run_20260216T182303Z.json @@ -0,0 +1,831 @@ +{ + "fetch_records": [], + "generated_at_utc": "2026-02-16T18:23:03.906832+00:00", + "page_analyses": [ + { + "articles_range": null, + "field_presence": { + "authors_text": 0, + "citation_count": 0, + "cluster_id": 0, + "title": 0, + "venue_text": 0, + "year": 0 + }, + "has_operation_error_banner": false, + "has_show_more_button": false, + "marker_counts": { + "gs_gray": 0, + "gsc_a_ac": 0, + "gsc_a_at": 0, + "gsc_a_h": 0, + "gsc_a_tr": 0, + "gsc_a_y": 0, + "gsc_prf_in": 0, + "gsc_rsb_st": 0 + }, + "parse_warnings": [ + "no_rows_detected" + ], + "profile_name": null, + "publication_count": 0, + "source": "profile_AAAAAAAAAAAA", + "status": "layout_changed" + }, + { + "articles_range": "Articles 1\u201312", + "field_presence": { + "authors_text": 12, + "citation_count": 12, + "cluster_id": 12, + "title": 12, + "venue_text": 10, + "year": 11 + }, + "has_operation_error_banner": false, + "has_show_more_button": true, + "marker_counts": { + "gs_gray": 26, + "gsc_a_ac": 18, + "gsc_a_at": 15, + "gsc_a_h": 31, + "gsc_a_tr": 32, + "gsc_a_y": 26, + "gsc_prf_in": 17, + "gsc_rsb_st": 30 + }, + "parse_warnings": [ + "possible_partial_page_show_more_present" + ], + "profile_name": "Doaa Elmatary", + "publication_count": 12, + "source": "profile_LZ5D_p4AAAAJ", + "status": "ok" + }, + { + "articles_range": "Articles 1\u20135", + "field_presence": { + "authors_text": 5, + "citation_count": 5, + "cluster_id": 5, + "title": 5, + "venue_text": 5, + "year": 5 + }, + "has_operation_error_banner": false, + "has_show_more_button": true, + "marker_counts": { + "gs_gray": 14, + "gsc_a_ac": 11, + "gsc_a_at": 8, + "gsc_a_h": 17, + "gsc_a_tr": 25, + "gsc_a_y": 19, + "gsc_prf_in": 21, + "gsc_rsb_st": 30 + }, + "parse_warnings": [ + "possible_partial_page_show_more_present" + ], + "profile_name": "WENRUI ZUO", + "publication_count": 5, + "source": "profile_P1RwlvoAAAAJ", + "status": "ok" + }, + { + "articles_range": "Articles 1\u201320", + "field_presence": { + "authors_text": 20, + "citation_count": 20, + "cluster_id": 20, + "title": 20, + "venue_text": 19, + "year": 20 + }, + "has_operation_error_banner": false, + "has_show_more_button": true, + "marker_counts": { + "gs_gray": 42, + "gsc_a_ac": 26, + "gsc_a_at": 23, + "gsc_a_h": 47, + "gsc_a_tr": 40, + "gsc_a_y": 34, + "gsc_prf_in": 20, + "gsc_rsb_st": 30 + }, + "parse_warnings": [ + "possible_partial_page_show_more_present" + ], + "profile_name": "K. Srinivasan", + "publication_count": 20, + "source": "profile_RxmmtT8AAAAJ", + "status": "ok" + }, + { + "articles_range": "Articles 1\u201320", + "field_presence": { + "authors_text": 20, + "citation_count": 20, + "cluster_id": 20, + "title": 20, + "venue_text": 20, + "year": 20 + }, + "has_operation_error_banner": false, + "has_show_more_button": true, + "marker_counts": { + "gs_gray": 42, + "gsc_a_ac": 26, + "gsc_a_at": 23, + "gsc_a_h": 47, + "gsc_a_tr": 40, + "gsc_a_y": 34, + "gsc_prf_in": 21, + "gsc_rsb_st": 30 + }, + "parse_warnings": [ + "possible_partial_page_show_more_present" + ], + "profile_name": "Bangar Raju Cherukuri", + "publication_count": 20, + "source": "profile_amIMrIEAAAAJ", + "status": "ok" + }, + { + "articles_range": null, + "field_presence": { + "authors_text": 0, + "citation_count": 0, + "cluster_id": 0, + "title": 0, + "venue_text": 0, + "year": 0 + }, + "has_operation_error_banner": false, + "has_show_more_button": false, + "marker_counts": { + "gs_gray": 0, + "gsc_a_ac": 0, + "gsc_a_at": 0, + "gsc_a_h": 0, + "gsc_a_tr": 0, + "gsc_a_y": 0, + "gsc_prf_in": 0, + "gsc_rsb_st": 0 + }, + "parse_warnings": [ + "no_rows_detected" + ], + "profile_name": null, + "publication_count": 0, + "source": "profile_AAAAAAAAAAAA", + "status": "layout_changed" + }, + { + "articles_range": "Articles 1\u201312", + "field_presence": { + "authors_text": 12, + "citation_count": 12, + "cluster_id": 12, + "title": 12, + "venue_text": 10, + "year": 11 + }, + "has_operation_error_banner": false, + "has_show_more_button": true, + "marker_counts": { + "gs_gray": 26, + "gsc_a_ac": 18, + "gsc_a_at": 15, + "gsc_a_h": 31, + "gsc_a_tr": 32, + "gsc_a_y": 26, + "gsc_prf_in": 17, + "gsc_rsb_st": 30 + }, + "parse_warnings": [ + "possible_partial_page_show_more_present" + ], + "profile_name": "Doaa Elmatary", + "publication_count": 12, + "source": "profile_LZ5D_p4AAAAJ", + "status": "ok" + }, + { + "articles_range": "Articles 1\u20135", + "field_presence": { + "authors_text": 5, + "citation_count": 5, + "cluster_id": 5, + "title": 5, + "venue_text": 5, + "year": 5 + }, + "has_operation_error_banner": false, + "has_show_more_button": true, + "marker_counts": { + "gs_gray": 14, + "gsc_a_ac": 11, + "gsc_a_at": 8, + "gsc_a_h": 17, + "gsc_a_tr": 25, + "gsc_a_y": 19, + "gsc_prf_in": 21, + "gsc_rsb_st": 30 + }, + "parse_warnings": [ + "possible_partial_page_show_more_present" + ], + "profile_name": "WENRUI ZUO", + "publication_count": 5, + "source": "profile_P1RwlvoAAAAJ", + "status": "ok" + }, + { + "articles_range": "Articles 1\u201320", + "field_presence": { + "authors_text": 20, + "citation_count": 20, + "cluster_id": 20, + "title": 20, + "venue_text": 19, + "year": 20 + }, + "has_operation_error_banner": false, + "has_show_more_button": true, + "marker_counts": { + "gs_gray": 42, + "gsc_a_ac": 26, + "gsc_a_at": 23, + "gsc_a_h": 47, + "gsc_a_tr": 40, + "gsc_a_y": 34, + "gsc_prf_in": 20, + "gsc_rsb_st": 30 + }, + "parse_warnings": [ + "possible_partial_page_show_more_present" + ], + "profile_name": "K. Srinivasan", + "publication_count": 20, + "source": "profile_RxmmtT8AAAAJ", + "status": "ok" + }, + { + "articles_range": "Articles 1\u201320", + "field_presence": { + "authors_text": 20, + "citation_count": 20, + "cluster_id": 20, + "title": 20, + "venue_text": 20, + "year": 20 + }, + "has_operation_error_banner": false, + "has_show_more_button": true, + "marker_counts": { + "gs_gray": 42, + "gsc_a_ac": 26, + "gsc_a_at": 23, + "gsc_a_h": 47, + "gsc_a_tr": 40, + "gsc_a_y": 34, + "gsc_prf_in": 21, + "gsc_rsb_st": 30 + }, + "parse_warnings": [ + "possible_partial_page_show_more_present" + ], + "profile_name": "Bangar Raju Cherukuri", + "publication_count": 20, + "source": "profile_amIMrIEAAAAJ", + "status": "ok" + } + ], + "publications_by_source": { + "profile_AAAAAAAAAAAA": [], + "profile_LZ5D_p4AAAAJ": [ + { + "authors_text": "EM Abd Allah, DE El-Matary, EM Eid, AST El Dien", + "citation_count": 25, + "cluster_id": "u-x6o8ySG0sC", + "title": "Performance comparison of various machine learning approaches to identify the best one in predicting heart disease", + "title_url": "/citations?view_op=view_citation&hl=en&user=LZ5D_p4AAAAJ&citation_for_view=LZ5D_p4AAAAJ:u-x6o8ySG0sC", + "venue_text": "Journal of Computer and Communications 10 (2), 1-18, 2022", + "year": 2022 + }, + { + "authors_text": "EAAA Hagras, D El-Saied, HH Aly", + "citation_count": 15, + "cluster_id": "YsMSGLbcyi4C", + "title": "Energy efficient key management scheme based on elliptic curve signcryption for wireless sensor networks", + "title_url": "/citations?view_op=view_citation&hl=en&user=LZ5D_p4AAAAJ&citation_for_view=LZ5D_p4AAAAJ:YsMSGLbcyi4C", + "venue_text": "2011 28th National Radio Science Conference (NRSC), 1-9, 2011", + "year": 2011 + }, + { + "authors_text": "EA Hagras, D El-Saied, HH Aly", + "citation_count": 6, + "cluster_id": "Y0pCki6q_DkC", + "title": "A new forward secure elliptic curve signcryption key management (fs-ecskm) scheme for heterogeneous wireless sensor networks", + "title_url": "/citations?view_op=view_citation&hl=en&user=LZ5D_p4AAAAJ&citation_for_view=LZ5D_p4AAAAJ:Y0pCki6q_DkC", + "venue_text": "International Journal of Computer Science and Technology 2 (2), 19-23, 2011", + "year": 2011 + }, + { + "authors_text": "DEE Matary, EA Hagras, HM Abdel-Kader", + "citation_count": 4, + "cluster_id": "d1gkVwhDpl0C", + "title": "Performance of polar codes for OFDM-based UWB channel", + "title_url": "/citations?view_op=view_citation&hl=en&user=LZ5D_p4AAAAJ&citation_for_view=LZ5D_p4AAAAJ:d1gkVwhDpl0C", + "venue_text": "Journal of Computer and Communications 6 (03), 102-117, 2018", + "year": 2018 + }, + { + "authors_text": "DEE Matary, EAA Hagras, HM Abdel-Kader", + "citation_count": 3, + "cluster_id": "u5HHmVD_uO8C", + "title": "Multi-user Communication Based OFDM-UWB System under Gaussian and Non-Gaussian Noisy Channel", + "title_url": "/citations?view_op=view_citation&hl=en&user=LZ5D_p4AAAAJ&citation_for_view=LZ5D_p4AAAAJ:u5HHmVD_uO8C", + "venue_text": "American Journal of Electrical and Electronic Engineering 5 (4), 136-143, 2017", + "year": 2017 + }, + { + "authors_text": "EAAA Hagras", + "citation_count": 2, + "cluster_id": "W7OEmFMy1HYC", + "title": "Doaa El-Saied, Dr. Hazem H. Aly,\u201cA New Forward Secure Elliptic Curve Signcryption Key Management (FS-ECSKM) Scheme for Heterogeneous Wireless Sensor Networks\u201d", + "title_url": "/citations?view_op=view_citation&hl=en&user=LZ5D_p4AAAAJ&citation_for_view=LZ5D_p4AAAAJ:W7OEmFMy1HYC", + "venue_text": "International Journal of Computer Science and Technology 2 (2), 19-23, 2011", + "year": 2011 + }, + { + "authors_text": "DE El-Matary, EAA Hagras, HM Abdel-Kader", + "citation_count": 1, + "cluster_id": "_FxGoFyzp5QC", + "title": "Performance Analysis of LDPC-IDMA-UWB Signals in Non-Gaussian Noisy Channel", + "title_url": "/citations?view_op=view_citation&hl=en&user=LZ5D_p4AAAAJ&citation_for_view=LZ5D_p4AAAAJ:_FxGoFyzp5QC", + "venue_text": null, + "year": 2013 + }, + { + "authors_text": "DE Elmatary", + "citation_count": 0, + "cluster_id": "ufrVoPGSRksC", + "title": "Intelligent Sign Multi-Language Real-Time Prediction System with Effective Data Preprocessing", + "title_url": "/citations?view_op=view_citation&hl=en&user=LZ5D_p4AAAAJ&citation_for_view=LZ5D_p4AAAAJ:ufrVoPGSRksC", + "venue_text": "Journal of Computer and Communications, 120-134, 2023", + "year": 2023 + }, + { + "authors_text": "E M AbdAllah, D E El Matary", + "citation_count": 0, + "cluster_id": "WF5omc3nYNoC", + "title": "Smart Healthcare System Based IoT with Frequency Analysis of Heart Rate Data across FFT Algorithmto be used During Covid-19 Pandemic", + "title_url": "/citations?view_op=view_citation&hl=en&user=LZ5D_p4AAAAJ&citation_for_view=LZ5D_p4AAAAJ:WF5omc3nYNoC", + "venue_text": "Engineering Research Journal (Shoubra) 51 (1), 88-94, 2022", + "year": 2022 + }, + { + "authors_text": "DE El Matary, EAA Hagras, HM Abdel-Kader", + "citation_count": 0, + "cluster_id": "9yKSN-GCB0IC", + "title": "Polar coded Interleave Division Multiple Access Ultra Wide Band (IDMA-UWB) communication system", + "title_url": "/citations?view_op=view_citation&hl=en&user=LZ5D_p4AAAAJ&citation_for_view=LZ5D_p4AAAAJ:9yKSN-GCB0IC", + "venue_text": "2018 35th National Radio Science Conference (NRSC), 283-292, 2018", + "year": 2018 + }, + { + "authors_text": "HMAK Doaa E.El-Matary, Esam A.A. Hagras", + "citation_count": 0, + "cluster_id": "zYLM7Y9cAGgC", + "title": "Performance Analysis of LDPC-IDMA-UWB Signals in Non-Gaussian Noisy Channel", + "title_url": "/citations?view_op=view_citation&hl=en&user=LZ5D_p4AAAAJ&citation_for_view=LZ5D_p4AAAAJ:zYLM7Y9cAGgC", + "venue_text": "International Journal of Scientific & Engineering Research, 2013", + "year": 2013 + }, + { + "authors_text": "DEE Matary, EAA Hagras, HM Abdel-Kader", + "citation_count": 0, + "cluster_id": "eQOLeE2rZwMC", + "title": "Non-Gaussian Noisy Channel Effect on Multi-user IDMA-UWB Communication System", + "title_url": "/citations?view_op=view_citation&hl=en&user=LZ5D_p4AAAAJ&citation_for_view=LZ5D_p4AAAAJ:eQOLeE2rZwMC", + "venue_text": null, + "year": null + } + ], + "profile_P1RwlvoAAAAJ": [ + { + "authors_text": "W Zuo, A Raman, RJ Mondrag\u00f3n, G Tyson", + "citation_count": 18, + "cluster_id": "u5HHmVD_uO8C", + "title": "Set in stone: Analysis of an immutable web3 social media platform", + "title_url": "/citations?view_op=view_citation&hl=en&user=P1RwlvoAAAAJ&citation_for_view=P1RwlvoAAAAJ:u5HHmVD_uO8C", + "venue_text": "Proceedings of the ACM Web Conference 2023, 1865-1874, 2023", + "year": 2023 + }, + { + "authors_text": "W Zuo, RJ Mondragon, A Raman, G Tyson", + "citation_count": 10, + "cluster_id": "d1gkVwhDpl0C", + "title": "Understanding and improving content moderation in web3 platforms", + "title_url": "/citations?view_op=view_citation&hl=en&user=P1RwlvoAAAAJ&citation_for_view=P1RwlvoAAAAJ:d1gkVwhDpl0C", + "venue_text": "Proceedings of the International AAAI Conference on Web and Social Media 18 \u2026, 2024", + "year": 2024 + }, + { + "authors_text": "W Zuo, A Raman, RJ Mondrag\u00d3N, G Tyson", + "citation_count": 7, + "cluster_id": "u-x6o8ySG0sC", + "title": "A first look at user-controlled moderation on web3 social media: The case of memo. cash", + "title_url": "/citations?view_op=view_citation&hl=en&user=P1RwlvoAAAAJ&citation_for_view=P1RwlvoAAAAJ:u-x6o8ySG0sC", + "venue_text": "Proceedings of the 3rd International Workshop on Open Challenges in Online \u2026, 2023", + "year": 2023 + }, + { + "authors_text": "Y Liu, RM Williams, G Xie, Y Wang, W Zuo", + "citation_count": 4, + "cluster_id": "9yKSN-GCB0IC", + "title": "Promoting the Culture of Qinhuai River Lantern Shadow Puppetry with a Digital Archive and Immersive Experience", + "title_url": "/citations?view_op=view_citation&hl=en&user=P1RwlvoAAAAJ&citation_for_view=P1RwlvoAAAAJ:9yKSN-GCB0IC", + "venue_text": "arXiv preprint arXiv:2410.03532, 2024", + "year": 2024 + }, + { + "authors_text": "W Zuo", + "citation_count": 0, + "cluster_id": "2osOgNQ5qMEC", + "title": "Understanding and Improving User-controlled Content Moderation Systems on Social Media", + "title_url": "/citations?view_op=view_citation&hl=en&user=P1RwlvoAAAAJ&citation_for_view=P1RwlvoAAAAJ:2osOgNQ5qMEC", + "venue_text": "Queen Mary University of London, 2025", + "year": 2025 + } + ], + "profile_RxmmtT8AAAAJ": [ + { + "authors_text": "K Srinivasan", + "citation_count": 1185, + "cluster_id": "qjMakFHDy7sC", + "title": "Black pepper and its pungent principle-piperine: a review of diverse physiological effects", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:qjMakFHDy7sC", + "venue_text": "Critical reviews in food science and nutrition 47 (8), 735-748, 2007", + "year": 2007 + }, + { + "authors_text": "K Platel, K Srinivasan", + "citation_count": 742, + "cluster_id": "W7OEmFMy1HYC", + "title": "Digestive stimulant action of spices: a myth or reality?", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:W7OEmFMy1HYC", + "venue_text": "Indian Journal of Medical Research 119 (5), 167, 2004", + "year": 2004 + }, + { + "authors_text": "K Platel, K Srinivasan", + "citation_count": 682, + "cluster_id": "u-x6o8ySG0sC", + "title": "Influence of dietary spices and their active principles on pancreatic digestive enzymes in albino rats", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:u-x6o8ySG0sC", + "venue_text": "Food/Nahrung 44 (1), 42-46, 2000", + "year": 2000 + }, + { + "authors_text": "K Srinivasan", + "citation_count": 663, + "cluster_id": "LkGwnXOMwfcC", + "title": "Fenugreek (Trigonella foenum-graecum): A Review of Health Beneficial Physiological Effects", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:LkGwnXOMwfcC", + "venue_text": "Food reviews international 22 (2), 203-224, 2006", + "year": 2006 + }, + { + "authors_text": "K Srinivasan", + "citation_count": 574, + "cluster_id": "FPJr55Dyh1AC", + "title": "Biological activities of red pepper (Capsicum annuum) and its pungent principle capsaicin: a review", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:FPJr55Dyh1AC", + "venue_text": "Critical reviews in food science and nutrition 56 (9), 1488-1500, 2016", + "year": 2016 + }, + { + "authors_text": "K Srinivasan", + "citation_count": 555, + "cluster_id": "9yKSN-GCB0IC", + "title": "Plant foods in the management of diabetes mellitus: spices as beneficial antidiabetic food adjuncts", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:9yKSN-GCB0IC", + "venue_text": "International journal of food sciences and nutrition 56 (6), 399-414, 2005", + "year": 2005 + }, + { + "authors_text": "PS Babu, K Srinivasan", + "citation_count": 539, + "cluster_id": "ZfRJV9d4-WMC", + "title": "Hypolipidemic action of curcumin, the active principle of turmeric (Curcuma longa) in streptozotocin induced diabetic rats", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:ZfRJV9d4-WMC", + "venue_text": "Molecular and cellular biochemistry 166 (1), 169-175, 1997", + "year": 1997 + }, + { + "authors_text": "K Srinivasan", + "citation_count": 513, + "cluster_id": "IjCSPb-OGe4C", + "title": "Spices as influencers of body metabolism: an overview of three decades of research", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:IjCSPb-OGe4C", + "venue_text": "Food Research International 38 (1), 77-86, 2005", + "year": 2005 + }, + { + "authors_text": "K Srinivasan", + "citation_count": 455, + "cluster_id": "Tyk-4Ss8FVUC", + "title": "Role of spices beyond food flavoring: Nutraceuticals with multiple health effects", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:Tyk-4Ss8FVUC", + "venue_text": "Food reviews international 21 (2), 167-188, 2005", + "year": 2005 + }, + { + "authors_text": "K Srinivasan", + "citation_count": 446, + "cluster_id": "tkaPQYYpVKoC", + "title": "Antioxidant potential of spices and their active constituents", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:tkaPQYYpVKoC", + "venue_text": "Critical Reviews in Food Science and Nutrition 54 (3), 252-272, 2013", + "year": 2013 + }, + { + "authors_text": "D Suresh, K Srinivasan", + "citation_count": 387, + "cluster_id": "5nxA0vEk-isC", + "title": "Tissue distribution and elimination of capsaicin, piperine & curcumin following oral intake in rats.", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:5nxA0vEk-isC", + "venue_text": "Indian Journal of Medical Research 131, 682-691, 2010", + "year": 2010 + }, + { + "authors_text": "R Ramakrishna Rao, K Platel, K Srinivasan", + "citation_count": 368, + "cluster_id": "YsMSGLbcyi4C", + "title": "In vitro influence of spices and spice\u2010active principles on digestive enzymes of rat pancreas and small intestine", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:YsMSGLbcyi4C", + "venue_text": "Food/Nahrung 47 (6), 408-412, 2003", + "year": 2003 + }, + { + "authors_text": "K Srinivasan", + "citation_count": 367, + "cluster_id": "zCSUwVk65WsC", + "title": "Cumin (Cuminum cyminum) and black cumin (Nigella sativa) seeds: traditional uses, chemical constituents, and nutraceutical effects", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:zCSUwVk65WsC", + "venue_text": "Food quality and safety 2 (1), 1-16, 2018", + "year": 2018 + }, + { + "authors_text": "K Srinivasan", + "citation_count": 364, + "cluster_id": "yqoGN6RLRZoC", + "title": "Ginger rhizomes (Zingiber officinale): A spice with multiple health beneficial potentials", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:yqoGN6RLRZoC", + "venue_text": "PharmaNutrition 5 (1), 18-28, 2017", + "year": 2017 + }, + { + "authors_text": "K Platel, K Srinivasan", + "citation_count": 364, + "cluster_id": "d1gkVwhDpl0C", + "title": "Influence of dietary spices or their active principles on digestive enzymes of small intestinal mucosa in rats", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:d1gkVwhDpl0C", + "venue_text": "International Journal of Food Sciences and Nutrition 47 (1), 55-59, 1996", + "year": 1996 + }, + { + "authors_text": "K Platel, K Srinivasan", + "citation_count": 296, + "cluster_id": "UeHWp8X0CEIC", + "title": "Plant foods in the management of diabetes mellitus: vegetables as potential hypoglycaemic agents", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:UeHWp8X0CEIC", + "venue_text": "Food/Nahrung 41 (2), 68-74, 1997", + "year": 1997 + }, + { + "authors_text": "K Srinivasan, K Sambaiah", + "citation_count": 292, + "cluster_id": "2osOgNQ5qMEC", + "title": "The effect of spices on cholesterol 7\u03b1-hydroxylase activity and on serum and hepatic cholesterol levels in the rat.", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:2osOgNQ5qMEC", + "venue_text": null, + "year": 1991 + }, + { + "authors_text": "S Hemalatha, K Platel, K Srinivasan", + "citation_count": 269, + "cluster_id": "UebtZRa9Y70C", + "title": "Zinc and iron contents and their bioaccessibility in cereals and pulses consumed in India", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:UebtZRa9Y70C", + "venue_text": "Food chemistry 102 (4), 1328-1336, 2007", + "year": 2007 + }, + { + "authors_text": "K Platel, K Srinivasan", + "citation_count": 258, + "cluster_id": "_FxGoFyzp5QC", + "title": "Studies on the influence of dietary spices on food transit time in experimental rats", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:_FxGoFyzp5QC", + "venue_text": "Nutrition Research 21 (9), 1309-1314, 2001", + "year": 2001 + }, + { + "authors_text": "K Platel, K Srinivasan", + "citation_count": 240, + "cluster_id": "hCrLmN-GePgC", + "title": "Bioavailability of micronutrients from plant foods: an update", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:hCrLmN-GePgC", + "venue_text": "Critical reviews in food science and nutrition 56 (10), 1608-1619, 2016", + "year": 2016 + } + ], + "profile_amIMrIEAAAAJ": [ + { + "authors_text": "BR Cherukuri, V Arulkumar", + "citation_count": 131, + "cluster_id": "u5HHmVD_uO8C", + "title": "Optimization of Data Structures and Trade-Offs with Concurrency Control in Multithread Software Structures Using Artificial Intelligence", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:u5HHmVD_uO8C", + "venue_text": "2024 IEEE International Conference on Computing, Power and Communication \u2026, 2024", + "year": 2024 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 81, + "cluster_id": "d1gkVwhDpl0C", + "title": "Development of Design Patterns with Adaptive User Interface for Cloud Native Microservice Architecture Using Deep Learning With IoT", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:d1gkVwhDpl0C", + "venue_text": "2024 IEEE International Conference on Computing, Power and Communication \u2026, 2024", + "year": 2024 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 33, + "cluster_id": "zYLM7Y9cAGgC", + "title": "AI-powered personalization: How machine learning is shaping the future of user experience", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:zYLM7Y9cAGgC", + "venue_text": "International Journal of Science and Research Archive (IJSRA) 12 (1), 3111\u20133126, 2024", + "year": 2024 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 30, + "cluster_id": "roLk4NBRz8UC", + "title": "Future of cloud computing: Innovations in multi-cloud and hybrid architectures", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:roLk4NBRz8UC", + "venue_text": "World Journal of Advanced Research and Reviews (WJARR) 1 (1), 068-081, 2019", + "year": 2019 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 27, + "cluster_id": "UebtZRa9Y70C", + "title": "Microservices and containerization: Accelerating web development cycles", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:UebtZRa9Y70C", + "venue_text": "World Journal of Advanced Research and Reviews (WJARR) 6 (01), 283-296, 2020", + "year": 2020 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 18, + "cluster_id": "hqOjcs7Dif8C", + "title": "Ethical AI in cloud: Mitigating risks in machine learning models", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:hqOjcs7Dif8C", + "venue_text": "World Journal of Advanced Engineering Technology and Sciences ((WJAETS) 1 \u2026, 2020", + "year": 2020 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 14, + "cluster_id": "0EnyYjriUFMC", + "title": "Quantum machine learning: Transforming cloud-based AI solutions", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:0EnyYjriUFMC", + "venue_text": "International Journal of Science and Research Archive (IJSRA) 1 (01), 110-122, 2020", + "year": 2020 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 13, + "cluster_id": "WF5omc3nYNoC", + "title": "Enhanced Trimodal Emotion Recognition Using Multibranch Fusion Attention with Epistemic Neural Networks and Fire Hawk Optimization", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:WF5omc3nYNoC", + "venue_text": "Journal of Machine and Computing 5 (1), 2025", + "year": 2025 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 13, + "cluster_id": "W7OEmFMy1HYC", + "title": "Containerization in cloud computing: comparing Docker and Kubernetes for scalable web applications", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:W7OEmFMy1HYC", + "venue_text": "International Journal of Science and Research Archive (IJSRA) 13 (1), 3302\u20133315, 2024", + "year": 2024 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 13, + "cluster_id": "IjCSPb-OGe4C", + "title": "Serverless computing: How to build and deploy applications without managing infrastructure", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:IjCSPb-OGe4C", + "venue_text": "World Journal of Advanced Engineering Technology and Sciences ((WJAETS) 11 \u2026, 2024", + "year": 2024 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 12, + "cluster_id": "YsMSGLbcyi4C", + "title": "Edge Computing vs. Cloud Computing: A Comparative Analysis for Real-Time AI Applications", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:YsMSGLbcyi4C", + "venue_text": "International Journal For Multidisciplinary Research (IJFMR) 6 (5), 1-17, 2024", + "year": 2024 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 9, + "cluster_id": "Se3iqnhoufwC", + "title": "Serverless revolution: Redefining application scalability and cost efficiency", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:Se3iqnhoufwC", + "venue_text": "World Journal of Advanced Research and Reviews (WJARR) 2 (03), 039-053, 2019", + "year": 2019 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 8, + "cluster_id": "Tyk-4Ss8FVUC", + "title": "Progressive Web Apps (PWAs): Enhancing User Experience through Modern Web Development", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:Tyk-4Ss8FVUC", + "venue_text": "International Journal of Science and Research (IJSR) 13 (10), 1550-1560, 2024", + "year": 2024 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 7, + "cluster_id": "UeHWp8X0CEIC", + "title": "Building Scalable Web Applications: Best Practices for Backend Architecture", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:UeHWp8X0CEIC", + "venue_text": "International Journal of Science and Research (IJSR) 13 (10), 126-139, 2024", + "year": 2024 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 5, + "cluster_id": "5nxA0vEk-isC", + "title": "Advanced Multi Class Cyber Security Attack Classification in IoT Based Wireless Sensor Networks Using Context Aware Depthwise Separable Convolutional Neural Network", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:5nxA0vEk-isC", + "venue_text": "Journal of Machine and Computing 5 (2), 2025", + "year": 2025 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 3, + "cluster_id": "u-x6o8ySG0sC", + "title": "Maintenance of Web Development Standard for Multiple Devices with Serverless Computing through Cross Browser Affinity Using Hybrid Optimization", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:u-x6o8ySG0sC", + "venue_text": "2024 IEEE International Conference on Computing, Power and Communication \u2026, 2024", + "year": 2024 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 3, + "cluster_id": "LkGwnXOMwfcC", + "title": "Developing Intelligent Chatbots for Real-Time Customer Support in E-Commerce", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:LkGwnXOMwfcC", + "venue_text": "International Journal of Science and Research (IJSR) 11 (01), 1709-1719, 2022", + "year": 2022 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 3, + "cluster_id": "_FxGoFyzp5QC", + "title": "Enhancing Web Application Performance with AI - Driven Optimization Techniques", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:_FxGoFyzp5QC", + "venue_text": "International Journal of Science and Research (IJSR) 10 (2), 1779-1788, 2021", + "year": 2021 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 1, + "cluster_id": "Y0pCki6q_DkC", + "title": "Federated Learning: Privacy-Preserving Machine Learning in Cloud Environments", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:Y0pCki6q_DkC", + "venue_text": "International Journal of Science and Research (IJSR) 13 (10), 1539-1549, 2024", + "year": 2024 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 1, + "cluster_id": "ufrVoPGSRksC", + "title": "Scalable machine learning model deployment using serverless cloud architectures", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:ufrVoPGSRksC", + "venue_text": "World Journal of Advanced Engineering Technology and Sciences ((WJAETS) 5 (1 \u2026, 2022", + "year": 2022 + } + ] + }, + "run_dir": "planning/scholar_probe_tmp/fixtures/run_20260216T182303Z" +} \ No newline at end of file diff --git a/planning/scholar_probe_tmp/notes/probe_report_run_20260216T182303Z.md b/planning/scholar_probe_tmp/notes/probe_report_run_20260216T182303Z.md new file mode 100644 index 0000000..4f58d08 --- /dev/null +++ b/planning/scholar_probe_tmp/notes/probe_report_run_20260216T182303Z.md @@ -0,0 +1,83 @@ +# Scholar Scrape Probe Report + +Generated UTC: `2026-02-16T18:23:03.906832+00:00` +Run fixtures dir: `planning/scholar_probe_tmp/fixtures/run_20260216T182303Z` + +## Robots Snapshot + +```text +User-agent: * +Disallow: /search +Disallow: /index.html +Disallow: /scholar +Disallow: /citations? +Allow: /citations?user= +Disallow: /citations?*cstart= +Disallow: /citations?user=*%40 +Disallow: /citations?user=*@ +Allow: /citations?view_op=list_classic_articles +Allow: /citations?view_op=mandates_leaderboard +Allow: /citations?view_op=metrics_intro +Allow: /citations?view_op=new_profile +Allow: /citations?view_op=sitemap +Allow: /citations?view_op=top_venues + +User-agent: Twitterbot +Disallow: + +User-agent: facebookexternalhit +Disallow: + +User-agent: PetalBot +Disallow: / +``` + +## Fetch Summary + +| Source | Status Code | Status/Error | Final URL | +| --- | --- | --- | --- | + +## Parse Summary + +| Source | Parse Status | Profile | Publications | Articles Range | Show More | Warnings | +| --- | --- | --- | --- | --- | --- | --- | +| `profile_AAAAAAAAAAAA` | `layout_changed` | - | 0 | - | no | no_rows_detected | +| `profile_LZ5D_p4AAAAJ` | `ok` | Doaa Elmatary | 12 | Articles 1–12 | yes | possible_partial_page_show_more_present | +| `profile_P1RwlvoAAAAJ` | `ok` | WENRUI ZUO | 5 | Articles 1–5 | yes | possible_partial_page_show_more_present | +| `profile_RxmmtT8AAAAJ` | `ok` | K. Srinivasan | 20 | Articles 1–20 | yes | possible_partial_page_show_more_present | +| `profile_amIMrIEAAAAJ` | `ok` | Bangar Raju Cherukuri | 20 | Articles 1–20 | yes | possible_partial_page_show_more_present | +| `profile_AAAAAAAAAAAA` | `layout_changed` | - | 0 | - | no | no_rows_detected | +| `profile_LZ5D_p4AAAAJ` | `ok` | Doaa Elmatary | 12 | Articles 1–12 | yes | possible_partial_page_show_more_present | +| `profile_P1RwlvoAAAAJ` | `ok` | WENRUI ZUO | 5 | Articles 1–5 | yes | possible_partial_page_show_more_present | +| `profile_RxmmtT8AAAAJ` | `ok` | K. Srinivasan | 20 | Articles 1–20 | yes | possible_partial_page_show_more_present | +| `profile_amIMrIEAAAAJ` | `ok` | Bangar Raju Cherukuri | 20 | Articles 1–20 | yes | possible_partial_page_show_more_present | + +## Field Coverage + +Total parsed publication rows: **57** + +| Field | Present | Coverage | +| --- | --- | --- | +| `title` | 57 | 100.0% | +| `cluster_id` | 57 | 100.0% | +| `year` | 56 | 98.2% | +| `citation_count` | 57 | 100.0% | +| `authors_text` | 57 | 100.0% | +| `venue_text` | 54 | 94.7% | + +## Parser Contract Recommendation + +- Primary row marker: `tr.gsc_a_tr`. +- Title anchor marker: `a.gsc_a_at`; derive `cluster_id` from `citation_for_view` query token. +- Metadata text markers: first/second `div.gs_gray` per row for authors and venue. +- Year marker fallback: classes containing `gsc_a_h` or `gsc_a_y` and 4-digit year regex. +- Failure states to persist: `ok`, `no_results`, `blocked_or_captcha`, `layout_changed`, `network_error`. + +## Future-Proofing Notes + +- Keep raw HTML fixture snapshots and update parser tests on DOM drift. +- Treat blocked pages as retriable with backoff, not parser errors. +- If `Show more` is present, treat first-page-only results as partial and surface that in run status/UI. +- Track robots policy changes because `/citations?*cstart=` is currently disallowed. +- Add marker-count assertions in CI to catch silent layout shifts early. +- Use explicit parse status per run/scholar so automation can degrade gracefully. diff --git a/planning/scholar_probe_tmp/notes/probe_report_run_20260216T182320Z.json b/planning/scholar_probe_tmp/notes/probe_report_run_20260216T182320Z.json new file mode 100644 index 0000000..7bd3e52 --- /dev/null +++ b/planning/scholar_probe_tmp/notes/probe_report_run_20260216T182320Z.json @@ -0,0 +1,681 @@ +{ + "fetch_records": [], + "generated_at_utc": "2026-02-16T18:23:20.768051+00:00", + "page_analyses": [ + { + "articles_range": null, + "field_presence": { + "authors_text": 0, + "citation_count": 0, + "cluster_id": 0, + "title": 0, + "venue_text": 0, + "year": 0 + }, + "has_operation_error_banner": false, + "has_show_more_button": false, + "marker_counts": { + "gs_gray": 0, + "gsc_a_ac": 0, + "gsc_a_at": 0, + "gsc_a_h": 0, + "gsc_a_tr": 0, + "gsc_a_y": 0, + "gsc_prf_in": 0, + "gsc_rsb_st": 0 + }, + "parse_warnings": [ + "no_rows_detected" + ], + "profile_name": null, + "publication_count": 0, + "source": "profile_AAAAAAAAAAAA", + "status": "layout_changed" + }, + { + "articles_range": "Articles 1\u201312", + "field_presence": { + "authors_text": 12, + "citation_count": 12, + "cluster_id": 12, + "title": 12, + "venue_text": 10, + "year": 11 + }, + "has_operation_error_banner": false, + "has_show_more_button": true, + "marker_counts": { + "gs_gray": 26, + "gsc_a_ac": 18, + "gsc_a_at": 15, + "gsc_a_h": 31, + "gsc_a_tr": 32, + "gsc_a_y": 26, + "gsc_prf_in": 17, + "gsc_rsb_st": 30 + }, + "parse_warnings": [ + "possible_partial_page_show_more_present" + ], + "profile_name": "Doaa Elmatary", + "publication_count": 12, + "source": "profile_LZ5D_p4AAAAJ", + "status": "ok" + }, + { + "articles_range": "Articles 1\u20135", + "field_presence": { + "authors_text": 5, + "citation_count": 5, + "cluster_id": 5, + "title": 5, + "venue_text": 5, + "year": 5 + }, + "has_operation_error_banner": false, + "has_show_more_button": true, + "marker_counts": { + "gs_gray": 14, + "gsc_a_ac": 11, + "gsc_a_at": 8, + "gsc_a_h": 17, + "gsc_a_tr": 25, + "gsc_a_y": 19, + "gsc_prf_in": 21, + "gsc_rsb_st": 30 + }, + "parse_warnings": [ + "possible_partial_page_show_more_present" + ], + "profile_name": "WENRUI ZUO", + "publication_count": 5, + "source": "profile_P1RwlvoAAAAJ", + "status": "ok" + }, + { + "articles_range": "Articles 1\u201320", + "field_presence": { + "authors_text": 20, + "citation_count": 20, + "cluster_id": 20, + "title": 20, + "venue_text": 19, + "year": 20 + }, + "has_operation_error_banner": false, + "has_show_more_button": true, + "marker_counts": { + "gs_gray": 42, + "gsc_a_ac": 26, + "gsc_a_at": 23, + "gsc_a_h": 47, + "gsc_a_tr": 40, + "gsc_a_y": 34, + "gsc_prf_in": 20, + "gsc_rsb_st": 30 + }, + "parse_warnings": [ + "possible_partial_page_show_more_present" + ], + "profile_name": "K. Srinivasan", + "publication_count": 20, + "source": "profile_RxmmtT8AAAAJ", + "status": "ok" + }, + { + "articles_range": "Articles 1\u201320", + "field_presence": { + "authors_text": 20, + "citation_count": 20, + "cluster_id": 20, + "title": 20, + "venue_text": 20, + "year": 20 + }, + "has_operation_error_banner": false, + "has_show_more_button": true, + "marker_counts": { + "gs_gray": 42, + "gsc_a_ac": 26, + "gsc_a_at": 23, + "gsc_a_h": 47, + "gsc_a_tr": 40, + "gsc_a_y": 34, + "gsc_prf_in": 21, + "gsc_rsb_st": 30 + }, + "parse_warnings": [ + "possible_partial_page_show_more_present" + ], + "profile_name": "Bangar Raju Cherukuri", + "publication_count": 20, + "source": "profile_amIMrIEAAAAJ", + "status": "ok" + } + ], + "publications_by_source": { + "profile_AAAAAAAAAAAA": [], + "profile_LZ5D_p4AAAAJ": [ + { + "authors_text": "EM Abd Allah, DE El-Matary, EM Eid, AST El Dien", + "citation_count": 25, + "cluster_id": "u-x6o8ySG0sC", + "title": "Performance comparison of various machine learning approaches to identify the best one in predicting heart disease", + "title_url": "/citations?view_op=view_citation&hl=en&user=LZ5D_p4AAAAJ&citation_for_view=LZ5D_p4AAAAJ:u-x6o8ySG0sC", + "venue_text": "Journal of Computer and Communications 10 (2), 1-18, 2022", + "year": 2022 + }, + { + "authors_text": "EAAA Hagras, D El-Saied, HH Aly", + "citation_count": 15, + "cluster_id": "YsMSGLbcyi4C", + "title": "Energy efficient key management scheme based on elliptic curve signcryption for wireless sensor networks", + "title_url": "/citations?view_op=view_citation&hl=en&user=LZ5D_p4AAAAJ&citation_for_view=LZ5D_p4AAAAJ:YsMSGLbcyi4C", + "venue_text": "2011 28th National Radio Science Conference (NRSC), 1-9, 2011", + "year": 2011 + }, + { + "authors_text": "EA Hagras, D El-Saied, HH Aly", + "citation_count": 6, + "cluster_id": "Y0pCki6q_DkC", + "title": "A new forward secure elliptic curve signcryption key management (fs-ecskm) scheme for heterogeneous wireless sensor networks", + "title_url": "/citations?view_op=view_citation&hl=en&user=LZ5D_p4AAAAJ&citation_for_view=LZ5D_p4AAAAJ:Y0pCki6q_DkC", + "venue_text": "International Journal of Computer Science and Technology 2 (2), 19-23, 2011", + "year": 2011 + }, + { + "authors_text": "DEE Matary, EA Hagras, HM Abdel-Kader", + "citation_count": 4, + "cluster_id": "d1gkVwhDpl0C", + "title": "Performance of polar codes for OFDM-based UWB channel", + "title_url": "/citations?view_op=view_citation&hl=en&user=LZ5D_p4AAAAJ&citation_for_view=LZ5D_p4AAAAJ:d1gkVwhDpl0C", + "venue_text": "Journal of Computer and Communications 6 (03), 102-117, 2018", + "year": 2018 + }, + { + "authors_text": "DEE Matary, EAA Hagras, HM Abdel-Kader", + "citation_count": 3, + "cluster_id": "u5HHmVD_uO8C", + "title": "Multi-user Communication Based OFDM-UWB System under Gaussian and Non-Gaussian Noisy Channel", + "title_url": "/citations?view_op=view_citation&hl=en&user=LZ5D_p4AAAAJ&citation_for_view=LZ5D_p4AAAAJ:u5HHmVD_uO8C", + "venue_text": "American Journal of Electrical and Electronic Engineering 5 (4), 136-143, 2017", + "year": 2017 + }, + { + "authors_text": "EAAA Hagras", + "citation_count": 2, + "cluster_id": "W7OEmFMy1HYC", + "title": "Doaa El-Saied, Dr. Hazem H. Aly,\u201cA New Forward Secure Elliptic Curve Signcryption Key Management (FS-ECSKM) Scheme for Heterogeneous Wireless Sensor Networks\u201d", + "title_url": "/citations?view_op=view_citation&hl=en&user=LZ5D_p4AAAAJ&citation_for_view=LZ5D_p4AAAAJ:W7OEmFMy1HYC", + "venue_text": "International Journal of Computer Science and Technology 2 (2), 19-23, 2011", + "year": 2011 + }, + { + "authors_text": "DE El-Matary, EAA Hagras, HM Abdel-Kader", + "citation_count": 1, + "cluster_id": "_FxGoFyzp5QC", + "title": "Performance Analysis of LDPC-IDMA-UWB Signals in Non-Gaussian Noisy Channel", + "title_url": "/citations?view_op=view_citation&hl=en&user=LZ5D_p4AAAAJ&citation_for_view=LZ5D_p4AAAAJ:_FxGoFyzp5QC", + "venue_text": null, + "year": 2013 + }, + { + "authors_text": "DE Elmatary", + "citation_count": 0, + "cluster_id": "ufrVoPGSRksC", + "title": "Intelligent Sign Multi-Language Real-Time Prediction System with Effective Data Preprocessing", + "title_url": "/citations?view_op=view_citation&hl=en&user=LZ5D_p4AAAAJ&citation_for_view=LZ5D_p4AAAAJ:ufrVoPGSRksC", + "venue_text": "Journal of Computer and Communications, 120-134, 2023", + "year": 2023 + }, + { + "authors_text": "E M AbdAllah, D E El Matary", + "citation_count": 0, + "cluster_id": "WF5omc3nYNoC", + "title": "Smart Healthcare System Based IoT with Frequency Analysis of Heart Rate Data across FFT Algorithmto be used During Covid-19 Pandemic", + "title_url": "/citations?view_op=view_citation&hl=en&user=LZ5D_p4AAAAJ&citation_for_view=LZ5D_p4AAAAJ:WF5omc3nYNoC", + "venue_text": "Engineering Research Journal (Shoubra) 51 (1), 88-94, 2022", + "year": 2022 + }, + { + "authors_text": "DE El Matary, EAA Hagras, HM Abdel-Kader", + "citation_count": 0, + "cluster_id": "9yKSN-GCB0IC", + "title": "Polar coded Interleave Division Multiple Access Ultra Wide Band (IDMA-UWB) communication system", + "title_url": "/citations?view_op=view_citation&hl=en&user=LZ5D_p4AAAAJ&citation_for_view=LZ5D_p4AAAAJ:9yKSN-GCB0IC", + "venue_text": "2018 35th National Radio Science Conference (NRSC), 283-292, 2018", + "year": 2018 + }, + { + "authors_text": "HMAK Doaa E.El-Matary, Esam A.A. Hagras", + "citation_count": 0, + "cluster_id": "zYLM7Y9cAGgC", + "title": "Performance Analysis of LDPC-IDMA-UWB Signals in Non-Gaussian Noisy Channel", + "title_url": "/citations?view_op=view_citation&hl=en&user=LZ5D_p4AAAAJ&citation_for_view=LZ5D_p4AAAAJ:zYLM7Y9cAGgC", + "venue_text": "International Journal of Scientific & Engineering Research, 2013", + "year": 2013 + }, + { + "authors_text": "DEE Matary, EAA Hagras, HM Abdel-Kader", + "citation_count": 0, + "cluster_id": "eQOLeE2rZwMC", + "title": "Non-Gaussian Noisy Channel Effect on Multi-user IDMA-UWB Communication System", + "title_url": "/citations?view_op=view_citation&hl=en&user=LZ5D_p4AAAAJ&citation_for_view=LZ5D_p4AAAAJ:eQOLeE2rZwMC", + "venue_text": null, + "year": null + } + ], + "profile_P1RwlvoAAAAJ": [ + { + "authors_text": "W Zuo, A Raman, RJ Mondrag\u00f3n, G Tyson", + "citation_count": 18, + "cluster_id": "u5HHmVD_uO8C", + "title": "Set in stone: Analysis of an immutable web3 social media platform", + "title_url": "/citations?view_op=view_citation&hl=en&user=P1RwlvoAAAAJ&citation_for_view=P1RwlvoAAAAJ:u5HHmVD_uO8C", + "venue_text": "Proceedings of the ACM Web Conference 2023, 1865-1874, 2023", + "year": 2023 + }, + { + "authors_text": "W Zuo, RJ Mondragon, A Raman, G Tyson", + "citation_count": 10, + "cluster_id": "d1gkVwhDpl0C", + "title": "Understanding and improving content moderation in web3 platforms", + "title_url": "/citations?view_op=view_citation&hl=en&user=P1RwlvoAAAAJ&citation_for_view=P1RwlvoAAAAJ:d1gkVwhDpl0C", + "venue_text": "Proceedings of the International AAAI Conference on Web and Social Media 18 \u2026, 2024", + "year": 2024 + }, + { + "authors_text": "W Zuo, A Raman, RJ Mondrag\u00d3N, G Tyson", + "citation_count": 7, + "cluster_id": "u-x6o8ySG0sC", + "title": "A first look at user-controlled moderation on web3 social media: The case of memo. cash", + "title_url": "/citations?view_op=view_citation&hl=en&user=P1RwlvoAAAAJ&citation_for_view=P1RwlvoAAAAJ:u-x6o8ySG0sC", + "venue_text": "Proceedings of the 3rd International Workshop on Open Challenges in Online \u2026, 2023", + "year": 2023 + }, + { + "authors_text": "Y Liu, RM Williams, G Xie, Y Wang, W Zuo", + "citation_count": 4, + "cluster_id": "9yKSN-GCB0IC", + "title": "Promoting the Culture of Qinhuai River Lantern Shadow Puppetry with a Digital Archive and Immersive Experience", + "title_url": "/citations?view_op=view_citation&hl=en&user=P1RwlvoAAAAJ&citation_for_view=P1RwlvoAAAAJ:9yKSN-GCB0IC", + "venue_text": "arXiv preprint arXiv:2410.03532, 2024", + "year": 2024 + }, + { + "authors_text": "W Zuo", + "citation_count": 0, + "cluster_id": "2osOgNQ5qMEC", + "title": "Understanding and Improving User-controlled Content Moderation Systems on Social Media", + "title_url": "/citations?view_op=view_citation&hl=en&user=P1RwlvoAAAAJ&citation_for_view=P1RwlvoAAAAJ:2osOgNQ5qMEC", + "venue_text": "Queen Mary University of London, 2025", + "year": 2025 + } + ], + "profile_RxmmtT8AAAAJ": [ + { + "authors_text": "K Srinivasan", + "citation_count": 1185, + "cluster_id": "qjMakFHDy7sC", + "title": "Black pepper and its pungent principle-piperine: a review of diverse physiological effects", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:qjMakFHDy7sC", + "venue_text": "Critical reviews in food science and nutrition 47 (8), 735-748, 2007", + "year": 2007 + }, + { + "authors_text": "K Platel, K Srinivasan", + "citation_count": 742, + "cluster_id": "W7OEmFMy1HYC", + "title": "Digestive stimulant action of spices: a myth or reality?", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:W7OEmFMy1HYC", + "venue_text": "Indian Journal of Medical Research 119 (5), 167, 2004", + "year": 2004 + }, + { + "authors_text": "K Platel, K Srinivasan", + "citation_count": 682, + "cluster_id": "u-x6o8ySG0sC", + "title": "Influence of dietary spices and their active principles on pancreatic digestive enzymes in albino rats", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:u-x6o8ySG0sC", + "venue_text": "Food/Nahrung 44 (1), 42-46, 2000", + "year": 2000 + }, + { + "authors_text": "K Srinivasan", + "citation_count": 663, + "cluster_id": "LkGwnXOMwfcC", + "title": "Fenugreek (Trigonella foenum-graecum): A Review of Health Beneficial Physiological Effects", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:LkGwnXOMwfcC", + "venue_text": "Food reviews international 22 (2), 203-224, 2006", + "year": 2006 + }, + { + "authors_text": "K Srinivasan", + "citation_count": 574, + "cluster_id": "FPJr55Dyh1AC", + "title": "Biological activities of red pepper (Capsicum annuum) and its pungent principle capsaicin: a review", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:FPJr55Dyh1AC", + "venue_text": "Critical reviews in food science and nutrition 56 (9), 1488-1500, 2016", + "year": 2016 + }, + { + "authors_text": "K Srinivasan", + "citation_count": 555, + "cluster_id": "9yKSN-GCB0IC", + "title": "Plant foods in the management of diabetes mellitus: spices as beneficial antidiabetic food adjuncts", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:9yKSN-GCB0IC", + "venue_text": "International journal of food sciences and nutrition 56 (6), 399-414, 2005", + "year": 2005 + }, + { + "authors_text": "PS Babu, K Srinivasan", + "citation_count": 539, + "cluster_id": "ZfRJV9d4-WMC", + "title": "Hypolipidemic action of curcumin, the active principle of turmeric (Curcuma longa) in streptozotocin induced diabetic rats", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:ZfRJV9d4-WMC", + "venue_text": "Molecular and cellular biochemistry 166 (1), 169-175, 1997", + "year": 1997 + }, + { + "authors_text": "K Srinivasan", + "citation_count": 513, + "cluster_id": "IjCSPb-OGe4C", + "title": "Spices as influencers of body metabolism: an overview of three decades of research", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:IjCSPb-OGe4C", + "venue_text": "Food Research International 38 (1), 77-86, 2005", + "year": 2005 + }, + { + "authors_text": "K Srinivasan", + "citation_count": 455, + "cluster_id": "Tyk-4Ss8FVUC", + "title": "Role of spices beyond food flavoring: Nutraceuticals with multiple health effects", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:Tyk-4Ss8FVUC", + "venue_text": "Food reviews international 21 (2), 167-188, 2005", + "year": 2005 + }, + { + "authors_text": "K Srinivasan", + "citation_count": 446, + "cluster_id": "tkaPQYYpVKoC", + "title": "Antioxidant potential of spices and their active constituents", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:tkaPQYYpVKoC", + "venue_text": "Critical Reviews in Food Science and Nutrition 54 (3), 252-272, 2013", + "year": 2013 + }, + { + "authors_text": "D Suresh, K Srinivasan", + "citation_count": 387, + "cluster_id": "5nxA0vEk-isC", + "title": "Tissue distribution and elimination of capsaicin, piperine & curcumin following oral intake in rats.", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:5nxA0vEk-isC", + "venue_text": "Indian Journal of Medical Research 131, 682-691, 2010", + "year": 2010 + }, + { + "authors_text": "R Ramakrishna Rao, K Platel, K Srinivasan", + "citation_count": 368, + "cluster_id": "YsMSGLbcyi4C", + "title": "In vitro influence of spices and spice\u2010active principles on digestive enzymes of rat pancreas and small intestine", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:YsMSGLbcyi4C", + "venue_text": "Food/Nahrung 47 (6), 408-412, 2003", + "year": 2003 + }, + { + "authors_text": "K Srinivasan", + "citation_count": 367, + "cluster_id": "zCSUwVk65WsC", + "title": "Cumin (Cuminum cyminum) and black cumin (Nigella sativa) seeds: traditional uses, chemical constituents, and nutraceutical effects", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:zCSUwVk65WsC", + "venue_text": "Food quality and safety 2 (1), 1-16, 2018", + "year": 2018 + }, + { + "authors_text": "K Srinivasan", + "citation_count": 364, + "cluster_id": "yqoGN6RLRZoC", + "title": "Ginger rhizomes (Zingiber officinale): A spice with multiple health beneficial potentials", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:yqoGN6RLRZoC", + "venue_text": "PharmaNutrition 5 (1), 18-28, 2017", + "year": 2017 + }, + { + "authors_text": "K Platel, K Srinivasan", + "citation_count": 364, + "cluster_id": "d1gkVwhDpl0C", + "title": "Influence of dietary spices or their active principles on digestive enzymes of small intestinal mucosa in rats", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:d1gkVwhDpl0C", + "venue_text": "International Journal of Food Sciences and Nutrition 47 (1), 55-59, 1996", + "year": 1996 + }, + { + "authors_text": "K Platel, K Srinivasan", + "citation_count": 296, + "cluster_id": "UeHWp8X0CEIC", + "title": "Plant foods in the management of diabetes mellitus: vegetables as potential hypoglycaemic agents", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:UeHWp8X0CEIC", + "venue_text": "Food/Nahrung 41 (2), 68-74, 1997", + "year": 1997 + }, + { + "authors_text": "K Srinivasan, K Sambaiah", + "citation_count": 292, + "cluster_id": "2osOgNQ5qMEC", + "title": "The effect of spices on cholesterol 7\u03b1-hydroxylase activity and on serum and hepatic cholesterol levels in the rat.", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:2osOgNQ5qMEC", + "venue_text": null, + "year": 1991 + }, + { + "authors_text": "S Hemalatha, K Platel, K Srinivasan", + "citation_count": 269, + "cluster_id": "UebtZRa9Y70C", + "title": "Zinc and iron contents and their bioaccessibility in cereals and pulses consumed in India", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:UebtZRa9Y70C", + "venue_text": "Food chemistry 102 (4), 1328-1336, 2007", + "year": 2007 + }, + { + "authors_text": "K Platel, K Srinivasan", + "citation_count": 258, + "cluster_id": "_FxGoFyzp5QC", + "title": "Studies on the influence of dietary spices on food transit time in experimental rats", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:_FxGoFyzp5QC", + "venue_text": "Nutrition Research 21 (9), 1309-1314, 2001", + "year": 2001 + }, + { + "authors_text": "K Platel, K Srinivasan", + "citation_count": 240, + "cluster_id": "hCrLmN-GePgC", + "title": "Bioavailability of micronutrients from plant foods: an update", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:hCrLmN-GePgC", + "venue_text": "Critical reviews in food science and nutrition 56 (10), 1608-1619, 2016", + "year": 2016 + } + ], + "profile_amIMrIEAAAAJ": [ + { + "authors_text": "BR Cherukuri, V Arulkumar", + "citation_count": 131, + "cluster_id": "u5HHmVD_uO8C", + "title": "Optimization of Data Structures and Trade-Offs with Concurrency Control in Multithread Software Structures Using Artificial Intelligence", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:u5HHmVD_uO8C", + "venue_text": "2024 IEEE International Conference on Computing, Power and Communication \u2026, 2024", + "year": 2024 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 81, + "cluster_id": "d1gkVwhDpl0C", + "title": "Development of Design Patterns with Adaptive User Interface for Cloud Native Microservice Architecture Using Deep Learning With IoT", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:d1gkVwhDpl0C", + "venue_text": "2024 IEEE International Conference on Computing, Power and Communication \u2026, 2024", + "year": 2024 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 33, + "cluster_id": "zYLM7Y9cAGgC", + "title": "AI-powered personalization: How machine learning is shaping the future of user experience", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:zYLM7Y9cAGgC", + "venue_text": "International Journal of Science and Research Archive (IJSRA) 12 (1), 3111\u20133126, 2024", + "year": 2024 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 30, + "cluster_id": "roLk4NBRz8UC", + "title": "Future of cloud computing: Innovations in multi-cloud and hybrid architectures", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:roLk4NBRz8UC", + "venue_text": "World Journal of Advanced Research and Reviews (WJARR) 1 (1), 068-081, 2019", + "year": 2019 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 27, + "cluster_id": "UebtZRa9Y70C", + "title": "Microservices and containerization: Accelerating web development cycles", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:UebtZRa9Y70C", + "venue_text": "World Journal of Advanced Research and Reviews (WJARR) 6 (01), 283-296, 2020", + "year": 2020 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 18, + "cluster_id": "hqOjcs7Dif8C", + "title": "Ethical AI in cloud: Mitigating risks in machine learning models", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:hqOjcs7Dif8C", + "venue_text": "World Journal of Advanced Engineering Technology and Sciences ((WJAETS) 1 \u2026, 2020", + "year": 2020 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 14, + "cluster_id": "0EnyYjriUFMC", + "title": "Quantum machine learning: Transforming cloud-based AI solutions", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:0EnyYjriUFMC", + "venue_text": "International Journal of Science and Research Archive (IJSRA) 1 (01), 110-122, 2020", + "year": 2020 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 13, + "cluster_id": "WF5omc3nYNoC", + "title": "Enhanced Trimodal Emotion Recognition Using Multibranch Fusion Attention with Epistemic Neural Networks and Fire Hawk Optimization", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:WF5omc3nYNoC", + "venue_text": "Journal of Machine and Computing 5 (1), 2025", + "year": 2025 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 13, + "cluster_id": "W7OEmFMy1HYC", + "title": "Containerization in cloud computing: comparing Docker and Kubernetes for scalable web applications", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:W7OEmFMy1HYC", + "venue_text": "International Journal of Science and Research Archive (IJSRA) 13 (1), 3302\u20133315, 2024", + "year": 2024 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 13, + "cluster_id": "IjCSPb-OGe4C", + "title": "Serverless computing: How to build and deploy applications without managing infrastructure", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:IjCSPb-OGe4C", + "venue_text": "World Journal of Advanced Engineering Technology and Sciences ((WJAETS) 11 \u2026, 2024", + "year": 2024 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 12, + "cluster_id": "YsMSGLbcyi4C", + "title": "Edge Computing vs. Cloud Computing: A Comparative Analysis for Real-Time AI Applications", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:YsMSGLbcyi4C", + "venue_text": "International Journal For Multidisciplinary Research (IJFMR) 6 (5), 1-17, 2024", + "year": 2024 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 9, + "cluster_id": "Se3iqnhoufwC", + "title": "Serverless revolution: Redefining application scalability and cost efficiency", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:Se3iqnhoufwC", + "venue_text": "World Journal of Advanced Research and Reviews (WJARR) 2 (03), 039-053, 2019", + "year": 2019 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 8, + "cluster_id": "Tyk-4Ss8FVUC", + "title": "Progressive Web Apps (PWAs): Enhancing User Experience through Modern Web Development", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:Tyk-4Ss8FVUC", + "venue_text": "International Journal of Science and Research (IJSR) 13 (10), 1550-1560, 2024", + "year": 2024 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 7, + "cluster_id": "UeHWp8X0CEIC", + "title": "Building Scalable Web Applications: Best Practices for Backend Architecture", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:UeHWp8X0CEIC", + "venue_text": "International Journal of Science and Research (IJSR) 13 (10), 126-139, 2024", + "year": 2024 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 5, + "cluster_id": "5nxA0vEk-isC", + "title": "Advanced Multi Class Cyber Security Attack Classification in IoT Based Wireless Sensor Networks Using Context Aware Depthwise Separable Convolutional Neural Network", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:5nxA0vEk-isC", + "venue_text": "Journal of Machine and Computing 5 (2), 2025", + "year": 2025 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 3, + "cluster_id": "u-x6o8ySG0sC", + "title": "Maintenance of Web Development Standard for Multiple Devices with Serverless Computing through Cross Browser Affinity Using Hybrid Optimization", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:u-x6o8ySG0sC", + "venue_text": "2024 IEEE International Conference on Computing, Power and Communication \u2026, 2024", + "year": 2024 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 3, + "cluster_id": "LkGwnXOMwfcC", + "title": "Developing Intelligent Chatbots for Real-Time Customer Support in E-Commerce", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:LkGwnXOMwfcC", + "venue_text": "International Journal of Science and Research (IJSR) 11 (01), 1709-1719, 2022", + "year": 2022 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 3, + "cluster_id": "_FxGoFyzp5QC", + "title": "Enhancing Web Application Performance with AI - Driven Optimization Techniques", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:_FxGoFyzp5QC", + "venue_text": "International Journal of Science and Research (IJSR) 10 (2), 1779-1788, 2021", + "year": 2021 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 1, + "cluster_id": "Y0pCki6q_DkC", + "title": "Federated Learning: Privacy-Preserving Machine Learning in Cloud Environments", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:Y0pCki6q_DkC", + "venue_text": "International Journal of Science and Research (IJSR) 13 (10), 1539-1549, 2024", + "year": 2024 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 1, + "cluster_id": "ufrVoPGSRksC", + "title": "Scalable machine learning model deployment using serverless cloud architectures", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:ufrVoPGSRksC", + "venue_text": "World Journal of Advanced Engineering Technology and Sciences ((WJAETS) 5 (1 \u2026, 2022", + "year": 2022 + } + ] + }, + "run_dir": "planning/scholar_probe_tmp/fixtures/run_20260216T182320Z" +} \ No newline at end of file diff --git a/planning/scholar_probe_tmp/notes/probe_report_run_20260216T182320Z.md b/planning/scholar_probe_tmp/notes/probe_report_run_20260216T182320Z.md new file mode 100644 index 0000000..d59ec1d --- /dev/null +++ b/planning/scholar_probe_tmp/notes/probe_report_run_20260216T182320Z.md @@ -0,0 +1,78 @@ +# Scholar Scrape Probe Report + +Generated UTC: `2026-02-16T18:23:20.768051+00:00` +Run fixtures dir: `planning/scholar_probe_tmp/fixtures/run_20260216T182320Z` + +## Robots Snapshot + +```text +User-agent: * +Disallow: /search +Disallow: /index.html +Disallow: /scholar +Disallow: /citations? +Allow: /citations?user= +Disallow: /citations?*cstart= +Disallow: /citations?user=*%40 +Disallow: /citations?user=*@ +Allow: /citations?view_op=list_classic_articles +Allow: /citations?view_op=mandates_leaderboard +Allow: /citations?view_op=metrics_intro +Allow: /citations?view_op=new_profile +Allow: /citations?view_op=sitemap +Allow: /citations?view_op=top_venues + +User-agent: Twitterbot +Disallow: + +User-agent: facebookexternalhit +Disallow: + +User-agent: PetalBot +Disallow: / +``` + +## Fetch Summary + +| Source | Status Code | Status/Error | Final URL | +| --- | --- | --- | --- | + +## Parse Summary + +| Source | Parse Status | Profile | Publications | Articles Range | Show More | Warnings | +| --- | --- | --- | --- | --- | --- | --- | +| `profile_AAAAAAAAAAAA` | `layout_changed` | - | 0 | - | no | no_rows_detected | +| `profile_LZ5D_p4AAAAJ` | `ok` | Doaa Elmatary | 12 | Articles 1–12 | yes | possible_partial_page_show_more_present | +| `profile_P1RwlvoAAAAJ` | `ok` | WENRUI ZUO | 5 | Articles 1–5 | yes | possible_partial_page_show_more_present | +| `profile_RxmmtT8AAAAJ` | `ok` | K. Srinivasan | 20 | Articles 1–20 | yes | possible_partial_page_show_more_present | +| `profile_amIMrIEAAAAJ` | `ok` | Bangar Raju Cherukuri | 20 | Articles 1–20 | yes | possible_partial_page_show_more_present | + +## Field Coverage + +Total parsed publication rows: **57** + +| Field | Present | Coverage | +| --- | --- | --- | +| `title` | 57 | 100.0% | +| `cluster_id` | 57 | 100.0% | +| `year` | 56 | 98.2% | +| `citation_count` | 57 | 100.0% | +| `authors_text` | 57 | 100.0% | +| `venue_text` | 54 | 94.7% | + +## Parser Contract Recommendation + +- Primary row marker: `tr.gsc_a_tr`. +- Title anchor marker: `a.gsc_a_at`; derive `cluster_id` from `citation_for_view` query token. +- Metadata text markers: first/second `div.gs_gray` per row for authors and venue. +- Year marker fallback: classes containing `gsc_a_h` or `gsc_a_y` and 4-digit year regex. +- Failure states to persist: `ok`, `no_results`, `blocked_or_captcha`, `layout_changed`, `network_error`. + +## Future-Proofing Notes + +- Keep raw HTML fixture snapshots and update parser tests on DOM drift. +- Treat blocked pages as retriable with backoff, not parser errors. +- If `Show more` is present, treat first-page-only results as partial and surface that in run status/UI. +- Track robots policy changes because `/citations?*cstart=` is currently disallowed. +- Add marker-count assertions in CI to catch silent layout shifts early. +- Use explicit parse status per run/scholar so automation can degrade gracefully. diff --git a/planning/scholar_probe_tmp/notes/probe_report_run_20260216T182334Z.json b/planning/scholar_probe_tmp/notes/probe_report_run_20260216T182334Z.json new file mode 100644 index 0000000..051a952 --- /dev/null +++ b/planning/scholar_probe_tmp/notes/probe_report_run_20260216T182334Z.json @@ -0,0 +1,742 @@ +{ + "fetch_records": [ + { + "elapsed_seconds": 0.054, + "error": null, + "fetched_at_utc": "2026-02-16T18:23:34.228993+00:00", + "file_name": "robots.txt", + "final_url": "https://scholar.google.com/robots.txt", + "source": "robots", + "status_code": 200, + "url": "https://scholar.google.com/robots.txt" + }, + { + "elapsed_seconds": 0.792, + "error": null, + "fetched_at_utc": "2026-02-16T18:23:35.021068+00:00", + "file_name": "profile_amIMrIEAAAAJ.html", + "final_url": "https://scholar.google.com/citations?hl=en&user=amIMrIEAAAAJ", + "source": "profile_amIMrIEAAAAJ", + "status_code": 200, + "url": "https://scholar.google.com/citations?hl=en&user=amIMrIEAAAAJ" + }, + { + "elapsed_seconds": 0.293, + "error": null, + "fetched_at_utc": "2026-02-16T18:23:39.900012+00:00", + "file_name": "profile_P1RwlvoAAAAJ.html", + "final_url": "https://scholar.google.com/citations?hl=en&user=P1RwlvoAAAAJ", + "source": "profile_P1RwlvoAAAAJ", + "status_code": 200, + "url": "https://scholar.google.com/citations?hl=en&user=P1RwlvoAAAAJ" + }, + { + "elapsed_seconds": 0.808, + "error": null, + "fetched_at_utc": "2026-02-16T18:23:45.650527+00:00", + "file_name": "profile_RxmmtT8AAAAJ.html", + "final_url": "https://scholar.google.com/citations?hl=en&user=RxmmtT8AAAAJ", + "source": "profile_RxmmtT8AAAAJ", + "status_code": 200, + "url": "https://scholar.google.com/citations?hl=en&user=RxmmtT8AAAAJ" + }, + { + "elapsed_seconds": 0.795, + "error": null, + "fetched_at_utc": "2026-02-16T18:23:50.907480+00:00", + "file_name": "profile_LZ5D_p4AAAAJ.html", + "final_url": "https://scholar.google.com/citations?hl=en&user=LZ5D_p4AAAAJ", + "source": "profile_LZ5D_p4AAAAJ", + "status_code": 200, + "url": "https://scholar.google.com/citations?hl=en&user=LZ5D_p4AAAAJ" + }, + { + "elapsed_seconds": 0.995, + "error": null, + "fetched_at_utc": "2026-02-16T18:23:56.367359+00:00", + "file_name": "profile_AAAAAAAAAAAA.html", + "final_url": "https://accounts.google.com/v3/signin/identifier?continue=https%3A%2F%2Fscholar.google.com%2Fcitations%3Fhl%3Den%26user%3DAAAAAAAAAAAA&dsh=S-1576812270%3A1771266236087781&hl=en&ifkv=ASfE1-pcNcR5S-R4fUZcbEWwaMAwsKrYMzr8-7CFvIajHFoW7EhUN90SrdKqfzzkHfWDxL97xvHq&service=citations&flowName=GlifWebSignIn&flowEntry=ServiceLogin", + "source": "profile_AAAAAAAAAAAA", + "status_code": 200, + "url": "https://scholar.google.com/citations?hl=en&user=AAAAAAAAAAAA" + } + ], + "generated_at_utc": "2026-02-16T18:23:56.436305+00:00", + "page_analyses": [ + { + "articles_range": null, + "field_presence": { + "authors_text": 0, + "citation_count": 0, + "cluster_id": 0, + "title": 0, + "venue_text": 0, + "year": 0 + }, + "has_operation_error_banner": false, + "has_show_more_button": false, + "marker_counts": { + "gs_gray": 0, + "gsc_a_ac": 0, + "gsc_a_at": 0, + "gsc_a_h": 0, + "gsc_a_tr": 0, + "gsc_a_y": 0, + "gsc_prf_in": 0, + "gsc_rsb_st": 0 + }, + "parse_warnings": [ + "no_rows_detected" + ], + "profile_name": null, + "publication_count": 0, + "source": "profile_AAAAAAAAAAAA", + "status": "blocked_or_captcha" + }, + { + "articles_range": "Articles 1\u201312", + "field_presence": { + "authors_text": 12, + "citation_count": 12, + "cluster_id": 12, + "title": 12, + "venue_text": 10, + "year": 11 + }, + "has_operation_error_banner": false, + "has_show_more_button": true, + "marker_counts": { + "gs_gray": 26, + "gsc_a_ac": 18, + "gsc_a_at": 15, + "gsc_a_h": 31, + "gsc_a_tr": 32, + "gsc_a_y": 26, + "gsc_prf_in": 17, + "gsc_rsb_st": 30 + }, + "parse_warnings": [ + "possible_partial_page_show_more_present" + ], + "profile_name": "Doaa Elmatary", + "publication_count": 12, + "source": "profile_LZ5D_p4AAAAJ", + "status": "ok" + }, + { + "articles_range": "Articles 1\u20135", + "field_presence": { + "authors_text": 5, + "citation_count": 5, + "cluster_id": 5, + "title": 5, + "venue_text": 5, + "year": 5 + }, + "has_operation_error_banner": false, + "has_show_more_button": true, + "marker_counts": { + "gs_gray": 14, + "gsc_a_ac": 11, + "gsc_a_at": 8, + "gsc_a_h": 17, + "gsc_a_tr": 25, + "gsc_a_y": 19, + "gsc_prf_in": 21, + "gsc_rsb_st": 30 + }, + "parse_warnings": [ + "possible_partial_page_show_more_present" + ], + "profile_name": "WENRUI ZUO", + "publication_count": 5, + "source": "profile_P1RwlvoAAAAJ", + "status": "ok" + }, + { + "articles_range": "Articles 1\u201320", + "field_presence": { + "authors_text": 20, + "citation_count": 20, + "cluster_id": 20, + "title": 20, + "venue_text": 19, + "year": 20 + }, + "has_operation_error_banner": false, + "has_show_more_button": true, + "marker_counts": { + "gs_gray": 42, + "gsc_a_ac": 26, + "gsc_a_at": 23, + "gsc_a_h": 47, + "gsc_a_tr": 40, + "gsc_a_y": 34, + "gsc_prf_in": 20, + "gsc_rsb_st": 30 + }, + "parse_warnings": [ + "possible_partial_page_show_more_present" + ], + "profile_name": "K. Srinivasan", + "publication_count": 20, + "source": "profile_RxmmtT8AAAAJ", + "status": "ok" + }, + { + "articles_range": "Articles 1\u201320", + "field_presence": { + "authors_text": 20, + "citation_count": 20, + "cluster_id": 20, + "title": 20, + "venue_text": 20, + "year": 20 + }, + "has_operation_error_banner": false, + "has_show_more_button": true, + "marker_counts": { + "gs_gray": 42, + "gsc_a_ac": 26, + "gsc_a_at": 23, + "gsc_a_h": 47, + "gsc_a_tr": 40, + "gsc_a_y": 34, + "gsc_prf_in": 21, + "gsc_rsb_st": 30 + }, + "parse_warnings": [ + "possible_partial_page_show_more_present" + ], + "profile_name": "Bangar Raju Cherukuri", + "publication_count": 20, + "source": "profile_amIMrIEAAAAJ", + "status": "ok" + } + ], + "publications_by_source": { + "profile_AAAAAAAAAAAA": [], + "profile_LZ5D_p4AAAAJ": [ + { + "authors_text": "EM Abd Allah, DE El-Matary, EM Eid, AST El Dien", + "citation_count": 25, + "cluster_id": "u-x6o8ySG0sC", + "title": "Performance comparison of various machine learning approaches to identify the best one in predicting heart disease", + "title_url": "/citations?view_op=view_citation&hl=en&user=LZ5D_p4AAAAJ&citation_for_view=LZ5D_p4AAAAJ:u-x6o8ySG0sC", + "venue_text": "Journal of Computer and Communications 10 (2), 1-18, 2022", + "year": 2022 + }, + { + "authors_text": "EAAA Hagras, D El-Saied, HH Aly", + "citation_count": 15, + "cluster_id": "YsMSGLbcyi4C", + "title": "Energy efficient key management scheme based on elliptic curve signcryption for wireless sensor networks", + "title_url": "/citations?view_op=view_citation&hl=en&user=LZ5D_p4AAAAJ&citation_for_view=LZ5D_p4AAAAJ:YsMSGLbcyi4C", + "venue_text": "2011 28th National Radio Science Conference (NRSC), 1-9, 2011", + "year": 2011 + }, + { + "authors_text": "EA Hagras, D El-Saied, HH Aly", + "citation_count": 6, + "cluster_id": "Y0pCki6q_DkC", + "title": "A new forward secure elliptic curve signcryption key management (fs-ecskm) scheme for heterogeneous wireless sensor networks", + "title_url": "/citations?view_op=view_citation&hl=en&user=LZ5D_p4AAAAJ&citation_for_view=LZ5D_p4AAAAJ:Y0pCki6q_DkC", + "venue_text": "International Journal of Computer Science and Technology 2 (2), 19-23, 2011", + "year": 2011 + }, + { + "authors_text": "DEE Matary, EA Hagras, HM Abdel-Kader", + "citation_count": 4, + "cluster_id": "d1gkVwhDpl0C", + "title": "Performance of polar codes for OFDM-based UWB channel", + "title_url": "/citations?view_op=view_citation&hl=en&user=LZ5D_p4AAAAJ&citation_for_view=LZ5D_p4AAAAJ:d1gkVwhDpl0C", + "venue_text": "Journal of Computer and Communications 6 (03), 102-117, 2018", + "year": 2018 + }, + { + "authors_text": "DEE Matary, EAA Hagras, HM Abdel-Kader", + "citation_count": 3, + "cluster_id": "u5HHmVD_uO8C", + "title": "Multi-user Communication Based OFDM-UWB System under Gaussian and Non-Gaussian Noisy Channel", + "title_url": "/citations?view_op=view_citation&hl=en&user=LZ5D_p4AAAAJ&citation_for_view=LZ5D_p4AAAAJ:u5HHmVD_uO8C", + "venue_text": "American Journal of Electrical and Electronic Engineering 5 (4), 136-143, 2017", + "year": 2017 + }, + { + "authors_text": "EAAA Hagras", + "citation_count": 2, + "cluster_id": "W7OEmFMy1HYC", + "title": "Doaa El-Saied, Dr. Hazem H. Aly,\u201cA New Forward Secure Elliptic Curve Signcryption Key Management (FS-ECSKM) Scheme for Heterogeneous Wireless Sensor Networks\u201d", + "title_url": "/citations?view_op=view_citation&hl=en&user=LZ5D_p4AAAAJ&citation_for_view=LZ5D_p4AAAAJ:W7OEmFMy1HYC", + "venue_text": "International Journal of Computer Science and Technology 2 (2), 19-23, 2011", + "year": 2011 + }, + { + "authors_text": "DE El-Matary, EAA Hagras, HM Abdel-Kader", + "citation_count": 1, + "cluster_id": "_FxGoFyzp5QC", + "title": "Performance Analysis of LDPC-IDMA-UWB Signals in Non-Gaussian Noisy Channel", + "title_url": "/citations?view_op=view_citation&hl=en&user=LZ5D_p4AAAAJ&citation_for_view=LZ5D_p4AAAAJ:_FxGoFyzp5QC", + "venue_text": null, + "year": 2013 + }, + { + "authors_text": "DE Elmatary", + "citation_count": 0, + "cluster_id": "ufrVoPGSRksC", + "title": "Intelligent Sign Multi-Language Real-Time Prediction System with Effective Data Preprocessing", + "title_url": "/citations?view_op=view_citation&hl=en&user=LZ5D_p4AAAAJ&citation_for_view=LZ5D_p4AAAAJ:ufrVoPGSRksC", + "venue_text": "Journal of Computer and Communications, 120-134, 2023", + "year": 2023 + }, + { + "authors_text": "E M AbdAllah, D E El Matary", + "citation_count": 0, + "cluster_id": "WF5omc3nYNoC", + "title": "Smart Healthcare System Based IoT with Frequency Analysis of Heart Rate Data across FFT Algorithmto be used During Covid-19 Pandemic", + "title_url": "/citations?view_op=view_citation&hl=en&user=LZ5D_p4AAAAJ&citation_for_view=LZ5D_p4AAAAJ:WF5omc3nYNoC", + "venue_text": "Engineering Research Journal (Shoubra) 51 (1), 88-94, 2022", + "year": 2022 + }, + { + "authors_text": "DE El Matary, EAA Hagras, HM Abdel-Kader", + "citation_count": 0, + "cluster_id": "9yKSN-GCB0IC", + "title": "Polar coded Interleave Division Multiple Access Ultra Wide Band (IDMA-UWB) communication system", + "title_url": "/citations?view_op=view_citation&hl=en&user=LZ5D_p4AAAAJ&citation_for_view=LZ5D_p4AAAAJ:9yKSN-GCB0IC", + "venue_text": "2018 35th National Radio Science Conference (NRSC), 283-292, 2018", + "year": 2018 + }, + { + "authors_text": "HMAK Doaa E.El-Matary, Esam A.A. Hagras", + "citation_count": 0, + "cluster_id": "zYLM7Y9cAGgC", + "title": "Performance Analysis of LDPC-IDMA-UWB Signals in Non-Gaussian Noisy Channel", + "title_url": "/citations?view_op=view_citation&hl=en&user=LZ5D_p4AAAAJ&citation_for_view=LZ5D_p4AAAAJ:zYLM7Y9cAGgC", + "venue_text": "International Journal of Scientific & Engineering Research, 2013", + "year": 2013 + }, + { + "authors_text": "DEE Matary, EAA Hagras, HM Abdel-Kader", + "citation_count": 0, + "cluster_id": "eQOLeE2rZwMC", + "title": "Non-Gaussian Noisy Channel Effect on Multi-user IDMA-UWB Communication System", + "title_url": "/citations?view_op=view_citation&hl=en&user=LZ5D_p4AAAAJ&citation_for_view=LZ5D_p4AAAAJ:eQOLeE2rZwMC", + "venue_text": null, + "year": null + } + ], + "profile_P1RwlvoAAAAJ": [ + { + "authors_text": "W Zuo, A Raman, RJ Mondrag\u00f3n, G Tyson", + "citation_count": 18, + "cluster_id": "u5HHmVD_uO8C", + "title": "Set in stone: Analysis of an immutable web3 social media platform", + "title_url": "/citations?view_op=view_citation&hl=en&user=P1RwlvoAAAAJ&citation_for_view=P1RwlvoAAAAJ:u5HHmVD_uO8C", + "venue_text": "Proceedings of the ACM Web Conference 2023, 1865-1874, 2023", + "year": 2023 + }, + { + "authors_text": "W Zuo, RJ Mondragon, A Raman, G Tyson", + "citation_count": 10, + "cluster_id": "d1gkVwhDpl0C", + "title": "Understanding and improving content moderation in web3 platforms", + "title_url": "/citations?view_op=view_citation&hl=en&user=P1RwlvoAAAAJ&citation_for_view=P1RwlvoAAAAJ:d1gkVwhDpl0C", + "venue_text": "Proceedings of the International AAAI Conference on Web and Social Media 18 \u2026, 2024", + "year": 2024 + }, + { + "authors_text": "W Zuo, A Raman, RJ Mondrag\u00d3N, G Tyson", + "citation_count": 7, + "cluster_id": "u-x6o8ySG0sC", + "title": "A first look at user-controlled moderation on web3 social media: The case of memo. cash", + "title_url": "/citations?view_op=view_citation&hl=en&user=P1RwlvoAAAAJ&citation_for_view=P1RwlvoAAAAJ:u-x6o8ySG0sC", + "venue_text": "Proceedings of the 3rd International Workshop on Open Challenges in Online \u2026, 2023", + "year": 2023 + }, + { + "authors_text": "Y Liu, RM Williams, G Xie, Y Wang, W Zuo", + "citation_count": 4, + "cluster_id": "9yKSN-GCB0IC", + "title": "Promoting the Culture of Qinhuai River Lantern Shadow Puppetry with a Digital Archive and Immersive Experience", + "title_url": "/citations?view_op=view_citation&hl=en&user=P1RwlvoAAAAJ&citation_for_view=P1RwlvoAAAAJ:9yKSN-GCB0IC", + "venue_text": "arXiv preprint arXiv:2410.03532, 2024", + "year": 2024 + }, + { + "authors_text": "W Zuo", + "citation_count": 0, + "cluster_id": "2osOgNQ5qMEC", + "title": "Understanding and Improving User-controlled Content Moderation Systems on Social Media", + "title_url": "/citations?view_op=view_citation&hl=en&user=P1RwlvoAAAAJ&citation_for_view=P1RwlvoAAAAJ:2osOgNQ5qMEC", + "venue_text": "Queen Mary University of London, 2025", + "year": 2025 + } + ], + "profile_RxmmtT8AAAAJ": [ + { + "authors_text": "K Srinivasan", + "citation_count": 1185, + "cluster_id": "qjMakFHDy7sC", + "title": "Black pepper and its pungent principle-piperine: a review of diverse physiological effects", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:qjMakFHDy7sC", + "venue_text": "Critical reviews in food science and nutrition 47 (8), 735-748, 2007", + "year": 2007 + }, + { + "authors_text": "K Platel, K Srinivasan", + "citation_count": 742, + "cluster_id": "W7OEmFMy1HYC", + "title": "Digestive stimulant action of spices: a myth or reality?", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:W7OEmFMy1HYC", + "venue_text": "Indian Journal of Medical Research 119 (5), 167, 2004", + "year": 2004 + }, + { + "authors_text": "K Platel, K Srinivasan", + "citation_count": 682, + "cluster_id": "u-x6o8ySG0sC", + "title": "Influence of dietary spices and their active principles on pancreatic digestive enzymes in albino rats", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:u-x6o8ySG0sC", + "venue_text": "Food/Nahrung 44 (1), 42-46, 2000", + "year": 2000 + }, + { + "authors_text": "K Srinivasan", + "citation_count": 663, + "cluster_id": "LkGwnXOMwfcC", + "title": "Fenugreek (Trigonella foenum-graecum): A Review of Health Beneficial Physiological Effects", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:LkGwnXOMwfcC", + "venue_text": "Food reviews international 22 (2), 203-224, 2006", + "year": 2006 + }, + { + "authors_text": "K Srinivasan", + "citation_count": 574, + "cluster_id": "FPJr55Dyh1AC", + "title": "Biological activities of red pepper (Capsicum annuum) and its pungent principle capsaicin: a review", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:FPJr55Dyh1AC", + "venue_text": "Critical reviews in food science and nutrition 56 (9), 1488-1500, 2016", + "year": 2016 + }, + { + "authors_text": "K Srinivasan", + "citation_count": 555, + "cluster_id": "9yKSN-GCB0IC", + "title": "Plant foods in the management of diabetes mellitus: spices as beneficial antidiabetic food adjuncts", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:9yKSN-GCB0IC", + "venue_text": "International journal of food sciences and nutrition 56 (6), 399-414, 2005", + "year": 2005 + }, + { + "authors_text": "PS Babu, K Srinivasan", + "citation_count": 539, + "cluster_id": "ZfRJV9d4-WMC", + "title": "Hypolipidemic action of curcumin, the active principle of turmeric (Curcuma longa) in streptozotocin induced diabetic rats", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:ZfRJV9d4-WMC", + "venue_text": "Molecular and cellular biochemistry 166 (1), 169-175, 1997", + "year": 1997 + }, + { + "authors_text": "K Srinivasan", + "citation_count": 513, + "cluster_id": "IjCSPb-OGe4C", + "title": "Spices as influencers of body metabolism: an overview of three decades of research", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:IjCSPb-OGe4C", + "venue_text": "Food Research International 38 (1), 77-86, 2005", + "year": 2005 + }, + { + "authors_text": "K Srinivasan", + "citation_count": 455, + "cluster_id": "Tyk-4Ss8FVUC", + "title": "Role of spices beyond food flavoring: Nutraceuticals with multiple health effects", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:Tyk-4Ss8FVUC", + "venue_text": "Food reviews international 21 (2), 167-188, 2005", + "year": 2005 + }, + { + "authors_text": "K Srinivasan", + "citation_count": 446, + "cluster_id": "tkaPQYYpVKoC", + "title": "Antioxidant potential of spices and their active constituents", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:tkaPQYYpVKoC", + "venue_text": "Critical Reviews in Food Science and Nutrition 54 (3), 252-272, 2013", + "year": 2013 + }, + { + "authors_text": "D Suresh, K Srinivasan", + "citation_count": 387, + "cluster_id": "5nxA0vEk-isC", + "title": "Tissue distribution and elimination of capsaicin, piperine & curcumin following oral intake in rats.", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:5nxA0vEk-isC", + "venue_text": "Indian Journal of Medical Research 131, 682-691, 2010", + "year": 2010 + }, + { + "authors_text": "R Ramakrishna Rao, K Platel, K Srinivasan", + "citation_count": 368, + "cluster_id": "YsMSGLbcyi4C", + "title": "In vitro influence of spices and spice\u2010active principles on digestive enzymes of rat pancreas and small intestine", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:YsMSGLbcyi4C", + "venue_text": "Food/Nahrung 47 (6), 408-412, 2003", + "year": 2003 + }, + { + "authors_text": "K Srinivasan", + "citation_count": 367, + "cluster_id": "zCSUwVk65WsC", + "title": "Cumin (Cuminum cyminum) and black cumin (Nigella sativa) seeds: traditional uses, chemical constituents, and nutraceutical effects", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:zCSUwVk65WsC", + "venue_text": "Food quality and safety 2 (1), 1-16, 2018", + "year": 2018 + }, + { + "authors_text": "K Srinivasan", + "citation_count": 364, + "cluster_id": "yqoGN6RLRZoC", + "title": "Ginger rhizomes (Zingiber officinale): A spice with multiple health beneficial potentials", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:yqoGN6RLRZoC", + "venue_text": "PharmaNutrition 5 (1), 18-28, 2017", + "year": 2017 + }, + { + "authors_text": "K Platel, K Srinivasan", + "citation_count": 364, + "cluster_id": "d1gkVwhDpl0C", + "title": "Influence of dietary spices or their active principles on digestive enzymes of small intestinal mucosa in rats", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:d1gkVwhDpl0C", + "venue_text": "International Journal of Food Sciences and Nutrition 47 (1), 55-59, 1996", + "year": 1996 + }, + { + "authors_text": "K Platel, K Srinivasan", + "citation_count": 296, + "cluster_id": "UeHWp8X0CEIC", + "title": "Plant foods in the management of diabetes mellitus: vegetables as potential hypoglycaemic agents", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:UeHWp8X0CEIC", + "venue_text": "Food/Nahrung 41 (2), 68-74, 1997", + "year": 1997 + }, + { + "authors_text": "K Srinivasan, K Sambaiah", + "citation_count": 292, + "cluster_id": "2osOgNQ5qMEC", + "title": "The effect of spices on cholesterol 7\u03b1-hydroxylase activity and on serum and hepatic cholesterol levels in the rat.", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:2osOgNQ5qMEC", + "venue_text": null, + "year": 1991 + }, + { + "authors_text": "S Hemalatha, K Platel, K Srinivasan", + "citation_count": 269, + "cluster_id": "UebtZRa9Y70C", + "title": "Zinc and iron contents and their bioaccessibility in cereals and pulses consumed in India", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:UebtZRa9Y70C", + "venue_text": "Food chemistry 102 (4), 1328-1336, 2007", + "year": 2007 + }, + { + "authors_text": "K Platel, K Srinivasan", + "citation_count": 258, + "cluster_id": "_FxGoFyzp5QC", + "title": "Studies on the influence of dietary spices on food transit time in experimental rats", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:_FxGoFyzp5QC", + "venue_text": "Nutrition Research 21 (9), 1309-1314, 2001", + "year": 2001 + }, + { + "authors_text": "K Platel, K Srinivasan", + "citation_count": 240, + "cluster_id": "hCrLmN-GePgC", + "title": "Bioavailability of micronutrients from plant foods: an update", + "title_url": "/citations?view_op=view_citation&hl=en&user=RxmmtT8AAAAJ&citation_for_view=RxmmtT8AAAAJ:hCrLmN-GePgC", + "venue_text": "Critical reviews in food science and nutrition 56 (10), 1608-1619, 2016", + "year": 2016 + } + ], + "profile_amIMrIEAAAAJ": [ + { + "authors_text": "BR Cherukuri, V Arulkumar", + "citation_count": 131, + "cluster_id": "u5HHmVD_uO8C", + "title": "Optimization of Data Structures and Trade-Offs with Concurrency Control in Multithread Software Structures Using Artificial Intelligence", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:u5HHmVD_uO8C", + "venue_text": "2024 IEEE International Conference on Computing, Power and Communication \u2026, 2024", + "year": 2024 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 81, + "cluster_id": "d1gkVwhDpl0C", + "title": "Development of Design Patterns with Adaptive User Interface for Cloud Native Microservice Architecture Using Deep Learning With IoT", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:d1gkVwhDpl0C", + "venue_text": "2024 IEEE International Conference on Computing, Power and Communication \u2026, 2024", + "year": 2024 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 33, + "cluster_id": "zYLM7Y9cAGgC", + "title": "AI-powered personalization: How machine learning is shaping the future of user experience", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:zYLM7Y9cAGgC", + "venue_text": "International Journal of Science and Research Archive (IJSRA) 12 (1), 3111\u20133126, 2024", + "year": 2024 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 30, + "cluster_id": "roLk4NBRz8UC", + "title": "Future of cloud computing: Innovations in multi-cloud and hybrid architectures", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:roLk4NBRz8UC", + "venue_text": "World Journal of Advanced Research and Reviews (WJARR) 1 (1), 068-081, 2019", + "year": 2019 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 27, + "cluster_id": "UebtZRa9Y70C", + "title": "Microservices and containerization: Accelerating web development cycles", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:UebtZRa9Y70C", + "venue_text": "World Journal of Advanced Research and Reviews (WJARR) 6 (01), 283-296, 2020", + "year": 2020 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 18, + "cluster_id": "hqOjcs7Dif8C", + "title": "Ethical AI in cloud: Mitigating risks in machine learning models", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:hqOjcs7Dif8C", + "venue_text": "World Journal of Advanced Engineering Technology and Sciences ((WJAETS) 1 \u2026, 2020", + "year": 2020 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 14, + "cluster_id": "0EnyYjriUFMC", + "title": "Quantum machine learning: Transforming cloud-based AI solutions", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:0EnyYjriUFMC", + "venue_text": "International Journal of Science and Research Archive (IJSRA) 1 (01), 110-122, 2020", + "year": 2020 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 13, + "cluster_id": "WF5omc3nYNoC", + "title": "Enhanced Trimodal Emotion Recognition Using Multibranch Fusion Attention with Epistemic Neural Networks and Fire Hawk Optimization", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:WF5omc3nYNoC", + "venue_text": "Journal of Machine and Computing 5 (1), 2025", + "year": 2025 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 13, + "cluster_id": "W7OEmFMy1HYC", + "title": "Containerization in cloud computing: comparing Docker and Kubernetes for scalable web applications", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:W7OEmFMy1HYC", + "venue_text": "International Journal of Science and Research Archive (IJSRA) 13 (1), 3302\u20133315, 2024", + "year": 2024 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 13, + "cluster_id": "IjCSPb-OGe4C", + "title": "Serverless computing: How to build and deploy applications without managing infrastructure", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:IjCSPb-OGe4C", + "venue_text": "World Journal of Advanced Engineering Technology and Sciences ((WJAETS) 11 \u2026, 2024", + "year": 2024 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 12, + "cluster_id": "YsMSGLbcyi4C", + "title": "Edge Computing vs. Cloud Computing: A Comparative Analysis for Real-Time AI Applications", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:YsMSGLbcyi4C", + "venue_text": "International Journal For Multidisciplinary Research (IJFMR) 6 (5), 1-17, 2024", + "year": 2024 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 9, + "cluster_id": "Se3iqnhoufwC", + "title": "Serverless revolution: Redefining application scalability and cost efficiency", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:Se3iqnhoufwC", + "venue_text": "World Journal of Advanced Research and Reviews (WJARR) 2 (03), 039-053, 2019", + "year": 2019 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 8, + "cluster_id": "Tyk-4Ss8FVUC", + "title": "Progressive Web Apps (PWAs): Enhancing User Experience through Modern Web Development", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:Tyk-4Ss8FVUC", + "venue_text": "International Journal of Science and Research (IJSR) 13 (10), 1550-1560, 2024", + "year": 2024 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 7, + "cluster_id": "UeHWp8X0CEIC", + "title": "Building Scalable Web Applications: Best Practices for Backend Architecture", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:UeHWp8X0CEIC", + "venue_text": "International Journal of Science and Research (IJSR) 13 (10), 126-139, 2024", + "year": 2024 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 5, + "cluster_id": "5nxA0vEk-isC", + "title": "Advanced Multi Class Cyber Security Attack Classification in IoT Based Wireless Sensor Networks Using Context Aware Depthwise Separable Convolutional Neural Network", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:5nxA0vEk-isC", + "venue_text": "Journal of Machine and Computing 5 (2), 2025", + "year": 2025 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 3, + "cluster_id": "u-x6o8ySG0sC", + "title": "Maintenance of Web Development Standard for Multiple Devices with Serverless Computing through Cross Browser Affinity Using Hybrid Optimization", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:u-x6o8ySG0sC", + "venue_text": "2024 IEEE International Conference on Computing, Power and Communication \u2026, 2024", + "year": 2024 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 3, + "cluster_id": "LkGwnXOMwfcC", + "title": "Developing Intelligent Chatbots for Real-Time Customer Support in E-Commerce", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:LkGwnXOMwfcC", + "venue_text": "International Journal of Science and Research (IJSR) 11 (01), 1709-1719, 2022", + "year": 2022 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 3, + "cluster_id": "_FxGoFyzp5QC", + "title": "Enhancing Web Application Performance with AI - Driven Optimization Techniques", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:_FxGoFyzp5QC", + "venue_text": "International Journal of Science and Research (IJSR) 10 (2), 1779-1788, 2021", + "year": 2021 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 1, + "cluster_id": "Y0pCki6q_DkC", + "title": "Federated Learning: Privacy-Preserving Machine Learning in Cloud Environments", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:Y0pCki6q_DkC", + "venue_text": "International Journal of Science and Research (IJSR) 13 (10), 1539-1549, 2024", + "year": 2024 + }, + { + "authors_text": "BR Cherukuri", + "citation_count": 1, + "cluster_id": "ufrVoPGSRksC", + "title": "Scalable machine learning model deployment using serverless cloud architectures", + "title_url": "/citations?view_op=view_citation&hl=en&user=amIMrIEAAAAJ&citation_for_view=amIMrIEAAAAJ:ufrVoPGSRksC", + "venue_text": "World Journal of Advanced Engineering Technology and Sciences ((WJAETS) 5 (1 \u2026, 2022", + "year": 2022 + } + ] + }, + "run_dir": "planning/scholar_probe_tmp/fixtures/run_20260216T182334Z" +} \ No newline at end of file diff --git a/planning/scholar_probe_tmp/notes/probe_report_run_20260216T182334Z.md b/planning/scholar_probe_tmp/notes/probe_report_run_20260216T182334Z.md new file mode 100644 index 0000000..75804d8 --- /dev/null +++ b/planning/scholar_probe_tmp/notes/probe_report_run_20260216T182334Z.md @@ -0,0 +1,84 @@ +# Scholar Scrape Probe Report + +Generated UTC: `2026-02-16T18:23:56.436305+00:00` +Run fixtures dir: `planning/scholar_probe_tmp/fixtures/run_20260216T182334Z` + +## Robots Snapshot + +```text +User-agent: * +Disallow: /search +Disallow: /index.html +Disallow: /scholar +Disallow: /citations? +Allow: /citations?user= +Disallow: /citations?*cstart= +Disallow: /citations?user=*%40 +Disallow: /citations?user=*@ +Allow: /citations?view_op=list_classic_articles +Allow: /citations?view_op=mandates_leaderboard +Allow: /citations?view_op=metrics_intro +Allow: /citations?view_op=new_profile +Allow: /citations?view_op=sitemap +Allow: /citations?view_op=top_venues + +User-agent: Twitterbot +Disallow: + +User-agent: facebookexternalhit +Disallow: + +User-agent: PetalBot +Disallow: / +``` + +## Fetch Summary + +| Source | Status Code | Status/Error | Final URL | +| --- | --- | --- | --- | +| `robots` | 200 | ok | `https://scholar.google.com/robots.txt` | +| `profile_amIMrIEAAAAJ` | 200 | ok | `https://scholar.google.com/citations?hl=en&user=amIMrIEAAAAJ` | +| `profile_P1RwlvoAAAAJ` | 200 | ok | `https://scholar.google.com/citations?hl=en&user=P1RwlvoAAAAJ` | +| `profile_RxmmtT8AAAAJ` | 200 | ok | `https://scholar.google.com/citations?hl=en&user=RxmmtT8AAAAJ` | +| `profile_LZ5D_p4AAAAJ` | 200 | ok | `https://scholar.google.com/citations?hl=en&user=LZ5D_p4AAAAJ` | +| `profile_AAAAAAAAAAAA` | 200 | ok | `https://accounts.google.com/v3/signin/identifier?continue=https%3A%2F%2Fscholar.google.com%2Fcitations%3Fhl%3Den%26user%3DAAAAAAAAAAAA&dsh=S-1576812270%3A1771266236087781&hl=en&ifkv=ASfE1-pcNcR5S-R4fUZcbEWwaMAwsKrYMzr8-7CFvIajHFoW7EhUN90SrdKqfzzkHfWDxL97xvHq&service=citations&flowName=GlifWebSignIn&flowEntry=ServiceLogin` | + +## Parse Summary + +| Source | Parse Status | Profile | Publications | Articles Range | Show More | Warnings | +| --- | --- | --- | --- | --- | --- | --- | +| `profile_AAAAAAAAAAAA` | `blocked_or_captcha` | - | 0 | - | no | no_rows_detected | +| `profile_LZ5D_p4AAAAJ` | `ok` | Doaa Elmatary | 12 | Articles 1–12 | yes | possible_partial_page_show_more_present | +| `profile_P1RwlvoAAAAJ` | `ok` | WENRUI ZUO | 5 | Articles 1–5 | yes | possible_partial_page_show_more_present | +| `profile_RxmmtT8AAAAJ` | `ok` | K. Srinivasan | 20 | Articles 1–20 | yes | possible_partial_page_show_more_present | +| `profile_amIMrIEAAAAJ` | `ok` | Bangar Raju Cherukuri | 20 | Articles 1–20 | yes | possible_partial_page_show_more_present | + +## Field Coverage + +Total parsed publication rows: **57** + +| Field | Present | Coverage | +| --- | --- | --- | +| `title` | 57 | 100.0% | +| `cluster_id` | 57 | 100.0% | +| `year` | 56 | 98.2% | +| `citation_count` | 57 | 100.0% | +| `authors_text` | 57 | 100.0% | +| `venue_text` | 54 | 94.7% | + +## Parser Contract Recommendation + +- Primary row marker: `tr.gsc_a_tr`. +- Title anchor marker: `a.gsc_a_at`; derive `cluster_id` from `citation_for_view` query token. +- Metadata text markers: first/second `div.gs_gray` per row for authors and venue. +- Year marker fallback: classes containing `gsc_a_h` or `gsc_a_y` and 4-digit year regex. +- Failure states to persist: `ok`, `no_results`, `blocked_or_captcha`, `layout_changed`, `network_error`. + +## Future-Proofing Notes + +- Keep raw HTML fixture snapshots and update parser tests on DOM drift. +- Treat blocked pages as retriable with backoff, not parser errors. +- If `Show more` is present, treat first-page-only results as partial and surface that in run status/UI. +- Track robots policy changes because `/citations?*cstart=` is currently disallowed. +- Add marker-count assertions in CI to catch silent layout shifts early. +- Use explicit parse status per run/scholar so automation can degrade gracefully. diff --git a/planning/scholar_probe_tmp/notes/seed_scholar_ids.txt b/planning/scholar_probe_tmp/notes/seed_scholar_ids.txt new file mode 100644 index 0000000..3ecab65 --- /dev/null +++ b/planning/scholar_probe_tmp/notes/seed_scholar_ids.txt @@ -0,0 +1,6 @@ +# Seed IDs for probe capture (public profiles) +amIMrIEAAAAJ +P1RwlvoAAAAJ +RxmmtT8AAAAJ +LZ5D_p4AAAAJ +AAAAAAAAAAAA diff --git a/planning/scholar_probe_tmp/scripts/scholar_probe.py b/planning/scholar_probe_tmp/scripts/scholar_probe.py new file mode 100755 index 0000000..9216a3d --- /dev/null +++ b/planning/scholar_probe_tmp/scripts/scholar_probe.py @@ -0,0 +1,777 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import json +import random +import re +import time +from dataclasses import dataclass, asdict +from datetime import datetime, timezone +from html import unescape +from html.parser import HTMLParser +from pathlib import Path +from typing import Any +from urllib.error import HTTPError, URLError +from urllib.parse import parse_qs, urlencode, urlparse +from urllib.request import Request, urlopen + +ROBOTS_URL = "https://scholar.google.com/robots.txt" +PROFILE_URL = "https://scholar.google.com/citations" + +DEFAULT_USER_AGENTS = [ + ( + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36" + ), + ( + "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_7) AppleWebKit/605.1.15 " + "(KHTML, like Gecko) Version/18.1 Safari/605.1.15" + ), + ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:131.0) " + "Gecko/20100101 Firefox/131.0" + ), +] + +BLOCKED_KEYWORDS = [ + "unusual traffic", + "sorry/index", + "not a robot", + "our systems have detected", + "automated queries", +] + +NO_RESULTS_KEYWORDS = [ + "didn't match any articles", + "did not match any articles", + "no articles", + "no documents", +] + +MARKER_KEYS = [ + "gsc_a_tr", + "gsc_a_at", + "gsc_a_ac", + "gsc_a_h", + "gsc_a_y", + "gs_gray", + "gsc_prf_in", + "gsc_rsb_st", +] + + +@dataclass +class FetchRecord: + source: str + url: str + file_name: str + status_code: int | None + fetched_at_utc: str + elapsed_seconds: float + final_url: str | None + error: str | None + + +@dataclass +class PublicationCandidate: + title: str + title_url: str | None + cluster_id: str | None + year: int | None + citation_count: int | None + authors_text: str | None + venue_text: str | None + + +@dataclass +class PageAnalysis: + source: str + status: str + profile_name: str | None + publication_count: int + articles_range: str | None + has_show_more_button: bool + has_operation_error_banner: bool + marker_counts: dict[str, int] + field_presence: dict[str, int] + parse_warnings: list[str] + + +def normalize_space(value: str) -> str: + return " ".join(unescape(value).split()) + + +TAG_RE = re.compile(r"<[^>]+>", re.S) + + +def strip_tags(value: str) -> str: + return normalize_space(TAG_RE.sub(" ", value)) + + +def attr_class(attrs: list[tuple[str, str | None]]) -> str: + for name, raw_value in attrs: + if name.lower() == "class": + return raw_value or "" + return "" + + +def attr_href(attrs: list[tuple[str, str | None]]) -> str | None: + for name, raw_value in attrs: + if name.lower() == "href": + return raw_value + return None + + +class ScholarRowParser(HTMLParser): + def __init__(self) -> None: + super().__init__(convert_charrefs=True) + self.title_href: str | None = None + self.title_parts: list[str] = [] + self.citation_parts: list[str] = [] + self.year_parts: list[str] = [] + self.gray_texts: list[str] = [] + + self._title_depth = 0 + self._citation_depth = 0 + self._year_depth = 0 + self._gray_stack: list[dict[str, Any]] = [] + + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + if self._title_depth > 0: + self._title_depth += 1 + if self._citation_depth > 0: + self._citation_depth += 1 + if self._year_depth > 0: + self._year_depth += 1 + if self._gray_stack: + self._gray_stack[-1]["depth"] += 1 + + classes = attr_class(attrs) + + if tag == "a" and "gsc_a_at" in classes: + self._title_depth = 1 + self.title_href = attr_href(attrs) + return + + if tag == "a" and "gsc_a_ac" in classes: + self._citation_depth = 1 + return + + if tag in {"span", "a"} and ("gsc_a_h" in classes or "gsc_a_y" in classes): + self._year_depth = 1 + return + + if tag == "div" and "gs_gray" in classes: + self._gray_stack.append({"depth": 1, "parts": []}) + return + + def handle_data(self, data: str) -> None: + if self._title_depth > 0: + self.title_parts.append(data) + if self._citation_depth > 0: + self.citation_parts.append(data) + if self._year_depth > 0: + self.year_parts.append(data) + if self._gray_stack: + self._gray_stack[-1]["parts"].append(data) + + def handle_endtag(self, _tag: str) -> None: + if self._title_depth > 0: + self._title_depth -= 1 + if self._citation_depth > 0: + self._citation_depth -= 1 + if self._year_depth > 0: + self._year_depth -= 1 + if self._gray_stack: + self._gray_stack[-1]["depth"] -= 1 + if self._gray_stack[-1]["depth"] == 0: + text = normalize_space("".join(self._gray_stack[-1]["parts"])) + if text: + self.gray_texts.append(text) + self._gray_stack.pop() + + +def extract_rows(html: str) -> list[str]: + pattern = re.compile( + r"]*\bclass\s*=\s*['\"][^'\"]*\bgsc_a_tr\b[^'\"]*['\"])[^>]*>(.*?)", + re.I | re.S, + ) + return [match.group(1) for match in pattern.finditer(html)] + + +def parse_cluster_id_from_href(href: str | None) -> str | None: + if not href: + return None + parsed = urlparse(href) + query = parse_qs(parsed.query) + + citation_for_view = query.get("citation_for_view") + if citation_for_view: + token = citation_for_view[0].strip() + if token: + if ":" in token: + return token.rsplit(":", 1)[-1] or None + return token + + cluster = query.get("cluster") + if cluster: + token = cluster[0].strip() + if token: + return token + return None + + +def parse_year(parts: list[str]) -> int | None: + text = normalize_space(" ".join(parts)) + match = re.search(r"\b(19|20)\d{2}\b", text) + if not match: + return None + try: + return int(match.group(0)) + except ValueError: + return None + + +def parse_citation_count(parts: list[str]) -> int | None: + text = normalize_space(" ".join(parts)) + if not text: + return 0 + match = re.search(r"\d+", text) + if not match: + return None + return int(match.group(0)) + + +def parse_publications(html: str) -> tuple[list[PublicationCandidate], list[str]]: + rows = extract_rows(html) + warnings: list[str] = [] + publications: list[PublicationCandidate] = [] + + for row_html in rows: + parser = ScholarRowParser() + parser.feed(row_html) + + title = normalize_space("".join(parser.title_parts)) + if not title: + warnings.append("row_missing_title") + continue + + authors_text = parser.gray_texts[0] if len(parser.gray_texts) > 0 else None + venue_text = parser.gray_texts[1] if len(parser.gray_texts) > 1 else None + + publications.append( + PublicationCandidate( + title=title, + title_url=parser.title_href, + cluster_id=parse_cluster_id_from_href(parser.title_href), + year=parse_year(parser.year_parts), + citation_count=parse_citation_count(parser.citation_parts), + authors_text=authors_text, + venue_text=venue_text, + ) + ) + + if not rows: + warnings.append("no_rows_detected") + + return publications, sorted(set(warnings)) + + +def extract_profile_name(html: str) -> str | None: + pattern = re.compile( + r"<[^>]*\bid\s*=\s*['\"]gsc_prf_in['\"][^>]*>(.*?)]+>", + re.I | re.S, + ) + match = pattern.search(html) + if not match: + return None + value = strip_tags(match.group(1)) + return value or None + + +def extract_articles_range(html: str) -> str | None: + pattern = re.compile( + r"<[^>]*\bid\s*=\s*['\"]gsc_a_nn['\"][^>]*>(.*?)]+>", + re.I | re.S, + ) + match = pattern.search(html) + if not match: + return None + value = strip_tags(match.group(1)) + return value or None + + +def has_show_more_button(html: str) -> bool: + lowered = html.lower() + return "id=\"gsc_bpf_more\"" in lowered or "id='gsc_bpf_more'" in lowered + + +def has_operation_error_banner(html: str) -> bool: + lowered = html.lower() + if "id=\"gsc_a_err\"" not in lowered and "id='gsc_a_err'" not in lowered: + return False + return "can't perform the operation now" in lowered or "cannot perform the operation now" in lowered + + +def count_markers(html: str) -> dict[str, int]: + lowered = html.lower() + return {key: lowered.count(key.lower()) for key in MARKER_KEYS} + + +def detect_status( + *, + status_code: int | None, + final_url: str | None, + html: str, + publication_count: int, + marker_counts: dict[str, int], +) -> str: + if status_code is None: + return "network_error" + + lowered = html.lower() + final = (final_url or "").lower() + + if "accounts.google.com" in final and ("signin" in final or "servicelogin" in final): + return "blocked_or_captcha" + + if any(keyword in lowered for keyword in BLOCKED_KEYWORDS) or "sorry/index" in final: + return "blocked_or_captcha" + + if publication_count == 0 and any(keyword in lowered for keyword in NO_RESULTS_KEYWORDS): + return "no_results" + + if publication_count == 0: + has_profile_markers = marker_counts.get("gsc_prf_in", 0) > 0 + has_table_markers = marker_counts.get("gsc_a_tr", 0) > 0 or marker_counts.get("gsc_a_at", 0) > 0 + if not has_profile_markers and not has_table_markers: + return "layout_changed" + + return "ok" + + +def load_scholar_ids(id_file: Path | None, cli_ids: list[str]) -> list[str]: + seen: set[str] = set() + collected: list[str] = [] + + def maybe_add(raw: str) -> None: + item = raw.strip() + if not item: + return + if item.startswith("#"): + return + if item in seen: + return + seen.add(item) + collected.append(item) + + for item in cli_ids: + maybe_add(item) + + if id_file and id_file.exists(): + for line in id_file.read_text(encoding="utf-8").splitlines(): + maybe_add(line) + + return collected + + +def fetch_url(url: str, user_agent: str, timeout_seconds: float) -> tuple[int | None, str | None, str, str | None]: + request = Request( + url, + headers={ + "User-Agent": user_agent, + "Accept": "text/html,application/xhtml+xml", + "Accept-Language": "en-US,en;q=0.9", + "Connection": "close", + }, + ) + start = time.perf_counter() + + try: + with urlopen(request, timeout=timeout_seconds) as response: + body = response.read().decode("utf-8", errors="replace") + elapsed = time.perf_counter() - start + status = getattr(response, "status", 200) + final_url = response.geturl() + return status, final_url, body, None + except HTTPError as exc: + body = "" + try: + body = exc.read().decode("utf-8", errors="replace") + except Exception: + body = "" + elapsed = time.perf_counter() - start + _ = elapsed + return exc.code, exc.geturl(), body, str(exc) + except URLError as exc: + elapsed = time.perf_counter() - start + _ = elapsed + return None, None, "", str(exc) + + +def write_json(path: Path, payload: Any) -> None: + path.write_text(json.dumps(payload, indent=2, sort_keys=True), encoding="utf-8") + + +def render_markdown( + *, + generated_at: str, + run_dir: Path, + fetch_records: list[FetchRecord], + page_analyses: list[PageAnalysis], + publications_by_source: dict[str, list[PublicationCandidate]], + robots_excerpt: str, +) -> str: + lines: list[str] = [] + lines.append("# Scholar Scrape Probe Report") + lines.append("") + lines.append(f"Generated UTC: `{generated_at}`") + lines.append(f"Run fixtures dir: `{run_dir.as_posix()}`") + lines.append("") + + lines.append("## Robots Snapshot") + lines.append("") + lines.append("```text") + lines.append(robots_excerpt.rstrip() or "(robots fetch unavailable)") + lines.append("```") + lines.append("") + + lines.append("## Fetch Summary") + lines.append("") + lines.append("| Source | Status Code | Status/Error | Final URL |") + lines.append("| --- | --- | --- | --- |") + for record in fetch_records: + status = str(record.status_code) if record.status_code is not None else "-" + state = record.error or "ok" + final_url = record.final_url or "-" + lines.append(f"| `{record.source}` | {status} | {state} | `{final_url}` |") + lines.append("") + + lines.append("## Parse Summary") + lines.append("") + lines.append("| Source | Parse Status | Profile | Publications | Articles Range | Show More | Warnings |") + lines.append("| --- | --- | --- | --- | --- | --- | --- |") + for analysis in page_analyses: + warnings = ", ".join(analysis.parse_warnings) if analysis.parse_warnings else "-" + profile = analysis.profile_name or "-" + articles_range = analysis.articles_range or "-" + show_more = "yes" if analysis.has_show_more_button else "no" + lines.append( + f"| `{analysis.source}` | `{analysis.status}` | {profile} | {analysis.publication_count} | {articles_range} | {show_more} | {warnings} |" + ) + lines.append("") + + all_publications = [ + publication + for publications in publications_by_source.values() + for publication in publications + ] + + if all_publications: + total = len(all_publications) + def pct(count: int) -> str: + return f"{(count / total) * 100:.1f}%" + + title_count = sum(1 for item in all_publications if item.title) + cluster_count = sum(1 for item in all_publications if item.cluster_id) + year_count = sum(1 for item in all_publications if item.year is not None) + citation_count = sum(1 for item in all_publications if item.citation_count is not None) + author_count = sum(1 for item in all_publications if item.authors_text) + venue_count = sum(1 for item in all_publications if item.venue_text) + + lines.append("## Field Coverage") + lines.append("") + lines.append(f"Total parsed publication rows: **{total}**") + lines.append("") + lines.append("| Field | Present | Coverage |") + lines.append("| --- | --- | --- |") + lines.append(f"| `title` | {title_count} | {pct(title_count)} |") + lines.append(f"| `cluster_id` | {cluster_count} | {pct(cluster_count)} |") + lines.append(f"| `year` | {year_count} | {pct(year_count)} |") + lines.append(f"| `citation_count` | {citation_count} | {pct(citation_count)} |") + lines.append(f"| `authors_text` | {author_count} | {pct(author_count)} |") + lines.append(f"| `venue_text` | {venue_count} | {pct(venue_count)} |") + lines.append("") + + lines.append("## Parser Contract Recommendation") + lines.append("") + lines.append("- Primary row marker: `tr.gsc_a_tr`.") + lines.append("- Title anchor marker: `a.gsc_a_at`; derive `cluster_id` from `citation_for_view` query token.") + lines.append("- Metadata text markers: first/second `div.gs_gray` per row for authors and venue.") + lines.append("- Year marker fallback: classes containing `gsc_a_h` or `gsc_a_y` and 4-digit year regex.") + lines.append("- Failure states to persist: `ok`, `no_results`, `blocked_or_captcha`, `layout_changed`, `network_error`.") + lines.append("") + + lines.append("## Future-Proofing Notes") + lines.append("") + lines.append("- Keep raw HTML fixture snapshots and update parser tests on DOM drift.") + lines.append("- Treat blocked pages as retriable with backoff, not parser errors.") + lines.append("- If `Show more` is present, treat first-page-only results as partial and surface that in run status/UI.") + lines.append("- Track robots policy changes because `/citations?*cstart=` is currently disallowed.") + lines.append("- Add marker-count assertions in CI to catch silent layout shifts early.") + lines.append("- Use explicit parse status per run/scholar so automation can degrade gracefully.") + + lines.append("") + return "\n".join(lines) + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Temporary probe: capture and analyze Google Scholar profile HTML for parser planning." + ) + parser.add_argument( + "--output-root", + default="planning/scholar_probe_tmp", + help="Root directory of temporary probe workspace.", + ) + parser.add_argument( + "--scholar-id", + action="append", + default=[], + help="Scholar profile id (can be passed multiple times).", + ) + parser.add_argument( + "--id-file", + default="planning/scholar_probe_tmp/notes/seed_scholar_ids.txt", + help="Path to newline-delimited scholar ids.", + ) + parser.add_argument( + "--max-profiles", + type=int, + default=5, + help="Maximum number of scholar profiles to fetch in this probe run.", + ) + parser.add_argument( + "--request-delay-seconds", + type=float, + default=8.0, + help="Base delay between live requests.", + ) + parser.add_argument( + "--request-jitter-seconds", + type=float, + default=2.0, + help="Randomized additional delay in seconds.", + ) + parser.add_argument( + "--timeout-seconds", + type=float, + default=25.0, + help="HTTP timeout seconds.", + ) + parser.add_argument( + "--allow-live-fetch", + action="store_true", + help="Enable outbound fetch for robots and scholar profile pages.", + ) + parser.add_argument( + "--analyze-existing-fixtures", + action="store_true", + help="Analyze existing fixtures in output-root/fixtures even if live fetch is disabled.", + ) + return parser + + +def main() -> int: + args = build_parser().parse_args() + + output_root = Path(args.output_root) + fixtures_root = output_root / "fixtures" + notes_root = output_root / "notes" + fixtures_root.mkdir(parents=True, exist_ok=True) + notes_root.mkdir(parents=True, exist_ok=True) + + now = datetime.now(timezone.utc) + run_id = now.strftime("run_%Y%m%dT%H%M%SZ") + run_dir = fixtures_root / run_id + run_dir.mkdir(parents=True, exist_ok=True) + + id_file = Path(args.id_file) + scholar_ids = load_scholar_ids(id_file, args.scholar_id) + if args.max_profiles > 0: + scholar_ids = scholar_ids[: args.max_profiles] + + fetch_records: list[FetchRecord] = [] + robots_excerpt = "" + + if args.allow_live_fetch: + robots_start = time.perf_counter() + robots_status, robots_final_url, robots_body, robots_error = fetch_url( + ROBOTS_URL, + user_agent=random.choice(DEFAULT_USER_AGENTS), + timeout_seconds=args.timeout_seconds, + ) + robots_elapsed = time.perf_counter() - robots_start + + robots_path = run_dir / "robots.txt" + robots_path.write_text(robots_body or "", encoding="utf-8") + robots_excerpt = "\n".join((robots_body or "").splitlines()[:50]) + + fetch_records.append( + FetchRecord( + source="robots", + url=ROBOTS_URL, + file_name=robots_path.name, + status_code=robots_status, + fetched_at_utc=datetime.now(timezone.utc).isoformat(), + elapsed_seconds=round(robots_elapsed, 3), + final_url=robots_final_url, + error=robots_error, + ) + ) + + for index, scholar_id in enumerate(scholar_ids): + params = {"hl": "en", "user": scholar_id} + url = f"{PROFILE_URL}?{urlencode(params)}" + + user_agent = DEFAULT_USER_AGENTS[index % len(DEFAULT_USER_AGENTS)] + start = time.perf_counter() + status_code, final_url, body, error = fetch_url( + url, + user_agent=user_agent, + timeout_seconds=args.timeout_seconds, + ) + elapsed = time.perf_counter() - start + + safe_source = f"profile_{scholar_id}" + html_name = f"{safe_source}.html" + html_path = run_dir / html_name + html_path.write_text(body or "", encoding="utf-8") + + fetch_records.append( + FetchRecord( + source=safe_source, + url=url, + file_name=html_name, + status_code=status_code, + fetched_at_utc=datetime.now(timezone.utc).isoformat(), + elapsed_seconds=round(elapsed, 3), + final_url=final_url, + error=error, + ) + ) + + if index < len(scholar_ids) - 1: + delay = max(0.0, args.request_delay_seconds) + random.uniform( + 0.0, + max(0.0, args.request_jitter_seconds), + ) + time.sleep(delay) + + if not robots_excerpt: + robots_candidates = sorted(fixtures_root.glob("run_*/robots.txt"), reverse=True) + if robots_candidates: + robots_excerpt = "\n".join( + robots_candidates[0].read_text(encoding="utf-8").splitlines()[:50] + ) + + html_files: list[Path] = [] + html_files.extend(run_dir.glob("profile_*.html")) + + if args.analyze_existing_fixtures: + latest_runs = sorted(fixtures_root.glob("run_*"), reverse=True) + for existing_run in latest_runs: + if existing_run == run_dir: + continue + html_files.extend(existing_run.glob("profile_*.html")) + + by_source: dict[str, Path] = {} + for candidate in sorted(html_files, reverse=True): + source = candidate.stem + if source in by_source: + continue + by_source[source] = candidate + deduped_files = [by_source[source] for source in sorted(by_source)] + + page_analyses: list[PageAnalysis] = [] + publications_by_source: dict[str, list[PublicationCandidate]] = {} + + fetch_record_by_source = {record.source: record for record in fetch_records} + + for html_path in deduped_files: + source = html_path.stem + html = html_path.read_text(encoding="utf-8") + publications, warnings = parse_publications(html) + markers = count_markers(html) + articles_range = extract_articles_range(html) + show_more = has_show_more_button(html) + operation_error_banner = has_operation_error_banner(html) + + if show_more: + warnings.append("possible_partial_page_show_more_present") + if operation_error_banner: + warnings.append("operation_error_banner_present") + warnings = sorted(set(warnings)) + + record = fetch_record_by_source.get(source) + status = detect_status( + status_code=record.status_code if record else 200, + final_url=record.final_url if record else None, + html=html, + publication_count=len(publications), + marker_counts=markers, + ) + + field_presence = { + "title": sum(1 for item in publications if item.title), + "cluster_id": sum(1 for item in publications if item.cluster_id), + "year": sum(1 for item in publications if item.year is not None), + "citation_count": sum(1 for item in publications if item.citation_count is not None), + "authors_text": sum(1 for item in publications if item.authors_text), + "venue_text": sum(1 for item in publications if item.venue_text), + } + + analysis = PageAnalysis( + source=source, + status=status, + profile_name=extract_profile_name(html), + publication_count=len(publications), + articles_range=articles_range, + has_show_more_button=show_more, + has_operation_error_banner=operation_error_banner, + marker_counts=markers, + field_presence=field_presence, + parse_warnings=warnings, + ) + page_analyses.append(analysis) + publications_by_source[source] = publications + + report_payload = { + "generated_at_utc": datetime.now(timezone.utc).isoformat(), + "run_dir": run_dir.as_posix(), + "fetch_records": [asdict(record) for record in fetch_records], + "page_analyses": [asdict(analysis) for analysis in page_analyses], + "publications_by_source": { + source: [asdict(item) for item in publications] + for source, publications in publications_by_source.items() + }, + } + + json_report_path = notes_root / f"probe_report_{run_id}.json" + write_json(json_report_path, report_payload) + + markdown_report = render_markdown( + generated_at=report_payload["generated_at_utc"], + run_dir=run_dir, + fetch_records=fetch_records, + page_analyses=page_analyses, + publications_by_source=publications_by_source, + robots_excerpt=robots_excerpt, + ) + markdown_report_path = notes_root / f"probe_report_{run_id}.md" + markdown_report_path.write_text(markdown_report, encoding="utf-8") + + summary = { + "run_id": run_id, + "run_dir": run_dir.as_posix(), + "json_report": json_report_path.as_posix(), + "markdown_report": markdown_report_path.as_posix(), + "profiles_analyzed": len(page_analyses), + } + print(json.dumps(summary, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..0669801 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,47 @@ +[build-system] +requires = ["setuptools>=69", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "scholarr" +version = "0.1.0" +description = "Self-hosted scholarr service" +readme = "README.md" +requires-python = ">=3.12" +dependencies = [ + "alembic>=1.14,<2.0", + "argon2-cffi>=25.1,<26.0", + "asyncpg>=0.30,<0.31", + "fastapi>=0.116,<0.117", + "itsdangerous>=2.2,<3.0", + "jinja2>=3.1,<3.2", + "python-multipart>=0.0.20,<0.0.21", + "sqlalchemy>=2.0,<2.1", + "uvicorn[standard]>=0.34,<0.35", +] + +[project.optional-dependencies] +dev = [ + "httpx>=0.28,<0.29", + "pytest>=8.3,<9.0", + "pytest-asyncio>=0.25,<0.26", +] + +[tool.pytest.ini_options] +addopts = "-q -m \"not integration\"" +asyncio_mode = "auto" +testpaths = ["tests"] +markers = [ + "integration: integration tests that require external services", + "db: tests that validate database behavior and constraints", + "migrations: tests focused on alembic schema migration correctness", + "schema: tests focused on multi-tenant schema invariants", + "smoke: smoke tests for containerized runtime", +] + +[tool.setuptools] +include-package-data = true + +[tool.setuptools.packages.find] +where = ["."] +include = ["app*"] diff --git a/scripts/bootstrap_admin.py b/scripts/bootstrap_admin.py new file mode 100644 index 0000000..55f3c0f --- /dev/null +++ b/scripts/bootstrap_admin.py @@ -0,0 +1,126 @@ +from __future__ import annotations + +import argparse +import asyncio +import logging +import os + +from sqlalchemy import select + +from app.auth.security import PasswordService +from app.db.models import User +from app.db.session import get_session_factory +from app.logging_config import configure_logging, parse_redact_fields +from app.settings import settings + +configure_logging( + level=settings.log_level, + log_format=settings.log_format, + redact_fields=parse_redact_fields(settings.log_redact_fields), + include_uvicorn_access=settings.log_uvicorn_access, +) + +logger = logging.getLogger(__name__) + + +def _build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="Create or update the initial admin user.") + parser.add_argument("--email", default=os.getenv("BOOTSTRAP_ADMIN_EMAIL")) + parser.add_argument("--password", default=os.getenv("BOOTSTRAP_ADMIN_PASSWORD")) + parser.add_argument( + "--force-password", + action="store_true", + default=os.getenv("BOOTSTRAP_ADMIN_FORCE_PASSWORD", "0") in {"1", "true", "yes"}, + help="When user exists, replace password with provided value.", + ) + return parser + + +async def bootstrap_admin(*, email: str | None, password: str | None, force_password: bool) -> int: + if not email: + logger.info( + "admin.bootstrap_skipped", + extra={"event": "admin.bootstrap_skipped"}, + ) + return 0 + if not password: + logger.error( + "admin.bootstrap_missing_password", + extra={"event": "admin.bootstrap_missing_password"}, + ) + return 1 + + normalized_email = email.strip().lower() + if not normalized_email: + logger.error( + "admin.bootstrap_invalid_email", + extra={"event": "admin.bootstrap_invalid_email"}, + ) + return 1 + if len(password.strip()) < 8: + logger.error( + "admin.bootstrap_invalid_password", + extra={"event": "admin.bootstrap_invalid_password"}, + ) + return 1 + + session_factory = get_session_factory() + password_service = PasswordService() + async with session_factory() as session: + result = await session.execute(select(User).where(User.email == normalized_email)) + user = result.scalar_one_or_none() + + if user is None: + user = User( + email=normalized_email, + password_hash=password_service.hash_password(password.strip()), + is_admin=True, + is_active=True, + ) + session.add(user) + await session.commit() + logger.info( + "admin.bootstrap_created", + extra={"event": "admin.bootstrap_created", "email": normalized_email}, + ) + return 0 + + changed = False + if not user.is_admin: + user.is_admin = True + changed = True + if not user.is_active: + user.is_active = True + changed = True + if force_password: + user.password_hash = password_service.hash_password(password.strip()) + changed = True + + if changed: + await session.commit() + logger.info( + "admin.bootstrap_updated", + extra={"event": "admin.bootstrap_updated", "email": normalized_email}, + ) + else: + logger.info( + "admin.bootstrap_already_configured", + extra={"event": "admin.bootstrap_already_configured", "email": normalized_email}, + ) + return 0 + + +def main() -> int: + parser = _build_parser() + args = parser.parse_args() + return asyncio.run( + bootstrap_admin( + email=args.email, + password=args.password, + force_password=args.force_password, + ) + ) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/entrypoint.sh b/scripts/entrypoint.sh new file mode 100755 index 0000000..2408bc1 --- /dev/null +++ b/scripts/entrypoint.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env sh +set -eu + +uv run python /app/scripts/wait_for_db.py + +if [ "${MIGRATE_ON_START:-1}" = "1" ]; then + uv run alembic upgrade head +fi + +if [ "${BOOTSTRAP_ADMIN_ON_START:-0}" = "1" ]; then + uv run python /app/scripts/bootstrap_admin.py +fi + +if [ "$#" -eq 0 ]; then + if [ "${APP_RELOAD:-0}" = "1" ]; then + set -- uv run uvicorn app.main:app --host "${APP_HOST:-0.0.0.0}" --port "${APP_PORT:-8000}" --reload + else + set -- uv run uvicorn app.main:app --host "${APP_HOST:-0.0.0.0}" --port "${APP_PORT:-8000}" --workers "${UVICORN_WORKERS:-1}" + fi +fi + +exec "$@" diff --git a/scripts/smoke_compose.sh b/scripts/smoke_compose.sh new file mode 100755 index 0000000..c1b669e --- /dev/null +++ b/scripts/smoke_compose.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +set -euo pipefail + +export COMPOSE_PROJECT_NAME="${COMPOSE_PROJECT_NAME:-scholarr-smoke}" +export APP_PORT="${APP_PORT:-8000}" +export APP_HOST_PORT="${APP_HOST_PORT:-18000}" +export POSTGRES_PORT="${POSTGRES_PORT:-15432}" + +cleanup() { + docker compose down -v --remove-orphans +} + +trap cleanup EXIT + +docker compose up -d --build + +echo "Waiting for application health check..." +for _ in {1..45}; do + if curl -fsS "http://localhost:${APP_HOST_PORT}/healthz" >/dev/null; then + break + fi + sleep 2 +done + +curl -fsS "http://localhost:${APP_HOST_PORT}/healthz" >/dev/null + +docker compose run --rm app uv run pytest tests/smoke -m "integration and smoke" diff --git a/scripts/wait_for_db.py b/scripts/wait_for_db.py new file mode 100644 index 0000000..e2175a3 --- /dev/null +++ b/scripts/wait_for_db.py @@ -0,0 +1,68 @@ +import asyncio +import logging +import os + +from sqlalchemy import text +from sqlalchemy.ext.asyncio import create_async_engine + +from app.logging_config import configure_logging, parse_redact_fields +from app.settings import settings + +configure_logging( + level=settings.log_level, + log_format=settings.log_format, + redact_fields=parse_redact_fields(settings.log_redact_fields), + include_uvicorn_access=settings.log_uvicorn_access, +) + +logger = logging.getLogger(__name__) + + +async def can_connect(database_url: str) -> bool: + engine = create_async_engine(database_url, pool_pre_ping=True) + try: + async with engine.connect() as conn: + result = await conn.execute(text("SELECT 1")) + return result.scalar_one() == 1 + except Exception: + return False + finally: + await engine.dispose() + + +async def wait_for_database() -> int: + database_url = os.getenv("DATABASE_URL") + if not database_url: + logger.error("db.wait_missing_database_url", extra={"event": "db.wait_missing_database_url"}) + return 1 + + timeout_seconds = int(os.getenv("DB_WAIT_TIMEOUT_SECONDS", "60")) + interval_seconds = int(os.getenv("DB_WAIT_INTERVAL_SECONDS", "2")) + retries = max(timeout_seconds // max(interval_seconds, 1), 1) + + for attempt in range(1, retries + 1): + if await can_connect(database_url): + logger.info("db.wait_ready", extra={"event": "db.wait_ready"}) + return 0 + logger.info( + "db.wait_retry", + extra={ + "event": "db.wait_retry", + "attempt": attempt, + "retries": retries, + }, + ) + await asyncio.sleep(interval_seconds) + + logger.error( + "db.wait_timeout", + extra={ + "event": "db.wait_timeout", + "retries": retries, + }, + ) + return 1 + + +if __name__ == "__main__": + raise SystemExit(asyncio.run(wait_for_database())) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..c9c8ae7 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1,2 @@ +# Tests package for shared helper imports. + diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..344278f --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,171 @@ +from __future__ import annotations + +import asyncio +import os +import re +from collections.abc import AsyncIterator, Iterator + +import pytest +from alembic import command +from alembic.config import Config +from sqlalchemy.engine import make_url +from sqlalchemy import text +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine + +from app.auth.deps import get_login_rate_limiter +from app.db.session import close_engine +from app.settings import settings + +RESET_SQL = text( + """ + TRUNCATE TABLE + ingestion_queue_items, + scholar_publications, + crawl_runs, + scholar_profiles, + publications, + user_settings, + users + RESTART IDENTITY CASCADE + """ +) + +DB_NAME_SAFE_RE = re.compile(r"^[A-Za-z0-9_]+$") + + +def _resolve_test_database_url() -> str | None: + explicit = (os.getenv("TEST_DATABASE_URL") or "").strip() + if explicit: + return explicit + + base = (os.getenv("DATABASE_URL") or "").strip() + if not base: + return None + + parsed = make_url(base) + if not parsed.database: + return None + derived_database = ( + parsed.database if parsed.database.endswith("_test") else f"{parsed.database}_test" + ) + return parsed.set(database=derived_database).render_as_string(hide_password=False) + + +def pytest_runtest_setup(item: pytest.Item) -> None: + if "integration" in item.keywords and not _resolve_test_database_url(): + pytest.skip("DATABASE_URL (or TEST_DATABASE_URL) is required for integration tests") + + +@pytest.fixture(scope="session") +def database_url() -> str: + value = _resolve_test_database_url() + if not value: + pytest.skip("DATABASE_URL (or TEST_DATABASE_URL) is required for database tests") + return value + + +@pytest.fixture(scope="session") +def alembic_config(database_url: str) -> Config: + config = Config("alembic.ini") + config.set_main_option("sqlalchemy.url", database_url) + return config + + +@pytest.fixture(scope="session") +def ensure_test_database_exists(database_url: str) -> Iterator[None]: + parsed = make_url(database_url) + database_name = parsed.database + if not database_name: + raise RuntimeError("TEST_DATABASE_URL must include a database name.") + if not DB_NAME_SAFE_RE.fullmatch(database_name): + raise RuntimeError( + "TEST_DATABASE_URL database name must match [A-Za-z0-9_]+ for safe auto-provisioning." + ) + + admin_name = "postgres" if database_name != "postgres" else "template1" + admin_url = parsed.set(database=admin_name) + admin_url_rendered = admin_url.render_as_string(hide_password=False) + + async def _ensure_database() -> None: + engine = create_async_engine( + admin_url_rendered, + pool_pre_ping=True, + isolation_level="AUTOCOMMIT", + ) + try: + async with engine.connect() as connection: + exists_result = await connection.execute( + text("SELECT 1 FROM pg_database WHERE datname = :database_name"), + {"database_name": database_name}, + ) + if exists_result.scalar_one_or_none() == 1: + return + try: + await connection.execute(text(f'CREATE DATABASE "{database_name}"')) + except Exception as exc: + raise RuntimeError( + "Unable to auto-create test database. " + "Create it manually or grant CREATEDB to the database user." + ) from exc + finally: + await engine.dispose() + + asyncio.run(_ensure_database()) + yield + + +@pytest.fixture(scope="session") +def migrated_database( + alembic_config: Config, + ensure_test_database_exists: None, + database_url: str, +) -> Iterator[None]: + previous_env_database_url = os.getenv("DATABASE_URL") + previous_settings_database_url = settings.database_url + + os.environ["DATABASE_URL"] = database_url + object.__setattr__(settings, "database_url", database_url) + asyncio.run(close_engine()) + command.upgrade(alembic_config, "head") + + try: + yield + finally: + if previous_env_database_url is None: + os.environ.pop("DATABASE_URL", None) + else: + os.environ["DATABASE_URL"] = previous_env_database_url + object.__setattr__(settings, "database_url", previous_settings_database_url) + asyncio.run(close_engine()) + + +@pytest.fixture +async def db_session( + migrated_database: None, + database_url: str, +) -> AsyncIterator[AsyncSession]: + engine = create_async_engine(database_url, pool_pre_ping=True) + async with engine.begin() as connection: + await connection.execute(RESET_SQL) + + session_factory = async_sessionmaker(engine, expire_on_commit=False) + async with session_factory() as session: + yield session + await session.rollback() + + await engine.dispose() + + +@pytest.fixture(autouse=True) +def reset_rate_limiter_state() -> Iterator[None]: + limiter = get_login_rate_limiter() + limiter.clear_all() + yield + limiter.clear_all() + + +@pytest.fixture(autouse=True) +async def reset_app_engine() -> AsyncIterator[None]: + await close_engine() + yield + await close_engine() diff --git a/tests/fixtures/scholar/profile_ok_amIMrIEAAAAJ.html b/tests/fixtures/scholar/profile_ok_amIMrIEAAAAJ.html new file mode 100644 index 0000000..dfa4a70 --- /dev/null +++ b/tests/fixtures/scholar/profile_ok_amIMrIEAAAAJ.html @@ -0,0 +1,76 @@ +‪Bangar Raju Cherukuri‬ - ‪Google Scholar‬
Follow
Bangar Raju Cherukuri
Bangar Raju Cherukuri
Senior .NET Web Developer
Verified email at dc.gov
Title
Cited by
Cited by
Year
Optimization of Data Structures and Trade-Offs with Concurrency Control in Multithread Software Structures Using Artificial Intelligence
BR Cherukuri, V Arulkumar
2024 IEEE International Conference on Computing, Power and Communication …, 2024
1312024
Development of Design Patterns with Adaptive User Interface for Cloud Native Microservice Architecture Using Deep Learning With IoT
BR Cherukuri
2024 IEEE International Conference on Computing, Power and Communication …, 2024
812024
AI-powered personalization: How machine learning is shaping the future of user experience
BR Cherukuri
International Journal of Science and Research Archive (IJSRA) 12 (1), 3111–3126, 2024
332024
Future of cloud computing: Innovations in multi-cloud and hybrid architectures
BR Cherukuri
World Journal of Advanced Research and Reviews (WJARR) 1 (1), 068-081, 2019
302019
Microservices and containerization: Accelerating web development cycles
BR Cherukuri
World Journal of Advanced Research and Reviews (WJARR) 6 (01), 283-296, 2020
272020
Ethical AI in cloud: Mitigating risks in machine learning models
BR Cherukuri
World Journal of Advanced Engineering Technology and Sciences ((WJAETS) 1 …, 2020
182020
Quantum machine learning: Transforming cloud-based AI solutions
BR Cherukuri
International Journal of Science and Research Archive (IJSRA) 1 (01), 110-122, 2020
142020
Enhanced Trimodal Emotion Recognition Using Multibranch Fusion Attention with Epistemic Neural Networks and Fire Hawk Optimization
BR Cherukuri
Journal of Machine and Computing 5 (1), 2025
132025
Containerization in cloud computing: comparing Docker and Kubernetes for scalable web applications
BR Cherukuri
International Journal of Science and Research Archive (IJSRA) 13 (1), 3302–3315, 2024
132024
Serverless computing: How to build and deploy applications without managing infrastructure
BR Cherukuri
World Journal of Advanced Engineering Technology and Sciences ((WJAETS) 11 …, 2024
132024
Edge Computing vs. Cloud Computing: A Comparative Analysis for Real-Time AI Applications
BR Cherukuri
International Journal For Multidisciplinary Research (IJFMR) 6 (5), 1-17, 2024
122024
Serverless revolution: Redefining application scalability and cost efficiency
BR Cherukuri
World Journal of Advanced Research and Reviews (WJARR) 2 (03), 039-053, 2019
92019
Progressive Web Apps (PWAs): Enhancing User Experience through Modern Web Development
BR Cherukuri
International Journal of Science and Research (IJSR) 13 (10), 1550-1560, 2024
82024
Building Scalable Web Applications: Best Practices for Backend Architecture
BR Cherukuri
International Journal of Science and Research (IJSR) 13 (10), 126-139, 2024
72024
Advanced Multi Class Cyber Security Attack Classification in IoT Based Wireless Sensor Networks Using Context Aware Depthwise Separable Convolutional Neural Network
BR Cherukuri
Journal of Machine and Computing 5 (2), 2025
52025
Maintenance of Web Development Standard for Multiple Devices with Serverless Computing through Cross Browser Affinity Using Hybrid Optimization
BR Cherukuri
2024 IEEE International Conference on Computing, Power and Communication …, 2024
32024
Developing Intelligent Chatbots for Real-Time Customer Support in E-Commerce
BR Cherukuri
International Journal of Science and Research (IJSR) 11 (01), 1709-1719, 2022
32022
Enhancing Web Application Performance with AI - Driven Optimization Techniques
BR Cherukuri
International Journal of Science and Research (IJSR) 10 (2), 1779-1788, 2021
32021
Federated Learning: Privacy-Preserving Machine Learning in Cloud Environments
BR Cherukuri
International Journal of Science and Research (IJSR) 13 (10), 1539-1549, 2024
12024
Scalable machine learning model deployment using serverless cloud architectures
BR Cherukuri
World Journal of Advanced Engineering Technology and Sciences ((WJAETS) 5 (1 …, 2022
12022
The system can't perform the operation now. Try again later.
Articles 1–20
\ No newline at end of file diff --git a/tests/fixtures/scholar/regression/profile_AAAAAAAAAAAA.html b/tests/fixtures/scholar/regression/profile_AAAAAAAAAAAA.html new file mode 100644 index 0000000..b3707e0 --- /dev/null +++ b/tests/fixtures/scholar/regression/profile_AAAAAAAAAAAA.html @@ -0,0 +1,421 @@ +Google Scholar Citations

Sign in

to continue to Google Scholar Citations
CAPTCHA image of text used to distinguish humans from robots
Not your computer? Use Private Browsing windows to sign in. Learn more about using Guest mode
  • Afrikaans
  • azərbaycan
  • bosanski
  • català
  • Čeština
  • Cymraeg
  • Dansk
  • Deutsch
  • eesti
  • English (United Kingdom)
  • English (United States)
  • Español (España)
  • Español (Latinoamérica)
  • euskara
  • Filipino
  • Français (Canada)
  • Français (France)
  • Gaeilge
  • galego
  • Hrvatski
  • Indonesia
  • isiZulu
  • íslenska
  • Italiano
  • Kiswahili
  • latviešu
  • lietuvių
  • magyar
  • Melayu
  • Nederlands
  • norsk
  • o‘zbek
  • polski
  • Português (Brasil)
  • Português (Portugal)
  • română
  • shqip
  • Slovenčina
  • slovenščina
  • srpski (latinica)
  • Suomi
  • Svenska
  • Tiếng Việt
  • Türkçe
  • Ελληνικά
  • беларуская
  • български
  • кыргызча
  • қазақ тілі
  • македонски
  • монгол
  • Русский
  • српски (ћирилица)
  • Українська
  • ქართული
  • հայերեն
  • ‫עברית‬‎
  • ‫اردو‬‎
  • ‫العربية‬‎
  • ‫فارسی‬‎
  • አማርኛ
  • नेपाली
  • मराठी
  • हिन्दी
  • অসমীয়া
  • বাংলা
  • ਪੰਜਾਬੀ
  • ગુજરાતી
  • ଓଡ଼ିଆ
  • தமிழ்
  • తెలుగు
  • ಕನ್ನಡ
  • മലയാളം
  • සිංහල
  • ไทย
  • ລາວ
  • မြန်မာ
  • ខ្មែរ
  • 한국어
  • 中文（香港）
  • 日本語
  • 简体中文
  • 繁體中文
\ No newline at end of file diff --git a/tests/fixtures/scholar/regression/profile_LZ5D_p4AAAAJ.html b/tests/fixtures/scholar/regression/profile_LZ5D_p4AAAAJ.html new file mode 100644 index 0000000..034d74e --- /dev/null +++ b/tests/fixtures/scholar/regression/profile_LZ5D_p4AAAAJ.html @@ -0,0 +1,76 @@ +‪Doaa Elmatary‬ - ‪Google Scholar‬
Follow
Doaa Elmatary
Doaa Elmatary
مدرس هندسة الاتصالات في معهد الصفوة العالي للهندسة
Verified email at alsafwa.edu.eg
Title
Cited by
Cited by
Year
Performance comparison of various machine learning approaches to identify the best one in predicting heart disease
EM Abd Allah, DE El-Matary, EM Eid, AST El Dien
Journal of Computer and Communications 10 (2), 1-18, 2022
252022
Energy efficient key management scheme based on elliptic curve signcryption for wireless sensor networks
EAAA Hagras, D El-Saied, HH Aly
2011 28th National Radio Science Conference (NRSC), 1-9, 2011
152011
A new forward secure elliptic curve signcryption key management (fs-ecskm) scheme for heterogeneous wireless sensor networks
EA Hagras, D El-Saied, HH Aly
International Journal of Computer Science and Technology 2 (2), 19-23, 2011
62011
Performance of polar codes for OFDM-based UWB channel
DEE Matary, EA Hagras, HM Abdel-Kader
Journal of Computer and Communications 6 (03), 102-117, 2018
42018
Multi-user Communication Based OFDM-UWB System under Gaussian and Non-Gaussian Noisy Channel
DEE Matary, EAA Hagras, HM Abdel-Kader
American Journal of Electrical and Electronic Engineering 5 (4), 136-143, 2017
32017
Doaa El-Saied, Dr. Hazem H. Aly,“A New Forward Secure Elliptic Curve Signcryption Key Management (FS-ECSKM) Scheme for Heterogeneous Wireless Sensor Networks”
EAAA Hagras
International Journal of Computer Science and Technology 2 (2), 19-23, 2011
22011
Performance Analysis of LDPC-IDMA-UWB Signals in Non-Gaussian Noisy Channel
DE El-Matary, EAA Hagras, HM Abdel-Kader
12013
Intelligent Sign Multi-Language Real-Time Prediction System with Effective Data Preprocessing
DE Elmatary
Journal of Computer and Communications, 120-134, 2023
2023
Smart Healthcare System Based IoT with Frequency Analysis of Heart Rate Data across FFT Algorithmto be used During Covid-19 Pandemic
E M AbdAllah, D E El Matary
Engineering Research Journal (Shoubra) 51 (1), 88-94, 2022
2022
Polar coded Interleave Division Multiple Access Ultra Wide Band (IDMA-UWB) communication system
DE El Matary, EAA Hagras, HM Abdel-Kader
2018 35th National Radio Science Conference (NRSC), 283-292, 2018
2018
Performance Analysis of LDPC-IDMA-UWB Signals in Non-Gaussian Noisy Channel
HMAK Doaa E.El-Matary, Esam A.A. Hagras
International Journal of Scientific & Engineering Research, 2013
2013
Non-Gaussian Noisy Channel Effect on Multi-user IDMA-UWB Communication System
DEE Matary, EAA Hagras, HM Abdel-Kader
The system can't perform the operation now. Try again later.
Articles 1–12
\ No newline at end of file diff --git a/tests/fixtures/scholar/regression/profile_P1RwlvoAAAAJ.html b/tests/fixtures/scholar/regression/profile_P1RwlvoAAAAJ.html new file mode 100644 index 0000000..47072eb --- /dev/null +++ b/tests/fixtures/scholar/regression/profile_P1RwlvoAAAAJ.html @@ -0,0 +1,76 @@ +‪WENRUI ZUO‬ - ‪Google Scholar‬
Follow
WENRUI ZUO
Title
Cited by
Cited by
Year
Set in stone: Analysis of an immutable web3 social media platform
W Zuo, A Raman, RJ Mondragón, G Tyson
Proceedings of the ACM Web Conference 2023, 1865-1874, 2023
182023
Understanding and improving content moderation in web3 platforms
W Zuo, RJ Mondragon, A Raman, G Tyson
Proceedings of the International AAAI Conference on Web and Social Media 18 …, 2024
102024
A first look at user-controlled moderation on web3 social media: The case of memo. cash
W Zuo, A Raman, RJ MondragÓN, G Tyson
Proceedings of the 3rd International Workshop on Open Challenges in Online …, 2023
72023
Promoting the Culture of Qinhuai River Lantern Shadow Puppetry with a Digital Archive and Immersive Experience
Y Liu, RM Williams, G Xie, Y Wang, W Zuo
arXiv preprint arXiv:2410.03532, 2024
42024
Understanding and Improving User-controlled Content Moderation Systems on Social Media
W Zuo
Queen Mary University of London, 2025
2025
The system can't perform the operation now. Try again later.
Articles 1–5
\ No newline at end of file diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py new file mode 100644 index 0000000..88b5764 --- /dev/null +++ b/tests/integration/__init__.py @@ -0,0 +1,2 @@ +# Integration tests package. + diff --git a/tests/integration/helpers.py b/tests/integration/helpers.py new file mode 100644 index 0000000..49cf32f --- /dev/null +++ b/tests/integration/helpers.py @@ -0,0 +1,63 @@ +from __future__ import annotations + +import re + +from fastapi.testclient import TestClient +from sqlalchemy import text +from sqlalchemy.ext.asyncio import AsyncSession + +from app.auth.security import PasswordService + +CSRF_TOKEN_PATTERN = re.compile(r'name="csrf_token" value="([^"]+)"') + + +def extract_csrf_token(html: str) -> str: + match = CSRF_TOKEN_PATTERN.search(html) + assert match is not None + return match.group(1) + + +def login_user(client: TestClient, *, email: str, password: str) -> None: + login_page = client.get("/login") + csrf_token = extract_csrf_token(login_page.text) + response = client.post( + "/login", + data={ + "email": email, + "password": password, + "csrf_token": csrf_token, + }, + follow_redirects=False, + ) + assert response.status_code == 303 + assert response.headers["location"] == "/" + + +async def insert_user( + db_session: AsyncSession, + *, + email: str, + password: str, + is_admin: bool = False, + is_active: bool = True, +) -> int: + password_service = PasswordService() + result = await db_session.execute( + text( + """ + INSERT INTO users (email, password_hash, is_active, is_admin) + VALUES (:email, :password_hash, :is_active, :is_admin) + RETURNING id + """ + ), + { + "email": email, + "password_hash": password_service.hash_password(password), + "is_active": is_active, + "is_admin": is_admin, + }, + ) + user_id = int(result.scalar_one()) + await db_session.commit() + return user_id + diff --git a/tests/integration/test_admin_user_management.py b/tests/integration/test_admin_user_management.py new file mode 100644 index 0000000..7553317 --- /dev/null +++ b/tests/integration/test_admin_user_management.py @@ -0,0 +1,200 @@ +import pytest +from fastapi.testclient import TestClient +from sqlalchemy import text +from sqlalchemy.ext.asyncio import AsyncSession + +from app.main import app +from tests.integration.helpers import extract_csrf_token, insert_user, login_user + + +@pytest.mark.integration +@pytest.mark.db +@pytest.mark.asyncio +async def test_users_page_is_admin_only(db_session: AsyncSession) -> None: + await insert_user( + db_session, + email="member@example.com", + password="member-pass", + is_admin=False, + ) + + client = TestClient(app) + login_user(client, email="member@example.com", password="member-pass") + + response = client.get("/users") + + assert response.status_code == 403 + assert response.json()["detail"] == "Admin access required." + + +@pytest.mark.integration +@pytest.mark.db +@pytest.mark.asyncio +async def test_non_admin_dashboard_hides_users_nav(db_session: AsyncSession) -> None: + await insert_user( + db_session, + email="member@example.com", + password="member-pass", + is_admin=False, + ) + + client = TestClient(app) + login_user(client, email="member@example.com", password="member-pass") + dashboard = client.get("/") + + assert dashboard.status_code == 200 + assert ">Users<" not in dashboard.text + + +@pytest.mark.integration +@pytest.mark.db +@pytest.mark.asyncio +async def test_admin_dashboard_shows_users_nav(db_session: AsyncSession) -> None: + await insert_user( + db_session, + email="admin@example.com", + password="admin-pass", + is_admin=True, + ) + + client = TestClient(app) + login_user(client, email="admin@example.com", password="admin-pass") + dashboard = client.get("/") + + assert dashboard.status_code == 200 + assert ">Users<" in dashboard.text + + +@pytest.mark.integration +@pytest.mark.db +@pytest.mark.asyncio +async def test_admin_can_create_and_deactivate_user(db_session: AsyncSession) -> None: + await insert_user( + db_session, + email="admin@example.com", + password="admin-pass", + is_admin=True, + ) + + client = TestClient(app) + login_user(client, email="admin@example.com", password="admin-pass") + + users_page = client.get("/users") + csrf_token = extract_csrf_token(users_page.text) + create_response = client.post( + "/users", + data={ + "email": "new-user@example.com", + "password": "new-user-pass", + "csrf_token": csrf_token, + }, + follow_redirects=False, + ) + assert create_response.status_code == 303 + assert create_response.headers["location"].startswith("/users") + + created_user_id_result = await db_session.execute( + text("SELECT id FROM users WHERE email = 'new-user@example.com'") + ) + created_user_id = int(created_user_id_result.scalar_one()) + + users_page_after_create = client.get("/users") + csrf_after_create = extract_csrf_token(users_page_after_create.text) + toggle_response = client.post( + f"/users/{created_user_id}/toggle-active", + data={"csrf_token": csrf_after_create}, + follow_redirects=False, + ) + assert toggle_response.status_code == 303 + + status_result = await db_session.execute( + text("SELECT is_active FROM users WHERE id = :user_id"), + {"user_id": created_user_id}, + ) + assert status_result.scalar_one() is False + + +@pytest.mark.integration +@pytest.mark.db +@pytest.mark.asyncio +async def test_admin_can_reset_user_password(db_session: AsyncSession) -> None: + target_user_id = await insert_user( + db_session, + email="target@example.com", + password="old-password", + is_admin=False, + ) + await insert_user( + db_session, + email="admin@example.com", + password="admin-pass", + is_admin=True, + ) + + client = TestClient(app) + login_user(client, email="admin@example.com", password="admin-pass") + + users_page = client.get("/users") + csrf_token = extract_csrf_token(users_page.text) + reset_response = client.post( + f"/users/{target_user_id}/reset-password", + data={ + "new_password": "new-password", + "csrf_token": csrf_token, + }, + follow_redirects=False, + ) + assert reset_response.status_code == 303 + + users_page_after_reset = client.get("/users") + logout_csrf = extract_csrf_token(users_page_after_reset.text) + client.post( + "/logout", + data={"csrf_token": logout_csrf}, + follow_redirects=False, + ) + + failed_login_page = client.get("/login") + failed_login_csrf = extract_csrf_token(failed_login_page.text) + failed_login = client.post( + "/login", + data={ + "email": "target@example.com", + "password": "old-password", + "csrf_token": failed_login_csrf, + }, + follow_redirects=False, + ) + assert failed_login.status_code == 401 + + login_user(client, email="target@example.com", password="new-password") + + +@pytest.mark.integration +@pytest.mark.db +@pytest.mark.asyncio +async def test_admin_cannot_deactivate_self(db_session: AsyncSession) -> None: + admin_user_id = await insert_user( + db_session, + email="admin@example.com", + password="admin-pass", + is_admin=True, + ) + client = TestClient(app) + login_user(client, email="admin@example.com", password="admin-pass") + + users_page = client.get("/users") + csrf_token = extract_csrf_token(users_page.text) + response = client.post( + f"/users/{admin_user_id}/toggle-active", + data={"csrf_token": csrf_token}, + follow_redirects=False, + ) + + assert response.status_code == 303 + assert response.headers["location"].startswith("/users") + result = await db_session.execute( + text("SELECT is_active FROM users WHERE id = :user_id"), + {"user_id": admin_user_id}, + ) + assert result.scalar_one() is True diff --git a/tests/integration/test_api_v1.py b/tests/integration/test_api_v1.py new file mode 100644 index 0000000..11e8b19 --- /dev/null +++ b/tests/integration/test_api_v1.py @@ -0,0 +1,518 @@ +from __future__ import annotations + +import pytest +from fastapi.testclient import TestClient +from sqlalchemy import text +from sqlalchemy.ext.asyncio import AsyncSession + +from app.main import app +from tests.integration.helpers import insert_user, login_user + + +def _api_bootstrap_csrf_headers(client: TestClient) -> dict[str, str]: + bootstrap_response = client.get("/api/v1/auth/csrf") + assert bootstrap_response.status_code == 200 + payload = bootstrap_response.json()["data"] + token = payload["csrf_token"] + assert isinstance(token, str) and token + return {"X-CSRF-Token": token} + + +def _api_csrf_headers(client: TestClient) -> dict[str, str]: + me_response = client.get("/api/v1/auth/me") + assert me_response.status_code == 200 + body = me_response.json() + token = body["data"]["csrf_token"] + assert isinstance(token, str) and token + return {"X-CSRF-Token": token} + + +@pytest.mark.integration +@pytest.mark.db +@pytest.mark.asyncio +async def test_api_me_requires_authentication() -> None: + client = TestClient(app) + + response = client.get("/api/v1/auth/me") + + assert response.status_code == 401 + payload = response.json() + assert payload["error"]["code"] == "auth_required" + assert payload["error"]["message"] == "Authentication required." + assert "request_id" in payload["meta"] + + +@pytest.mark.integration +@pytest.mark.db +@pytest.mark.asyncio +async def test_api_csrf_bootstrap_returns_token_for_anonymous_and_authenticated( + db_session: AsyncSession, +) -> None: + await insert_user( + db_session, + email="api-bootstrap@example.com", + password="api-password", + ) + client = TestClient(app) + + anonymous_response = client.get("/api/v1/auth/csrf") + assert anonymous_response.status_code == 200 + anonymous_payload = anonymous_response.json()["data"] + assert isinstance(anonymous_payload["csrf_token"], str) + assert anonymous_payload["authenticated"] is False + + login_user(client, email="api-bootstrap@example.com", password="api-password") + authenticated_response = client.get("/api/v1/auth/csrf") + assert authenticated_response.status_code == 200 + authenticated_payload = authenticated_response.json()["data"] + assert isinstance(authenticated_payload["csrf_token"], str) + assert authenticated_payload["authenticated"] is True + + +@pytest.mark.integration +@pytest.mark.db +@pytest.mark.asyncio +async def test_api_me_returns_user_and_csrf_token(db_session: AsyncSession) -> None: + await insert_user( + db_session, + email="api-me@example.com", + password="api-password", + ) + + client = TestClient(app) + login_user(client, email="api-me@example.com", password="api-password") + + response = client.get("/api/v1/auth/me") + + assert response.status_code == 200 + payload = response.json() + assert payload["data"]["authenticated"] is True + assert payload["data"]["user"]["email"] == "api-me@example.com" + assert isinstance(payload["data"]["csrf_token"], str) + assert payload["meta"]["request_id"] + + +@pytest.mark.integration +@pytest.mark.db +@pytest.mark.asyncio +async def test_api_login_and_change_password_flow(db_session: AsyncSession) -> None: + await insert_user( + db_session, + email="api-login@example.com", + password="old-password", + ) + client = TestClient(app) + + missing_csrf = client.post( + "/api/v1/auth/login", + json={"email": "api-login@example.com", "password": "old-password"}, + ) + assert missing_csrf.status_code == 403 + assert missing_csrf.json()["error"]["code"] == "csrf_missing" + + login_headers = _api_bootstrap_csrf_headers(client) + login_response = client.post( + "/api/v1/auth/login", + json={"email": "api-login@example.com", "password": "old-password"}, + headers=login_headers, + ) + assert login_response.status_code == 200 + login_payload = login_response.json()["data"] + assert login_payload["authenticated"] is True + assert login_payload["user"]["email"] == "api-login@example.com" + assert isinstance(login_payload["csrf_token"], str) + + bad_change_response = client.post( + "/api/v1/auth/change-password", + json={ + "current_password": "not-correct", + "new_password": "new-password", + "confirm_password": "new-password", + }, + headers={"X-CSRF-Token": login_payload["csrf_token"]}, + ) + assert bad_change_response.status_code == 400 + assert bad_change_response.json()["error"]["code"] == "invalid_current_password" + + change_response = client.post( + "/api/v1/auth/change-password", + json={ + "current_password": "old-password", + "new_password": "new-password", + "confirm_password": "new-password", + }, + headers={"X-CSRF-Token": login_payload["csrf_token"]}, + ) + assert change_response.status_code == 200 + assert change_response.json()["data"]["message"] == "Password updated successfully." + + logout_response = client.post( + "/api/v1/auth/logout", + headers={"X-CSRF-Token": login_payload["csrf_token"]}, + ) + assert logout_response.status_code == 200 + + relogin_headers = _api_bootstrap_csrf_headers(client) + old_password_login = client.post( + "/api/v1/auth/login", + json={"email": "api-login@example.com", "password": "old-password"}, + headers=relogin_headers, + ) + assert old_password_login.status_code == 401 + assert old_password_login.json()["error"]["code"] == "invalid_credentials" + + fresh_headers = _api_bootstrap_csrf_headers(client) + new_password_login = client.post( + "/api/v1/auth/login", + json={"email": "api-login@example.com", "password": "new-password"}, + headers=fresh_headers, + ) + assert new_password_login.status_code == 200 + assert new_password_login.json()["data"]["authenticated"] is True + + +@pytest.mark.integration +@pytest.mark.db +@pytest.mark.asyncio +async def test_api_admin_user_management_endpoints(db_session: AsyncSession) -> None: + admin_user_id = await insert_user( + db_session, + email="api-admin@example.com", + password="admin-password", + is_admin=True, + ) + target_user_id = await insert_user( + db_session, + email="api-target@example.com", + password="target-password", + is_admin=False, + ) + await insert_user( + db_session, + email="api-member@example.com", + password="member-password", + is_admin=False, + ) + + client = TestClient(app) + login_user(client, email="api-admin@example.com", password="admin-password") + headers = _api_csrf_headers(client) + + list_response = client.get("/api/v1/admin/users") + assert list_response.status_code == 200 + users = list_response.json()["data"]["users"] + assert any(item["email"] == "api-target@example.com" for item in users) + + create_response = client.post( + "/api/v1/admin/users", + json={ + "email": "api-created@example.com", + "password": "created-password", + "is_admin": True, + }, + headers=headers, + ) + assert create_response.status_code == 201 + created_user = create_response.json()["data"] + assert created_user["email"] == "api-created@example.com" + created_user_id = int(created_user["id"]) + + deactivate_response = client.patch( + f"/api/v1/admin/users/{target_user_id}/active", + json={"is_active": False}, + headers=headers, + ) + assert deactivate_response.status_code == 200 + assert deactivate_response.json()["data"]["is_active"] is False + + reactivate_response = client.patch( + f"/api/v1/admin/users/{target_user_id}/active", + json={"is_active": True}, + headers=headers, + ) + assert reactivate_response.status_code == 200 + assert reactivate_response.json()["data"]["is_active"] is True + + reset_response = client.post( + f"/api/v1/admin/users/{target_user_id}/reset-password", + json={"new_password": "target-password-updated"}, + headers=headers, + ) + assert reset_response.status_code == 200 + assert "Password reset" in reset_response.json()["data"]["message"] + + self_deactivate = client.patch( + f"/api/v1/admin/users/{admin_user_id}/active", + json={"is_active": False}, + headers=headers, + ) + assert self_deactivate.status_code == 400 + assert self_deactivate.json()["error"]["code"] == "cannot_deactivate_self" + + logout_response = client.post("/api/v1/auth/logout", headers=headers) + assert logout_response.status_code == 200 + + target_headers = _api_bootstrap_csrf_headers(client) + target_login = client.post( + "/api/v1/auth/login", + json={"email": "api-target@example.com", "password": "target-password-updated"}, + headers=target_headers, + ) + assert target_login.status_code == 200 + + non_admin_headers = {"X-CSRF-Token": target_login.json()["data"]["csrf_token"]} + forbidden_response = client.get("/api/v1/admin/users") + assert forbidden_response.status_code == 403 + assert forbidden_response.json()["error"]["code"] == "forbidden" + + forbidden_create = client.post( + "/api/v1/admin/users", + json={ + "email": "should-not-work@example.com", + "password": "password-123", + "is_admin": False, + }, + headers=non_admin_headers, + ) + assert forbidden_create.status_code == 403 + + created_exists = await db_session.execute( + text("SELECT COUNT(*) FROM users WHERE id = :user_id"), + {"user_id": created_user_id}, + ) + assert created_exists.scalar_one() == 1 + + +@pytest.mark.integration +@pytest.mark.db +@pytest.mark.asyncio +async def test_api_scholars_crud_flow(db_session: AsyncSession) -> None: + await insert_user( + db_session, + email="api-scholars@example.com", + password="api-password", + ) + + client = TestClient(app) + login_user(client, email="api-scholars@example.com", password="api-password") + headers = _api_csrf_headers(client) + + missing_csrf = client.post( + "/api/v1/scholars", + json={"scholar_id": "abcDEF123456", "display_name": "Ada"}, + ) + assert missing_csrf.status_code == 403 + assert missing_csrf.json()["error"]["code"] == "csrf_invalid" + + create_response = client.post( + "/api/v1/scholars", + json={"scholar_id": "abcDEF123456", "display_name": "Ada"}, + headers=headers, + ) + assert create_response.status_code == 201 + created = create_response.json()["data"] + assert created["scholar_id"] == "abcDEF123456" + scholar_profile_id = int(created["id"]) + + list_response = client.get("/api/v1/scholars") + assert list_response.status_code == 200 + scholars = list_response.json()["data"]["scholars"] + assert any(int(item["id"]) == scholar_profile_id for item in scholars) + + toggle_response = client.patch( + f"/api/v1/scholars/{scholar_profile_id}/toggle", + headers=headers, + ) + assert toggle_response.status_code == 200 + assert toggle_response.json()["data"]["is_enabled"] is False + + delete_response = client.delete( + f"/api/v1/scholars/{scholar_profile_id}", + headers=headers, + ) + assert delete_response.status_code == 200 + assert delete_response.json()["data"]["message"] == "Scholar deleted." + + +@pytest.mark.integration +@pytest.mark.db +@pytest.mark.asyncio +async def test_api_settings_get_and_update(db_session: AsyncSession) -> None: + await insert_user( + db_session, + email="api-settings@example.com", + password="api-password", + ) + + client = TestClient(app) + login_user(client, email="api-settings@example.com", password="api-password") + headers = _api_csrf_headers(client) + + get_response = client.get("/api/v1/settings") + assert get_response.status_code == 200 + assert "request_delay_seconds" in get_response.json()["data"] + + update_response = client.put( + "/api/v1/settings", + json={ + "auto_run_enabled": True, + "run_interval_minutes": 45, + "request_delay_seconds": 6, + }, + headers=headers, + ) + assert update_response.status_code == 200 + updated = update_response.json()["data"] + assert updated["auto_run_enabled"] is True + assert updated["run_interval_minutes"] == 45 + assert updated["request_delay_seconds"] == 6 + + +@pytest.mark.integration +@pytest.mark.db +@pytest.mark.asyncio +async def test_api_runs_manual_and_queue_actions(db_session: AsyncSession) -> None: + user_id = await insert_user( + db_session, + email="api-runs@example.com", + password="api-password", + ) + + client = TestClient(app) + login_user(client, email="api-runs@example.com", password="api-password") + headers = _api_csrf_headers(client) + + run_response = client.post( + "/api/v1/runs/manual", + headers={**headers, "Idempotency-Key": "manual-run-0001"}, + ) + assert run_response.status_code == 200 + run_payload = run_response.json()["data"] + assert "run_id" in run_payload + assert run_payload["status"] in {"success", "partial_failure", "failed"} + assert run_payload["reused_existing_run"] is False + assert run_payload["idempotency_key"] == "manual-run-0001" + + replay_response = client.post( + "/api/v1/runs/manual", + headers={**headers, "Idempotency-Key": "manual-run-0001"}, + ) + assert replay_response.status_code == 200 + replay_payload = replay_response.json()["data"] + assert replay_payload["run_id"] == run_payload["run_id"] + assert replay_payload["reused_existing_run"] is True + + runs_response = client.get("/api/v1/runs") + assert runs_response.status_code == 200 + assert len(runs_response.json()["data"]["runs"]) >= 1 + run_id = int(run_payload["run_id"]) + + run_detail_response = client.get(f"/api/v1/runs/{run_id}") + assert run_detail_response.status_code == 200 + detail_payload = run_detail_response.json()["data"] + assert "summary" in detail_payload + assert isinstance(detail_payload["scholar_results"], list) + + scholar_result = await db_session.execute( + text( + """ + INSERT INTO scholar_profiles (user_id, scholar_id, display_name, is_enabled) + VALUES (:user_id, :scholar_id, :display_name, true) + RETURNING id + """ + ), + { + "user_id": user_id, + "scholar_id": "abcDEF123456", + "display_name": "Queue Scholar", + }, + ) + scholar_profile_id = int(scholar_result.scalar_one()) + queue_result = await db_session.execute( + text( + """ + INSERT INTO ingestion_queue_items ( + user_id, + scholar_profile_id, + resume_cstart, + reason, + status, + attempt_count, + next_attempt_dt, + dropped_reason, + dropped_at + ) + VALUES ( + :user_id, + :scholar_profile_id, + 7, + 'dropped', + 'dropped', + 2, + NOW(), + 'manual_drop', + NOW() + ) + RETURNING id + """ + ), + {"user_id": user_id, "scholar_profile_id": scholar_profile_id}, + ) + queue_item_id = int(queue_result.scalar_one()) + await db_session.commit() + + queue_list_response = client.get("/api/v1/runs/queue/items") + assert queue_list_response.status_code == 200 + assert any( + int(item["id"]) == queue_item_id + for item in queue_list_response.json()["data"]["queue_items"] + ) + + retry_response = client.post(f"/api/v1/runs/queue/{queue_item_id}/retry", headers=headers) + assert retry_response.status_code == 200 + assert retry_response.json()["data"]["status"] == "queued" + + retry_again_response = client.post( + f"/api/v1/runs/queue/{queue_item_id}/retry", + headers=headers, + ) + assert retry_again_response.status_code == 409 + assert retry_again_response.json()["error"]["code"] == "queue_item_already_queued" + + drop_response = client.post(f"/api/v1/runs/queue/{queue_item_id}/drop", headers=headers) + assert drop_response.status_code == 200 + assert drop_response.json()["data"]["status"] == "dropped" + + clear_response = client.request( + "DELETE", + f"/api/v1/runs/queue/{queue_item_id}", + headers=headers, + ) + assert clear_response.status_code == 200 + assert clear_response.json()["data"]["status"] == "cleared" + assert clear_response.json()["data"]["message"] == "Queue item cleared." + + +@pytest.mark.integration +@pytest.mark.db +@pytest.mark.asyncio +async def test_api_publications_list_and_mark_read(db_session: AsyncSession) -> None: + await insert_user( + db_session, + email="api-pubs@example.com", + password="api-password", + ) + + client = TestClient(app) + login_user(client, email="api-pubs@example.com", password="api-password") + headers = _api_csrf_headers(client) + + list_response = client.get("/api/v1/publications?mode=all") + assert list_response.status_code == 200 + data = list_response.json()["data"] + assert data["mode"] == "all" + assert isinstance(data["publications"], list) + + mark_response = client.post("/api/v1/publications/mark-all-read", headers=headers) + assert mark_response.status_code == 200 + assert "updated_count" in mark_response.json()["data"] diff --git a/tests/integration/test_auth_flow.py b/tests/integration/test_auth_flow.py new file mode 100644 index 0000000..2455b80 --- /dev/null +++ b/tests/integration/test_auth_flow.py @@ -0,0 +1,109 @@ +import re + +import pytest +from sqlalchemy import text +from sqlalchemy.ext.asyncio import AsyncSession +from fastapi.testclient import TestClient + +from app.auth.security import PasswordService +from app.main import app + +CSRF_TOKEN_PATTERN = re.compile(r'name="csrf_token" value="([^"]+)"') + + +def _extract_csrf_token(html: str) -> str: + match = CSRF_TOKEN_PATTERN.search(html) + assert match is not None + return match.group(1) + + +@pytest.mark.integration +def test_dashboard_requires_authentication() -> None: + client = TestClient(app) + + response = client.get("/", follow_redirects=False) + + assert response.status_code == 303 + assert response.headers["location"] == "/login" + + +@pytest.mark.integration +@pytest.mark.db +@pytest.mark.asyncio +async def test_login_with_valid_credentials_allows_access(db_session: AsyncSession) -> None: + password_service = PasswordService() + await db_session.execute( + text( + """ + INSERT INTO users (email, password_hash, is_active, is_admin) + VALUES (:email, :password_hash, :is_active, :is_admin) + """ + ), + { + "email": "reader@example.com", + "password_hash": password_service.hash_password("correct-password"), + "is_active": True, + "is_admin": False, + }, + ) + await db_session.commit() + + client = TestClient(app) + login_page = client.get("/login") + csrf_token = _extract_csrf_token(login_page.text) + login_response = client.post( + "/login", + data={ + "email": "reader@example.com", + "password": "correct-password", + "csrf_token": csrf_token, + }, + follow_redirects=False, + ) + + assert login_response.status_code == 303 + assert login_response.headers["location"] == "/" + + dashboard_response = client.get("/") + assert dashboard_response.status_code == 200 + assert "reader@example.com" in dashboard_response.text + assert 'data-test="home-hero"' in dashboard_response.text + + +@pytest.mark.integration +@pytest.mark.db +@pytest.mark.asyncio +async def test_login_rejects_inactive_user(db_session: AsyncSession) -> None: + password_service = PasswordService() + await db_session.execute( + text( + """ + INSERT INTO users (email, password_hash, is_active, is_admin) + VALUES (:email, :password_hash, :is_active, :is_admin) + """ + ), + { + "email": "inactive@example.com", + "password_hash": password_service.hash_password("correct-password"), + "is_active": False, + "is_admin": False, + }, + ) + await db_session.commit() + + client = TestClient(app) + login_page = client.get("/login") + csrf_token = _extract_csrf_token(login_page.text) + login_response = client.post( + "/login", + data={ + "email": "inactive@example.com", + "password": "correct-password", + "csrf_token": csrf_token, + }, + follow_redirects=False, + ) + + assert login_response.status_code == 401 + assert "Invalid email or password." in login_response.text + diff --git a/tests/integration/test_manual_ingestion_flow.py b/tests/integration/test_manual_ingestion_flow.py new file mode 100644 index 0000000..25de49a --- /dev/null +++ b/tests/integration/test_manual_ingestion_flow.py @@ -0,0 +1,1151 @@ +from __future__ import annotations + +import pytest +from fastapi.testclient import TestClient +from sqlalchemy import text +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine + +from app.main import app, get_scholar_source +from app.db.models import RunTriggerType +from app.services.ingestion import RUN_LOCK_NAMESPACE, RunAlreadyInProgressError, ScholarIngestionService +from app.services.scheduler import SchedulerService +from app.services.scholar_source import FetchResult +from tests.integration.helpers import extract_csrf_token, insert_user, login_user + +HTML_BASELINE = """ + +
Fixture Scholar
+ Articles 1-1 + + + + + + +
+ Paper One +
A Author
+
Venue One, 2024
+ 		32024
+ +""" + +HTML_INCREMENTAL = """ + +
Fixture Scholar
+ Articles 1-2 + + + + + + + + + + + +
+ Paper One +
A Author
+
Venue One, 2024
+ 		42024
+ Paper Two +
B Author
+
Venue Two, 2025
+ 		12025
+ +""" + +HTML_PAGED_ONE = """ + +
Paged Scholar
+ Articles 1-1 + + + + + + +
+ Paged Paper One +
P Author
+
Paged Venue, 2023
+ 		52023
+
+ +
+ +""" + +HTML_PAGED_TWO = """ + +
Paged Scholar
+ Articles 2-2 + + + + + + +
+ Paged Paper Two +
P Author
+
Paged Venue, 2024
+ 		22024
+ +""" + +HTML_STALLED_TAIL_ONE = """ + +
Tail Scholar
+ Articles 1-1 + + + + + + +
+ Tail Paper One +
T Author
+
Tail Venue, 2022
+ 		92022
+
+ +""" + +HTML_STALLED_TAIL_TWO = """ + +
Tail Scholar
+
No documents. Your search didn't match any articles.
+
+ +""" + + +class StubScholarSource: + def __init__(self, html_bodies: list[str]) -> None: + self._html_bodies = html_bodies + self._index = 0 + + async def fetch_profile_html(self, scholar_id: str) -> FetchResult: + if self._index >= len(self._html_bodies): + body = self._html_bodies[-1] + else: + body = self._html_bodies[self._index] + self._index += 1 + + url = f"https://scholar.google.com/citations?hl=en&user={scholar_id}" + return FetchResult( + requested_url=url, + status_code=200, + final_url=url, + body=body, + error=None, + ) + + +class StubScholarSourceResults: + def __init__(self, results: list[FetchResult]) -> None: + self._results = results + self._index = 0 + + async def fetch_profile_html(self, scholar_id: str) -> FetchResult: + if self._index >= len(self._results): + result = self._results[-1] + else: + result = self._results[self._index] + self._index += 1 + return result + + +class CountingScholarSourceResults(StubScholarSourceResults): + def __init__(self, results: list[FetchResult]) -> None: + super().__init__(results) + self.calls = 0 + + async def fetch_profile_html(self, scholar_id: str) -> FetchResult: + self.calls += 1 + return await super().fetch_profile_html(scholar_id) + + +class StubPagedScholarSource: + def __init__(self, pages: dict[int, str]) -> None: + self._pages = pages + self.calls: list[tuple[int, int]] = [] + + async def fetch_profile_html(self, scholar_id: str) -> FetchResult: + return await self.fetch_profile_page_html( + scholar_id, + cstart=0, + pagesize=100, + ) + + async def fetch_profile_page_html( + self, + scholar_id: str, + *, + cstart: int, + pagesize: int, + ) -> FetchResult: + self.calls.append((cstart, pagesize)) + body = self._pages.get(cstart, "") + url = ( + "https://scholar.google.com/citations" + f"?hl=en&user={scholar_id}&cstart={cstart}&pagesize={pagesize}" + ) + return FetchResult( + requested_url=url, + status_code=200, + final_url=url, + body=body, + error=None, + ) + + +@pytest.mark.integration +@pytest.mark.db +@pytest.mark.asyncio +async def test_manual_ingestion_sets_baseline_then_adds_unread(db_session: AsyncSession) -> None: + user_id = await insert_user( + db_session, + email="ingest@example.com", + password="ingest-password", + ) + await db_session.execute( + text( + """ + INSERT INTO scholar_profiles (user_id, scholar_id, display_name, is_enabled) + VALUES (:user_id, :scholar_id, :display_name, :is_enabled) + """ + ), + { + "user_id": user_id, + "scholar_id": "abcDEF123456", + "display_name": "Fixture Scholar", + "is_enabled": True, + }, + ) + await db_session.commit() + + stub_source = StubScholarSource([HTML_BASELINE, HTML_INCREMENTAL]) + app.dependency_overrides[get_scholar_source] = lambda: stub_source + + client = TestClient(app) + try: + login_user(client, email="ingest@example.com", password="ingest-password") + + dashboard = client.get("/") + csrf = extract_csrf_token(dashboard.text) + run_one = client.post( + "/runs/manual", + data={"csrf_token": csrf}, + follow_redirects=False, + ) + assert run_one.status_code == 303 + assert run_one.headers["location"].startswith("/") + + baseline_status = await db_session.execute( + text("SELECT baseline_completed FROM scholar_profiles WHERE user_id = :user_id"), + {"user_id": user_id}, + ) + assert baseline_status.scalar_one() is True + + run_one_stats = await db_session.execute( + text( + """ + SELECT status::text, new_pub_count + FROM crawl_runs + WHERE user_id = :user_id + ORDER BY id DESC + LIMIT 1 + """ + ), + {"user_id": user_id}, + ) + assert run_one_stats.one() == ("success", 1) + + unread_after_baseline = await db_session.execute( + text( + """ + SELECT COUNT(*) + FROM scholar_publications sp + JOIN scholar_profiles s ON s.id = sp.scholar_profile_id + WHERE s.user_id = :user_id AND sp.is_read = false + """ + ), + {"user_id": user_id}, + ) + assert unread_after_baseline.scalar_one() == 1 + + dashboard_again = client.get("/") + csrf_two = extract_csrf_token(dashboard_again.text) + run_two = client.post( + "/runs/manual", + data={"csrf_token": csrf_two}, + follow_redirects=False, + ) + assert run_two.status_code == 303 + + run_two_stats = await db_session.execute( + text( + """ + SELECT status::text, new_pub_count + FROM crawl_runs + WHERE user_id = :user_id + ORDER BY id DESC + LIMIT 1 + """ + ), + {"user_id": user_id}, + ) + assert run_two_stats.one() == ("success", 1) + + unread_after_second = await db_session.execute( + text( + """ + SELECT COUNT(*) + FROM scholar_publications sp + JOIN scholar_profiles s ON s.id = sp.scholar_profile_id + WHERE s.user_id = :user_id AND sp.is_read = false + """ + ), + {"user_id": user_id}, + ) + assert unread_after_second.scalar_one() == 2 + + publications_new_page = client.get("/publications?mode=new") + assert publications_new_page.status_code == 200 + assert "Paper Two" in publications_new_page.text + assert "Paper One" not in publications_new_page.text + + publications_all_page = client.get("/publications?mode=all") + assert publications_all_page.status_code == 200 + assert "Paper One" in publications_all_page.text + assert "Paper Two" in publications_all_page.text + + dashboard_mark = client.get("/") + csrf_mark = extract_csrf_token(dashboard_mark.text) + mark_response = client.post( + "/publications/mark-all-read", + data={"csrf_token": csrf_mark}, + follow_redirects=False, + ) + assert mark_response.status_code == 303 + + unread_after_mark = await db_session.execute( + text( + """ + SELECT COUNT(*) + FROM scholar_publications sp + JOIN scholar_profiles s ON s.id = sp.scholar_profile_id + WHERE s.user_id = :user_id AND sp.is_read = false + """ + ), + {"user_id": user_id}, + ) + assert unread_after_mark.scalar_one() == 0 + + publications_new_after_mark = client.get("/publications?mode=new") + assert publications_new_after_mark.status_code == 200 + assert "Paper Two" in publications_new_after_mark.text + assert "Paper One" not in publications_new_after_mark.text + finally: + app.dependency_overrides.clear() + + +@pytest.mark.integration +@pytest.mark.db +@pytest.mark.asyncio +async def test_publications_page_supports_scholar_filtering_navigation( + db_session: AsyncSession, +) -> None: + user_id = await insert_user( + db_session, + email="filtering@example.com", + password="filter-password", + ) + + scholar_one_result = await db_session.execute( + text( + """ + INSERT INTO scholar_profiles (user_id, scholar_id, display_name, is_enabled) + VALUES (:user_id, :scholar_id, :display_name, :is_enabled) + RETURNING id + """ + ), + { + "user_id": user_id, + "scholar_id": "abcDEF123456", + "display_name": "Scholar One", + "is_enabled": True, + }, + ) + scholar_one_id = int(scholar_one_result.scalar_one()) + + scholar_two_result = await db_session.execute( + text( + """ + INSERT INTO scholar_profiles (user_id, scholar_id, display_name, is_enabled) + VALUES (:user_id, :scholar_id, :display_name, :is_enabled) + RETURNING id + """ + ), + { + "user_id": user_id, + "scholar_id": "uvwXYZ987654", + "display_name": "Scholar Two", + "is_enabled": True, + }, + ) + scholar_two_id = int(scholar_two_result.scalar_one()) + + run_result = await db_session.execute( + text( + """ + INSERT INTO crawl_runs ( + user_id, + trigger_type, + status, + start_dt, + end_dt, + scholar_count, + new_pub_count, + error_log + ) + VALUES ( + :user_id, + 'manual', + 'success', + NOW(), + NOW(), + 2, + 2, + '{}'::jsonb + ) + RETURNING id + """ + ), + {"user_id": user_id}, + ) + run_id = int(run_result.scalar_one()) + + publication_one_result = await db_session.execute( + text( + """ + INSERT INTO publications ( + cluster_id, + fingerprint_sha256, + title_raw, + title_normalized, + year, + citation_count, + author_text, + venue_text, + pub_url, + pdf_url + ) + VALUES ( + NULL, + :fingerprint, + :title_raw, + :title_normalized, + :year, + 0, + NULL, + NULL, + NULL, + NULL + ) + RETURNING id + """ + ), + { + "fingerprint": "f" * 64, + "title_raw": "Scholar One Paper", + "title_normalized": "scholar one paper", + "year": 2024, + }, + ) + publication_one_id = int(publication_one_result.scalar_one()) + + publication_two_result = await db_session.execute( + text( + """ + INSERT INTO publications ( + cluster_id, + fingerprint_sha256, + title_raw, + title_normalized, + year, + citation_count, + author_text, + venue_text, + pub_url, + pdf_url + ) + VALUES ( + NULL, + :fingerprint, + :title_raw, + :title_normalized, + :year, + 0, + NULL, + NULL, + NULL, + NULL + ) + RETURNING id + """ + ), + { + "fingerprint": "e" * 64, + "title_raw": "Scholar Two Paper", + "title_normalized": "scholar two paper", + "year": 2025, + }, + ) + publication_two_id = int(publication_two_result.scalar_one()) + + await db_session.execute( + text( + """ + INSERT INTO scholar_publications (scholar_profile_id, publication_id, is_read, first_seen_run_id) + VALUES (:scholar_profile_id, :publication_id, :is_read, :first_seen_run_id) + """ + ), + { + "scholar_profile_id": scholar_one_id, + "publication_id": publication_one_id, + "is_read": False, + "first_seen_run_id": run_id, + }, + ) + await db_session.execute( + text( + """ + INSERT INTO scholar_publications (scholar_profile_id, publication_id, is_read, first_seen_run_id) + VALUES (:scholar_profile_id, :publication_id, :is_read, :first_seen_run_id) + """ + ), + { + "scholar_profile_id": scholar_two_id, + "publication_id": publication_two_id, + "is_read": False, + "first_seen_run_id": run_id, + }, + ) + await db_session.commit() + + client = TestClient(app) + login_user(client, email="filtering@example.com", password="filter-password") + + scholars_page = client.get("/scholars") + assert scholars_page.status_code == 200 + assert f"/publications?mode=new&scholar_profile_id={scholar_one_id}" in scholars_page.text + assert f"/publications?mode=all&scholar_profile_id={scholar_two_id}" in scholars_page.text + + filtered_all = client.get(f"/publications?mode=all&scholar_profile_id={scholar_one_id}") + assert filtered_all.status_code == 200 + assert "Scholar One Paper" in filtered_all.text + assert "Scholar Two Paper" not in filtered_all.text + + filtered_new = client.get(f"/publications?mode=new&scholar_profile_id={scholar_two_id}") + assert filtered_new.status_code == 200 + assert "Scholar Two Paper" in filtered_new.text + assert "Scholar One Paper" not in filtered_new.text + + +@pytest.mark.integration +@pytest.mark.db +@pytest.mark.asyncio +async def test_manual_ingestion_persists_failure_debug_context(db_session: AsyncSession) -> None: + user_id = await insert_user( + db_session, + email="ingest-failure@example.com", + password="ingest-password", + ) + await db_session.execute( + text( + """ + INSERT INTO scholar_profiles (user_id, scholar_id, display_name, is_enabled) + VALUES (:user_id, :scholar_id, :display_name, :is_enabled) + """ + ), + { + "user_id": user_id, + "scholar_id": "abcDEF123456", + "display_name": "Failure Fixture", + "is_enabled": True, + }, + ) + await db_session.commit() + + stub_source = StubScholarSourceResults( + [ + FetchResult( + requested_url="https://scholar.google.com/citations?hl=en&user=abcDEF123456", + status_code=None, + final_url=None, + body="", + error="timed out", + ) + ] + ) + app.dependency_overrides[get_scholar_source] = lambda: stub_source + + client = TestClient(app) + try: + login_user(client, email="ingest-failure@example.com", password="ingest-password") + dashboard = client.get("/") + csrf = extract_csrf_token(dashboard.text) + run_response = client.post( + "/runs/manual", + data={"csrf_token": csrf}, + follow_redirects=False, + ) + assert run_response.status_code == 303 + + run_result = await db_session.execute( + text( + """ + SELECT status::text, error_log + FROM crawl_runs + WHERE user_id = :user_id + ORDER BY id DESC + LIMIT 1 + """ + ), + {"user_id": user_id}, + ) + status_text, error_log = run_result.one() + assert status_text == "failed" + assert error_log["summary"]["failed_count"] == 1 + assert error_log["summary"]["failed_state_counts"] == {"network_error": 1} + assert error_log["summary"]["failed_reason_counts"] == { + "network_error_missing_status_code": 1 + } + + scholar_result = error_log["scholar_results"][0] + assert scholar_result["state"] == "network_error" + assert scholar_result["state_reason"] == "network_error_missing_status_code" + debug = scholar_result["debug"] + assert debug["status_code"] is None + assert debug["fetch_error"] == "timed out" + assert debug["requested_url"].endswith("user=abcDEF123456") + assert debug["body_excerpt"] is None + + runs_page = client.get("/runs") + assert runs_page.status_code == 200 + assert "Run History" in runs_page.text + assert "failed /" in runs_page.text + + failed_only_page = client.get("/runs?failed_only=1") + assert failed_only_page.status_code == 200 + assert "Run History" in failed_only_page.text + + run_id_result = await db_session.execute( + text( + """ + SELECT id + FROM crawl_runs + WHERE user_id = :user_id + ORDER BY id DESC + LIMIT 1 + """ + ), + {"user_id": user_id}, + ) + run_id = run_id_result.scalar_one() + + detail_page = client.get(f"/runs/{run_id}") + assert detail_page.status_code == 200 + assert f"Run #{run_id}" in detail_page.text + assert "network_error_missing_status_code" in detail_page.text + finally: + app.dependency_overrides.clear() + + +@pytest.mark.integration +@pytest.mark.db +@pytest.mark.asyncio +async def test_manual_ingestion_fetches_all_pages_for_scholar(db_session: AsyncSession) -> None: + user_id = await insert_user( + db_session, + email="paged@example.com", + password="ingest-password", + ) + await db_session.execute( + text( + """ + INSERT INTO scholar_profiles (user_id, scholar_id, display_name, is_enabled) + VALUES (:user_id, :scholar_id, :display_name, :is_enabled) + """ + ), + { + "user_id": user_id, + "scholar_id": "abcDEF123456", + "display_name": "Paged Scholar", + "is_enabled": True, + }, + ) + await db_session.commit() + + source = StubPagedScholarSource( + { + 0: HTML_PAGED_ONE, + 1: HTML_PAGED_TWO, + } + ) + app.dependency_overrides[get_scholar_source] = lambda: source + + client = TestClient(app) + try: + login_user(client, email="paged@example.com", password="ingest-password") + dashboard = client.get("/") + csrf = extract_csrf_token(dashboard.text) + run_response = client.post( + "/runs/manual", + data={"csrf_token": csrf}, + follow_redirects=False, + ) + assert run_response.status_code == 303 + + assert source.calls[0][0] == 0 + assert source.calls[1][0] == 1 + + publications_count = await db_session.execute( + text( + """ + SELECT COUNT(*) + FROM scholar_publications sp + JOIN scholar_profiles s ON s.id = sp.scholar_profile_id + WHERE s.user_id = :user_id + """ + ), + {"user_id": user_id}, + ) + assert publications_count.scalar_one() == 2 + + run_result = await db_session.execute( + text( + """ + SELECT status::text, new_pub_count, error_log + FROM crawl_runs + WHERE user_id = :user_id + ORDER BY id DESC + LIMIT 1 + """ + ), + {"user_id": user_id}, + ) + status_text, new_pub_count, error_log = run_result.one() + assert status_text == "success" + assert new_pub_count == 2 + scholar_result = error_log["scholar_results"][0] + assert scholar_result["publication_count"] == 2 + assert scholar_result["pages_fetched"] == 2 + assert scholar_result["has_more_remaining"] is False + assert scholar_result["pagination_truncated_reason"] is None + finally: + app.dependency_overrides.clear() + + +@pytest.mark.integration +@pytest.mark.db +@pytest.mark.asyncio +async def test_manual_ingestion_handles_empty_no_results_tail_without_partial( + db_session: AsyncSession, +) -> None: + user_id = await insert_user( + db_session, + email="tail@example.com", + password="ingest-password", + ) + await db_session.execute( + text( + """ + INSERT INTO scholar_profiles (user_id, scholar_id, display_name, is_enabled) + VALUES (:user_id, :scholar_id, :display_name, :is_enabled) + """ + ), + { + "user_id": user_id, + "scholar_id": "abcDEF123456", + "display_name": "Tail Scholar", + "is_enabled": True, + }, + ) + await db_session.commit() + + source = StubPagedScholarSource( + { + 0: HTML_STALLED_TAIL_ONE, + 1: HTML_STALLED_TAIL_TWO, + } + ) + app.dependency_overrides[get_scholar_source] = lambda: source + + client = TestClient(app) + try: + login_user(client, email="tail@example.com", password="ingest-password") + dashboard = client.get("/") + csrf = extract_csrf_token(dashboard.text) + run_response = client.post( + "/runs/manual", + data={"csrf_token": csrf}, + follow_redirects=False, + ) + assert run_response.status_code == 303 + assert source.calls[0][0] == 0 + assert source.calls[1][0] == 1 + + run_result = await db_session.execute( + text( + """ + SELECT status::text, new_pub_count, error_log + FROM crawl_runs + WHERE user_id = :user_id + ORDER BY id DESC + LIMIT 1 + """ + ), + {"user_id": user_id}, + ) + status_text, new_pub_count, error_log = run_result.one() + assert status_text == "success" + assert new_pub_count == 1 + scholar_result = error_log["scholar_results"][0] + assert scholar_result["outcome"] == "success" + assert scholar_result["pagination_truncated_reason"] is None + + queue_count_result = await db_session.execute( + text( + """ + SELECT COUNT(*) + FROM ingestion_queue_items + WHERE user_id = :user_id + """ + ), + {"user_id": user_id}, + ) + assert queue_count_result.scalar_one() == 0 + finally: + app.dependency_overrides.clear() + + +@pytest.mark.integration +@pytest.mark.db +@pytest.mark.asyncio +async def test_ingestion_enqueues_continuation_when_max_pages_reached( + db_session: AsyncSession, +) -> None: + user_id = await insert_user( + db_session, + email="queued@example.com", + password="ingest-password", + ) + await db_session.execute( + text( + """ + INSERT INTO scholar_profiles (user_id, scholar_id, display_name, is_enabled) + VALUES (:user_id, :scholar_id, :display_name, :is_enabled) + """ + ), + { + "user_id": user_id, + "scholar_id": "abcDEF123456", + "display_name": "Queued Scholar", + "is_enabled": True, + }, + ) + await db_session.commit() + + source = StubPagedScholarSource({0: HTML_PAGED_ONE, 1: HTML_PAGED_TWO}) + service = ScholarIngestionService(source=source) + + summary = await service.run_for_user( + db_session, + user_id=user_id, + trigger_type=RunTriggerType.MANUAL, + request_delay_seconds=0, + network_error_retries=0, + retry_backoff_seconds=0, + max_pages_per_scholar=1, + page_size=100, + auto_queue_continuations=True, + queue_delay_seconds=0, + ) + + assert summary.status.value == "partial_failure" + queue_result = await db_session.execute( + text( + """ + SELECT reason, resume_cstart, attempt_count + FROM ingestion_queue_items + WHERE user_id = :user_id + """ + ), + {"user_id": user_id}, + ) + reason, resume_cstart, attempt_count = queue_result.one() + assert reason == "max_pages_reached" + assert int(resume_cstart) == 1 + assert int(attempt_count) == 0 + + +@pytest.mark.integration +@pytest.mark.db +@pytest.mark.asyncio +async def test_scheduler_processes_queued_continuation_items( + db_session: AsyncSession, +) -> None: + user_id = await insert_user( + db_session, + email="queue-scheduler@example.com", + password="ingest-password", + ) + scholar_result = await db_session.execute( + text( + """ + INSERT INTO scholar_profiles (user_id, scholar_id, display_name, is_enabled) + VALUES (:user_id, :scholar_id, :display_name, :is_enabled) + RETURNING id + """ + ), + { + "user_id": user_id, + "scholar_id": "abcDEF123456", + "display_name": "Queue Scheduler Scholar", + "is_enabled": True, + }, + ) + scholar_profile_id = int(scholar_result.scalar_one()) + await db_session.execute( + text( + """ + INSERT INTO ingestion_queue_items ( + user_id, + scholar_profile_id, + resume_cstart, + reason, + attempt_count, + next_attempt_dt + ) + VALUES ( + :user_id, + :scholar_profile_id, + :resume_cstart, + :reason, + :attempt_count, + NOW() - INTERVAL '1 minute' + ) + """ + ), + { + "user_id": user_id, + "scholar_profile_id": scholar_profile_id, + "resume_cstart": 1, + "reason": "max_pages_reached", + "attempt_count": 0, + }, + ) + await db_session.commit() + + source = StubPagedScholarSource({1: HTML_PAGED_TWO}) + scheduler = SchedulerService( + enabled=True, + tick_seconds=60, + network_error_retries=0, + retry_backoff_seconds=0, + max_pages_per_scholar=30, + page_size=100, + continuation_queue_enabled=True, + continuation_base_delay_seconds=1, + continuation_max_delay_seconds=60, + continuation_max_attempts=4, + queue_batch_size=10, + ) + scheduler._source = source + + await scheduler._tick_once() + + queue_count_result = await db_session.execute( + text( + """ + SELECT COUNT(*) + FROM ingestion_queue_items + WHERE user_id = :user_id + """ + ), + {"user_id": user_id}, + ) + assert queue_count_result.scalar_one() == 0 + + publications_count = await db_session.execute( + text( + """ + SELECT COUNT(*) + FROM scholar_publications sp + JOIN scholar_profiles s ON s.id = sp.scholar_profile_id + WHERE s.user_id = :user_id + """ + ), + {"user_id": user_id}, + ) + assert publications_count.scalar_one() == 1 + assert source.calls and source.calls[0][0] == 1 + + +@pytest.mark.integration +@pytest.mark.db +@pytest.mark.asyncio +async def test_ingestion_retries_network_error_and_recovers(db_session: AsyncSession) -> None: + user_id = await insert_user( + db_session, + email="retry@example.com", + password="ingest-password", + ) + await db_session.execute( + text( + """ + INSERT INTO scholar_profiles (user_id, scholar_id, display_name, is_enabled) + VALUES (:user_id, :scholar_id, :display_name, :is_enabled) + """ + ), + { + "user_id": user_id, + "scholar_id": "abcDEF123456", + "display_name": "Retry Fixture", + "is_enabled": True, + }, + ) + await db_session.commit() + + source = CountingScholarSourceResults( + [ + FetchResult( + requested_url="https://scholar.google.com/citations?hl=en&user=abcDEF123456", + status_code=None, + final_url=None, + body="", + error="temporary timeout", + ), + FetchResult( + requested_url="https://scholar.google.com/citations?hl=en&user=abcDEF123456", + status_code=200, + final_url="https://scholar.google.com/citations?hl=en&user=abcDEF123456", + body=HTML_BASELINE, + error=None, + ), + ] + ) + service = ScholarIngestionService(source=source) + + summary = await service.run_for_user( + db_session, + user_id=user_id, + trigger_type=RunTriggerType.MANUAL, + request_delay_seconds=0, + network_error_retries=1, + retry_backoff_seconds=0, + ) + + assert source.calls == 2 + assert summary.status.value == "success" + assert summary.failed_count == 0 + assert summary.succeeded_count == 1 + + run_error_log_result = await db_session.execute( + text( + """ + SELECT error_log + FROM crawl_runs + WHERE user_id = :user_id + ORDER BY id DESC + LIMIT 1 + """ + ), + {"user_id": user_id}, + ) + run_error_log = run_error_log_result.scalar_one() + scholar_result = run_error_log["scholar_results"][0] + assert scholar_result["attempt_count"] == 2 + assert scholar_result["state"] == "ok" + assert scholar_result["state_reason"] == "publications_extracted" + + +@pytest.mark.integration +@pytest.mark.db +@pytest.mark.asyncio +async def test_ingestion_rejects_overlapping_run_lock( + db_session: AsyncSession, + database_url: str, +) -> None: + user_id = await insert_user( + db_session, + email="lock@example.com", + password="ingest-password", + ) + await db_session.execute( + text( + """ + INSERT INTO scholar_profiles (user_id, scholar_id, display_name, is_enabled) + VALUES (:user_id, :scholar_id, :display_name, :is_enabled) + """ + ), + { + "user_id": user_id, + "scholar_id": "abcDEF123456", + "display_name": "Lock Fixture", + "is_enabled": True, + }, + ) + await db_session.commit() + + engine = create_async_engine(database_url, pool_pre_ping=True) + factory = async_sessionmaker(engine, expire_on_commit=False) + lock_session = factory() + try: + await lock_session.execute( + text("SELECT pg_advisory_lock(:namespace, :user_key)"), + { + "namespace": RUN_LOCK_NAMESPACE, + "user_key": user_id, + }, + ) + + service = ScholarIngestionService( + source=CountingScholarSourceResults( + [ + FetchResult( + requested_url="https://scholar.google.com/citations?hl=en&user=abcDEF123456", + status_code=200, + final_url="https://scholar.google.com/citations?hl=en&user=abcDEF123456", + body=HTML_BASELINE, + error=None, + ) + ] + ) + ) + with pytest.raises(RunAlreadyInProgressError): + await service.run_for_user( + db_session, + user_id=user_id, + trigger_type=RunTriggerType.MANUAL, + request_delay_seconds=0, + network_error_retries=0, + retry_backoff_seconds=0, + ) + finally: + await lock_session.execute( + text("SELECT pg_advisory_unlock(:namespace, :user_key)"), + { + "namespace": RUN_LOCK_NAMESPACE, + "user_key": user_id, + }, + ) + await lock_session.close() + await engine.dispose() diff --git a/tests/integration/test_migrations.py b/tests/integration/test_migrations.py new file mode 100644 index 0000000..4835706 --- /dev/null +++ b/tests/integration/test_migrations.py @@ -0,0 +1,93 @@ +import pytest +from sqlalchemy import text +from sqlalchemy.ext.asyncio import AsyncSession + +EXPECTED_TABLES = { + "alembic_version", + "users", + "user_settings", + "scholar_profiles", + "publications", + "scholar_publications", + "crawl_runs", + "ingestion_queue_items", +} + +EXPECTED_ENUMS = {"run_status", "run_trigger_type"} +EXPECTED_REVISION = "20260217_0004" + + +@pytest.mark.integration +@pytest.mark.db +@pytest.mark.migrations +@pytest.mark.asyncio +async def test_migration_creates_expected_tables(db_session: AsyncSession) -> None: + result = await db_session.execute( + text("SELECT tablename FROM pg_tables WHERE schemaname = 'public'") + ) + table_names = {row[0] for row in result} + assert EXPECTED_TABLES.issubset(table_names) + + +@pytest.mark.integration +@pytest.mark.db +@pytest.mark.migrations +@pytest.mark.asyncio +async def test_migration_registers_expected_enums(db_session: AsyncSession) -> None: + result = await db_session.execute( + text( + """ + SELECT t.typname + FROM pg_type t + JOIN pg_namespace n ON n.oid = t.typnamespace + WHERE n.nspname = 'public' + """ + ) + ) + enum_names = {row[0] for row in result} + assert EXPECTED_ENUMS.issubset(enum_names) + + +@pytest.mark.integration +@pytest.mark.db +@pytest.mark.migrations +@pytest.mark.asyncio +async def test_migration_head_revision_is_applied(db_session: AsyncSession) -> None: + result = await db_session.execute(text("SELECT version_num FROM alembic_version")) + assert result.scalar_one() == EXPECTED_REVISION + + +@pytest.mark.integration +@pytest.mark.db +@pytest.mark.migrations +@pytest.mark.asyncio +async def test_users_table_has_is_admin_column(db_session: AsyncSession) -> None: + result = await db_session.execute( + text( + """ + SELECT 1 + FROM information_schema.columns + WHERE table_name = 'users' AND column_name = 'is_admin' + """ + ) + ) + assert result.scalar_one() == 1 + + +@pytest.mark.integration +@pytest.mark.db +@pytest.mark.migrations +@pytest.mark.asyncio +async def test_ingestion_queue_table_has_status_and_drop_columns(db_session: AsyncSession) -> None: + result = await db_session.execute( + text( + """ + SELECT column_name + FROM information_schema.columns + WHERE table_name = 'ingestion_queue_items' + AND column_name IN ('status', 'dropped_reason', 'dropped_at') + """ + ) + ) + columns = {row[0] for row in result} + assert columns == {"status", "dropped_reason", "dropped_at"} diff --git a/tests/integration/test_multi_user_schema.py b/tests/integration/test_multi_user_schema.py new file mode 100644 index 0000000..263a83a --- /dev/null +++ b/tests/integration/test_multi_user_schema.py @@ -0,0 +1,241 @@ +import pytest +from sqlalchemy import text +from sqlalchemy.exc import IntegrityError +from sqlalchemy.ext.asyncio import AsyncSession + + +async def _insert_user(db_session: AsyncSession, email: str) -> int: + result = await db_session.execute( + text( + """ + INSERT INTO users (email, password_hash) + VALUES (:email, :password_hash) + RETURNING id + """ + ), + {"email": email, "password_hash": "argon2id$placeholder"}, + ) + return int(result.scalar_one()) + + +@pytest.mark.integration +@pytest.mark.db +@pytest.mark.schema +@pytest.mark.asyncio +async def test_scholar_id_uniqueness_is_scoped_to_user(db_session: AsyncSession) -> None: + user_a = await _insert_user(db_session, "owner-a@example.com") + user_b = await _insert_user(db_session, "owner-b@example.com") + await db_session.commit() + + await db_session.execute( + text( + """ + INSERT INTO scholar_profiles (user_id, scholar_id, display_name) + VALUES (:user_id, :scholar_id, :display_name) + """ + ), + {"user_id": user_a, "scholar_id": "abcDEF123456", "display_name": "Alpha"}, + ) + await db_session.execute( + text( + """ + INSERT INTO scholar_profiles (user_id, scholar_id, display_name) + VALUES (:user_id, :scholar_id, :display_name) + """ + ), + {"user_id": user_b, "scholar_id": "abcDEF123456", "display_name": "Beta"}, + ) + await db_session.commit() + + with pytest.raises(IntegrityError): + await db_session.execute( + text( + """ + INSERT INTO scholar_profiles (user_id, scholar_id, display_name) + VALUES (:user_id, :scholar_id, :display_name) + """ + ), + {"user_id": user_a, "scholar_id": "abcDEF123456", "display_name": "Gamma"}, + ) + await db_session.commit() + await db_session.rollback() + + +@pytest.mark.integration +@pytest.mark.db +@pytest.mark.schema +@pytest.mark.asyncio +async def test_user_settings_allow_only_one_row_per_user(db_session: AsyncSession) -> None: + user_id = await _insert_user(db_session, "settings-owner@example.com") + await db_session.commit() + + await db_session.execute( + text( + """ + INSERT INTO user_settings (user_id, auto_run_enabled, run_interval_minutes, request_delay_seconds) + VALUES (:user_id, :auto_run_enabled, :run_interval_minutes, :request_delay_seconds) + """ + ), + { + "user_id": user_id, + "auto_run_enabled": True, + "run_interval_minutes": 60, + "request_delay_seconds": 10, + }, + ) + await db_session.commit() + + with pytest.raises(IntegrityError): + await db_session.execute( + text( + """ + INSERT INTO user_settings (user_id, auto_run_enabled, run_interval_minutes, request_delay_seconds) + VALUES (:user_id, :auto_run_enabled, :run_interval_minutes, :request_delay_seconds) + """ + ), + { + "user_id": user_id, + "auto_run_enabled": False, + "run_interval_minutes": 30, + "request_delay_seconds": 5, + }, + ) + await db_session.commit() + await db_session.rollback() + + +@pytest.mark.integration +@pytest.mark.db +@pytest.mark.schema +@pytest.mark.asyncio +async def test_crawl_run_requires_owner_user(db_session: AsyncSession) -> None: + with pytest.raises(IntegrityError): + await db_session.execute( + text( + """ + INSERT INTO crawl_runs (user_id, trigger_type, status) + VALUES (:user_id, :trigger_type, :status) + """ + ), + {"user_id": None, "trigger_type": "manual", "status": "running"}, + ) + await db_session.commit() + await db_session.rollback() + + +@pytest.mark.integration +@pytest.mark.db +@pytest.mark.schema +@pytest.mark.asyncio +async def test_read_state_is_isolated_across_users(db_session: AsyncSession) -> None: + user_a = await _insert_user(db_session, "reader-a@example.com") + user_b = await _insert_user(db_session, "reader-b@example.com") + + scholar_a = await db_session.execute( + text( + """ + INSERT INTO scholar_profiles (user_id, scholar_id, display_name) + VALUES (:user_id, :scholar_id, :display_name) + RETURNING id + """ + ), + {"user_id": user_a, "scholar_id": "qwerty123456", "display_name": "Reader A"}, + ) + scholar_a_id = int(scholar_a.scalar_one()) + + scholar_b = await db_session.execute( + text( + """ + INSERT INTO scholar_profiles (user_id, scholar_id, display_name) + VALUES (:user_id, :scholar_id, :display_name) + RETURNING id + """ + ), + {"user_id": user_b, "scholar_id": "zxcvbn654321", "display_name": "Reader B"}, + ) + scholar_b_id = int(scholar_b.scalar_one()) + + publication = await db_session.execute( + text( + """ + INSERT INTO publications (fingerprint_sha256, title_raw, title_normalized, year) + VALUES (:fingerprint_sha256, :title_raw, :title_normalized, :year) + RETURNING id + """ + ), + { + "fingerprint_sha256": "f" * 64, + "title_raw": "A Shared Paper", + "title_normalized": "a shared paper", + "year": 2026, + }, + ) + publication_id = int(publication.scalar_one()) + + run_a = await db_session.execute( + text( + """ + INSERT INTO crawl_runs (user_id, trigger_type, status) + VALUES (:user_id, :trigger_type, :status) + RETURNING id + """ + ), + {"user_id": user_a, "trigger_type": "manual", "status": "success"}, + ) + run_a_id = int(run_a.scalar_one()) + + run_b = await db_session.execute( + text( + """ + INSERT INTO crawl_runs (user_id, trigger_type, status) + VALUES (:user_id, :trigger_type, :status) + RETURNING id + """ + ), + {"user_id": user_b, "trigger_type": "manual", "status": "success"}, + ) + run_b_id = int(run_b.scalar_one()) + + await db_session.execute( + text( + """ + INSERT INTO scholar_publications (scholar_profile_id, publication_id, is_read, first_seen_run_id) + VALUES (:scholar_profile_id, :publication_id, :is_read, :first_seen_run_id) + """ + ), + { + "scholar_profile_id": scholar_a_id, + "publication_id": publication_id, + "is_read": False, + "first_seen_run_id": run_a_id, + }, + ) + await db_session.execute( + text( + """ + INSERT INTO scholar_publications (scholar_profile_id, publication_id, is_read, first_seen_run_id) + VALUES (:scholar_profile_id, :publication_id, :is_read, :first_seen_run_id) + """ + ), + { + "scholar_profile_id": scholar_b_id, + "publication_id": publication_id, + "is_read": True, + "first_seen_run_id": run_b_id, + }, + ) + await db_session.commit() + + result = await db_session.execute( + text( + """ + SELECT sp.user_id, spp.is_read + FROM scholar_publications spp + JOIN scholar_profiles sp ON sp.id = spp.scholar_profile_id + ORDER BY sp.user_id + """ + ) + ) + rows = result.all() + assert rows == [(user_a, False), (user_b, True)] + diff --git a/tests/integration/test_queue_ui.py b/tests/integration/test_queue_ui.py new file mode 100644 index 0000000..602ed60 --- /dev/null +++ b/tests/integration/test_queue_ui.py @@ -0,0 +1,227 @@ +from __future__ import annotations + +import pytest +from fastapi.testclient import TestClient +from sqlalchemy import text +from sqlalchemy.ext.asyncio import AsyncSession + +from app.main import app +from tests.integration.helpers import extract_csrf_token, insert_user, login_user + + +@pytest.mark.integration +@pytest.mark.db +@pytest.mark.asyncio +async def test_runs_page_queue_actions_lifecycle(db_session: AsyncSession) -> None: + user_id = await insert_user( + db_session, + email="queue-ui@example.com", + password="queue-ui-password", + ) + scholar_result = await db_session.execute( + text( + """ + INSERT INTO scholar_profiles (user_id, scholar_id, display_name, is_enabled) + VALUES (:user_id, :scholar_id, :display_name, true) + RETURNING id + """ + ), + { + "user_id": user_id, + "scholar_id": "abcDEF123456", + "display_name": "Queue UI Scholar", + }, + ) + scholar_profile_id = int(scholar_result.scalar_one()) + + queue_result = await db_session.execute( + text( + """ + INSERT INTO ingestion_queue_items ( + user_id, + scholar_profile_id, + resume_cstart, + reason, + status, + attempt_count, + next_attempt_dt, + last_error, + dropped_reason, + dropped_at + ) + VALUES ( + :user_id, + :scholar_profile_id, + 200, + 'dropped', + 'dropped', + 3, + NOW() + INTERVAL '30 minutes', + 'captcha challenge', + 'max_attempts_after_run', + NOW() - INTERVAL '1 minute' + ) + RETURNING id + """ + ), + { + "user_id": user_id, + "scholar_profile_id": scholar_profile_id, + }, + ) + queue_item_id = int(queue_result.scalar_one()) + await db_session.commit() + + client = TestClient(app) + login_user(client, email="queue-ui@example.com", password="queue-ui-password") + + runs_page = client.get("/runs") + assert runs_page.status_code == 200 + assert "Continuation Queue" in runs_page.text + assert "Queue UI Scholar" in runs_page.text + assert "dropped" in runs_page.text + assert "max_attempts_after_run" in runs_page.text + + csrf_retry = extract_csrf_token(runs_page.text) + retry_response = client.post( + f"/runs/queue/{queue_item_id}/retry", + data={"csrf_token": csrf_retry}, + follow_redirects=False, + ) + assert retry_response.status_code == 303 + + queue_after_retry = await db_session.execute( + text( + """ + SELECT status, reason, attempt_count + FROM ingestion_queue_items + WHERE id = :queue_item_id + """ + ), + {"queue_item_id": queue_item_id}, + ) + assert queue_after_retry.one() == ("queued", "manual_retry", 0) + + runs_page_after_retry = client.get("/runs") + csrf_drop = extract_csrf_token(runs_page_after_retry.text) + drop_response = client.post( + f"/runs/queue/{queue_item_id}/drop", + data={"csrf_token": csrf_drop}, + follow_redirects=False, + ) + assert drop_response.status_code == 303 + + queue_after_drop = await db_session.execute( + text( + """ + SELECT status, reason, dropped_reason + FROM ingestion_queue_items + WHERE id = :queue_item_id + """ + ), + {"queue_item_id": queue_item_id}, + ) + assert queue_after_drop.one() == ("dropped", "dropped", "manual_drop") + + runs_page_after_drop = client.get("/runs") + csrf_clear = extract_csrf_token(runs_page_after_drop.text) + clear_response = client.post( + f"/runs/queue/{queue_item_id}/clear", + data={"csrf_token": csrf_clear}, + follow_redirects=False, + ) + assert clear_response.status_code == 303 + + queue_after_clear = await db_session.execute( + text("SELECT COUNT(*) FROM ingestion_queue_items WHERE id = :queue_item_id"), + {"queue_item_id": queue_item_id}, + ) + assert queue_after_clear.scalar_one() == 0 + + +@pytest.mark.integration +@pytest.mark.db +@pytest.mark.asyncio +async def test_runs_queue_actions_are_tenant_scoped(db_session: AsyncSession) -> None: + user_a_id = await insert_user( + db_session, + email="queue-owner-a@example.com", + password="queue-owner-a-password", + ) + user_b_id = await insert_user( + db_session, + email="queue-owner-b@example.com", + password="queue-owner-b-password", + ) + + scholar_result = await db_session.execute( + text( + """ + INSERT INTO scholar_profiles (user_id, scholar_id, display_name, is_enabled) + VALUES (:user_id, :scholar_id, :display_name, true) + RETURNING id + """ + ), + { + "user_id": user_b_id, + "scholar_id": "zxyWVU654321", + "display_name": "Owner B Queue Scholar", + }, + ) + scholar_profile_id = int(scholar_result.scalar_one()) + queue_result = await db_session.execute( + text( + """ + INSERT INTO ingestion_queue_items ( + user_id, + scholar_profile_id, + resume_cstart, + reason, + status, + attempt_count, + next_attempt_dt + ) + VALUES ( + :user_id, + :scholar_profile_id, + 0, + 'max_pages_reached', + 'queued', + 0, + NOW() + ) + RETURNING id + """ + ), + { + "user_id": user_b_id, + "scholar_profile_id": scholar_profile_id, + }, + ) + queue_item_id = int(queue_result.scalar_one()) + await db_session.commit() + + client = TestClient(app) + login_user(client, email="queue-owner-a@example.com", password="queue-owner-a-password") + + runs_page = client.get("/runs") + csrf_token = extract_csrf_token(runs_page.text) + forbidden_retry = client.post( + f"/runs/queue/{queue_item_id}/retry", + data={"csrf_token": csrf_token}, + follow_redirects=False, + ) + assert forbidden_retry.status_code == 404 + + unchanged = await db_session.execute( + text( + """ + SELECT status, reason, user_id + FROM ingestion_queue_items + WHERE id = :queue_item_id + """ + ), + {"queue_item_id": queue_item_id}, + ) + assert unchanged.one() == ("queued", "max_pages_reached", user_b_id) + assert user_a_id != user_b_id diff --git a/tests/integration/test_tenant_routes.py b/tests/integration/test_tenant_routes.py new file mode 100644 index 0000000..e5f7d59 --- /dev/null +++ b/tests/integration/test_tenant_routes.py @@ -0,0 +1,197 @@ +import pytest +from fastapi.testclient import TestClient +from sqlalchemy import text +from sqlalchemy.ext.asyncio import AsyncSession + +from app.main import app +from tests.integration.helpers import extract_csrf_token, insert_user, login_user + + +@pytest.mark.integration +@pytest.mark.db +@pytest.mark.asyncio +async def test_user_can_change_own_password(db_session: AsyncSession) -> None: + await insert_user( + db_session, + email="reader@example.com", + password="old-password", + ) + + client = TestClient(app) + login_user(client, email="reader@example.com", password="old-password") + + account_page = client.get("/account/password") + csrf_token = extract_csrf_token(account_page.text) + wrong_current = client.post( + "/account/password", + data={ + "current_password": "wrong-password", + "new_password": "new-password", + "confirm_password": "new-password", + "csrf_token": csrf_token, + }, + follow_redirects=False, + ) + assert wrong_current.status_code == 303 + assert wrong_current.headers["location"].startswith("/account/password") + + account_page_retry = client.get("/account/password") + retry_csrf = extract_csrf_token(account_page_retry.text) + success = client.post( + "/account/password", + data={ + "current_password": "old-password", + "new_password": "new-password", + "confirm_password": "new-password", + "csrf_token": retry_csrf, + }, + follow_redirects=False, + ) + assert success.status_code == 303 + + account_page_after = client.get("/account/password") + logout_csrf = extract_csrf_token(account_page_after.text) + client.post("/logout", data={"csrf_token": logout_csrf}, follow_redirects=False) + + failed_login_page = client.get("/login") + failed_login_csrf = extract_csrf_token(failed_login_page.text) + failed_login = client.post( + "/login", + data={ + "email": "reader@example.com", + "password": "old-password", + "csrf_token": failed_login_csrf, + }, + follow_redirects=False, + ) + assert failed_login.status_code == 401 + + login_user(client, email="reader@example.com", password="new-password") + + +@pytest.mark.integration +@pytest.mark.db +@pytest.mark.asyncio +async def test_scholar_routes_are_tenant_scoped(db_session: AsyncSession) -> None: + user_a_id = await insert_user( + db_session, + email="owner-a@example.com", + password="owner-a-password", + ) + user_b_id = await insert_user( + db_session, + email="owner-b@example.com", + password="owner-b-password", + ) + + client = TestClient(app) + login_user(client, email="owner-a@example.com", password="owner-a-password") + + scholars_page = client.get("/scholars") + csrf_token = extract_csrf_token(scholars_page.text) + create_response = client.post( + "/scholars", + data={ + "scholar_id": "abcDEF123456", + "display_name": "Owner A Scholar", + "csrf_token": csrf_token, + }, + follow_redirects=False, + ) + assert create_response.status_code == 303 + + owner_a_row = await db_session.execute( + text( + """ + SELECT user_id + FROM scholar_profiles + WHERE scholar_id = 'abcDEF123456' + """ + ) + ) + assert owner_a_row.scalar_one() == user_a_id + + owner_b_profile = await db_session.execute( + text( + """ + INSERT INTO scholar_profiles (user_id, scholar_id, display_name, is_enabled) + VALUES (:user_id, :scholar_id, :display_name, :is_enabled) + RETURNING id + """ + ), + { + "user_id": user_b_id, + "scholar_id": "zxcvbn654321", + "display_name": "Owner B Scholar", + "is_enabled": True, + }, + ) + owner_b_profile_id = int(owner_b_profile.scalar_one()) + await db_session.commit() + + scholars_page_after_insert = client.get("/scholars") + csrf_after_insert = extract_csrf_token(scholars_page_after_insert.text) + forbidden_toggle = client.post( + f"/scholars/{owner_b_profile_id}/toggle", + data={"csrf_token": csrf_after_insert}, + follow_redirects=False, + ) + assert forbidden_toggle.status_code == 404 + + owner_b_status = await db_session.execute( + text("SELECT is_enabled FROM scholar_profiles WHERE id = :profile_id"), + {"profile_id": owner_b_profile_id}, + ) + assert owner_b_status.scalar_one() is True + + +@pytest.mark.integration +@pytest.mark.db +@pytest.mark.asyncio +async def test_settings_updates_are_user_scoped(db_session: AsyncSession) -> None: + user_a_id = await insert_user( + db_session, + email="settings-a@example.com", + password="settings-a-password", + ) + user_b_id = await insert_user( + db_session, + email="settings-b@example.com", + password="settings-b-password", + ) + + client = TestClient(app) + login_user(client, email="settings-a@example.com", password="settings-a-password") + + settings_page = client.get("/settings") + csrf_token = extract_csrf_token(settings_page.text) + response = client.post( + "/settings", + data={ + "auto_run_enabled": "on", + "run_interval_minutes": "45", + "request_delay_seconds": "7", + "csrf_token": csrf_token, + }, + follow_redirects=False, + ) + assert response.status_code == 303 + + user_a_settings = await db_session.execute( + text( + """ + SELECT auto_run_enabled, run_interval_minutes, request_delay_seconds + FROM user_settings + WHERE user_id = :user_id + """ + ), + {"user_id": user_a_id}, + ) + assert user_a_settings.one() == (True, 45, 7) + + user_b_settings = await db_session.execute( + text("SELECT COUNT(*) FROM user_settings WHERE user_id = :user_id"), + {"user_id": user_b_id}, + ) + assert user_b_settings.scalar_one() == 0 + diff --git a/tests/smoke/test_db_connectivity.py b/tests/smoke/test_db_connectivity.py new file mode 100644 index 0000000..848a034 --- /dev/null +++ b/tests/smoke/test_db_connectivity.py @@ -0,0 +1,23 @@ +import os + +import pytest +from sqlalchemy import text +from sqlalchemy.ext.asyncio import create_async_engine + + +@pytest.mark.integration +@pytest.mark.smoke +@pytest.mark.asyncio +async def test_database_connectivity_from_database_url() -> None: + database_url = os.getenv("DATABASE_URL") + if not database_url: + pytest.skip("DATABASE_URL is not set") + + engine = create_async_engine(database_url, pool_pre_ping=True) + try: + async with engine.connect() as connection: + result = await connection.execute(text("SELECT 1")) + assert result.scalar_one() == 1 + finally: + await engine.dispose() + diff --git a/tests/smoke/test_migration_schema.py b/tests/smoke/test_migration_schema.py new file mode 100644 index 0000000..a56503e --- /dev/null +++ b/tests/smoke/test_migration_schema.py @@ -0,0 +1,46 @@ +import pytest +from sqlalchemy import text +from sqlalchemy.ext.asyncio import AsyncSession + + +@pytest.mark.integration +@pytest.mark.smoke +@pytest.mark.db +@pytest.mark.asyncio +async def test_schema_head_revision_is_available(db_session: AsyncSession) -> None: + result = await db_session.execute(text("SELECT version_num FROM alembic_version")) + assert result.scalar_one() == "20260217_0004" + + +@pytest.mark.integration +@pytest.mark.smoke +@pytest.mark.db +@pytest.mark.asyncio +async def test_user_table_exists_in_public_schema(db_session: AsyncSession) -> None: + result = await db_session.execute( + text( + """ + SELECT 1 + FROM pg_tables + WHERE schemaname = 'public' AND tablename = 'users' + """ + ) + ) + assert result.scalar_one() == 1 + + +@pytest.mark.integration +@pytest.mark.smoke +@pytest.mark.db +@pytest.mark.asyncio +async def test_users_table_includes_admin_column(db_session: AsyncSession) -> None: + result = await db_session.execute( + text( + """ + SELECT 1 + FROM information_schema.columns + WHERE table_name = 'users' AND column_name = 'is_admin' + """ + ) + ) + assert result.scalar_one() == 1 diff --git a/tests/unit/test_auth.py b/tests/unit/test_auth.py new file mode 100644 index 0000000..d471996 --- /dev/null +++ b/tests/unit/test_auth.py @@ -0,0 +1,204 @@ +import re +from collections.abc import AsyncIterator +from types import SimpleNamespace +from unittest.mock import AsyncMock + +from fastapi.testclient import TestClient +import pytest + +from app.auth.deps import get_auth_service, get_login_rate_limiter +from app.auth.rate_limit import SlidingWindowRateLimiter +from app.db.session import get_db_session +from app.main import app + +CSRF_TOKEN_PATTERN = re.compile(r'name="csrf_token" value="([^"]+)"') + + +class StubAuthService: + def __init__(self, *, user: object | None) -> None: + self._user = user + + async def authenticate_user(self, _db_session, *, email: str, password: str): + if self._user is None: + return None + if email.strip().lower() != str(self._user.email): + return None + if password != "correct-password": + return None + return self._user + + +def _extract_csrf_token(html: str) -> str: + match = CSRF_TOKEN_PATTERN.search(html) + assert match is not None + return match.group(1) + + +async def _override_db_session() -> AsyncIterator[object]: + yield object() + + +@pytest.fixture(autouse=True) +def clear_dependency_overrides() -> AsyncIterator[None]: + app.dependency_overrides.clear() + yield + app.dependency_overrides.clear() + + +def test_login_requires_csrf_token() -> None: + client = TestClient(app) + + response = client.post( + "/login", + data={"email": "user@example.com", "password": "correct-password"}, + follow_redirects=False, + ) + + assert response.status_code == 403 + assert response.text == "CSRF token missing." + + +def test_successful_login_creates_session_and_allows_dashboard(monkeypatch) -> None: + limiter = SlidingWindowRateLimiter(max_attempts=5, window_seconds=60) + app.dependency_overrides[get_db_session] = _override_db_session + app.dependency_overrides[get_auth_service] = lambda: StubAuthService( + user=SimpleNamespace(id=1, email="user@example.com", is_admin=False) + ) + app.dependency_overrides[get_login_rate_limiter] = lambda: limiter + client = TestClient(app) + monkeypatch.setattr( + "app.web.common.get_authenticated_user", + AsyncMock( + return_value=SimpleNamespace( + id=1, + email="user@example.com", + is_admin=False, + ) + ), + ) + monkeypatch.setattr( + "app.web.routers.dashboard.publication_service.list_new_for_latest_run_for_user", + AsyncMock(return_value=[]), + ) + monkeypatch.setattr( + "app.web.routers.dashboard.run_service.list_recent_runs_for_user", + AsyncMock(return_value=[]), + ) + monkeypatch.setattr( + "app.web.routers.dashboard.run_service.queue_status_counts_for_user", + AsyncMock(return_value={"queued": 0, "retrying": 0, "dropped": 0}), + ) + monkeypatch.setattr( + "app.web.routers.dashboard.user_settings_service.get_or_create_settings", + AsyncMock(return_value=SimpleNamespace(request_delay_seconds=0)), + ) + + login_page = client.get("/login") + csrf_token = _extract_csrf_token(login_page.text) + + login_response = client.post( + "/login", + data={ + "email": "user@example.com", + "password": "correct-password", + "csrf_token": csrf_token, + }, + follow_redirects=False, + ) + + assert login_response.status_code == 303 + assert login_response.headers["location"] == "/" + + dashboard_response = client.get("/") + assert dashboard_response.status_code == 200 + assert 'data-test="home-hero"' in dashboard_response.text + assert 'data-test="session-user"' in dashboard_response.text + assert "user@example.com" in dashboard_response.text + + +def test_login_rate_limiting_returns_429_after_threshold() -> None: + limiter = SlidingWindowRateLimiter(max_attempts=2, window_seconds=60) + app.dependency_overrides[get_db_session] = _override_db_session + app.dependency_overrides[get_auth_service] = lambda: StubAuthService(user=None) + app.dependency_overrides[get_login_rate_limiter] = lambda: limiter + client = TestClient(app) + + login_page = client.get("/login") + csrf_token = _extract_csrf_token(login_page.text) + payload = { + "email": "user@example.com", + "password": "wrong-password", + "csrf_token": csrf_token, + } + + first = client.post("/login", data=payload, follow_redirects=False) + second = client.post("/login", data=payload, follow_redirects=False) + third = client.post("/login", data=payload, follow_redirects=False) + + assert first.status_code == 401 + assert second.status_code == 401 + assert third.status_code == 429 + assert third.headers["Retry-After"] == "60" + + +def test_logout_requires_csrf_token_and_clears_session(monkeypatch) -> None: + limiter = SlidingWindowRateLimiter(max_attempts=5, window_seconds=60) + app.dependency_overrides[get_db_session] = _override_db_session + app.dependency_overrides[get_auth_service] = lambda: StubAuthService( + user=SimpleNamespace(id=1, email="user@example.com", is_admin=False) + ) + app.dependency_overrides[get_login_rate_limiter] = lambda: limiter + client = TestClient(app) + monkeypatch.setattr( + "app.web.common.get_authenticated_user", + AsyncMock( + side_effect=[ + SimpleNamespace(id=1, email="user@example.com", is_admin=False), + None, + ] + ), + ) + monkeypatch.setattr( + "app.web.routers.dashboard.publication_service.list_new_for_latest_run_for_user", + AsyncMock(return_value=[]), + ) + monkeypatch.setattr( + "app.web.routers.dashboard.run_service.list_recent_runs_for_user", + AsyncMock(return_value=[]), + ) + monkeypatch.setattr( + "app.web.routers.dashboard.run_service.queue_status_counts_for_user", + AsyncMock(return_value={"queued": 0, "retrying": 0, "dropped": 0}), + ) + monkeypatch.setattr( + "app.web.routers.dashboard.user_settings_service.get_or_create_settings", + AsyncMock(return_value=SimpleNamespace(request_delay_seconds=0)), + ) + + login_page = client.get("/login") + csrf_token = _extract_csrf_token(login_page.text) + client.post( + "/login", + data={ + "email": "user@example.com", + "password": "correct-password", + "csrf_token": csrf_token, + }, + follow_redirects=False, + ) + + failed_logout = client.post("/logout", data={}, follow_redirects=False) + assert failed_logout.status_code == 403 + assert failed_logout.text == "CSRF token invalid." + + dashboard_page = client.get("/") + logout_token = _extract_csrf_token(dashboard_page.text) + successful_logout = client.post( + "/logout", + data={"csrf_token": logout_token}, + follow_redirects=False, + ) + + assert successful_logout.status_code == 303 + assert successful_logout.headers["location"] == "/login" + assert client.get("/", follow_redirects=False).headers["location"] == "/login" diff --git a/tests/unit/test_auth_primitives.py b/tests/unit/test_auth_primitives.py new file mode 100644 index 0000000..4e0816c --- /dev/null +++ b/tests/unit/test_auth_primitives.py @@ -0,0 +1,34 @@ +from app.auth.rate_limit import SlidingWindowRateLimiter +from app.auth.security import PasswordService + + +def test_password_service_hash_and_verify_round_trip() -> None: + password_service = PasswordService() + password_hash = password_service.hash_password("example-password") + + assert password_hash.startswith("$argon2id$") + assert password_service.verify_password(password_hash, "example-password") + assert not password_service.verify_password(password_hash, "wrong-password") + + +def test_rate_limiter_enforces_windowed_attempt_budget() -> None: + clock = {"now": 0.0} + limiter = SlidingWindowRateLimiter( + max_attempts=2, + window_seconds=10, + now=lambda: clock["now"], + ) + key = "127.0.0.1:test@example.com" + + assert limiter.check(key).allowed + limiter.record_failure(key) + assert limiter.check(key).allowed + limiter.record_failure(key) + + decision = limiter.check(key) + assert not decision.allowed + assert decision.retry_after_seconds == 10 + + clock["now"] = 11.0 + assert limiter.check(key).allowed + diff --git a/tests/unit/test_dashboard_view_model.py b/tests/unit/test_dashboard_view_model.py new file mode 100644 index 0000000..70bb34a --- /dev/null +++ b/tests/unit/test_dashboard_view_model.py @@ -0,0 +1,88 @@ +from __future__ import annotations + +from datetime import datetime, timezone + +from app.db.models import CrawlRun, RunStatus, RunTriggerType +from app.presentation.dashboard import SECTION_TEMPLATES, build_dashboard_view_model +from app.services.publications import UnreadPublicationItem + + +def test_build_dashboard_view_model_maps_sections_and_unread_items() -> None: + unread_items = [ + UnreadPublicationItem( + publication_id=10, + scholar_profile_id=3, + scholar_label="Ada Lovelace", + title="Analytical Engine Notes", + year=None, + citation_count=42, + venue_text="Computing Letters", + pub_url="https://example.test/pub-10", + ) + ] + runs = [ + CrawlRun( + id=77, + user_id=1, + trigger_type=RunTriggerType.MANUAL, + status=RunStatus.SUCCESS, + start_dt=datetime(2026, 2, 16, 18, 0, tzinfo=timezone.utc), + scholar_count=1, + new_pub_count=1, + error_log={}, + ) + ] + + vm = build_dashboard_view_model( + unread_publications=unread_items, + recent_runs=runs, + request_delay_seconds=7, + queue_counts={"queued": 2, "retrying": 1, "dropped": 3}, + ) + + assert vm.section_templates == SECTION_TEMPLATES + assert vm.run_controls.request_delay_seconds == 7 + assert vm.run_controls.queue_queued_count == 2 + assert vm.run_controls.queue_retrying_count == 1 + assert vm.run_controls.queue_dropped_count == 3 + assert vm.unread_publications[0].title == "Analytical Engine Notes" + assert vm.unread_publications[0].year_display == "-" + assert vm.unread_publications[0].citation_count == 42 + assert vm.run_history[0].status_label == "success" + assert vm.run_history[0].status_badge == "ok" + assert vm.run_history[0].started_at_display == "2026-02-16 18:00 UTC" + assert vm.run_history[0].detail_url == "/runs/77" + + +def test_build_dashboard_view_model_maps_failed_and_partial_statuses() -> None: + runs = [ + CrawlRun( + id=11, + user_id=1, + trigger_type=RunTriggerType.MANUAL, + status=RunStatus.FAILED, + start_dt=datetime(2026, 2, 16, 19, 0, tzinfo=timezone.utc), + scholar_count=2, + new_pub_count=0, + error_log={}, + ), + CrawlRun( + id=12, + user_id=1, + trigger_type=RunTriggerType.MANUAL, + status=RunStatus.PARTIAL_FAILURE, + start_dt=datetime(2026, 2, 16, 20, 0, tzinfo=timezone.utc), + scholar_count=2, + new_pub_count=1, + error_log={}, + ), + ] + + vm = build_dashboard_view_model( + unread_publications=[], + recent_runs=runs, + request_delay_seconds=1, + ) + + assert [item.status_badge for item in vm.run_history] == ["danger", "warn"] + assert [item.status_label for item in vm.run_history] == ["failed", "partial_failure"] diff --git a/tests/unit/test_healthz.py b/tests/unit/test_healthz.py new file mode 100644 index 0000000..a1776c3 --- /dev/null +++ b/tests/unit/test_healthz.py @@ -0,0 +1,25 @@ +from unittest.mock import AsyncMock + +from fastapi.testclient import TestClient + +from app.main import app + + +def test_healthz_returns_200_when_database_is_available(monkeypatch) -> None: + monkeypatch.setattr("app.web.routers.health.check_database", AsyncMock(return_value=True)) + client = TestClient(app) + + response = client.get("/healthz") + + assert response.status_code == 200 + assert response.json() == {"status": "ok"} + + +def test_healthz_returns_500_when_database_is_unavailable(monkeypatch) -> None: + monkeypatch.setattr("app.web.routers.health.check_database", AsyncMock(return_value=False)) + client = TestClient(app) + + response = client.get("/healthz") + + assert response.status_code == 500 + assert response.json()["detail"] == "database unavailable" diff --git a/tests/unit/test_logging.py b/tests/unit/test_logging.py new file mode 100644 index 0000000..ccd9293 --- /dev/null +++ b/tests/unit/test_logging.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +import json +import logging +import re + +from fastapi.testclient import TestClient + +from app.logging_config import JsonLogFormatter, parse_redact_fields +from app.main import app +from app.web.middleware import REQUEST_ID_HEADER, parse_skip_paths + + +def test_json_log_formatter_redacts_sensitive_fields() -> None: + formatter = JsonLogFormatter(redact_fields=parse_redact_fields("api_key")) + record = logging.makeLogRecord( + { + "name": "tests.logging", + "levelno": logging.INFO, + "levelname": "INFO", + "msg": "test.event", + "args": (), + "password": "very-secret", + "payload": { + "csrf_token": "token-value", + "safe": "ok", + }, + "color_message": "ANSI-noise", + } + ) + + payload = json.loads(formatter.format(record)) + + assert payload["event"] == "test.event" + assert re.fullmatch(r"\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}Z", payload["timestamp"]) + assert payload["password"] == "[REDACTED]" + assert payload["payload"]["csrf_token"] == "[REDACTED]" + assert payload["payload"]["safe"] == "ok" + assert "color_message" not in payload + + +def test_request_logging_middleware_sets_request_id_header() -> None: + client = TestClient(app) + + response = client.get("/login", headers={REQUEST_ID_HEADER: "request-123"}) + + assert response.status_code == 200 + assert response.headers[REQUEST_ID_HEADER] == "request-123" + + +def test_parse_skip_paths_trims_and_discards_empty_segments() -> None: + assert parse_skip_paths(" /healthz , , /static/ ") == ("/healthz", "/static/") diff --git a/tests/unit/test_pages.py b/tests/unit/test_pages.py new file mode 100644 index 0000000..2f0a003 --- /dev/null +++ b/tests/unit/test_pages.py @@ -0,0 +1,54 @@ +from fastapi.testclient import TestClient + +from app.main import app + + +def test_home_page_redirects_to_login_when_unauthenticated() -> None: + client = TestClient(app) + + response = client.get("/", follow_redirects=False) + + assert response.status_code == 303 + assert response.headers["location"] == "/login" + + +def test_dev_ui_page_is_removed() -> None: + client = TestClient(app) + + response = client.get("/dev/ui", follow_redirects=False) + + assert response.status_code == 404 + + +def test_login_page_renders_html() -> None: + client = TestClient(app) + + response = client.get("/login") + + assert response.status_code == 200 + assert response.headers["content-type"].startswith("text/html") + assert "scholarr" in response.text + assert 'data-test="login-form"' in response.text + assert 'name="csrf_token"' in response.text + assert 'data-theme-control' in response.text + set_cookie = response.headers["set-cookie"].lower() + assert "httponly" in set_cookie + assert "samesite=lax" in set_cookie + + +def test_theme_query_parameter_accepts_supported_theme() -> None: + client = TestClient(app) + + response = client.get("/login?theme=spruce") + + assert response.status_code == 200 + assert 'data-theme="spruce"' in response.text + + +def test_theme_query_parameter_falls_back_for_unknown_theme() -> None: + client = TestClient(app) + + response = client.get("/login?theme=not-a-theme") + + assert response.status_code == 200 + assert 'data-theme="terracotta"' in response.text diff --git a/tests/unit/test_phase2_pages.py b/tests/unit/test_phase2_pages.py new file mode 100644 index 0000000..cf2a308 --- /dev/null +++ b/tests/unit/test_phase2_pages.py @@ -0,0 +1,20 @@ +from fastapi.testclient import TestClient + +from app.main import app + + +def test_phase2_pages_redirect_to_login_when_unauthenticated() -> None: + client = TestClient(app) + + for path in ( + "/users", + "/runs", + "/runs/1", + "/publications", + "/scholars", + "/settings", + "/account/password", + ): + response = client.get(path, follow_redirects=False) + assert response.status_code == 303 + assert response.headers["location"] == "/login" diff --git a/tests/unit/test_scholar_parser.py b/tests/unit/test_scholar_parser.py new file mode 100644 index 0000000..8c246e4 --- /dev/null +++ b/tests/unit/test_scholar_parser.py @@ -0,0 +1,240 @@ +from __future__ import annotations + +from pathlib import Path + +from app.services.scholar_parser import ParseState, parse_profile_page +from app.services.scholar_source import FetchResult + + +def _fixture(name: str) -> str: + path = Path("tests/fixtures/scholar") / name + return path.read_text(encoding="utf-8") + + +def _regression_fixture(name: str) -> str: + path = Path("tests/fixtures/scholar/regression") / name + return path.read_text(encoding="utf-8") + + +def test_parse_profile_page_extracts_core_fields_from_fixture() -> None: + fetch_result = FetchResult( + requested_url="https://scholar.google.com/citations?hl=en&user=amIMrIEAAAAJ", + status_code=200, + final_url="https://scholar.google.com/citations?hl=en&user=amIMrIEAAAAJ", + body=_fixture("profile_ok_amIMrIEAAAAJ.html"), + error=None, + ) + + parsed = parse_profile_page(fetch_result) + + assert parsed.state == ParseState.OK + assert parsed.state_reason == "publications_extracted" + assert parsed.profile_name == "Bangar Raju Cherukuri" + assert len(parsed.publications) >= 10 + assert parsed.has_show_more_button is True + assert parsed.articles_range is not None + first = parsed.publications[0] + assert first.title + assert first.cluster_id + assert first.citation_count is not None + + +def test_parse_profile_page_classifies_accounts_redirect_as_blocked() -> None: + fetch_result = FetchResult( + requested_url="https://scholar.google.com/citations?hl=en&user=AAAAAAAAAAAA", + status_code=200, + final_url="https://accounts.google.com/v3/signin/identifier?continue=...", + body="Sign in", + error=None, + ) + + parsed = parse_profile_page(fetch_result) + + assert parsed.state == ParseState.BLOCKED_OR_CAPTCHA + assert parsed.state_reason == "blocked_accounts_redirect" + assert len(parsed.publications) == 0 + + +def test_parse_profile_page_handles_missing_optional_metadata() -> None: + html = """ + +
Test Author
+ Articles 1-1 + + + + + + + + +
+ A Test Paper +
A Person
+ 		7
+ + """ + fetch_result = FetchResult( + requested_url="https://scholar.google.com/citations?hl=en&user=abcDEF123456", + status_code=200, + final_url="https://scholar.google.com/citations?hl=en&user=abcDEF123456", + body=html, + error=None, + ) + + parsed = parse_profile_page(fetch_result) + + assert parsed.state == ParseState.OK + assert parsed.state_reason == "publications_extracted" + assert len(parsed.publications) == 1 + publication = parsed.publications[0] + assert publication.year is None + assert publication.venue_text is None + + +def test_parse_profile_page_detects_layout_change_when_markers_absent() -> None: + fetch_result = FetchResult( + requested_url="https://scholar.google.com/citations?hl=en&user=abcDEF123456", + status_code=200, + final_url="https://scholar.google.com/citations?hl=en&user=abcDEF123456", + body="

Unexpected page

", + error=None, + ) + + parsed = parse_profile_page(fetch_result) + + assert parsed.state == ParseState.LAYOUT_CHANGED + assert parsed.state_reason == "layout_markers_missing" + assert "no_rows_detected" in parsed.warnings + + +def test_parse_profile_page_reports_network_reason_when_status_missing() -> None: + fetch_result = FetchResult( + requested_url="https://scholar.google.com/citations?hl=en&user=abcDEF123456", + status_code=None, + final_url=None, + body="", + error="timed out", + ) + + parsed = parse_profile_page(fetch_result) + + assert parsed.state == ParseState.NETWORK_ERROR + assert parsed.state_reason == "network_error_missing_status_code" + + +def test_parse_profile_page_ignores_no_results_keyword_inside_script_blocks() -> None: + html = """ + + +
Scripted Author
+
+ + """ + fetch_result = FetchResult( + requested_url="https://scholar.google.com/citations?hl=en&user=abcDEF123456", + status_code=200, + final_url="https://scholar.google.com/citations?hl=en&user=abcDEF123456", + body=html, + error=None, + ) + + parsed = parse_profile_page(fetch_result) + + assert parsed.state == ParseState.OK + assert parsed.state_reason == "no_rows_with_known_markers" + + +def test_parse_profile_page_treats_disabled_show_more_button_as_absent() -> None: + html = """ + +
Disabled Show More
+ Articles 1-1 + + + + + + +
+ Paper + 12024
+ + + """ + fetch_result = FetchResult( + requested_url="https://scholar.google.com/citations?hl=en&user=abcDEF123456", + status_code=200, + final_url="https://scholar.google.com/citations?hl=en&user=abcDEF123456", + body=html, + error=None, + ) + + parsed = parse_profile_page(fetch_result) + + assert parsed.state == ParseState.OK + assert parsed.has_show_more_button is False + + +def test_parse_profile_page_regression_fixture_profile_p1rwlvo() -> None: + fetch_result = FetchResult( + requested_url="https://scholar.google.com/citations?hl=en&user=P1RwlvoAAAAJ", + status_code=200, + final_url="https://scholar.google.com/citations?hl=en&user=P1RwlvoAAAAJ", + body=_regression_fixture("profile_P1RwlvoAAAAJ.html"), + error=None, + ) + + parsed = parse_profile_page(fetch_result) + + assert parsed.state == ParseState.OK + assert parsed.state_reason == "publications_extracted" + assert parsed.profile_name == "WENRUI ZUO" + assert len(parsed.publications) == 5 + assert parsed.has_show_more_button is False + assert parsed.articles_range in {"Articles 1-5", "Articles 1–5"} + assert "possible_partial_page_show_more_present" not in parsed.warnings + assert all(item.cluster_id for item in parsed.publications) + + +def test_parse_profile_page_regression_fixture_profile_lz5d() -> None: + fetch_result = FetchResult( + requested_url="https://scholar.google.com/citations?hl=en&user=LZ5D_p4AAAAJ", + status_code=200, + final_url="https://scholar.google.com/citations?hl=en&user=LZ5D_p4AAAAJ", + body=_regression_fixture("profile_LZ5D_p4AAAAJ.html"), + error=None, + ) + + parsed = parse_profile_page(fetch_result) + + assert parsed.state == ParseState.OK + assert parsed.state_reason == "publications_extracted" + assert parsed.profile_name == "Doaa Elmatary" + assert len(parsed.publications) == 12 + assert parsed.has_show_more_button is False + assert parsed.articles_range in {"Articles 1-12", "Articles 1–12"} + assert "possible_partial_page_show_more_present" not in parsed.warnings + assert any(item.venue_text is None for item in parsed.publications) + + +def test_parse_profile_page_regression_fixture_blocked_redirect() -> None: + fetch_result = FetchResult( + requested_url="https://scholar.google.com/citations?hl=en&user=AAAAAAAAAAAA", + status_code=200, + final_url=( + "https://accounts.google.com/v3/signin/identifier" + "?continue=https%3A%2F%2Fscholar.google.com%2Fcitations%3Fhl%3Den%26user%3DAAAAAAAAAAAA" + ), + body=_regression_fixture("profile_AAAAAAAAAAAA.html"), + error=None, + ) + + parsed = parse_profile_page(fetch_result) + + assert parsed.state == ParseState.BLOCKED_OR_CAPTCHA + assert parsed.state_reason == "blocked_accounts_redirect" + assert parsed.profile_name is None + assert len(parsed.publications) == 0 diff --git a/uv.lock b/uv.lock new file mode 100644 index 0000000..571b45e --- /dev/null +++ b/uv.lock @@ -0,0 +1,931 @@ +version = 1 +revision = 1 +requires-python = ">=3.12" +resolution-markers = [ + "python_full_version >= '3.14'", + "python_full_version < '3.14'", +] + +[[package]] +name = "alembic" +version = "1.18.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mako" }, + { name = "sqlalchemy" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/94/13/8b084e0f2efb0275a1d534838844926f798bd766566b1375174e2448cd31/alembic-1.18.4.tar.gz", hash = "sha256:cb6e1fd84b6174ab8dbb2329f86d631ba9559dd78df550b57804d607672cedbc", size = 2056725 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d2/29/6533c317b74f707ea28f8d633734dbda2119bbadfc61b2f3640ba835d0f7/alembic-1.18.4-py3-none-any.whl", hash = "sha256:a5ed4adcf6d8a4cb575f3d759f071b03cd6e5c7618eb796cb52497be25bfe19a", size = 263893 }, +] + +[[package]] +name = "annotated-types" +version = "0.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ee/67/531ea369ba64dcff5ec9c3402f9f51bf748cec26dde048a2f973a4eea7f5/annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89", size = 16081 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53", size = 13643 }, +] + +[[package]] +name = "anyio" +version = "4.12.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "idna" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/96/f0/5eb65b2bb0d09ac6776f2eb54adee6abe8228ea05b20a5ad0e4945de8aac/anyio-4.12.1.tar.gz", hash = "sha256:41cfcc3a4c85d3f05c932da7c26d0201ac36f72abd4435ba90d0464a3ffed703", size = 228685 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/38/0e/27be9fdef66e72d64c0cdc3cc2823101b80585f8119b5c112c2e8f5f7dab/anyio-4.12.1-py3-none-any.whl", hash = "sha256:d405828884fc140aa80a3c667b8beed277f1dfedec42ba031bd6ac3db606ab6c", size = 113592 }, +] + +[[package]] +name = "argon2-cffi" +version = "25.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "argon2-cffi-bindings" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0e/89/ce5af8a7d472a67cc819d5d998aa8c82c5d860608c4db9f46f1162d7dab9/argon2_cffi-25.1.0.tar.gz", hash = "sha256:694ae5cc8a42f4c4e2bf2ca0e64e51e23a040c6a517a85074683d3959e1346c1", size = 45706 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4f/d3/a8b22fa575b297cd6e3e3b0155c7e25db170edf1c74783d6a31a2490b8d9/argon2_cffi-25.1.0-py3-none-any.whl", hash = "sha256:fdc8b074db390fccb6eb4a3604ae7231f219aa669a2652e0f20e16ba513d5741", size = 14657 }, +] + +[[package]] +name = "argon2-cffi-bindings" +version = "25.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5c/2d/db8af0df73c1cf454f71b2bbe5e356b8c1f8041c979f505b3d3186e520a9/argon2_cffi_bindings-25.1.0.tar.gz", hash = "sha256:b957f3e6ea4d55d820e40ff76f450952807013d361a65d7f28acc0acbf29229d", size = 1783441 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/60/97/3c0a35f46e52108d4707c44b95cfe2afcafc50800b5450c197454569b776/argon2_cffi_bindings-25.1.0-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:3d3f05610594151994ca9ccb3c771115bdb4daef161976a266f0dd8aa9996b8f", size = 54393 }, + { url = "https://files.pythonhosted.org/packages/9d/f4/98bbd6ee89febd4f212696f13c03ca302b8552e7dbf9c8efa11ea4a388c3/argon2_cffi_bindings-25.1.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:8b8efee945193e667a396cbc7b4fb7d357297d6234d30a489905d96caabde56b", size = 29328 }, + { url = "https://files.pythonhosted.org/packages/43/24/90a01c0ef12ac91a6be05969f29944643bc1e5e461155ae6559befa8f00b/argon2_cffi_bindings-25.1.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:3c6702abc36bf3ccba3f802b799505def420a1b7039862014a65db3205967f5a", size = 31269 }, + { url = "https://files.pythonhosted.org/packages/d4/d3/942aa10782b2697eee7af5e12eeff5ebb325ccfb86dd8abda54174e377e4/argon2_cffi_bindings-25.1.0-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a1c70058c6ab1e352304ac7e3b52554daadacd8d453c1752e547c76e9c99ac44", size = 86558 }, + { url = "https://files.pythonhosted.org/packages/0d/82/b484f702fec5536e71836fc2dbc8c5267b3f6e78d2d539b4eaa6f0db8bf8/argon2_cffi_bindings-25.1.0-cp314-cp314t-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e2fd3bfbff3c5d74fef31a722f729bf93500910db650c925c2d6ef879a7e51cb", size = 92364 }, + { url = "https://files.pythonhosted.org/packages/c9/c1/a606ff83b3f1735f3759ad0f2cd9e038a0ad11a3de3b6c673aa41c24bb7b/argon2_cffi_bindings-25.1.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:c4f9665de60b1b0e99bcd6be4f17d90339698ce954cfd8d9cf4f91c995165a92", size = 85637 }, + { url = "https://files.pythonhosted.org/packages/44/b4/678503f12aceb0262f84fa201f6027ed77d71c5019ae03b399b97caa2f19/argon2_cffi_bindings-25.1.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ba92837e4a9aa6a508c8d2d7883ed5a8f6c308c89a4790e1e447a220deb79a85", size = 91934 }, + { url = "https://files.pythonhosted.org/packages/f0/c7/f36bd08ef9bd9f0a9cff9428406651f5937ce27b6c5b07b92d41f91ae541/argon2_cffi_bindings-25.1.0-cp314-cp314t-win32.whl", hash = "sha256:84a461d4d84ae1295871329b346a97f68eade8c53b6ed9a7ca2d7467f3c8ff6f", size = 28158 }, + { url = "https://files.pythonhosted.org/packages/b3/80/0106a7448abb24a2c467bf7d527fe5413b7fdfa4ad6d6a96a43a62ef3988/argon2_cffi_bindings-25.1.0-cp314-cp314t-win_amd64.whl", hash = "sha256:b55aec3565b65f56455eebc9b9f34130440404f27fe21c3b375bf1ea4d8fbae6", size = 32597 }, + { url = "https://files.pythonhosted.org/packages/05/b8/d663c9caea07e9180b2cb662772865230715cbd573ba3b5e81793d580316/argon2_cffi_bindings-25.1.0-cp314-cp314t-win_arm64.whl", hash = "sha256:87c33a52407e4c41f3b70a9c2d3f6056d88b10dad7695be708c5021673f55623", size = 28231 }, + { url = "https://files.pythonhosted.org/packages/1d/57/96b8b9f93166147826da5f90376e784a10582dd39a393c99bb62cfcf52f0/argon2_cffi_bindings-25.1.0-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:aecba1723ae35330a008418a91ea6cfcedf6d31e5fbaa056a166462ff066d500", size = 54121 }, + { url = "https://files.pythonhosted.org/packages/0a/08/a9bebdb2e0e602dde230bdde8021b29f71f7841bd54801bcfd514acb5dcf/argon2_cffi_bindings-25.1.0-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:2630b6240b495dfab90aebe159ff784d08ea999aa4b0d17efa734055a07d2f44", size = 29177 }, + { url = "https://files.pythonhosted.org/packages/b6/02/d297943bcacf05e4f2a94ab6f462831dc20158614e5d067c35d4e63b9acb/argon2_cffi_bindings-25.1.0-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:7aef0c91e2c0fbca6fc68e7555aa60ef7008a739cbe045541e438373bc54d2b0", size = 31090 }, + { url = "https://files.pythonhosted.org/packages/c1/93/44365f3d75053e53893ec6d733e4a5e3147502663554b4d864587c7828a7/argon2_cffi_bindings-25.1.0-cp39-abi3-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1e021e87faa76ae0d413b619fe2b65ab9a037f24c60a1e6cc43457ae20de6dc6", size = 81246 }, + { url = "https://files.pythonhosted.org/packages/09/52/94108adfdd6e2ddf58be64f959a0b9c7d4ef2fa71086c38356d22dc501ea/argon2_cffi_bindings-25.1.0-cp39-abi3-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d3e924cfc503018a714f94a49a149fdc0b644eaead5d1f089330399134fa028a", size = 87126 }, + { url = "https://files.pythonhosted.org/packages/72/70/7a2993a12b0ffa2a9271259b79cc616e2389ed1a4d93842fac5a1f923ffd/argon2_cffi_bindings-25.1.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:c87b72589133f0346a1cb8d5ecca4b933e3c9b64656c9d175270a000e73b288d", size = 80343 }, + { url = "https://files.pythonhosted.org/packages/78/9a/4e5157d893ffc712b74dbd868c7f62365618266982b64accab26bab01edc/argon2_cffi_bindings-25.1.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:1db89609c06afa1a214a69a462ea741cf735b29a57530478c06eb81dd403de99", size = 86777 }, + { url = "https://files.pythonhosted.org/packages/74/cd/15777dfde1c29d96de7f18edf4cc94c385646852e7c7b0320aa91ccca583/argon2_cffi_bindings-25.1.0-cp39-abi3-win32.whl", hash = "sha256:473bcb5f82924b1becbb637b63303ec8d10e84c8d241119419897a26116515d2", size = 27180 }, + { url = "https://files.pythonhosted.org/packages/e2/c6/a759ece8f1829d1f162261226fbfd2c6832b3ff7657384045286d2afa384/argon2_cffi_bindings-25.1.0-cp39-abi3-win_amd64.whl", hash = "sha256:a98cd7d17e9f7ce244c0803cad3c23a7d379c301ba618a5fa76a67d116618b98", size = 31715 }, + { url = "https://files.pythonhosted.org/packages/42/b9/f8d6fa329ab25128b7e98fd83a3cb34d9db5b059a9847eddb840a0af45dd/argon2_cffi_bindings-25.1.0-cp39-abi3-win_arm64.whl", hash = "sha256:b0fdbcf513833809c882823f98dc2f931cf659d9a1429616ac3adebb49f5db94", size = 27149 }, +] + +[[package]] +name = "asyncpg" +version = "0.30.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2f/4c/7c991e080e106d854809030d8584e15b2e996e26f16aee6d757e387bc17d/asyncpg-0.30.0.tar.gz", hash = "sha256:c551e9928ab6707602f44811817f82ba3c446e018bfe1d3abecc8ba5f3eac851", size = 957746 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4b/64/9d3e887bb7b01535fdbc45fbd5f0a8447539833b97ee69ecdbb7a79d0cb4/asyncpg-0.30.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c902a60b52e506d38d7e80e0dd5399f657220f24635fee368117b8b5fce1142e", size = 673162 }, + { url = "https://files.pythonhosted.org/packages/6e/eb/8b236663f06984f212a087b3e849731f917ab80f84450e943900e8ca4052/asyncpg-0.30.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:aca1548e43bbb9f0f627a04666fedaca23db0a31a84136ad1f868cb15deb6e3a", size = 637025 }, + { url = "https://files.pythonhosted.org/packages/cc/57/2dc240bb263d58786cfaa60920779af6e8d32da63ab9ffc09f8312bd7a14/asyncpg-0.30.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6c2a2ef565400234a633da0eafdce27e843836256d40705d83ab7ec42074efb3", size = 3496243 }, + { url = "https://files.pythonhosted.org/packages/f4/40/0ae9d061d278b10713ea9021ef6b703ec44698fe32178715a501ac696c6b/asyncpg-0.30.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1292b84ee06ac8a2ad8e51c7475aa309245874b61333d97411aab835c4a2f737", size = 3575059 }, + { url = "https://files.pythonhosted.org/packages/c3/75/d6b895a35a2c6506952247640178e5f768eeb28b2e20299b6a6f1d743ba0/asyncpg-0.30.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:0f5712350388d0cd0615caec629ad53c81e506b1abaaf8d14c93f54b35e3595a", size = 3473596 }, + { url = "https://files.pythonhosted.org/packages/c8/e7/3693392d3e168ab0aebb2d361431375bd22ffc7b4a586a0fc060d519fae7/asyncpg-0.30.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:db9891e2d76e6f425746c5d2da01921e9a16b5a71a1c905b13f30e12a257c4af", size = 3641632 }, + { url = "https://files.pythonhosted.org/packages/32/ea/15670cea95745bba3f0352341db55f506a820b21c619ee66b7d12ea7867d/asyncpg-0.30.0-cp312-cp312-win32.whl", hash = "sha256:68d71a1be3d83d0570049cd1654a9bdfe506e794ecc98ad0873304a9f35e411e", size = 560186 }, + { url = "https://files.pythonhosted.org/packages/7e/6b/fe1fad5cee79ca5f5c27aed7bd95baee529c1bf8a387435c8ba4fe53d5c1/asyncpg-0.30.0-cp312-cp312-win_amd64.whl", hash = "sha256:9a0292c6af5c500523949155ec17b7fe01a00ace33b68a476d6b5059f9630305", size = 621064 }, + { url = "https://files.pythonhosted.org/packages/3a/22/e20602e1218dc07692acf70d5b902be820168d6282e69ef0d3cb920dc36f/asyncpg-0.30.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:05b185ebb8083c8568ea8a40e896d5f7af4b8554b64d7719c0eaa1eb5a5c3a70", size = 670373 }, + { url = "https://files.pythonhosted.org/packages/3d/b3/0cf269a9d647852a95c06eb00b815d0b95a4eb4b55aa2d6ba680971733b9/asyncpg-0.30.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:c47806b1a8cbb0a0db896f4cd34d89942effe353a5035c62734ab13b9f938da3", size = 634745 }, + { url = "https://files.pythonhosted.org/packages/8e/6d/a4f31bf358ce8491d2a31bfe0d7bcf25269e80481e49de4d8616c4295a34/asyncpg-0.30.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9b6fde867a74e8c76c71e2f64f80c64c0f3163e687f1763cfaf21633ec24ec33", size = 3512103 }, + { url = "https://files.pythonhosted.org/packages/96/19/139227a6e67f407b9c386cb594d9628c6c78c9024f26df87c912fabd4368/asyncpg-0.30.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:46973045b567972128a27d40001124fbc821c87a6cade040cfcd4fa8a30bcdc4", size = 3592471 }, + { url = "https://files.pythonhosted.org/packages/67/e4/ab3ca38f628f53f0fd28d3ff20edff1c975dd1cb22482e0061916b4b9a74/asyncpg-0.30.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:9110df111cabc2ed81aad2f35394a00cadf4f2e0635603db6ebbd0fc896f46a4", size = 3496253 }, + { url = "https://files.pythonhosted.org/packages/ef/5f/0bf65511d4eeac3a1f41c54034a492515a707c6edbc642174ae79034d3ba/asyncpg-0.30.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:04ff0785ae7eed6cc138e73fc67b8e51d54ee7a3ce9b63666ce55a0bf095f7ba", size = 3662720 }, + { url = "https://files.pythonhosted.org/packages/e7/31/1513d5a6412b98052c3ed9158d783b1e09d0910f51fbe0e05f56cc370bc4/asyncpg-0.30.0-cp313-cp313-win32.whl", hash = "sha256:ae374585f51c2b444510cdf3595b97ece4f233fde739aa14b50e0d64e8a7a590", size = 560404 }, + { url = "https://files.pythonhosted.org/packages/c8/a4/cec76b3389c4c5ff66301cd100fe88c318563ec8a520e0b2e792b5b84972/asyncpg-0.30.0-cp313-cp313-win_amd64.whl", hash = "sha256:f59b430b8e27557c3fb9869222559f7417ced18688375825f8f12302c34e915e", size = 621623 }, +] + +[[package]] +name = "certifi" +version = "2026.1.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e0/2d/a891ca51311197f6ad14a7ef42e2399f36cf2f9bd44752b3dc4eab60fdc5/certifi-2026.1.4.tar.gz", hash = "sha256:ac726dd470482006e014ad384921ed6438c457018f4b3d204aea4281258b2120", size = 154268 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e6/ad/3cc14f097111b4de0040c83a525973216457bbeeb63739ef1ed275c1c021/certifi-2026.1.4-py3-none-any.whl", hash = "sha256:9943707519e4add1115f44c2bc244f782c0249876bf51b6599fee1ffbedd685c", size = 152900 }, +] + +[[package]] +name = "cffi" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pycparser", marker = "implementation_name != 'PyPy'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/eb/56/b1ba7935a17738ae8453301356628e8147c79dbb825bcbc73dc7401f9846/cffi-2.0.0.tar.gz", hash = "sha256:44d1b5909021139fe36001ae048dbdde8214afa20200eda0f64c068cac5d5529", size = 523588 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ea/47/4f61023ea636104d4f16ab488e268b93008c3d0bb76893b1b31db1f96802/cffi-2.0.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6d02d6655b0e54f54c4ef0b94eb6be0607b70853c45ce98bd278dc7de718be5d", size = 185271 }, + { url = "https://files.pythonhosted.org/packages/df/a2/781b623f57358e360d62cdd7a8c681f074a71d445418a776eef0aadb4ab4/cffi-2.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8eca2a813c1cb7ad4fb74d368c2ffbbb4789d377ee5bb8df98373c2cc0dee76c", size = 181048 }, + { url = "https://files.pythonhosted.org/packages/ff/df/a4f0fbd47331ceeba3d37c2e51e9dfc9722498becbeec2bd8bc856c9538a/cffi-2.0.0-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:21d1152871b019407d8ac3985f6775c079416c282e431a4da6afe7aefd2bccbe", size = 212529 }, + { url = "https://files.pythonhosted.org/packages/d5/72/12b5f8d3865bf0f87cf1404d8c374e7487dcf097a1c91c436e72e6badd83/cffi-2.0.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b21e08af67b8a103c71a250401c78d5e0893beff75e28c53c98f4de42f774062", size = 220097 }, + { url = "https://files.pythonhosted.org/packages/c2/95/7a135d52a50dfa7c882ab0ac17e8dc11cec9d55d2c18dda414c051c5e69e/cffi-2.0.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:1e3a615586f05fc4065a8b22b8152f0c1b00cdbc60596d187c2a74f9e3036e4e", size = 207983 }, + { url = "https://files.pythonhosted.org/packages/3a/c8/15cb9ada8895957ea171c62dc78ff3e99159ee7adb13c0123c001a2546c1/cffi-2.0.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:81afed14892743bbe14dacb9e36d9e0e504cd204e0b165062c488942b9718037", size = 206519 }, + { url = "https://files.pythonhosted.org/packages/78/2d/7fa73dfa841b5ac06c7b8855cfc18622132e365f5b81d02230333ff26e9e/cffi-2.0.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3e17ed538242334bf70832644a32a7aae3d83b57567f9fd60a26257e992b79ba", size = 219572 }, + { url = "https://files.pythonhosted.org/packages/07/e0/267e57e387b4ca276b90f0434ff88b2c2241ad72b16d31836adddfd6031b/cffi-2.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3925dd22fa2b7699ed2617149842d2e6adde22b262fcbfada50e3d195e4b3a94", size = 222963 }, + { url = "https://files.pythonhosted.org/packages/b6/75/1f2747525e06f53efbd878f4d03bac5b859cbc11c633d0fb81432d98a795/cffi-2.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2c8f814d84194c9ea681642fd164267891702542f028a15fc97d4674b6206187", size = 221361 }, + { url = "https://files.pythonhosted.org/packages/7b/2b/2b6435f76bfeb6bbf055596976da087377ede68df465419d192acf00c437/cffi-2.0.0-cp312-cp312-win32.whl", hash = "sha256:da902562c3e9c550df360bfa53c035b2f241fed6d9aef119048073680ace4a18", size = 172932 }, + { url = "https://files.pythonhosted.org/packages/f8/ed/13bd4418627013bec4ed6e54283b1959cf6db888048c7cf4b4c3b5b36002/cffi-2.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:da68248800ad6320861f129cd9c1bf96ca849a2771a59e0344e88681905916f5", size = 183557 }, + { url = "https://files.pythonhosted.org/packages/95/31/9f7f93ad2f8eff1dbc1c3656d7ca5bfd8fb52c9d786b4dcf19b2d02217fa/cffi-2.0.0-cp312-cp312-win_arm64.whl", hash = "sha256:4671d9dd5ec934cb9a73e7ee9676f9362aba54f7f34910956b84d727b0d73fb6", size = 177762 }, + { url = "https://files.pythonhosted.org/packages/4b/8d/a0a47a0c9e413a658623d014e91e74a50cdd2c423f7ccfd44086ef767f90/cffi-2.0.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:00bdf7acc5f795150faa6957054fbbca2439db2f775ce831222b66f192f03beb", size = 185230 }, + { url = "https://files.pythonhosted.org/packages/4a/d2/a6c0296814556c68ee32009d9c2ad4f85f2707cdecfd7727951ec228005d/cffi-2.0.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:45d5e886156860dc35862657e1494b9bae8dfa63bf56796f2fb56e1679fc0bca", size = 181043 }, + { url = "https://files.pythonhosted.org/packages/b0/1e/d22cc63332bd59b06481ceaac49d6c507598642e2230f201649058a7e704/cffi-2.0.0-cp313-cp313-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:07b271772c100085dd28b74fa0cd81c8fb1a3ba18b21e03d7c27f3436a10606b", size = 212446 }, + { url = "https://files.pythonhosted.org/packages/a9/f5/a2c23eb03b61a0b8747f211eb716446c826ad66818ddc7810cc2cc19b3f2/cffi-2.0.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d48a880098c96020b02d5a1f7d9251308510ce8858940e6fa99ece33f610838b", size = 220101 }, + { url = "https://files.pythonhosted.org/packages/f2/7f/e6647792fc5850d634695bc0e6ab4111ae88e89981d35ac269956605feba/cffi-2.0.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:f93fd8e5c8c0a4aa1f424d6173f14a892044054871c771f8566e4008eaa359d2", size = 207948 }, + { url = "https://files.pythonhosted.org/packages/cb/1e/a5a1bd6f1fb30f22573f76533de12a00bf274abcdc55c8edab639078abb6/cffi-2.0.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:dd4f05f54a52fb558f1ba9f528228066954fee3ebe629fc1660d874d040ae5a3", size = 206422 }, + { url = "https://files.pythonhosted.org/packages/98/df/0a1755e750013a2081e863e7cd37e0cdd02664372c754e5560099eb7aa44/cffi-2.0.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c8d3b5532fc71b7a77c09192b4a5a200ea992702734a2e9279a37f2478236f26", size = 219499 }, + { url = "https://files.pythonhosted.org/packages/50/e1/a969e687fcf9ea58e6e2a928ad5e2dd88cc12f6f0ab477e9971f2309b57c/cffi-2.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d9b29c1f0ae438d5ee9acb31cadee00a58c46cc9c0b2f9038c6b0b3470877a8c", size = 222928 }, + { url = "https://files.pythonhosted.org/packages/36/54/0362578dd2c9e557a28ac77698ed67323ed5b9775ca9d3fe73fe191bb5d8/cffi-2.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6d50360be4546678fc1b79ffe7a66265e28667840010348dd69a314145807a1b", size = 221302 }, + { url = "https://files.pythonhosted.org/packages/eb/6d/bf9bda840d5f1dfdbf0feca87fbdb64a918a69bca42cfa0ba7b137c48cb8/cffi-2.0.0-cp313-cp313-win32.whl", hash = "sha256:74a03b9698e198d47562765773b4a8309919089150a0bb17d829ad7b44b60d27", size = 172909 }, + { url = "https://files.pythonhosted.org/packages/37/18/6519e1ee6f5a1e579e04b9ddb6f1676c17368a7aba48299c3759bbc3c8b3/cffi-2.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:19f705ada2530c1167abacb171925dd886168931e0a7b78f5bffcae5c6b5be75", size = 183402 }, + { url = "https://files.pythonhosted.org/packages/cb/0e/02ceeec9a7d6ee63bb596121c2c8e9b3a9e150936f4fbef6ca1943e6137c/cffi-2.0.0-cp313-cp313-win_arm64.whl", hash = "sha256:256f80b80ca3853f90c21b23ee78cd008713787b1b1e93eae9f3d6a7134abd91", size = 177780 }, + { url = "https://files.pythonhosted.org/packages/92/c4/3ce07396253a83250ee98564f8d7e9789fab8e58858f35d07a9a2c78de9f/cffi-2.0.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:fc33c5141b55ed366cfaad382df24fe7dcbc686de5be719b207bb248e3053dc5", size = 185320 }, + { url = "https://files.pythonhosted.org/packages/59/dd/27e9fa567a23931c838c6b02d0764611c62290062a6d4e8ff7863daf9730/cffi-2.0.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c654de545946e0db659b3400168c9ad31b5d29593291482c43e3564effbcee13", size = 181487 }, + { url = "https://files.pythonhosted.org/packages/d6/43/0e822876f87ea8a4ef95442c3d766a06a51fc5298823f884ef87aaad168c/cffi-2.0.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:24b6f81f1983e6df8db3adc38562c83f7d4a0c36162885ec7f7b77c7dcbec97b", size = 220049 }, + { url = "https://files.pythonhosted.org/packages/b4/89/76799151d9c2d2d1ead63c2429da9ea9d7aac304603de0c6e8764e6e8e70/cffi-2.0.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:12873ca6cb9b0f0d3a0da705d6086fe911591737a59f28b7936bdfed27c0d47c", size = 207793 }, + { url = "https://files.pythonhosted.org/packages/bb/dd/3465b14bb9e24ee24cb88c9e3730f6de63111fffe513492bf8c808a3547e/cffi-2.0.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:d9b97165e8aed9272a6bb17c01e3cc5871a594a446ebedc996e2397a1c1ea8ef", size = 206300 }, + { url = "https://files.pythonhosted.org/packages/47/d9/d83e293854571c877a92da46fdec39158f8d7e68da75bf73581225d28e90/cffi-2.0.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:afb8db5439b81cf9c9d0c80404b60c3cc9c3add93e114dcae767f1477cb53775", size = 219244 }, + { url = "https://files.pythonhosted.org/packages/2b/0f/1f177e3683aead2bb00f7679a16451d302c436b5cbf2505f0ea8146ef59e/cffi-2.0.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:737fe7d37e1a1bffe70bd5754ea763a62a066dc5913ca57e957824b72a85e205", size = 222828 }, + { url = "https://files.pythonhosted.org/packages/c6/0f/cafacebd4b040e3119dcb32fed8bdef8dfe94da653155f9d0b9dc660166e/cffi-2.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:38100abb9d1b1435bc4cc340bb4489635dc2f0da7456590877030c9b3d40b0c1", size = 220926 }, + { url = "https://files.pythonhosted.org/packages/3e/aa/df335faa45b395396fcbc03de2dfcab242cd61a9900e914fe682a59170b1/cffi-2.0.0-cp314-cp314-win32.whl", hash = "sha256:087067fa8953339c723661eda6b54bc98c5625757ea62e95eb4898ad5e776e9f", size = 175328 }, + { url = "https://files.pythonhosted.org/packages/bb/92/882c2d30831744296ce713f0feb4c1cd30f346ef747b530b5318715cc367/cffi-2.0.0-cp314-cp314-win_amd64.whl", hash = "sha256:203a48d1fb583fc7d78a4c6655692963b860a417c0528492a6bc21f1aaefab25", size = 185650 }, + { url = "https://files.pythonhosted.org/packages/9f/2c/98ece204b9d35a7366b5b2c6539c350313ca13932143e79dc133ba757104/cffi-2.0.0-cp314-cp314-win_arm64.whl", hash = "sha256:dbd5c7a25a7cb98f5ca55d258b103a2054f859a46ae11aaf23134f9cc0d356ad", size = 180687 }, + { url = "https://files.pythonhosted.org/packages/3e/61/c768e4d548bfa607abcda77423448df8c471f25dbe64fb2ef6d555eae006/cffi-2.0.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:9a67fc9e8eb39039280526379fb3a70023d77caec1852002b4da7e8b270c4dd9", size = 188773 }, + { url = "https://files.pythonhosted.org/packages/2c/ea/5f76bce7cf6fcd0ab1a1058b5af899bfbef198bea4d5686da88471ea0336/cffi-2.0.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7a66c7204d8869299919db4d5069a82f1561581af12b11b3c9f48c584eb8743d", size = 185013 }, + { url = "https://files.pythonhosted.org/packages/be/b4/c56878d0d1755cf9caa54ba71e5d049479c52f9e4afc230f06822162ab2f/cffi-2.0.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7cc09976e8b56f8cebd752f7113ad07752461f48a58cbba644139015ac24954c", size = 221593 }, + { url = "https://files.pythonhosted.org/packages/e0/0d/eb704606dfe8033e7128df5e90fee946bbcb64a04fcdaa97321309004000/cffi-2.0.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:92b68146a71df78564e4ef48af17551a5ddd142e5190cdf2c5624d0c3ff5b2e8", size = 209354 }, + { url = "https://files.pythonhosted.org/packages/d8/19/3c435d727b368ca475fb8742ab97c9cb13a0de600ce86f62eab7fa3eea60/cffi-2.0.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:b1e74d11748e7e98e2f426ab176d4ed720a64412b6a15054378afdb71e0f37dc", size = 208480 }, + { url = "https://files.pythonhosted.org/packages/d0/44/681604464ed9541673e486521497406fadcc15b5217c3e326b061696899a/cffi-2.0.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:28a3a209b96630bca57cce802da70c266eb08c6e97e5afd61a75611ee6c64592", size = 221584 }, + { url = "https://files.pythonhosted.org/packages/25/8e/342a504ff018a2825d395d44d63a767dd8ebc927ebda557fecdaca3ac33a/cffi-2.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:7553fb2090d71822f02c629afe6042c299edf91ba1bf94951165613553984512", size = 224443 }, + { url = "https://files.pythonhosted.org/packages/e1/5e/b666bacbbc60fbf415ba9988324a132c9a7a0448a9a8f125074671c0f2c3/cffi-2.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6c6c373cfc5c83a975506110d17457138c8c63016b563cc9ed6e056a82f13ce4", size = 223437 }, + { url = "https://files.pythonhosted.org/packages/a0/1d/ec1a60bd1a10daa292d3cd6bb0b359a81607154fb8165f3ec95fe003b85c/cffi-2.0.0-cp314-cp314t-win32.whl", hash = "sha256:1fc9ea04857caf665289b7a75923f2c6ed559b8298a1b8c49e59f7dd95c8481e", size = 180487 }, + { url = "https://files.pythonhosted.org/packages/bf/41/4c1168c74fac325c0c8156f04b6749c8b6a8f405bbf91413ba088359f60d/cffi-2.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:d68b6cef7827e8641e8ef16f4494edda8b36104d79773a334beaa1e3521430f6", size = 191726 }, + { url = "https://files.pythonhosted.org/packages/ae/3a/dbeec9d1ee0844c679f6bb5d6ad4e9f198b1224f4e7a32825f47f6192b0c/cffi-2.0.0-cp314-cp314t-win_arm64.whl", hash = "sha256:0a1527a803f0a659de1af2e1fd700213caba79377e27e4693648c2923da066f9", size = 184195 }, +] + +[[package]] +name = "click" +version = "8.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3d/fa/656b739db8587d7b5dfa22e22ed02566950fbfbcdc20311993483657a5c0/click-8.3.1.tar.gz", hash = "sha256:12ff4785d337a1bb490bb7e9c2b1ee5da3112e94a8622f26a6c77f5d2fc6842a", size = 295065 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/98/78/01c019cdb5d6498122777c1a43056ebb3ebfeef2076d9d026bfe15583b2b/click-8.3.1-py3-none-any.whl", hash = "sha256:981153a64e25f12d547d3426c367a4857371575ee7ad18df2a6183ab0545b2a6", size = 108274 }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335 }, +] + +[[package]] +name = "fastapi" +version = "0.116.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pydantic" }, + { name = "starlette" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/01/64/1296f46d6b9e3b23fb22e5d01af3f104ef411425531376212f1eefa2794d/fastapi-0.116.2.tar.gz", hash = "sha256:231a6af2fe21cfa2c32730170ad8514985fc250bec16c9b242d3b94c835ef529", size = 298595 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/32/e4/c543271a8018874b7f682bf6156863c416e1334b8ed3e51a69495c5d4360/fastapi-0.116.2-py3-none-any.whl", hash = "sha256:c3a7a8fb830b05f7e087d920e0d786ca1fc9892eb4e9a84b227be4c1bc7569db", size = 95670 }, +] + +[[package]] +name = "greenlet" +version = "3.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8a/99/1cd3411c56a410994669062bd73dd58270c00cc074cac15f385a1fd91f8a/greenlet-3.3.1.tar.gz", hash = "sha256:41848f3230b58c08bb43dee542e74a2a2e34d3c59dc3076cec9151aeeedcae98", size = 184690 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f9/c8/9d76a66421d1ae24340dfae7e79c313957f6e3195c144d2c73333b5bfe34/greenlet-3.3.1-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:7e806ca53acf6d15a888405880766ec84721aa4181261cd11a457dfe9a7a4975", size = 276443 }, + { url = "https://files.pythonhosted.org/packages/81/99/401ff34bb3c032d1f10477d199724f5e5f6fbfb59816ad1455c79c1eb8e7/greenlet-3.3.1-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d842c94b9155f1c9b3058036c24ffb8ff78b428414a19792b2380be9cecf4f36", size = 597359 }, + { url = "https://files.pythonhosted.org/packages/2b/bc/4dcc0871ed557792d304f50be0f7487a14e017952ec689effe2180a6ff35/greenlet-3.3.1-cp312-cp312-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:20fedaadd422fa02695f82093f9a98bad3dab5fcda793c658b945fcde2ab27ba", size = 607805 }, + { url = "https://files.pythonhosted.org/packages/3b/cd/7a7ca57588dac3389e97f7c9521cb6641fd8b6602faf1eaa4188384757df/greenlet-3.3.1-cp312-cp312-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c620051669fd04ac6b60ebc70478210119c56e2d5d5df848baec4312e260e4ca", size = 622363 }, + { url = "https://files.pythonhosted.org/packages/cf/05/821587cf19e2ce1f2b24945d890b164401e5085f9d09cbd969b0c193cd20/greenlet-3.3.1-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:14194f5f4305800ff329cbf02c5fcc88f01886cadd29941b807668a45f0d2336", size = 609947 }, + { url = "https://files.pythonhosted.org/packages/a4/52/ee8c46ed9f8babaa93a19e577f26e3d28a519feac6350ed6f25f1afee7e9/greenlet-3.3.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:7b2fe4150a0cf59f847a67db8c155ac36aed89080a6a639e9f16df5d6c6096f1", size = 1567487 }, + { url = "https://files.pythonhosted.org/packages/8f/7c/456a74f07029597626f3a6db71b273a3632aecb9afafeeca452cfa633197/greenlet-3.3.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:49f4ad195d45f4a66a0eb9c1ba4832bb380570d361912fa3554746830d332149", size = 1636087 }, + { url = "https://files.pythonhosted.org/packages/34/2f/5e0e41f33c69655300a5e54aeb637cf8ff57f1786a3aba374eacc0228c1d/greenlet-3.3.1-cp312-cp312-win_amd64.whl", hash = "sha256:cc98b9c4e4870fa983436afa999d4eb16b12872fab7071423d5262fa7120d57a", size = 227156 }, + { url = "https://files.pythonhosted.org/packages/c8/ab/717c58343cf02c5265b531384b248787e04d8160b8afe53d9eec053d7b44/greenlet-3.3.1-cp312-cp312-win_arm64.whl", hash = "sha256:bfb2d1763d777de5ee495c85309460f6fd8146e50ec9d0ae0183dbf6f0a829d1", size = 226403 }, + { url = "https://files.pythonhosted.org/packages/ec/ab/d26750f2b7242c2b90ea2ad71de70cfcd73a948a49513188a0fc0d6fc15a/greenlet-3.3.1-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:7ab327905cabb0622adca5971e488064e35115430cec2c35a50fd36e72a315b3", size = 275205 }, + { url = "https://files.pythonhosted.org/packages/10/d3/be7d19e8fad7c5a78eeefb2d896a08cd4643e1e90c605c4be3b46264998f/greenlet-3.3.1-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:65be2f026ca6a176f88fb935ee23c18333ccea97048076aef4db1ef5bc0713ac", size = 599284 }, + { url = "https://files.pythonhosted.org/packages/ae/21/fe703aaa056fdb0f17e5afd4b5c80195bbdab701208918938bd15b00d39b/greenlet-3.3.1-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7a3ae05b3d225b4155bda56b072ceb09d05e974bc74be6c3fc15463cf69f33fd", size = 610274 }, + { url = "https://files.pythonhosted.org/packages/06/00/95df0b6a935103c0452dad2203f5be8377e551b8466a29650c4c5a5af6cc/greenlet-3.3.1-cp313-cp313-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:12184c61e5d64268a160226fb4818af4df02cfead8379d7f8b99a56c3a54ff3e", size = 624375 }, + { url = "https://files.pythonhosted.org/packages/cb/86/5c6ab23bb3c28c21ed6bebad006515cfe08b04613eb105ca0041fecca852/greenlet-3.3.1-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6423481193bbbe871313de5fd06a082f2649e7ce6e08015d2a76c1e9186ca5b3", size = 612904 }, + { url = "https://files.pythonhosted.org/packages/c2/f3/7949994264e22639e40718c2daf6f6df5169bf48fb038c008a489ec53a50/greenlet-3.3.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:33a956fe78bbbda82bfc95e128d61129b32d66bcf0a20a1f0c08aa4839ffa951", size = 1567316 }, + { url = "https://files.pythonhosted.org/packages/8d/6e/d73c94d13b6465e9f7cd6231c68abde838bb22408596c05d9059830b7872/greenlet-3.3.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4b065d3284be43728dd280f6f9a13990b56470b81be20375a207cdc814a983f2", size = 1636549 }, + { url = "https://files.pythonhosted.org/packages/5e/b3/c9c23a6478b3bcc91f979ce4ca50879e4d0b2bd7b9a53d8ecded719b92e2/greenlet-3.3.1-cp313-cp313-win_amd64.whl", hash = "sha256:27289986f4e5b0edec7b5a91063c109f0276abb09a7e9bdab08437525977c946", size = 227042 }, + { url = "https://files.pythonhosted.org/packages/90/e7/824beda656097edee36ab15809fd063447b200cc03a7f6a24c34d520bc88/greenlet-3.3.1-cp313-cp313-win_arm64.whl", hash = "sha256:2f080e028001c5273e0b42690eaf359aeef9cb1389da0f171ea51a5dc3c7608d", size = 226294 }, + { url = "https://files.pythonhosted.org/packages/ae/fb/011c7c717213182caf78084a9bea51c8590b0afda98001f69d9f853a495b/greenlet-3.3.1-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:bd59acd8529b372775cd0fcbc5f420ae20681c5b045ce25bd453ed8455ab99b5", size = 275737 }, + { url = "https://files.pythonhosted.org/packages/41/2e/a3a417d620363fdbb08a48b1dd582956a46a61bf8fd27ee8164f9dfe87c2/greenlet-3.3.1-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b31c05dd84ef6871dd47120386aed35323c944d86c3d91a17c4b8d23df62f15b", size = 646422 }, + { url = "https://files.pythonhosted.org/packages/b4/09/c6c4a0db47defafd2d6bab8ddfe47ad19963b4e30f5bed84d75328059f8c/greenlet-3.3.1-cp314-cp314-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:02925a0bfffc41e542c70aa14c7eda3593e4d7e274bfcccca1827e6c0875902e", size = 658219 }, + { url = "https://files.pythonhosted.org/packages/e2/89/b95f2ddcc5f3c2bc09c8ee8d77be312df7f9e7175703ab780f2014a0e781/greenlet-3.3.1-cp314-cp314-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3e0f3878ca3a3ff63ab4ea478585942b53df66ddde327b59ecb191b19dbbd62d", size = 671455 }, + { url = "https://files.pythonhosted.org/packages/80/38/9d42d60dffb04b45f03dbab9430898352dba277758640751dc5cc316c521/greenlet-3.3.1-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:34a729e2e4e4ffe9ae2408d5ecaf12f944853f40ad724929b7585bca808a9d6f", size = 660237 }, + { url = "https://files.pythonhosted.org/packages/96/61/373c30b7197f9e756e4c81ae90a8d55dc3598c17673f91f4d31c3c689c3f/greenlet-3.3.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:aec9ab04e82918e623415947921dea15851b152b822661cce3f8e4393c3df683", size = 1615261 }, + { url = "https://files.pythonhosted.org/packages/fd/d3/ca534310343f5945316f9451e953dcd89b36fe7a19de652a1dc5a0eeef3f/greenlet-3.3.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:71c767cf281a80d02b6c1bdc41c9468e1f5a494fb11bc8688c360524e273d7b1", size = 1683719 }, + { url = "https://files.pythonhosted.org/packages/52/cb/c21a3fd5d2c9c8b622e7bede6d6d00e00551a5ee474ea6d831b5f567a8b4/greenlet-3.3.1-cp314-cp314-win_amd64.whl", hash = "sha256:96aff77af063b607f2489473484e39a0bbae730f2ea90c9e5606c9b73c44174a", size = 228125 }, + { url = "https://files.pythonhosted.org/packages/6a/8e/8a2db6d11491837af1de64b8aff23707c6e85241be13c60ed399a72e2ef8/greenlet-3.3.1-cp314-cp314-win_arm64.whl", hash = "sha256:b066e8b50e28b503f604fa538adc764a638b38cf8e81e025011d26e8a627fa79", size = 227519 }, + { url = "https://files.pythonhosted.org/packages/28/24/cbbec49bacdcc9ec652a81d3efef7b59f326697e7edf6ed775a5e08e54c2/greenlet-3.3.1-cp314-cp314t-macosx_11_0_universal2.whl", hash = "sha256:3e63252943c921b90abb035ebe9de832c436401d9c45f262d80e2d06cc659242", size = 282706 }, + { url = "https://files.pythonhosted.org/packages/86/2e/4f2b9323c144c4fe8842a4e0d92121465485c3c2c5b9e9b30a52e80f523f/greenlet-3.3.1-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:76e39058e68eb125de10c92524573924e827927df5d3891fbc97bd55764a8774", size = 651209 }, + { url = "https://files.pythonhosted.org/packages/d9/87/50ca60e515f5bb55a2fbc5f0c9b5b156de7d2fc51a0a69abc9d23914a237/greenlet-3.3.1-cp314-cp314t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c9f9d5e7a9310b7a2f416dd13d2e3fd8b42d803968ea580b7c0f322ccb389b97", size = 654300 }, + { url = "https://files.pythonhosted.org/packages/7c/25/c51a63f3f463171e09cb586eb64db0861eb06667ab01a7968371a24c4f3b/greenlet-3.3.1-cp314-cp314t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4b9721549a95db96689458a1e0ae32412ca18776ed004463df3a9299c1b257ab", size = 662574 }, + { url = "https://files.pythonhosted.org/packages/1d/94/74310866dfa2b73dd08659a3d18762f83985ad3281901ba0ee9a815194fb/greenlet-3.3.1-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:92497c78adf3ac703b57f1e3813c2d874f27f71a178f9ea5887855da413cd6d2", size = 653842 }, + { url = "https://files.pythonhosted.org/packages/97/43/8bf0ffa3d498eeee4c58c212a3905dd6146c01c8dc0b0a046481ca29b18c/greenlet-3.3.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ed6b402bc74d6557a705e197d47f9063733091ed6357b3de33619d8a8d93ac53", size = 1614917 }, + { url = "https://files.pythonhosted.org/packages/89/90/a3be7a5f378fc6e84abe4dcfb2ba32b07786861172e502388b4c90000d1b/greenlet-3.3.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:59913f1e5ada20fde795ba906916aea25d442abcc0593fba7e26c92b7ad76249", size = 1676092 }, + { url = "https://files.pythonhosted.org/packages/e1/2b/98c7f93e6db9977aaee07eb1e51ca63bd5f779b900d362791d3252e60558/greenlet-3.3.1-cp314-cp314t-win_amd64.whl", hash = "sha256:301860987846c24cb8964bdec0e31a96ad4a2a801b41b4ef40963c1b44f33451", size = 233181 }, +] + +[[package]] +name = "h11" +version = "0.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/ee/02a2c011bdab74c6fb3c75474d40b3052059d95df7e73351460c8588d963/h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1", size = 101250 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515 }, +] + +[[package]] +name = "httpcore" +version = "1.0.9" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "h11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/94/82699a10bca87a5556c9c59b5963f2d039dbd239f25bc2a63907a05a14cb/httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8", size = 85484 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784 }, +] + +[[package]] +name = "httptools" +version = "0.7.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b5/46/120a669232c7bdedb9d52d4aeae7e6c7dfe151e99dc70802e2fc7a5e1993/httptools-0.7.1.tar.gz", hash = "sha256:abd72556974f8e7c74a259655924a717a2365b236c882c3f6f8a45fe94703ac9", size = 258961 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/53/7f/403e5d787dc4942316e515e949b0c8a013d84078a915910e9f391ba9b3ed/httptools-0.7.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:38e0c83a2ea9746ebbd643bdfb521b9aa4a91703e2cd705c20443405d2fd16a5", size = 206280 }, + { url = "https://files.pythonhosted.org/packages/2a/0d/7f3fd28e2ce311ccc998c388dd1c53b18120fda3b70ebb022b135dc9839b/httptools-0.7.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f25bbaf1235e27704f1a7b86cd3304eabc04f569c828101d94a0e605ef7205a5", size = 110004 }, + { url = "https://files.pythonhosted.org/packages/84/a6/b3965e1e146ef5762870bbe76117876ceba51a201e18cc31f5703e454596/httptools-0.7.1-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:2c15f37ef679ab9ecc06bfc4e6e8628c32a8e4b305459de7cf6785acd57e4d03", size = 517655 }, + { url = "https://files.pythonhosted.org/packages/11/7d/71fee6f1844e6fa378f2eddde6c3e41ce3a1fb4b2d81118dd544e3441ec0/httptools-0.7.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7fe6e96090df46b36ccfaf746f03034e5ab723162bc51b0a4cf58305324036f2", size = 511440 }, + { url = "https://files.pythonhosted.org/packages/22/a5/079d216712a4f3ffa24af4a0381b108aa9c45b7a5cc6eb141f81726b1823/httptools-0.7.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f72fdbae2dbc6e68b8239defb48e6a5937b12218e6ffc2c7846cc37befa84362", size = 495186 }, + { url = "https://files.pythonhosted.org/packages/e9/9e/025ad7b65278745dee3bd0ebf9314934c4592560878308a6121f7f812084/httptools-0.7.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e99c7b90a29fd82fea9ef57943d501a16f3404d7b9ee81799d41639bdaae412c", size = 499192 }, + { url = "https://files.pythonhosted.org/packages/6d/de/40a8f202b987d43afc4d54689600ff03ce65680ede2f31df348d7f368b8f/httptools-0.7.1-cp312-cp312-win_amd64.whl", hash = "sha256:3e14f530fefa7499334a79b0cf7e7cd2992870eb893526fb097d51b4f2d0f321", size = 86694 }, + { url = "https://files.pythonhosted.org/packages/09/8f/c77b1fcbfd262d422f12da02feb0d218fa228d52485b77b953832105bb90/httptools-0.7.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:6babce6cfa2a99545c60bfef8bee0cc0545413cb0018f617c8059a30ad985de3", size = 202889 }, + { url = "https://files.pythonhosted.org/packages/0a/1a/22887f53602feaa066354867bc49a68fc295c2293433177ee90870a7d517/httptools-0.7.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:601b7628de7504077dd3dcb3791c6b8694bbd967148a6d1f01806509254fb1ca", size = 108180 }, + { url = "https://files.pythonhosted.org/packages/32/6a/6aaa91937f0010d288d3d124ca2946d48d60c3a5ee7ca62afe870e3ea011/httptools-0.7.1-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:04c6c0e6c5fb0739c5b8a9eb046d298650a0ff38cf42537fc372b28dc7e4472c", size = 478596 }, + { url = "https://files.pythonhosted.org/packages/6d/70/023d7ce117993107be88d2cbca566a7c1323ccbaf0af7eabf2064fe356f6/httptools-0.7.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:69d4f9705c405ae3ee83d6a12283dc9feba8cc6aaec671b412917e644ab4fa66", size = 473268 }, + { url = "https://files.pythonhosted.org/packages/32/4d/9dd616c38da088e3f436e9a616e1d0cc66544b8cdac405cc4e81c8679fc7/httptools-0.7.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:44c8f4347d4b31269c8a9205d8a5ee2df5322b09bbbd30f8f862185bb6b05346", size = 455517 }, + { url = "https://files.pythonhosted.org/packages/1d/3a/a6c595c310b7df958e739aae88724e24f9246a514d909547778d776799be/httptools-0.7.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:465275d76db4d554918aba40bf1cbebe324670f3dfc979eaffaa5d108e2ed650", size = 458337 }, + { url = "https://files.pythonhosted.org/packages/fd/82/88e8d6d2c51edc1cc391b6e044c6c435b6aebe97b1abc33db1b0b24cd582/httptools-0.7.1-cp313-cp313-win_amd64.whl", hash = "sha256:322d00c2068d125bd570f7bf78b2d367dad02b919d8581d7476d8b75b294e3e6", size = 85743 }, + { url = "https://files.pythonhosted.org/packages/34/50/9d095fcbb6de2d523e027a2f304d4551855c2f46e0b82befd718b8b20056/httptools-0.7.1-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:c08fe65728b8d70b6923ce31e3956f859d5e1e8548e6f22ec520a962c6757270", size = 203619 }, + { url = "https://files.pythonhosted.org/packages/07/f0/89720dc5139ae54b03f861b5e2c55a37dba9a5da7d51e1e824a1f343627f/httptools-0.7.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:7aea2e3c3953521c3c51106ee11487a910d45586e351202474d45472db7d72d3", size = 108714 }, + { url = "https://files.pythonhosted.org/packages/b3/cb/eea88506f191fb552c11787c23f9a405f4c7b0c5799bf73f2249cd4f5228/httptools-0.7.1-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:0e68b8582f4ea9166be62926077a3334064d422cf08ab87d8b74664f8e9058e1", size = 472909 }, + { url = "https://files.pythonhosted.org/packages/e0/4a/a548bdfae6369c0d078bab5769f7b66f17f1bfaa6fa28f81d6be6959066b/httptools-0.7.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:df091cf961a3be783d6aebae963cc9b71e00d57fa6f149025075217bc6a55a7b", size = 470831 }, + { url = "https://files.pythonhosted.org/packages/4d/31/14df99e1c43bd132eec921c2e7e11cda7852f65619bc0fc5bdc2d0cb126c/httptools-0.7.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:f084813239e1eb403ddacd06a30de3d3e09a9b76e7894dcda2b22f8a726e9c60", size = 452631 }, + { url = "https://files.pythonhosted.org/packages/22/d2/b7e131f7be8d854d48cb6d048113c30f9a46dca0c9a8b08fcb3fcd588cdc/httptools-0.7.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:7347714368fb2b335e9063bc2b96f2f87a9ceffcd9758ac295f8bbcd3ffbc0ca", size = 452910 }, + { url = "https://files.pythonhosted.org/packages/53/cf/878f3b91e4e6e011eff6d1fa9ca39f7eb17d19c9d7971b04873734112f30/httptools-0.7.1-cp314-cp314-win_amd64.whl", hash = "sha256:cfabda2a5bb85aa2a904ce06d974a3f30fb36cc63d7feaddec05d2050acede96", size = 88205 }, +] + +[[package]] +name = "httpx" +version = "0.28.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "certifi" }, + { name = "httpcore" }, + { name = "idna" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517 }, +] + +[[package]] +name = "idna" +version = "3.11" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6f/6d/0703ccc57f3a7233505399edb88de3cbd678da106337b9fcde432b65ed60/idna-3.11.tar.gz", hash = "sha256:795dafcc9c04ed0c1fb032c2aa73654d8e8c5023a7df64a53f39190ada629902", size = 194582 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0e/61/66938bbb5fc52dbdf84594873d5b51fb1f7c7794e9c0f5bd885f30bc507b/idna-3.11-py3-none-any.whl", hash = "sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea", size = 71008 }, +] + +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484 }, +] + +[[package]] +name = "itsdangerous" +version = "2.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9c/cb/8ac0172223afbccb63986cc25049b154ecfb5e85932587206f42317be31d/itsdangerous-2.2.0.tar.gz", hash = "sha256:e0050c0b7da1eea53ffaf149c0cfbb5c6e2e2b69c4bef22c81fa6eb73e5f6173", size = 54410 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/96/92447566d16df59b2a776c0fb82dbc4d9e07cd95062562af01e408583fc4/itsdangerous-2.2.0-py3-none-any.whl", hash = "sha256:c6242fc49e35958c8b15141343aa660db5fc54d4f13a1db01a3f5891b98700ef", size = 16234 }, +] + +[[package]] +name = "jinja2" +version = "3.1.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/df/bf/f7da0350254c0ed7c72f3e33cef02e048281fec7ecec5f032d4aac52226b/jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d", size = 245115 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899 }, +] + +[[package]] +name = "mako" +version = "1.3.10" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9e/38/bd5b78a920a64d708fe6bc8e0a2c075e1389d53bef8413725c63ba041535/mako-1.3.10.tar.gz", hash = "sha256:99579a6f39583fa7e5630a28c3c1f440e4e97a414b80372649c0ce338da2ea28", size = 392474 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/87/fb/99f81ac72ae23375f22b7afdb7642aba97c00a713c217124420147681a2f/mako-1.3.10-py3-none-any.whl", hash = "sha256:baef24a52fc4fc514a0887ac600f9f1cff3d82c61d4d700a1fa84d597b88db59", size = 78509 }, +] + +[[package]] +name = "markupsafe" +version = "3.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7e/99/7690b6d4034fffd95959cbe0c02de8deb3098cc577c67bb6a24fe5d7caa7/markupsafe-3.0.3.tar.gz", hash = "sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698", size = 80313 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5a/72/147da192e38635ada20e0a2e1a51cf8823d2119ce8883f7053879c2199b5/markupsafe-3.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d53197da72cc091b024dd97249dfc7794d6a56530370992a5e1a08983ad9230e", size = 11615 }, + { url = "https://files.pythonhosted.org/packages/9a/81/7e4e08678a1f98521201c3079f77db69fb552acd56067661f8c2f534a718/markupsafe-3.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1872df69a4de6aead3491198eaf13810b565bdbeec3ae2dc8780f14458ec73ce", size = 12020 }, + { url = "https://files.pythonhosted.org/packages/1e/2c/799f4742efc39633a1b54a92eec4082e4f815314869865d876824c257c1e/markupsafe-3.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3a7e8ae81ae39e62a41ec302f972ba6ae23a5c5396c8e60113e9066ef893da0d", size = 24332 }, + { url = "https://files.pythonhosted.org/packages/3c/2e/8d0c2ab90a8c1d9a24f0399058ab8519a3279d1bd4289511d74e909f060e/markupsafe-3.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d6dd0be5b5b189d31db7cda48b91d7e0a9795f31430b7f271219ab30f1d3ac9d", size = 22947 }, + { url = "https://files.pythonhosted.org/packages/2c/54/887f3092a85238093a0b2154bd629c89444f395618842e8b0c41783898ea/markupsafe-3.0.3-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:94c6f0bb423f739146aec64595853541634bde58b2135f27f61c1ffd1cd4d16a", size = 21962 }, + { url = "https://files.pythonhosted.org/packages/c9/2f/336b8c7b6f4a4d95e91119dc8521402461b74a485558d8f238a68312f11c/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:be8813b57049a7dc738189df53d69395eba14fb99345e0a5994914a3864c8a4b", size = 23760 }, + { url = "https://files.pythonhosted.org/packages/32/43/67935f2b7e4982ffb50a4d169b724d74b62a3964bc1a9a527f5ac4f1ee2b/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:83891d0e9fb81a825d9a6d61e3f07550ca70a076484292a70fde82c4b807286f", size = 21529 }, + { url = "https://files.pythonhosted.org/packages/89/e0/4486f11e51bbba8b0c041098859e869e304d1c261e59244baa3d295d47b7/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:77f0643abe7495da77fb436f50f8dab76dbc6e5fd25d39589a0f1fe6548bfa2b", size = 23015 }, + { url = "https://files.pythonhosted.org/packages/2f/e1/78ee7a023dac597a5825441ebd17170785a9dab23de95d2c7508ade94e0e/markupsafe-3.0.3-cp312-cp312-win32.whl", hash = "sha256:d88b440e37a16e651bda4c7c2b930eb586fd15ca7406cb39e211fcff3bf3017d", size = 14540 }, + { url = "https://files.pythonhosted.org/packages/aa/5b/bec5aa9bbbb2c946ca2733ef9c4ca91c91b6a24580193e891b5f7dbe8e1e/markupsafe-3.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:26a5784ded40c9e318cfc2bdb30fe164bdb8665ded9cd64d500a34fb42067b1c", size = 15105 }, + { url = "https://files.pythonhosted.org/packages/e5/f1/216fc1bbfd74011693a4fd837e7026152e89c4bcf3e77b6692fba9923123/markupsafe-3.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:35add3b638a5d900e807944a078b51922212fb3dedb01633a8defc4b01a3c85f", size = 13906 }, + { url = "https://files.pythonhosted.org/packages/38/2f/907b9c7bbba283e68f20259574b13d005c121a0fa4c175f9bed27c4597ff/markupsafe-3.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e1cf1972137e83c5d4c136c43ced9ac51d0e124706ee1c8aa8532c1287fa8795", size = 11622 }, + { url = "https://files.pythonhosted.org/packages/9c/d9/5f7756922cdd676869eca1c4e3c0cd0df60ed30199ffd775e319089cb3ed/markupsafe-3.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:116bb52f642a37c115f517494ea5feb03889e04df47eeff5b130b1808ce7c219", size = 12029 }, + { url = "https://files.pythonhosted.org/packages/00/07/575a68c754943058c78f30db02ee03a64b3c638586fba6a6dd56830b30a3/markupsafe-3.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:133a43e73a802c5562be9bbcd03d090aa5a1fe899db609c29e8c8d815c5f6de6", size = 24374 }, + { url = "https://files.pythonhosted.org/packages/a9/21/9b05698b46f218fc0e118e1f8168395c65c8a2c750ae2bab54fc4bd4e0e8/markupsafe-3.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ccfcd093f13f0f0b7fdd0f198b90053bf7b2f02a3927a30e63f3ccc9df56b676", size = 22980 }, + { url = "https://files.pythonhosted.org/packages/7f/71/544260864f893f18b6827315b988c146b559391e6e7e8f7252839b1b846a/markupsafe-3.0.3-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:509fa21c6deb7a7a273d629cf5ec029bc209d1a51178615ddf718f5918992ab9", size = 21990 }, + { url = "https://files.pythonhosted.org/packages/c2/28/b50fc2f74d1ad761af2f5dcce7492648b983d00a65b8c0e0cb457c82ebbe/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a4afe79fb3de0b7097d81da19090f4df4f8d3a2b3adaa8764138aac2e44f3af1", size = 23784 }, + { url = "https://files.pythonhosted.org/packages/ed/76/104b2aa106a208da8b17a2fb72e033a5a9d7073c68f7e508b94916ed47a9/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:795e7751525cae078558e679d646ae45574b47ed6e7771863fcc079a6171a0fc", size = 21588 }, + { url = "https://files.pythonhosted.org/packages/b5/99/16a5eb2d140087ebd97180d95249b00a03aa87e29cc224056274f2e45fd6/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8485f406a96febb5140bfeca44a73e3ce5116b2501ac54fe953e488fb1d03b12", size = 23041 }, + { url = "https://files.pythonhosted.org/packages/19/bc/e7140ed90c5d61d77cea142eed9f9c303f4c4806f60a1044c13e3f1471d0/markupsafe-3.0.3-cp313-cp313-win32.whl", hash = "sha256:bdd37121970bfd8be76c5fb069c7751683bdf373db1ed6c010162b2a130248ed", size = 14543 }, + { url = "https://files.pythonhosted.org/packages/05/73/c4abe620b841b6b791f2edc248f556900667a5a1cf023a6646967ae98335/markupsafe-3.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:9a1abfdc021a164803f4d485104931fb8f8c1efd55bc6b748d2f5774e78b62c5", size = 15113 }, + { url = "https://files.pythonhosted.org/packages/f0/3a/fa34a0f7cfef23cf9500d68cb7c32dd64ffd58a12b09225fb03dd37d5b80/markupsafe-3.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:7e68f88e5b8799aa49c85cd116c932a1ac15caaa3f5db09087854d218359e485", size = 13911 }, + { url = "https://files.pythonhosted.org/packages/e4/d7/e05cd7efe43a88a17a37b3ae96e79a19e846f3f456fe79c57ca61356ef01/markupsafe-3.0.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:218551f6df4868a8d527e3062d0fb968682fe92054e89978594c28e642c43a73", size = 11658 }, + { url = "https://files.pythonhosted.org/packages/99/9e/e412117548182ce2148bdeacdda3bb494260c0b0184360fe0d56389b523b/markupsafe-3.0.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:3524b778fe5cfb3452a09d31e7b5adefeea8c5be1d43c4f810ba09f2ceb29d37", size = 12066 }, + { url = "https://files.pythonhosted.org/packages/bc/e6/fa0ffcda717ef64a5108eaa7b4f5ed28d56122c9a6d70ab8b72f9f715c80/markupsafe-3.0.3-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4e885a3d1efa2eadc93c894a21770e4bc67899e3543680313b09f139e149ab19", size = 25639 }, + { url = "https://files.pythonhosted.org/packages/96/ec/2102e881fe9d25fc16cb4b25d5f5cde50970967ffa5dddafdb771237062d/markupsafe-3.0.3-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8709b08f4a89aa7586de0aadc8da56180242ee0ada3999749b183aa23df95025", size = 23569 }, + { url = "https://files.pythonhosted.org/packages/4b/30/6f2fce1f1f205fc9323255b216ca8a235b15860c34b6798f810f05828e32/markupsafe-3.0.3-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b8512a91625c9b3da6f127803b166b629725e68af71f8184ae7e7d54686a56d6", size = 23284 }, + { url = "https://files.pythonhosted.org/packages/58/47/4a0ccea4ab9f5dcb6f79c0236d954acb382202721e704223a8aafa38b5c8/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:9b79b7a16f7fedff2495d684f2b59b0457c3b493778c9eed31111be64d58279f", size = 24801 }, + { url = "https://files.pythonhosted.org/packages/6a/70/3780e9b72180b6fecb83a4814d84c3bf4b4ae4bf0b19c27196104149734c/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:12c63dfb4a98206f045aa9563db46507995f7ef6d83b2f68eda65c307c6829eb", size = 22769 }, + { url = "https://files.pythonhosted.org/packages/98/c5/c03c7f4125180fc215220c035beac6b9cb684bc7a067c84fc69414d315f5/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8f71bc33915be5186016f675cd83a1e08523649b0e33efdb898db577ef5bb009", size = 23642 }, + { url = "https://files.pythonhosted.org/packages/80/d6/2d1b89f6ca4bff1036499b1e29a1d02d282259f3681540e16563f27ebc23/markupsafe-3.0.3-cp313-cp313t-win32.whl", hash = "sha256:69c0b73548bc525c8cb9a251cddf1931d1db4d2258e9599c28c07ef3580ef354", size = 14612 }, + { url = "https://files.pythonhosted.org/packages/2b/98/e48a4bfba0a0ffcf9925fe2d69240bfaa19c6f7507b8cd09c70684a53c1e/markupsafe-3.0.3-cp313-cp313t-win_amd64.whl", hash = "sha256:1b4b79e8ebf6b55351f0d91fe80f893b4743f104bff22e90697db1590e47a218", size = 15200 }, + { url = "https://files.pythonhosted.org/packages/0e/72/e3cc540f351f316e9ed0f092757459afbc595824ca724cbc5a5d4263713f/markupsafe-3.0.3-cp313-cp313t-win_arm64.whl", hash = "sha256:ad2cf8aa28b8c020ab2fc8287b0f823d0a7d8630784c31e9ee5edea20f406287", size = 13973 }, + { url = "https://files.pythonhosted.org/packages/33/8a/8e42d4838cd89b7dde187011e97fe6c3af66d8c044997d2183fbd6d31352/markupsafe-3.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:eaa9599de571d72e2daf60164784109f19978b327a3910d3e9de8c97b5b70cfe", size = 11619 }, + { url = "https://files.pythonhosted.org/packages/b5/64/7660f8a4a8e53c924d0fa05dc3a55c9cee10bbd82b11c5afb27d44b096ce/markupsafe-3.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c47a551199eb8eb2121d4f0f15ae0f923d31350ab9280078d1e5f12b249e0026", size = 12029 }, + { url = "https://files.pythonhosted.org/packages/da/ef/e648bfd021127bef5fa12e1720ffed0c6cbb8310c8d9bea7266337ff06de/markupsafe-3.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f34c41761022dd093b4b6896d4810782ffbabe30f2d443ff5f083e0cbbb8c737", size = 24408 }, + { url = "https://files.pythonhosted.org/packages/41/3c/a36c2450754618e62008bf7435ccb0f88053e07592e6028a34776213d877/markupsafe-3.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:457a69a9577064c05a97c41f4e65148652db078a3a509039e64d3467b9e7ef97", size = 23005 }, + { url = "https://files.pythonhosted.org/packages/bc/20/b7fdf89a8456b099837cd1dc21974632a02a999ec9bf7ca3e490aacd98e7/markupsafe-3.0.3-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e8afc3f2ccfa24215f8cb28dcf43f0113ac3c37c2f0f0806d8c70e4228c5cf4d", size = 22048 }, + { url = "https://files.pythonhosted.org/packages/9a/a7/591f592afdc734f47db08a75793a55d7fbcc6902a723ae4cfbab61010cc5/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ec15a59cf5af7be74194f7ab02d0f59a62bdcf1a537677ce67a2537c9b87fcda", size = 23821 }, + { url = "https://files.pythonhosted.org/packages/7d/33/45b24e4f44195b26521bc6f1a82197118f74df348556594bd2262bda1038/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:0eb9ff8191e8498cca014656ae6b8d61f39da5f95b488805da4bb029cccbfbaf", size = 21606 }, + { url = "https://files.pythonhosted.org/packages/ff/0e/53dfaca23a69fbfbbf17a4b64072090e70717344c52eaaaa9c5ddff1e5f0/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2713baf880df847f2bece4230d4d094280f4e67b1e813eec43b4c0e144a34ffe", size = 23043 }, + { url = "https://files.pythonhosted.org/packages/46/11/f333a06fc16236d5238bfe74daccbca41459dcd8d1fa952e8fbd5dccfb70/markupsafe-3.0.3-cp314-cp314-win32.whl", hash = "sha256:729586769a26dbceff69f7a7dbbf59ab6572b99d94576a5592625d5b411576b9", size = 14747 }, + { url = "https://files.pythonhosted.org/packages/28/52/182836104b33b444e400b14f797212f720cbc9ed6ba34c800639d154e821/markupsafe-3.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:bdc919ead48f234740ad807933cdf545180bfbe9342c2bb451556db2ed958581", size = 15341 }, + { url = "https://files.pythonhosted.org/packages/6f/18/acf23e91bd94fd7b3031558b1f013adfa21a8e407a3fdb32745538730382/markupsafe-3.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:5a7d5dc5140555cf21a6fefbdbf8723f06fcd2f63ef108f2854de715e4422cb4", size = 14073 }, + { url = "https://files.pythonhosted.org/packages/3c/f0/57689aa4076e1b43b15fdfa646b04653969d50cf30c32a102762be2485da/markupsafe-3.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:1353ef0c1b138e1907ae78e2f6c63ff67501122006b0f9abad68fda5f4ffc6ab", size = 11661 }, + { url = "https://files.pythonhosted.org/packages/89/c3/2e67a7ca217c6912985ec766c6393b636fb0c2344443ff9d91404dc4c79f/markupsafe-3.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1085e7fbddd3be5f89cc898938f42c0b3c711fdcb37d75221de2666af647c175", size = 12069 }, + { url = "https://files.pythonhosted.org/packages/f0/00/be561dce4e6ca66b15276e184ce4b8aec61fe83662cce2f7d72bd3249d28/markupsafe-3.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1b52b4fb9df4eb9ae465f8d0c228a00624de2334f216f178a995ccdcf82c4634", size = 25670 }, + { url = "https://files.pythonhosted.org/packages/50/09/c419f6f5a92e5fadde27efd190eca90f05e1261b10dbd8cbcb39cd8ea1dc/markupsafe-3.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fed51ac40f757d41b7c48425901843666a6677e3e8eb0abcff09e4ba6e664f50", size = 23598 }, + { url = "https://files.pythonhosted.org/packages/22/44/a0681611106e0b2921b3033fc19bc53323e0b50bc70cffdd19f7d679bb66/markupsafe-3.0.3-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f190daf01f13c72eac4efd5c430a8de82489d9cff23c364c3ea822545032993e", size = 23261 }, + { url = "https://files.pythonhosted.org/packages/5f/57/1b0b3f100259dc9fffe780cfb60d4be71375510e435efec3d116b6436d43/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e56b7d45a839a697b5eb268c82a71bd8c7f6c94d6fd50c3d577fa39a9f1409f5", size = 24835 }, + { url = "https://files.pythonhosted.org/packages/26/6a/4bf6d0c97c4920f1597cc14dd720705eca0bf7c787aebc6bb4d1bead5388/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:f3e98bb3798ead92273dc0e5fd0f31ade220f59a266ffd8a4f6065e0a3ce0523", size = 22733 }, + { url = "https://files.pythonhosted.org/packages/14/c7/ca723101509b518797fedc2fdf79ba57f886b4aca8a7d31857ba3ee8281f/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5678211cb9333a6468fb8d8be0305520aa073f50d17f089b5b4b477ea6e67fdc", size = 23672 }, + { url = "https://files.pythonhosted.org/packages/fb/df/5bd7a48c256faecd1d36edc13133e51397e41b73bb77e1a69deab746ebac/markupsafe-3.0.3-cp314-cp314t-win32.whl", hash = "sha256:915c04ba3851909ce68ccc2b8e2cd691618c4dc4c4232fb7982bca3f41fd8c3d", size = 14819 }, + { url = "https://files.pythonhosted.org/packages/1a/8a/0402ba61a2f16038b48b39bccca271134be00c5c9f0f623208399333c448/markupsafe-3.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4faffd047e07c38848ce017e8725090413cd80cbc23d86e55c587bf979e579c9", size = 15426 }, + { url = "https://files.pythonhosted.org/packages/70/bc/6f1c2f612465f5fa89b95bead1f44dcb607670fd42891d8fdcd5d039f4f4/markupsafe-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa", size = 14146 }, +] + +[[package]] +name = "packaging" +version = "26.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/65/ee/299d360cdc32edc7d2cf530f3accf79c4fca01e96ffc950d8a52213bd8e4/packaging-26.0.tar.gz", hash = "sha256:00243ae351a257117b6a241061796684b084ed1c516a08c48a3f7e147a9d80b4", size = 143416 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/b9/c538f279a4e237a006a2c98387d081e9eb060d203d8ed34467cc0f0b9b53/packaging-26.0-py3-none-any.whl", hash = "sha256:b36f1fef9334a5588b4166f8bcd26a14e521f2b55e6b9de3aaa80d3ff7a37529", size = 74366 }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538 }, +] + +[[package]] +name = "pycparser" +version = "3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1b/7d/92392ff7815c21062bea51aa7b87d45576f649f16458d78b7cf94b9ab2e6/pycparser-3.0.tar.gz", hash = "sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29", size = 103492 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0c/c3/44f3fbbfa403ea2a7c779186dc20772604442dde72947e7d01069cbe98e3/pycparser-3.0-py3-none-any.whl", hash = "sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992", size = 48172 }, +] + +[[package]] +name = "pydantic" +version = "2.12.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-types" }, + { name = "pydantic-core" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/69/44/36f1a6e523abc58ae5f928898e4aca2e0ea509b5aa6f6f392a5d882be928/pydantic-2.12.5.tar.gz", hash = "sha256:4d351024c75c0f085a9febbb665ce8c0c6ec5d30e903bdb6394b7ede26aebb49", size = 821591 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5a/87/b70ad306ebb6f9b585f114d0ac2137d792b48be34d732d60e597c2f8465a/pydantic-2.12.5-py3-none-any.whl", hash = "sha256:e561593fccf61e8a20fc46dfc2dfe075b8be7d0188df33f221ad1f0139180f9d", size = 463580 }, +] + +[[package]] +name = "pydantic-core" +version = "2.41.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/71/70/23b021c950c2addd24ec408e9ab05d59b035b39d97cdc1130e1bce647bb6/pydantic_core-2.41.5.tar.gz", hash = "sha256:08daa51ea16ad373ffd5e7606252cc32f07bc72b28284b6bc9c6df804816476e", size = 460952 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5f/5d/5f6c63eebb5afee93bcaae4ce9a898f3373ca23df3ccaef086d0233a35a7/pydantic_core-2.41.5-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:f41a7489d32336dbf2199c8c0a215390a751c5b014c2c1c5366e817202e9cdf7", size = 2110990 }, + { url = "https://files.pythonhosted.org/packages/aa/32/9c2e8ccb57c01111e0fd091f236c7b371c1bccea0fa85247ac55b1e2b6b6/pydantic_core-2.41.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:070259a8818988b9a84a449a2a7337c7f430a22acc0859c6b110aa7212a6d9c0", size = 1896003 }, + { url = "https://files.pythonhosted.org/packages/68/b8/a01b53cb0e59139fbc9e4fda3e9724ede8de279097179be4ff31f1abb65a/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e96cea19e34778f8d59fe40775a7a574d95816eb150850a85a7a4c8f4b94ac69", size = 1919200 }, + { url = "https://files.pythonhosted.org/packages/38/de/8c36b5198a29bdaade07b5985e80a233a5ac27137846f3bc2d3b40a47360/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ed2e99c456e3fadd05c991f8f437ef902e00eedf34320ba2b0842bd1c3ca3a75", size = 2052578 }, + { url = "https://files.pythonhosted.org/packages/00/b5/0e8e4b5b081eac6cb3dbb7e60a65907549a1ce035a724368c330112adfdd/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:65840751b72fbfd82c3c640cff9284545342a4f1eb1586ad0636955b261b0b05", size = 2208504 }, + { url = "https://files.pythonhosted.org/packages/77/56/87a61aad59c7c5b9dc8caad5a41a5545cba3810c3e828708b3d7404f6cef/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e536c98a7626a98feb2d3eaf75944ef6f3dbee447e1f841eae16f2f0a72d8ddc", size = 2335816 }, + { url = "https://files.pythonhosted.org/packages/0d/76/941cc9f73529988688a665a5c0ecff1112b3d95ab48f81db5f7606f522d3/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eceb81a8d74f9267ef4081e246ffd6d129da5d87e37a77c9bde550cb04870c1c", size = 2075366 }, + { url = "https://files.pythonhosted.org/packages/d3/43/ebef01f69baa07a482844faaa0a591bad1ef129253ffd0cdaa9d8a7f72d3/pydantic_core-2.41.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d38548150c39b74aeeb0ce8ee1d8e82696f4a4e16ddc6de7b1d8823f7de4b9b5", size = 2171698 }, + { url = "https://files.pythonhosted.org/packages/b1/87/41f3202e4193e3bacfc2c065fab7706ebe81af46a83d3e27605029c1f5a6/pydantic_core-2.41.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:c23e27686783f60290e36827f9c626e63154b82b116d7fe9adba1fda36da706c", size = 2132603 }, + { url = "https://files.pythonhosted.org/packages/49/7d/4c00df99cb12070b6bccdef4a195255e6020a550d572768d92cc54dba91a/pydantic_core-2.41.5-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:482c982f814460eabe1d3bb0adfdc583387bd4691ef00b90575ca0d2b6fe2294", size = 2329591 }, + { url = "https://files.pythonhosted.org/packages/cc/6a/ebf4b1d65d458f3cda6a7335d141305dfa19bdc61140a884d165a8a1bbc7/pydantic_core-2.41.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:bfea2a5f0b4d8d43adf9d7b8bf019fb46fdd10a2e5cde477fbcb9d1fa08c68e1", size = 2319068 }, + { url = "https://files.pythonhosted.org/packages/49/3b/774f2b5cd4192d5ab75870ce4381fd89cf218af999515baf07e7206753f0/pydantic_core-2.41.5-cp312-cp312-win32.whl", hash = "sha256:b74557b16e390ec12dca509bce9264c3bbd128f8a2c376eaa68003d7f327276d", size = 1985908 }, + { url = "https://files.pythonhosted.org/packages/86/45/00173a033c801cacf67c190fef088789394feaf88a98a7035b0e40d53dc9/pydantic_core-2.41.5-cp312-cp312-win_amd64.whl", hash = "sha256:1962293292865bca8e54702b08a4f26da73adc83dd1fcf26fbc875b35d81c815", size = 2020145 }, + { url = "https://files.pythonhosted.org/packages/f9/22/91fbc821fa6d261b376a3f73809f907cec5ca6025642c463d3488aad22fb/pydantic_core-2.41.5-cp312-cp312-win_arm64.whl", hash = "sha256:1746d4a3d9a794cacae06a5eaaccb4b8643a131d45fbc9af23e353dc0a5ba5c3", size = 1976179 }, + { url = "https://files.pythonhosted.org/packages/87/06/8806241ff1f70d9939f9af039c6c35f2360cf16e93c2ca76f184e76b1564/pydantic_core-2.41.5-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:941103c9be18ac8daf7b7adca8228f8ed6bb7a1849020f643b3a14d15b1924d9", size = 2120403 }, + { url = "https://files.pythonhosted.org/packages/94/02/abfa0e0bda67faa65fef1c84971c7e45928e108fe24333c81f3bfe35d5f5/pydantic_core-2.41.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:112e305c3314f40c93998e567879e887a3160bb8689ef3d2c04b6cc62c33ac34", size = 1896206 }, + { url = "https://files.pythonhosted.org/packages/15/df/a4c740c0943e93e6500f9eb23f4ca7ec9bf71b19e608ae5b579678c8d02f/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0cbaad15cb0c90aa221d43c00e77bb33c93e8d36e0bf74760cd00e732d10a6a0", size = 1919307 }, + { url = "https://files.pythonhosted.org/packages/9a/e3/6324802931ae1d123528988e0e86587c2072ac2e5394b4bc2bc34b61ff6e/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:03ca43e12fab6023fc79d28ca6b39b05f794ad08ec2feccc59a339b02f2b3d33", size = 2063258 }, + { url = "https://files.pythonhosted.org/packages/c9/d4/2230d7151d4957dd79c3044ea26346c148c98fbf0ee6ebd41056f2d62ab5/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dc799088c08fa04e43144b164feb0c13f9a0bc40503f8df3e9fde58a3c0c101e", size = 2214917 }, + { url = "https://files.pythonhosted.org/packages/e6/9f/eaac5df17a3672fef0081b6c1bb0b82b33ee89aa5cec0d7b05f52fd4a1fa/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:97aeba56665b4c3235a0e52b2c2f5ae9cd071b8a8310ad27bddb3f7fb30e9aa2", size = 2332186 }, + { url = "https://files.pythonhosted.org/packages/cf/4e/35a80cae583a37cf15604b44240e45c05e04e86f9cfd766623149297e971/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:406bf18d345822d6c21366031003612b9c77b3e29ffdb0f612367352aab7d586", size = 2073164 }, + { url = "https://files.pythonhosted.org/packages/bf/e3/f6e262673c6140dd3305d144d032f7bd5f7497d3871c1428521f19f9efa2/pydantic_core-2.41.5-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b93590ae81f7010dbe380cdeab6f515902ebcbefe0b9327cc4804d74e93ae69d", size = 2179146 }, + { url = "https://files.pythonhosted.org/packages/75/c7/20bd7fc05f0c6ea2056a4565c6f36f8968c0924f19b7d97bbfea55780e73/pydantic_core-2.41.5-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:01a3d0ab748ee531f4ea6c3e48ad9dac84ddba4b0d82291f87248f2f9de8d740", size = 2137788 }, + { url = "https://files.pythonhosted.org/packages/3a/8d/34318ef985c45196e004bc46c6eab2eda437e744c124ef0dbe1ff2c9d06b/pydantic_core-2.41.5-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:6561e94ba9dacc9c61bce40e2d6bdc3bfaa0259d3ff36ace3b1e6901936d2e3e", size = 2340133 }, + { url = "https://files.pythonhosted.org/packages/9c/59/013626bf8c78a5a5d9350d12e7697d3d4de951a75565496abd40ccd46bee/pydantic_core-2.41.5-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:915c3d10f81bec3a74fbd4faebe8391013ba61e5a1a8d48c4455b923bdda7858", size = 2324852 }, + { url = "https://files.pythonhosted.org/packages/1a/d9/c248c103856f807ef70c18a4f986693a46a8ffe1602e5d361485da502d20/pydantic_core-2.41.5-cp313-cp313-win32.whl", hash = "sha256:650ae77860b45cfa6e2cdafc42618ceafab3a2d9a3811fcfbd3bbf8ac3c40d36", size = 1994679 }, + { url = "https://files.pythonhosted.org/packages/9e/8b/341991b158ddab181cff136acd2552c9f35bd30380422a639c0671e99a91/pydantic_core-2.41.5-cp313-cp313-win_amd64.whl", hash = "sha256:79ec52ec461e99e13791ec6508c722742ad745571f234ea6255bed38c6480f11", size = 2019766 }, + { url = "https://files.pythonhosted.org/packages/73/7d/f2f9db34af103bea3e09735bb40b021788a5e834c81eedb541991badf8f5/pydantic_core-2.41.5-cp313-cp313-win_arm64.whl", hash = "sha256:3f84d5c1b4ab906093bdc1ff10484838aca54ef08de4afa9de0f5f14d69639cd", size = 1981005 }, + { url = "https://files.pythonhosted.org/packages/ea/28/46b7c5c9635ae96ea0fbb779e271a38129df2550f763937659ee6c5dbc65/pydantic_core-2.41.5-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:3f37a19d7ebcdd20b96485056ba9e8b304e27d9904d233d7b1015db320e51f0a", size = 2119622 }, + { url = "https://files.pythonhosted.org/packages/74/1a/145646e5687e8d9a1e8d09acb278c8535ebe9e972e1f162ed338a622f193/pydantic_core-2.41.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1d1d9764366c73f996edd17abb6d9d7649a7eb690006ab6adbda117717099b14", size = 1891725 }, + { url = "https://files.pythonhosted.org/packages/23/04/e89c29e267b8060b40dca97bfc64a19b2a3cf99018167ea1677d96368273/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:25e1c2af0fce638d5f1988b686f3b3ea8cd7de5f244ca147c777769e798a9cd1", size = 1915040 }, + { url = "https://files.pythonhosted.org/packages/84/a3/15a82ac7bd97992a82257f777b3583d3e84bdb06ba6858f745daa2ec8a85/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:506d766a8727beef16b7adaeb8ee6217c64fc813646b424d0804d67c16eddb66", size = 2063691 }, + { url = "https://files.pythonhosted.org/packages/74/9b/0046701313c6ef08c0c1cf0e028c67c770a4e1275ca73131563c5f2a310a/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4819fa52133c9aa3c387b3328f25c1facc356491e6135b459f1de698ff64d869", size = 2213897 }, + { url = "https://files.pythonhosted.org/packages/8a/cd/6bac76ecd1b27e75a95ca3a9a559c643b3afcd2dd62086d4b7a32a18b169/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2b761d210c9ea91feda40d25b4efe82a1707da2ef62901466a42492c028553a2", size = 2333302 }, + { url = "https://files.pythonhosted.org/packages/4c/d2/ef2074dc020dd6e109611a8be4449b98cd25e1b9b8a303c2f0fca2f2bcf7/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:22f0fb8c1c583a3b6f24df2470833b40207e907b90c928cc8d3594b76f874375", size = 2064877 }, + { url = "https://files.pythonhosted.org/packages/18/66/e9db17a9a763d72f03de903883c057b2592c09509ccfe468187f2a2eef29/pydantic_core-2.41.5-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2782c870e99878c634505236d81e5443092fba820f0373997ff75f90f68cd553", size = 2180680 }, + { url = "https://files.pythonhosted.org/packages/d3/9e/3ce66cebb929f3ced22be85d4c2399b8e85b622db77dad36b73c5387f8f8/pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:0177272f88ab8312479336e1d777f6b124537d47f2123f89cb37e0accea97f90", size = 2138960 }, + { url = "https://files.pythonhosted.org/packages/a6/62/205a998f4327d2079326b01abee48e502ea739d174f0a89295c481a2272e/pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:63510af5e38f8955b8ee5687740d6ebf7c2a0886d15a6d65c32814613681bc07", size = 2339102 }, + { url = "https://files.pythonhosted.org/packages/3c/0d/f05e79471e889d74d3d88f5bd20d0ed189ad94c2423d81ff8d0000aab4ff/pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:e56ba91f47764cc14f1daacd723e3e82d1a89d783f0f5afe9c364b8bb491ccdb", size = 2326039 }, + { url = "https://files.pythonhosted.org/packages/ec/e1/e08a6208bb100da7e0c4b288eed624a703f4d129bde2da475721a80cab32/pydantic_core-2.41.5-cp314-cp314-win32.whl", hash = "sha256:aec5cf2fd867b4ff45b9959f8b20ea3993fc93e63c7363fe6851424c8a7e7c23", size = 1995126 }, + { url = "https://files.pythonhosted.org/packages/48/5d/56ba7b24e9557f99c9237e29f5c09913c81eeb2f3217e40e922353668092/pydantic_core-2.41.5-cp314-cp314-win_amd64.whl", hash = "sha256:8e7c86f27c585ef37c35e56a96363ab8de4e549a95512445b85c96d3e2f7c1bf", size = 2015489 }, + { url = "https://files.pythonhosted.org/packages/4e/bb/f7a190991ec9e3e0ba22e4993d8755bbc4a32925c0b5b42775c03e8148f9/pydantic_core-2.41.5-cp314-cp314-win_arm64.whl", hash = "sha256:e672ba74fbc2dc8eea59fb6d4aed6845e6905fc2a8afe93175d94a83ba2a01a0", size = 1977288 }, + { url = "https://files.pythonhosted.org/packages/92/ed/77542d0c51538e32e15afe7899d79efce4b81eee631d99850edc2f5e9349/pydantic_core-2.41.5-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:8566def80554c3faa0e65ac30ab0932b9e3a5cd7f8323764303d468e5c37595a", size = 2120255 }, + { url = "https://files.pythonhosted.org/packages/bb/3d/6913dde84d5be21e284439676168b28d8bbba5600d838b9dca99de0fad71/pydantic_core-2.41.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:b80aa5095cd3109962a298ce14110ae16b8c1aece8b72f9dafe81cf597ad80b3", size = 1863760 }, + { url = "https://files.pythonhosted.org/packages/5a/f0/e5e6b99d4191da102f2b0eb9687aaa7f5bea5d9964071a84effc3e40f997/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3006c3dd9ba34b0c094c544c6006cc79e87d8612999f1a5d43b769b89181f23c", size = 1878092 }, + { url = "https://files.pythonhosted.org/packages/71/48/36fb760642d568925953bcc8116455513d6e34c4beaa37544118c36aba6d/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:72f6c8b11857a856bcfa48c86f5368439f74453563f951e473514579d44aa612", size = 2053385 }, + { url = "https://files.pythonhosted.org/packages/20/25/92dc684dd8eb75a234bc1c764b4210cf2646479d54b47bf46061657292a8/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5cb1b2f9742240e4bb26b652a5aeb840aa4b417c7748b6f8387927bc6e45e40d", size = 2218832 }, + { url = "https://files.pythonhosted.org/packages/e2/09/f53e0b05023d3e30357d82eb35835d0f6340ca344720a4599cd663dca599/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bd3d54f38609ff308209bd43acea66061494157703364ae40c951f83ba99a1a9", size = 2327585 }, + { url = "https://files.pythonhosted.org/packages/aa/4e/2ae1aa85d6af35a39b236b1b1641de73f5a6ac4d5a7509f77b814885760c/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2ff4321e56e879ee8d2a879501c8e469414d948f4aba74a2d4593184eb326660", size = 2041078 }, + { url = "https://files.pythonhosted.org/packages/cd/13/2e215f17f0ef326fc72afe94776edb77525142c693767fc347ed6288728d/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d0d2568a8c11bf8225044aa94409e21da0cb09dcdafe9ecd10250b2baad531a9", size = 2173914 }, + { url = "https://files.pythonhosted.org/packages/02/7a/f999a6dcbcd0e5660bc348a3991c8915ce6599f4f2c6ac22f01d7a10816c/pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:a39455728aabd58ceabb03c90e12f71fd30fa69615760a075b9fec596456ccc3", size = 2129560 }, + { url = "https://files.pythonhosted.org/packages/3a/b1/6c990ac65e3b4c079a4fb9f5b05f5b013afa0f4ed6780a3dd236d2cbdc64/pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:239edca560d05757817c13dc17c50766136d21f7cd0fac50295499ae24f90fdf", size = 2329244 }, + { url = "https://files.pythonhosted.org/packages/d9/02/3c562f3a51afd4d88fff8dffb1771b30cfdfd79befd9883ee094f5b6c0d8/pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:2a5e06546e19f24c6a96a129142a75cee553cc018ffee48a460059b1185f4470", size = 2331955 }, + { url = "https://files.pythonhosted.org/packages/5c/96/5fb7d8c3c17bc8c62fdb031c47d77a1af698f1d7a406b0f79aaa1338f9ad/pydantic_core-2.41.5-cp314-cp314t-win32.whl", hash = "sha256:b4ececa40ac28afa90871c2cc2b9ffd2ff0bf749380fbdf57d165fd23da353aa", size = 1988906 }, + { url = "https://files.pythonhosted.org/packages/22/ed/182129d83032702912c2e2d8bbe33c036f342cc735737064668585dac28f/pydantic_core-2.41.5-cp314-cp314t-win_amd64.whl", hash = "sha256:80aa89cad80b32a912a65332f64a4450ed00966111b6615ca6816153d3585a8c", size = 1981607 }, + { url = "https://files.pythonhosted.org/packages/9f/ed/068e41660b832bb0b1aa5b58011dea2a3fe0ba7861ff38c4d4904c1c1a99/pydantic_core-2.41.5-cp314-cp314t-win_arm64.whl", hash = "sha256:35b44f37a3199f771c3eaa53051bc8a70cd7b54f333531c59e29fd4db5d15008", size = 1974769 }, +] + +[[package]] +name = "pygments" +version = "2.19.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b0/77/a5b8c569bf593b0140bde72ea885a803b82086995367bf2037de0159d924/pygments-2.19.2.tar.gz", hash = "sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887", size = 4968631 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b", size = 1225217 }, +] + +[[package]] +name = "pytest" +version = "8.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a3/5c/00a0e072241553e1a7496d638deababa67c5058571567b92a7eaa258397c/pytest-8.4.2.tar.gz", hash = "sha256:86c0d0b93306b961d58d62a4db4879f27fe25513d4b969df351abdddb3c30e01", size = 1519618 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a8/a4/20da314d277121d6534b3a980b29035dcd51e6744bd79075a6ce8fa4eb8d/pytest-8.4.2-py3-none-any.whl", hash = "sha256:872f880de3fc3a5bdc88a11b39c9710c3497a547cfa9320bc3c5e62fbf272e79", size = 365750 }, +] + +[[package]] +name = "pytest-asyncio" +version = "0.25.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f2/a8/ecbc8ede70921dd2f544ab1cadd3ff3bf842af27f87bbdea774c7baa1d38/pytest_asyncio-0.25.3.tar.gz", hash = "sha256:fc1da2cf9f125ada7e710b4ddad05518d4cee187ae9412e9ac9271003497f07a", size = 54239 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/67/17/3493c5624e48fd97156ebaec380dcaafee9506d7e2c46218ceebbb57d7de/pytest_asyncio-0.25.3-py3-none-any.whl", hash = "sha256:9e89518e0f9bd08928f97a3482fdc4e244df17529460bc038291ccaf8f85c7c3", size = 19467 }, +] + +[[package]] +name = "python-dotenv" +version = "1.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f0/26/19cadc79a718c5edbec86fd4919a6b6d3f681039a2f6d66d14be94e75fb9/python_dotenv-1.2.1.tar.gz", hash = "sha256:42667e897e16ab0d66954af0e60a9caa94f0fd4ecf3aaf6d2d260eec1aa36ad6", size = 44221 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/14/1b/a298b06749107c305e1fe0f814c6c74aea7b2f1e10989cb30f544a1b3253/python_dotenv-1.2.1-py3-none-any.whl", hash = "sha256:b81ee9561e9ca4004139c6cbba3a238c32b03e4894671e181b671e8cb8425d61", size = 21230 }, +] + +[[package]] +name = "python-multipart" +version = "0.0.20" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f3/87/f44d7c9f274c7ee665a29b885ec97089ec5dc034c7f3fafa03da9e39a09e/python_multipart-0.0.20.tar.gz", hash = "sha256:8dd0cab45b8e23064ae09147625994d090fa46f5b0d1e13af944c331a7fa9d13", size = 37158 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/45/58/38b5afbc1a800eeea951b9285d3912613f2603bdf897a4ab0f4bd7f405fc/python_multipart-0.0.20-py3-none-any.whl", hash = "sha256:8a62d3a8335e06589fe01f2a3e178cdcc632f3fbe0d492ad9ee0ec35aab1f104", size = 24546 }, +] + +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063 }, + { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973 }, + { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116 }, + { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011 }, + { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870 }, + { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089 }, + { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181 }, + { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658 }, + { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003 }, + { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344 }, + { url = "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", size = 181669 }, + { url = "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", size = 173252 }, + { url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081 }, + { url = "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", size = 841159 }, + { url = "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", size = 801626 }, + { url = "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", size = 753613 }, + { url = "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", size = 794115 }, + { url = "https://files.pythonhosted.org/packages/de/94/980b50a6531b3019e45ddeada0626d45fa85cbe22300844a7983285bed3b/pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", size = 137427 }, + { url = "https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", size = 154090 }, + { url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246 }, + { url = "https://files.pythonhosted.org/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", size = 181814 }, + { url = "https://files.pythonhosted.org/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", size = 173809 }, + { url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", size = 766454 }, + { url = "https://files.pythonhosted.org/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", size = 836355 }, + { url = "https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", size = 794175 }, + { url = "https://files.pythonhosted.org/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", size = 755228 }, + { url = "https://files.pythonhosted.org/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", size = 789194 }, + { url = "https://files.pythonhosted.org/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", size = 156429 }, + { url = "https://files.pythonhosted.org/packages/f4/f4/a4541072bb9422c8a883ab55255f918fa378ecf083f5b85e87fc2b4eda1b/pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", size = 143912 }, + { url = "https://files.pythonhosted.org/packages/7c/f9/07dd09ae774e4616edf6cda684ee78f97777bdd15847253637a6f052a62f/pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", size = 189108 }, + { url = "https://files.pythonhosted.org/packages/4e/78/8d08c9fb7ce09ad8c38ad533c1191cf27f7ae1effe5bb9400a46d9437fcf/pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", size = 183641 }, + { url = "https://files.pythonhosted.org/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", size = 831901 }, + { url = "https://files.pythonhosted.org/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", size = 861132 }, + { url = "https://files.pythonhosted.org/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", size = 839261 }, + { url = "https://files.pythonhosted.org/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", size = 805272 }, + { url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923 }, + { url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", size = 174062 }, + { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341 }, +] + +[[package]] +name = "scholarr" +version = "0.1.0" +source = { editable = "." } +dependencies = [ + { name = "alembic" }, + { name = "argon2-cffi" }, + { name = "asyncpg" }, + { name = "fastapi" }, + { name = "itsdangerous" }, + { name = "jinja2" }, + { name = "python-multipart" }, + { name = "sqlalchemy" }, + { name = "uvicorn", extra = ["standard"] }, +] + +[package.optional-dependencies] +dev = [ + { name = "httpx" }, + { name = "pytest" }, + { name = "pytest-asyncio" }, +] + +[package.metadata] +requires-dist = [ + { name = "alembic", specifier = ">=1.14,<2.0" }, + { name = "argon2-cffi", specifier = ">=25.1,<26.0" }, + { name = "asyncpg", specifier = ">=0.30,<0.31" }, + { name = "fastapi", specifier = ">=0.116,<0.117" }, + { name = "httpx", marker = "extra == 'dev'", specifier = ">=0.28,<0.29" }, + { name = "itsdangerous", specifier = ">=2.2,<3.0" }, + { name = "jinja2", specifier = ">=3.1,<3.2" }, + { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.3,<9.0" }, + { name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=0.25,<0.26" }, + { name = "python-multipart", specifier = ">=0.0.20,<0.0.21" }, + { name = "sqlalchemy", specifier = ">=2.0,<2.1" }, + { name = "uvicorn", extras = ["standard"], specifier = ">=0.34,<0.35" }, +] +provides-extras = ["dev"] + +[[package]] +name = "sqlalchemy" +version = "2.0.46" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "greenlet", marker = "platform_machine == 'AMD64' or platform_machine == 'WIN32' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'ppc64le' or platform_machine == 'win32' or platform_machine == 'x86_64'" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/aa/9ce0f3e7a9829ead5c8ce549392f33a12c4555a6c0609bb27d882e9c7ddf/sqlalchemy-2.0.46.tar.gz", hash = "sha256:cf36851ee7219c170bb0793dbc3da3e80c582e04a5437bc601bfe8c85c9216d7", size = 9865393 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b6/35/d16bfa235c8b7caba3730bba43e20b1e376d2224f407c178fbf59559f23e/sqlalchemy-2.0.46-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3a9a72b0da8387f15d5810f1facca8f879de9b85af8c645138cba61ea147968c", size = 2153405 }, + { url = "https://files.pythonhosted.org/packages/06/6c/3192e24486749862f495ddc6584ed730c0c994a67550ec395d872a2ad650/sqlalchemy-2.0.46-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2347c3f0efc4de367ba00218e0ae5c4ba2306e47216ef80d6e31761ac97cb0b9", size = 3334702 }, + { url = "https://files.pythonhosted.org/packages/ea/a2/b9f33c8d68a3747d972a0bb758c6b63691f8fb8a49014bc3379ba15d4274/sqlalchemy-2.0.46-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9094c8b3197db12aa6f05c51c05daaad0a92b8c9af5388569847b03b1007fb1b", size = 3347664 }, + { url = "https://files.pythonhosted.org/packages/aa/d2/3e59e2a91eaec9db7e8dc6b37b91489b5caeb054f670f32c95bcba98940f/sqlalchemy-2.0.46-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:37fee2164cf21417478b6a906adc1a91d69ae9aba8f9533e67ce882f4bb1de53", size = 3277372 }, + { url = "https://files.pythonhosted.org/packages/dd/dd/67bc2e368b524e2192c3927b423798deda72c003e73a1e94c21e74b20a85/sqlalchemy-2.0.46-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:b1e14b2f6965a685c7128bd315e27387205429c2e339eeec55cb75ca4ab0ea2e", size = 3312425 }, + { url = "https://files.pythonhosted.org/packages/43/82/0ecd68e172bfe62247e96cb47867c2d68752566811a4e8c9d8f6e7c38a65/sqlalchemy-2.0.46-cp312-cp312-win32.whl", hash = "sha256:412f26bb4ba942d52016edc8d12fb15d91d3cd46b0047ba46e424213ad407bcb", size = 2113155 }, + { url = "https://files.pythonhosted.org/packages/bc/2a/2821a45742073fc0331dc132552b30de68ba9563230853437cac54b2b53e/sqlalchemy-2.0.46-cp312-cp312-win_amd64.whl", hash = "sha256:ea3cd46b6713a10216323cda3333514944e510aa691c945334713fca6b5279ff", size = 2140078 }, + { url = "https://files.pythonhosted.org/packages/b3/4b/fa7838fe20bb752810feed60e45625a9a8b0102c0c09971e2d1d95362992/sqlalchemy-2.0.46-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:93a12da97cca70cea10d4b4fc602589c4511f96c1f8f6c11817620c021d21d00", size = 2150268 }, + { url = "https://files.pythonhosted.org/packages/46/c1/b34dccd712e8ea846edf396e00973dda82d598cb93762e55e43e6835eba9/sqlalchemy-2.0.46-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:af865c18752d416798dae13f83f38927c52f085c52e2f32b8ab0fef46fdd02c2", size = 3276511 }, + { url = "https://files.pythonhosted.org/packages/96/48/a04d9c94753e5d5d096c628c82a98c4793b9c08ca0e7155c3eb7d7db9f24/sqlalchemy-2.0.46-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8d679b5f318423eacb61f933a9a0f75535bfca7056daeadbf6bd5bcee6183aee", size = 3292881 }, + { url = "https://files.pythonhosted.org/packages/be/f4/06eda6e91476f90a7d8058f74311cb65a2fb68d988171aced81707189131/sqlalchemy-2.0.46-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:64901e08c33462acc9ec3bad27fc7a5c2b6491665f2aa57564e57a4f5d7c52ad", size = 3224559 }, + { url = "https://files.pythonhosted.org/packages/ab/a2/d2af04095412ca6345ac22b33b89fe8d6f32a481e613ffcb2377d931d8d0/sqlalchemy-2.0.46-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e8ac45e8f4eaac0f9f8043ea0e224158855c6a4329fd4ee37c45c61e3beb518e", size = 3262728 }, + { url = "https://files.pythonhosted.org/packages/31/48/1980c7caa5978a3b8225b4d230e69a2a6538a3562b8b31cea679b6933c83/sqlalchemy-2.0.46-cp313-cp313-win32.whl", hash = "sha256:8d3b44b3d0ab2f1319d71d9863d76eeb46766f8cf9e921ac293511804d39813f", size = 2111295 }, + { url = "https://files.pythonhosted.org/packages/2d/54/f8d65bbde3d877617c4720f3c9f60e99bb7266df0d5d78b6e25e7c149f35/sqlalchemy-2.0.46-cp313-cp313-win_amd64.whl", hash = "sha256:77f8071d8fbcbb2dd11b7fd40dedd04e8ebe2eb80497916efedba844298065ef", size = 2137076 }, + { url = "https://files.pythonhosted.org/packages/56/ba/9be4f97c7eb2b9d5544f2624adfc2853e796ed51d2bb8aec90bc94b7137e/sqlalchemy-2.0.46-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a1e8cc6cc01da346dc92d9509a63033b9b1bda4fed7a7a7807ed385c7dccdc10", size = 3556533 }, + { url = "https://files.pythonhosted.org/packages/20/a6/b1fc6634564dbb4415b7ed6419cdfeaadefd2c39cdab1e3aa07a5f2474c2/sqlalchemy-2.0.46-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:96c7cca1a4babaaf3bfff3e4e606e38578856917e52f0384635a95b226c87764", size = 3523208 }, + { url = "https://files.pythonhosted.org/packages/a1/d8/41e0bdfc0f930ff236f86fccd12962d8fa03713f17ed57332d38af6a3782/sqlalchemy-2.0.46-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:b2a9f9aee38039cf4755891a1e50e1effcc42ea6ba053743f452c372c3152b1b", size = 3464292 }, + { url = "https://files.pythonhosted.org/packages/f0/8b/9dcbec62d95bea85f5ecad9b8d65b78cc30fb0ffceeb3597961f3712549b/sqlalchemy-2.0.46-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:db23b1bf8cfe1f7fda19018e7207b20cdb5168f83c437ff7e95d19e39289c447", size = 3473497 }, + { url = "https://files.pythonhosted.org/packages/e9/f8/5ecdfc73383ec496de038ed1614de9e740a82db9ad67e6e4514ebc0708a3/sqlalchemy-2.0.46-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:56bdd261bfd0895452006d5316cbf35739c53b9bb71a170a331fa0ea560b2ada", size = 2152079 }, + { url = "https://files.pythonhosted.org/packages/e5/bf/eba3036be7663ce4d9c050bc3d63794dc29fbe01691f2bf5ccb64e048d20/sqlalchemy-2.0.46-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:33e462154edb9493f6c3ad2125931e273bbd0be8ae53f3ecd1c161ea9a1dd366", size = 3272216 }, + { url = "https://files.pythonhosted.org/packages/05/45/1256fb597bb83b58a01ddb600c59fe6fdf0e5afe333f0456ed75c0f8d7bd/sqlalchemy-2.0.46-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9bcdce05f056622a632f1d44bb47dbdb677f58cad393612280406ce37530eb6d", size = 3277208 }, + { url = "https://files.pythonhosted.org/packages/d9/a0/2053b39e4e63b5d7ceb3372cface0859a067c1ddbd575ea7e9985716f771/sqlalchemy-2.0.46-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:8e84b09a9b0f19accedcbeff5c2caf36e0dd537341a33aad8d680336152dc34e", size = 3221994 }, + { url = "https://files.pythonhosted.org/packages/1e/87/97713497d9502553c68f105a1cb62786ba1ee91dea3852ae4067ed956a50/sqlalchemy-2.0.46-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:4f52f7291a92381e9b4de9050b0a65ce5d6a763333406861e33906b8aa4906bf", size = 3243990 }, + { url = "https://files.pythonhosted.org/packages/a8/87/5d1b23548f420ff823c236f8bea36b1a997250fd2f892e44a3838ca424f4/sqlalchemy-2.0.46-cp314-cp314-win32.whl", hash = "sha256:70ed2830b169a9960193f4d4322d22be5c0925357d82cbf485b3369893350908", size = 2114215 }, + { url = "https://files.pythonhosted.org/packages/3a/20/555f39cbcf0c10cf452988b6a93c2a12495035f68b3dbd1a408531049d31/sqlalchemy-2.0.46-cp314-cp314-win_amd64.whl", hash = "sha256:3c32e993bc57be6d177f7d5d31edb93f30726d798ad86ff9066d75d9bf2e0b6b", size = 2139867 }, + { url = "https://files.pythonhosted.org/packages/3e/f0/f96c8057c982d9d8a7a68f45d69c674bc6f78cad401099692fe16521640a/sqlalchemy-2.0.46-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4dafb537740eef640c4d6a7c254611dca2df87eaf6d14d6a5fca9d1f4c3fc0fa", size = 3561202 }, + { url = "https://files.pythonhosted.org/packages/d7/53/3b37dda0a5b137f21ef608d8dfc77b08477bab0fe2ac9d3e0a66eaeab6fc/sqlalchemy-2.0.46-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:42a1643dc5427b69aca967dae540a90b0fbf57eaf248f13a90ea5930e0966863", size = 3526296 }, + { url = "https://files.pythonhosted.org/packages/33/75/f28622ba6dde79cd545055ea7bd4062dc934e0621f7b3be2891f8563f8de/sqlalchemy-2.0.46-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ff33c6e6ad006bbc0f34f5faf941cfc62c45841c64c0a058ac38c799f15b5ede", size = 3470008 }, + { url = "https://files.pythonhosted.org/packages/a9/42/4afecbbc38d5e99b18acef446453c76eec6fbd03db0a457a12a056836e22/sqlalchemy-2.0.46-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:82ec52100ec1e6ec671563bbd02d7c7c8d0b9e71a0723c72f22ecf52d1755330", size = 3476137 }, + { url = "https://files.pythonhosted.org/packages/fc/a1/9c4efa03300926601c19c18582531b45aededfb961ab3c3585f1e24f120b/sqlalchemy-2.0.46-py3-none-any.whl", hash = "sha256:f9c11766e7e7c0a2767dda5acb006a118640c9fc0a4104214b96269bfb78399e", size = 1937882 }, +] + +[[package]] +name = "starlette" +version = "0.48.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a7/a5/d6f429d43394057b67a6b5bbe6eae2f77a6bf7459d961fdb224bf206eee6/starlette-0.48.0.tar.gz", hash = "sha256:7e8cee469a8ab2352911528110ce9088fdc6a37d9876926e73da7ce4aa4c7a46", size = 2652949 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/be/72/2db2f49247d0a18b4f1bb9a5a39a0162869acf235f3a96418363947b3d46/starlette-0.48.0-py3-none-any.whl", hash = "sha256:0764ca97b097582558ecb498132ed0c7d942f233f365b86ba37770e026510659", size = 73736 }, +] + +[[package]] +name = "typing-extensions" +version = "4.15.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/94/1a15dd82efb362ac84269196e94cf00f187f7ed21c242792a923cdb1c61f/typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466", size = 109391 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614 }, +] + +[[package]] +name = "typing-inspection" +version = "0.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/55/e3/70399cb7dd41c10ac53367ae42139cf4b1ca5f36bb3dc6c9d33acdb43655/typing_inspection-0.4.2.tar.gz", hash = "sha256:ba561c48a67c5958007083d386c3295464928b01faa735ab8547c5692e87f464", size = 75949 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl", hash = "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7", size = 14611 }, +] + +[[package]] +name = "uvicorn" +version = "0.34.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "h11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/de/ad/713be230bcda622eaa35c28f0d328c3675c371238470abdea52417f17a8e/uvicorn-0.34.3.tar.gz", hash = "sha256:35919a9a979d7a59334b6b10e05d77c1d0d574c50e0fc98b8b1a0f165708b55a", size = 76631 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6d/0d/8adfeaa62945f90d19ddc461c55f4a50c258af7662d34b6a3d5d1f8646f6/uvicorn-0.34.3-py3-none-any.whl", hash = "sha256:16246631db62bdfbf069b0645177d6e8a77ba950cfedbfd093acef9444e4d885", size = 62431 }, +] + +[package.optional-dependencies] +standard = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "httptools" }, + { name = "python-dotenv" }, + { name = "pyyaml" }, + { name = "uvloop", marker = "platform_python_implementation != 'PyPy' and sys_platform != 'cygwin' and sys_platform != 'win32'" }, + { name = "watchfiles" }, + { name = "websockets" }, +] + +[[package]] +name = "uvloop" +version = "0.22.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/06/f0/18d39dbd1971d6d62c4629cc7fa67f74821b0dc1f5a77af43719de7936a7/uvloop-0.22.1.tar.gz", hash = "sha256:6c84bae345b9147082b17371e3dd5d42775bddce91f885499017f4607fdaf39f", size = 2443250 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3d/ff/7f72e8170be527b4977b033239a83a68d5c881cc4775fca255c677f7ac5d/uvloop-0.22.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:fe94b4564e865d968414598eea1a6de60adba0c040ba4ed05ac1300de402cd42", size = 1359936 }, + { url = "https://files.pythonhosted.org/packages/c3/c6/e5d433f88fd54d81ef4be58b2b7b0cea13c442454a1db703a1eea0db1a59/uvloop-0.22.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:51eb9bd88391483410daad430813d982010f9c9c89512321f5b60e2cddbdddd6", size = 752769 }, + { url = "https://files.pythonhosted.org/packages/24/68/a6ac446820273e71aa762fa21cdcc09861edd3536ff47c5cd3b7afb10eeb/uvloop-0.22.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:700e674a166ca5778255e0e1dc4e9d79ab2acc57b9171b79e65feba7184b3370", size = 4317413 }, + { url = "https://files.pythonhosted.org/packages/5f/6f/e62b4dfc7ad6518e7eff2516f680d02a0f6eb62c0c212e152ca708a0085e/uvloop-0.22.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7b5b1ac819a3f946d3b2ee07f09149578ae76066d70b44df3fa990add49a82e4", size = 4426307 }, + { url = "https://files.pythonhosted.org/packages/90/60/97362554ac21e20e81bcef1150cb2a7e4ffdaf8ea1e5b2e8bf7a053caa18/uvloop-0.22.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e047cc068570bac9866237739607d1313b9253c3051ad84738cbb095be0537b2", size = 4131970 }, + { url = "https://files.pythonhosted.org/packages/99/39/6b3f7d234ba3964c428a6e40006340f53ba37993f46ed6e111c6e9141d18/uvloop-0.22.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:512fec6815e2dd45161054592441ef76c830eddaad55c8aa30952e6fe1ed07c0", size = 4296343 }, + { url = "https://files.pythonhosted.org/packages/89/8c/182a2a593195bfd39842ea68ebc084e20c850806117213f5a299dfc513d9/uvloop-0.22.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:561577354eb94200d75aca23fbde86ee11be36b00e52a4eaf8f50fb0c86b7705", size = 1358611 }, + { url = "https://files.pythonhosted.org/packages/d2/14/e301ee96a6dc95224b6f1162cd3312f6d1217be3907b79173b06785f2fe7/uvloop-0.22.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:1cdf5192ab3e674ca26da2eada35b288d2fa49fdd0f357a19f0e7c4e7d5077c8", size = 751811 }, + { url = "https://files.pythonhosted.org/packages/b7/02/654426ce265ac19e2980bfd9ea6590ca96a56f10c76e63801a2df01c0486/uvloop-0.22.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6e2ea3d6190a2968f4a14a23019d3b16870dd2190cd69c8180f7c632d21de68d", size = 4288562 }, + { url = "https://files.pythonhosted.org/packages/15/c0/0be24758891ef825f2065cd5db8741aaddabe3e248ee6acc5e8a80f04005/uvloop-0.22.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0530a5fbad9c9e4ee3f2b33b148c6a64d47bbad8000ea63704fa8260f4cf728e", size = 4366890 }, + { url = "https://files.pythonhosted.org/packages/d2/53/8369e5219a5855869bcee5f4d317f6da0e2c669aecf0ef7d371e3d084449/uvloop-0.22.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:bc5ef13bbc10b5335792360623cc378d52d7e62c2de64660616478c32cd0598e", size = 4119472 }, + { url = "https://files.pythonhosted.org/packages/f8/ba/d69adbe699b768f6b29a5eec7b47dd610bd17a69de51b251126a801369ea/uvloop-0.22.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:1f38ec5e3f18c8a10ded09742f7fb8de0108796eb673f30ce7762ce1b8550cad", size = 4239051 }, + { url = "https://files.pythonhosted.org/packages/90/cd/b62bdeaa429758aee8de8b00ac0dd26593a9de93d302bff3d21439e9791d/uvloop-0.22.1-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:3879b88423ec7e97cd4eba2a443aa26ed4e59b45e6b76aabf13fe2f27023a142", size = 1362067 }, + { url = "https://files.pythonhosted.org/packages/0d/f8/a132124dfda0777e489ca86732e85e69afcd1ff7686647000050ba670689/uvloop-0.22.1-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:4baa86acedf1d62115c1dc6ad1e17134476688f08c6efd8a2ab076e815665c74", size = 752423 }, + { url = "https://files.pythonhosted.org/packages/a3/94/94af78c156f88da4b3a733773ad5ba0b164393e357cc4bd0ab2e2677a7d6/uvloop-0.22.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:297c27d8003520596236bdb2335e6b3f649480bd09e00d1e3a99144b691d2a35", size = 4272437 }, + { url = "https://files.pythonhosted.org/packages/b5/35/60249e9fd07b32c665192cec7af29e06c7cd96fa1d08b84f012a56a0b38e/uvloop-0.22.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c1955d5a1dd43198244d47664a5858082a3239766a839b2102a269aaff7a4e25", size = 4292101 }, + { url = "https://files.pythonhosted.org/packages/02/62/67d382dfcb25d0a98ce73c11ed1a6fba5037a1a1d533dcbb7cab033a2636/uvloop-0.22.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:b31dc2fccbd42adc73bc4e7cdbae4fc5086cf378979e53ca5d0301838c5682c6", size = 4114158 }, + { url = "https://files.pythonhosted.org/packages/f0/7a/f1171b4a882a5d13c8b7576f348acfe6074d72eaf52cccef752f748d4a9f/uvloop-0.22.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:93f617675b2d03af4e72a5333ef89450dfaa5321303ede6e67ba9c9d26878079", size = 4177360 }, + { url = "https://files.pythonhosted.org/packages/79/7b/b01414f31546caf0919da80ad57cbfe24c56b151d12af68cee1b04922ca8/uvloop-0.22.1-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:37554f70528f60cad66945b885eb01f1bb514f132d92b6eeed1c90fd54ed6289", size = 1454790 }, + { url = "https://files.pythonhosted.org/packages/d4/31/0bb232318dd838cad3fa8fb0c68c8b40e1145b32025581975e18b11fab40/uvloop-0.22.1-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:b76324e2dc033a0b2f435f33eb88ff9913c156ef78e153fb210e03c13da746b3", size = 796783 }, + { url = "https://files.pythonhosted.org/packages/42/38/c9b09f3271a7a723a5de69f8e237ab8e7803183131bc57c890db0b6bb872/uvloop-0.22.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:badb4d8e58ee08dad957002027830d5c3b06aea446a6a3744483c2b3b745345c", size = 4647548 }, + { url = "https://files.pythonhosted.org/packages/c1/37/945b4ca0ac27e3dc4952642d4c900edd030b3da6c9634875af6e13ae80e5/uvloop-0.22.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b91328c72635f6f9e0282e4a57da7470c7350ab1c9f48546c0f2866205349d21", size = 4467065 }, + { url = "https://files.pythonhosted.org/packages/97/cc/48d232f33d60e2e2e0b42f4e73455b146b76ebe216487e862700457fbf3c/uvloop-0.22.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:daf620c2995d193449393d6c62131b3fbd40a63bf7b307a1527856ace637fe88", size = 4328384 }, + { url = "https://files.pythonhosted.org/packages/e4/16/c1fd27e9549f3c4baf1dc9c20c456cd2f822dbf8de9f463824b0c0357e06/uvloop-0.22.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6cde23eeda1a25c75b2e07d39970f3374105d5eafbaab2a4482be82f272d5a5e", size = 4296730 }, +] + +[[package]] +name = "watchfiles" +version = "1.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c2/c9/8869df9b2a2d6c59d79220a4db37679e74f807c559ffe5265e08b227a210/watchfiles-1.1.1.tar.gz", hash = "sha256:a173cb5c16c4f40ab19cecf48a534c409f7ea983ab8fed0741304a1c0a31b3f2", size = 94440 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/74/d5/f039e7e3c639d9b1d09b07ea412a6806d38123f0508e5f9b48a87b0a76cc/watchfiles-1.1.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:8c89f9f2f740a6b7dcc753140dd5e1ab9215966f7a3530d0c0705c83b401bd7d", size = 404745 }, + { url = "https://files.pythonhosted.org/packages/a5/96/a881a13aa1349827490dab2d363c8039527060cfcc2c92cc6d13d1b1049e/watchfiles-1.1.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:bd404be08018c37350f0d6e34676bd1e2889990117a2b90070b3007f172d0610", size = 391769 }, + { url = "https://files.pythonhosted.org/packages/4b/5b/d3b460364aeb8da471c1989238ea0e56bec24b6042a68046adf3d9ddb01c/watchfiles-1.1.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8526e8f916bb5b9a0a777c8317c23ce65de259422bba5b31325a6fa6029d33af", size = 449374 }, + { url = "https://files.pythonhosted.org/packages/b9/44/5769cb62d4ed055cb17417c0a109a92f007114a4e07f30812a73a4efdb11/watchfiles-1.1.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2edc3553362b1c38d9f06242416a5d8e9fe235c204a4072e988ce2e5bb1f69f6", size = 459485 }, + { url = "https://files.pythonhosted.org/packages/19/0c/286b6301ded2eccd4ffd0041a1b726afda999926cf720aab63adb68a1e36/watchfiles-1.1.1-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:30f7da3fb3f2844259cba4720c3fc7138eb0f7b659c38f3bfa65084c7fc7abce", size = 488813 }, + { url = "https://files.pythonhosted.org/packages/c7/2b/8530ed41112dd4a22f4dcfdb5ccf6a1baad1ff6eed8dc5a5f09e7e8c41c7/watchfiles-1.1.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f8979280bdafff686ba5e4d8f97840f929a87ed9cdf133cbbd42f7766774d2aa", size = 594816 }, + { url = "https://files.pythonhosted.org/packages/ce/d2/f5f9fb49489f184f18470d4f99f4e862a4b3e9ac2865688eb2099e3d837a/watchfiles-1.1.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dcc5c24523771db3a294c77d94771abcfcb82a0e0ee8efd910c37c59ec1b31bb", size = 475186 }, + { url = "https://files.pythonhosted.org/packages/cf/68/5707da262a119fb06fbe214d82dd1fe4a6f4af32d2d14de368d0349eb52a/watchfiles-1.1.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1db5d7ae38ff20153d542460752ff397fcf5c96090c1230803713cf3147a6803", size = 456812 }, + { url = "https://files.pythonhosted.org/packages/66/ab/3cbb8756323e8f9b6f9acb9ef4ec26d42b2109bce830cc1f3468df20511d/watchfiles-1.1.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:28475ddbde92df1874b6c5c8aaeb24ad5be47a11f87cde5a28ef3835932e3e94", size = 630196 }, + { url = "https://files.pythonhosted.org/packages/78/46/7152ec29b8335f80167928944a94955015a345440f524d2dfe63fc2f437b/watchfiles-1.1.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:36193ed342f5b9842edd3532729a2ad55c4160ffcfa3700e0d54be496b70dd43", size = 622657 }, + { url = "https://files.pythonhosted.org/packages/0a/bf/95895e78dd75efe9a7f31733607f384b42eb5feb54bd2eb6ed57cc2e94f4/watchfiles-1.1.1-cp312-cp312-win32.whl", hash = "sha256:859e43a1951717cc8de7f4c77674a6d389b106361585951d9e69572823f311d9", size = 272042 }, + { url = "https://files.pythonhosted.org/packages/87/0a/90eb755f568de2688cb220171c4191df932232c20946966c27a59c400850/watchfiles-1.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:91d4c9a823a8c987cce8fa2690923b069966dabb196dd8d137ea2cede885fde9", size = 288410 }, + { url = "https://files.pythonhosted.org/packages/36/76/f322701530586922fbd6723c4f91ace21364924822a8772c549483abed13/watchfiles-1.1.1-cp312-cp312-win_arm64.whl", hash = "sha256:a625815d4a2bdca61953dbba5a39d60164451ef34c88d751f6c368c3ea73d404", size = 278209 }, + { url = "https://files.pythonhosted.org/packages/bb/f4/f750b29225fe77139f7ae5de89d4949f5a99f934c65a1f1c0b248f26f747/watchfiles-1.1.1-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:130e4876309e8686a5e37dba7d5e9bc77e6ed908266996ca26572437a5271e18", size = 404321 }, + { url = "https://files.pythonhosted.org/packages/2b/f9/f07a295cde762644aa4c4bb0f88921d2d141af45e735b965fb2e87858328/watchfiles-1.1.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:5f3bde70f157f84ece3765b42b4a52c6ac1a50334903c6eaf765362f6ccca88a", size = 391783 }, + { url = "https://files.pythonhosted.org/packages/bc/11/fc2502457e0bea39a5c958d86d2cb69e407a4d00b85735ca724bfa6e0d1a/watchfiles-1.1.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:14e0b1fe858430fc0251737ef3824c54027bedb8c37c38114488b8e131cf8219", size = 449279 }, + { url = "https://files.pythonhosted.org/packages/e3/1f/d66bc15ea0b728df3ed96a539c777acfcad0eb78555ad9efcaa1274688f0/watchfiles-1.1.1-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f27db948078f3823a6bb3b465180db8ebecf26dd5dae6f6180bd87383b6b4428", size = 459405 }, + { url = "https://files.pythonhosted.org/packages/be/90/9f4a65c0aec3ccf032703e6db02d89a157462fbb2cf20dd415128251cac0/watchfiles-1.1.1-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:059098c3a429f62fc98e8ec62b982230ef2c8df68c79e826e37b895bc359a9c0", size = 488976 }, + { url = "https://files.pythonhosted.org/packages/37/57/ee347af605d867f712be7029bb94c8c071732a4b44792e3176fa3c612d39/watchfiles-1.1.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bfb5862016acc9b869bb57284e6cb35fdf8e22fe59f7548858e2f971d045f150", size = 595506 }, + { url = "https://files.pythonhosted.org/packages/a8/78/cc5ab0b86c122047f75e8fc471c67a04dee395daf847d3e59381996c8707/watchfiles-1.1.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:319b27255aacd9923b8a276bb14d21a5f7ff82564c744235fc5eae58d95422ae", size = 474936 }, + { url = "https://files.pythonhosted.org/packages/62/da/def65b170a3815af7bd40a3e7010bf6ab53089ef1b75d05dd5385b87cf08/watchfiles-1.1.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c755367e51db90e75b19454b680903631d41f9e3607fbd941d296a020c2d752d", size = 456147 }, + { url = "https://files.pythonhosted.org/packages/57/99/da6573ba71166e82d288d4df0839128004c67d2778d3b566c138695f5c0b/watchfiles-1.1.1-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:c22c776292a23bfc7237a98f791b9ad3144b02116ff10d820829ce62dff46d0b", size = 630007 }, + { url = "https://files.pythonhosted.org/packages/a8/51/7439c4dd39511368849eb1e53279cd3454b4a4dbace80bab88feeb83c6b5/watchfiles-1.1.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:3a476189be23c3686bc2f4321dd501cb329c0a0469e77b7b534ee10129ae6374", size = 622280 }, + { url = "https://files.pythonhosted.org/packages/95/9c/8ed97d4bba5db6fdcdb2b298d3898f2dd5c20f6b73aee04eabe56c59677e/watchfiles-1.1.1-cp313-cp313-win32.whl", hash = "sha256:bf0a91bfb5574a2f7fc223cf95eeea79abfefa404bf1ea5e339c0c1560ae99a0", size = 272056 }, + { url = "https://files.pythonhosted.org/packages/1f/f3/c14e28429f744a260d8ceae18bf58c1d5fa56b50d006a7a9f80e1882cb0d/watchfiles-1.1.1-cp313-cp313-win_amd64.whl", hash = "sha256:52e06553899e11e8074503c8e716d574adeeb7e68913115c4b3653c53f9bae42", size = 288162 }, + { url = "https://files.pythonhosted.org/packages/dc/61/fe0e56c40d5cd29523e398d31153218718c5786b5e636d9ae8ae79453d27/watchfiles-1.1.1-cp313-cp313-win_arm64.whl", hash = "sha256:ac3cc5759570cd02662b15fbcd9d917f7ecd47efe0d6b40474eafd246f91ea18", size = 277909 }, + { url = "https://files.pythonhosted.org/packages/79/42/e0a7d749626f1e28c7108a99fb9bf524b501bbbeb9b261ceecde644d5a07/watchfiles-1.1.1-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:563b116874a9a7ce6f96f87cd0b94f7faf92d08d0021e837796f0a14318ef8da", size = 403389 }, + { url = "https://files.pythonhosted.org/packages/15/49/08732f90ce0fbbc13913f9f215c689cfc9ced345fb1bcd8829a50007cc8d/watchfiles-1.1.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:3ad9fe1dae4ab4212d8c91e80b832425e24f421703b5a42ef2e4a1e215aff051", size = 389964 }, + { url = "https://files.pythonhosted.org/packages/27/0d/7c315d4bd5f2538910491a0393c56bf70d333d51bc5b34bee8e68e8cea19/watchfiles-1.1.1-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ce70f96a46b894b36eba678f153f052967a0d06d5b5a19b336ab0dbbd029f73e", size = 448114 }, + { url = "https://files.pythonhosted.org/packages/c3/24/9e096de47a4d11bc4df41e9d1e61776393eac4cb6eb11b3e23315b78b2cc/watchfiles-1.1.1-cp313-cp313t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:cb467c999c2eff23a6417e58d75e5828716f42ed8289fe6b77a7e5a91036ca70", size = 460264 }, + { url = "https://files.pythonhosted.org/packages/cc/0f/e8dea6375f1d3ba5fcb0b3583e2b493e77379834c74fd5a22d66d85d6540/watchfiles-1.1.1-cp313-cp313t-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:836398932192dae4146c8f6f737d74baeac8b70ce14831a239bdb1ca882fc261", size = 487877 }, + { url = "https://files.pythonhosted.org/packages/ac/5b/df24cfc6424a12deb41503b64d42fbea6b8cb357ec62ca84a5a3476f654a/watchfiles-1.1.1-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:743185e7372b7bc7c389e1badcc606931a827112fbbd37f14c537320fca08620", size = 595176 }, + { url = "https://files.pythonhosted.org/packages/8f/b5/853b6757f7347de4e9b37e8cc3289283fb983cba1ab4d2d7144694871d9c/watchfiles-1.1.1-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:afaeff7696e0ad9f02cbb8f56365ff4686ab205fcf9c4c5b6fdfaaa16549dd04", size = 473577 }, + { url = "https://files.pythonhosted.org/packages/e1/f7/0a4467be0a56e80447c8529c9fce5b38eab4f513cb3d9bf82e7392a5696b/watchfiles-1.1.1-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3f7eb7da0eb23aa2ba036d4f616d46906013a68caf61b7fdbe42fc8b25132e77", size = 455425 }, + { url = "https://files.pythonhosted.org/packages/8e/e0/82583485ea00137ddf69bc84a2db88bd92ab4a6e3c405e5fb878ead8d0e7/watchfiles-1.1.1-cp313-cp313t-musllinux_1_1_aarch64.whl", hash = "sha256:831a62658609f0e5c64178211c942ace999517f5770fe9436be4c2faeba0c0ef", size = 628826 }, + { url = "https://files.pythonhosted.org/packages/28/9a/a785356fccf9fae84c0cc90570f11702ae9571036fb25932f1242c82191c/watchfiles-1.1.1-cp313-cp313t-musllinux_1_1_x86_64.whl", hash = "sha256:f9a2ae5c91cecc9edd47e041a930490c31c3afb1f5e6d71de3dc671bfaca02bf", size = 622208 }, + { url = "https://files.pythonhosted.org/packages/c3/f4/0872229324ef69b2c3edec35e84bd57a1289e7d3fe74588048ed8947a323/watchfiles-1.1.1-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:d1715143123baeeaeadec0528bb7441103979a1d5f6fd0e1f915383fea7ea6d5", size = 404315 }, + { url = "https://files.pythonhosted.org/packages/7b/22/16d5331eaed1cb107b873f6ae1b69e9ced582fcf0c59a50cd84f403b1c32/watchfiles-1.1.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:39574d6370c4579d7f5d0ad940ce5b20db0e4117444e39b6d8f99db5676c52fd", size = 390869 }, + { url = "https://files.pythonhosted.org/packages/b2/7e/5643bfff5acb6539b18483128fdc0ef2cccc94a5b8fbda130c823e8ed636/watchfiles-1.1.1-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7365b92c2e69ee952902e8f70f3ba6360d0d596d9299d55d7d386df84b6941fb", size = 449919 }, + { url = "https://files.pythonhosted.org/packages/51/2e/c410993ba5025a9f9357c376f48976ef0e1b1aefb73b97a5ae01a5972755/watchfiles-1.1.1-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bfff9740c69c0e4ed32416f013f3c45e2ae42ccedd1167ef2d805c000b6c71a5", size = 460845 }, + { url = "https://files.pythonhosted.org/packages/8e/a4/2df3b404469122e8680f0fcd06079317e48db58a2da2950fb45020947734/watchfiles-1.1.1-cp314-cp314-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b27cf2eb1dda37b2089e3907d8ea92922b673c0c427886d4edc6b94d8dfe5db3", size = 489027 }, + { url = "https://files.pythonhosted.org/packages/ea/84/4587ba5b1f267167ee715b7f66e6382cca6938e0a4b870adad93e44747e6/watchfiles-1.1.1-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:526e86aced14a65a5b0ec50827c745597c782ff46b571dbfe46192ab9e0b3c33", size = 595615 }, + { url = "https://files.pythonhosted.org/packages/6a/0f/c6988c91d06e93cd0bb3d4a808bcf32375ca1904609835c3031799e3ecae/watchfiles-1.1.1-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:04e78dd0b6352db95507fd8cb46f39d185cf8c74e4cf1e4fbad1d3df96faf510", size = 474836 }, + { url = "https://files.pythonhosted.org/packages/b4/36/ded8aebea91919485b7bbabbd14f5f359326cb5ec218cd67074d1e426d74/watchfiles-1.1.1-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5c85794a4cfa094714fb9c08d4a218375b2b95b8ed1666e8677c349906246c05", size = 455099 }, + { url = "https://files.pythonhosted.org/packages/98/e0/8c9bdba88af756a2fce230dd365fab2baf927ba42cd47521ee7498fd5211/watchfiles-1.1.1-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:74d5012b7630714b66be7b7b7a78855ef7ad58e8650c73afc4c076a1f480a8d6", size = 630626 }, + { url = "https://files.pythonhosted.org/packages/2a/84/a95db05354bf2d19e438520d92a8ca475e578c647f78f53197f5a2f17aaf/watchfiles-1.1.1-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:8fbe85cb3201c7d380d3d0b90e63d520f15d6afe217165d7f98c9c649654db81", size = 622519 }, + { url = "https://files.pythonhosted.org/packages/1d/ce/d8acdc8de545de995c339be67711e474c77d643555a9bb74a9334252bd55/watchfiles-1.1.1-cp314-cp314-win32.whl", hash = "sha256:3fa0b59c92278b5a7800d3ee7733da9d096d4aabcfabb9a928918bd276ef9b9b", size = 272078 }, + { url = "https://files.pythonhosted.org/packages/c4/c9/a74487f72d0451524be827e8edec251da0cc1fcf111646a511ae752e1a3d/watchfiles-1.1.1-cp314-cp314-win_amd64.whl", hash = "sha256:c2047d0b6cea13b3316bdbafbfa0c4228ae593d995030fda39089d36e64fc03a", size = 287664 }, + { url = "https://files.pythonhosted.org/packages/df/b8/8ac000702cdd496cdce998c6f4ee0ca1f15977bba51bdf07d872ebdfc34c/watchfiles-1.1.1-cp314-cp314-win_arm64.whl", hash = "sha256:842178b126593addc05acf6fce960d28bc5fae7afbaa2c6c1b3a7b9460e5be02", size = 277154 }, + { url = "https://files.pythonhosted.org/packages/47/a8/e3af2184707c29f0f14b1963c0aace6529f9d1b8582d5b99f31bbf42f59e/watchfiles-1.1.1-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:88863fbbc1a7312972f1c511f202eb30866370ebb8493aef2812b9ff28156a21", size = 403820 }, + { url = "https://files.pythonhosted.org/packages/c0/ec/e47e307c2f4bd75f9f9e8afbe3876679b18e1bcec449beca132a1c5ffb2d/watchfiles-1.1.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:55c7475190662e202c08c6c0f4d9e345a29367438cf8e8037f3155e10a88d5a5", size = 390510 }, + { url = "https://files.pythonhosted.org/packages/d5/a0/ad235642118090f66e7b2f18fd5c42082418404a79205cdfca50b6309c13/watchfiles-1.1.1-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3f53fa183d53a1d7a8852277c92b967ae99c2d4dcee2bfacff8868e6e30b15f7", size = 448408 }, + { url = "https://files.pythonhosted.org/packages/df/85/97fa10fd5ff3332ae17e7e40e20784e419e28521549780869f1413742e9d/watchfiles-1.1.1-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:6aae418a8b323732fa89721d86f39ec8f092fc2af67f4217a2b07fd3e93c6101", size = 458968 }, + { url = "https://files.pythonhosted.org/packages/47/c2/9059c2e8966ea5ce678166617a7f75ecba6164375f3b288e50a40dc6d489/watchfiles-1.1.1-cp314-cp314t-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f096076119da54a6080e8920cbdaac3dbee667eb91dcc5e5b78840b87415bd44", size = 488096 }, + { url = "https://files.pythonhosted.org/packages/94/44/d90a9ec8ac309bc26db808a13e7bfc0e4e78b6fc051078a554e132e80160/watchfiles-1.1.1-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:00485f441d183717038ed2e887a7c868154f216877653121068107b227a2f64c", size = 596040 }, + { url = "https://files.pythonhosted.org/packages/95/68/4e3479b20ca305cfc561db3ed207a8a1c745ee32bf24f2026a129d0ddb6e/watchfiles-1.1.1-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a55f3e9e493158d7bfdb60a1165035f1cf7d320914e7b7ea83fe22c6023b58fc", size = 473847 }, + { url = "https://files.pythonhosted.org/packages/4f/55/2af26693fd15165c4ff7857e38330e1b61ab8c37d15dc79118cdba115b7a/watchfiles-1.1.1-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8c91ed27800188c2ae96d16e3149f199d62f86c7af5f5f4d2c61a3ed8cd3666c", size = 455072 }, + { url = "https://files.pythonhosted.org/packages/66/1d/d0d200b10c9311ec25d2273f8aad8c3ef7cc7ea11808022501811208a750/watchfiles-1.1.1-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:311ff15a0bae3714ffb603e6ba6dbfba4065ab60865d15a6ec544133bdb21099", size = 629104 }, + { url = "https://files.pythonhosted.org/packages/e3/bd/fa9bb053192491b3867ba07d2343d9f2252e00811567d30ae8d0f78136fe/watchfiles-1.1.1-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:a916a2932da8f8ab582f242c065f5c81bed3462849ca79ee357dd9551b0e9b01", size = 622112 }, +] + +[[package]] +name = "websockets" +version = "16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/04/24/4b2031d72e840ce4c1ccb255f693b15c334757fc50023e4db9537080b8c4/websockets-16.0.tar.gz", hash = "sha256:5f6261a5e56e8d5c42a4497b364ea24d94d9563e8fbd44e78ac40879c60179b5", size = 179346 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/84/7b/bac442e6b96c9d25092695578dda82403c77936104b5682307bd4deb1ad4/websockets-16.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:71c989cbf3254fbd5e84d3bff31e4da39c43f884e64f2551d14bb3c186230f00", size = 177365 }, + { url = "https://files.pythonhosted.org/packages/b0/fe/136ccece61bd690d9c1f715baaeefd953bb2360134de73519d5df19d29ca/websockets-16.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:8b6e209ffee39ff1b6d0fa7bfef6de950c60dfb91b8fcead17da4ee539121a79", size = 175038 }, + { url = "https://files.pythonhosted.org/packages/40/1e/9771421ac2286eaab95b8575b0cb701ae3663abf8b5e1f64f1fd90d0a673/websockets-16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:86890e837d61574c92a97496d590968b23c2ef0aeb8a9bc9421d174cd378ae39", size = 175328 }, + { url = "https://files.pythonhosted.org/packages/18/29/71729b4671f21e1eaa5d6573031ab810ad2936c8175f03f97f3ff164c802/websockets-16.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:9b5aca38b67492ef518a8ab76851862488a478602229112c4b0d58d63a7a4d5c", size = 184915 }, + { url = "https://files.pythonhosted.org/packages/97/bb/21c36b7dbbafc85d2d480cd65df02a1dc93bf76d97147605a8e27ff9409d/websockets-16.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e0334872c0a37b606418ac52f6ab9cfd17317ac26365f7f65e203e2d0d0d359f", size = 186152 }, + { url = "https://files.pythonhosted.org/packages/4a/34/9bf8df0c0cf88fa7bfe36678dc7b02970c9a7d5e065a3099292db87b1be2/websockets-16.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a0b31e0b424cc6b5a04b8838bbaec1688834b2383256688cf47eb97412531da1", size = 185583 }, + { url = "https://files.pythonhosted.org/packages/47/88/4dd516068e1a3d6ab3c7c183288404cd424a9a02d585efbac226cb61ff2d/websockets-16.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:485c49116d0af10ac698623c513c1cc01c9446c058a4e61e3bf6c19dff7335a2", size = 184880 }, + { url = "https://files.pythonhosted.org/packages/91/d6/7d4553ad4bf1c0421e1ebd4b18de5d9098383b5caa1d937b63df8d04b565/websockets-16.0-cp312-cp312-win32.whl", hash = "sha256:eaded469f5e5b7294e2bdca0ab06becb6756ea86894a47806456089298813c89", size = 178261 }, + { url = "https://files.pythonhosted.org/packages/c3/f0/f3a17365441ed1c27f850a80b2bc680a0fa9505d733fe152fdf5e98c1c0b/websockets-16.0-cp312-cp312-win_amd64.whl", hash = "sha256:5569417dc80977fc8c2d43a86f78e0a5a22fee17565d78621b6bb264a115d4ea", size = 178693 }, + { url = "https://files.pythonhosted.org/packages/cc/9c/baa8456050d1c1b08dd0ec7346026668cbc6f145ab4e314d707bb845bf0d/websockets-16.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:878b336ac47938b474c8f982ac2f7266a540adc3fa4ad74ae96fea9823a02cc9", size = 177364 }, + { url = "https://files.pythonhosted.org/packages/7e/0c/8811fc53e9bcff68fe7de2bcbe75116a8d959ac699a3200f4847a8925210/websockets-16.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:52a0fec0e6c8d9a784c2c78276a48a2bdf099e4ccc2a4cad53b27718dbfd0230", size = 175039 }, + { url = "https://files.pythonhosted.org/packages/aa/82/39a5f910cb99ec0b59e482971238c845af9220d3ab9fa76dd9162cda9d62/websockets-16.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e6578ed5b6981005df1860a56e3617f14a6c307e6a71b4fff8c48fdc50f3ed2c", size = 175323 }, + { url = "https://files.pythonhosted.org/packages/bd/28/0a25ee5342eb5d5f297d992a77e56892ecb65e7854c7898fb7d35e9b33bd/websockets-16.0-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:95724e638f0f9c350bb1c2b0a7ad0e83d9cc0c9259f3ea94e40d7b02a2179ae5", size = 184975 }, + { url = "https://files.pythonhosted.org/packages/f9/66/27ea52741752f5107c2e41fda05e8395a682a1e11c4e592a809a90c6a506/websockets-16.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c0204dc62a89dc9d50d682412c10b3542d748260d743500a85c13cd1ee4bde82", size = 186203 }, + { url = "https://files.pythonhosted.org/packages/37/e5/8e32857371406a757816a2b471939d51c463509be73fa538216ea52b792a/websockets-16.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:52ac480f44d32970d66763115edea932f1c5b1312de36df06d6b219f6741eed8", size = 185653 }, + { url = "https://files.pythonhosted.org/packages/9b/67/f926bac29882894669368dc73f4da900fcdf47955d0a0185d60103df5737/websockets-16.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6e5a82b677f8f6f59e8dfc34ec06ca6b5b48bc4fcda346acd093694cc2c24d8f", size = 184920 }, + { url = "https://files.pythonhosted.org/packages/3c/a1/3d6ccdcd125b0a42a311bcd15a7f705d688f73b2a22d8cf1c0875d35d34a/websockets-16.0-cp313-cp313-win32.whl", hash = "sha256:abf050a199613f64c886ea10f38b47770a65154dc37181bfaff70c160f45315a", size = 178255 }, + { url = "https://files.pythonhosted.org/packages/6b/ae/90366304d7c2ce80f9b826096a9e9048b4bb760e44d3b873bb272cba696b/websockets-16.0-cp313-cp313-win_amd64.whl", hash = "sha256:3425ac5cf448801335d6fdc7ae1eb22072055417a96cc6b31b3861f455fbc156", size = 178689 }, + { url = "https://files.pythonhosted.org/packages/f3/1d/e88022630271f5bd349ed82417136281931e558d628dd52c4d8621b4a0b2/websockets-16.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:8cc451a50f2aee53042ac52d2d053d08bf89bcb31ae799cb4487587661c038a0", size = 177406 }, + { url = "https://files.pythonhosted.org/packages/f2/78/e63be1bf0724eeb4616efb1ae1c9044f7c3953b7957799abb5915bffd38e/websockets-16.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:daa3b6ff70a9241cf6c7fc9e949d41232d9d7d26fd3522b1ad2b4d62487e9904", size = 175085 }, + { url = "https://files.pythonhosted.org/packages/bb/f4/d3c9220d818ee955ae390cf319a7c7a467beceb24f05ee7aaaa2414345ba/websockets-16.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:fd3cb4adb94a2a6e2b7c0d8d05cb94e6f1c81a0cf9dc2694fb65c7e8d94c42e4", size = 175328 }, + { url = "https://files.pythonhosted.org/packages/63/bc/d3e208028de777087e6fb2b122051a6ff7bbcca0d6df9d9c2bf1dd869ae9/websockets-16.0-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:781caf5e8eee67f663126490c2f96f40906594cb86b408a703630f95550a8c3e", size = 185044 }, + { url = "https://files.pythonhosted.org/packages/ad/6e/9a0927ac24bd33a0a9af834d89e0abc7cfd8e13bed17a86407a66773cc0e/websockets-16.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:caab51a72c51973ca21fa8a18bd8165e1a0183f1ac7066a182ff27107b71e1a4", size = 186279 }, + { url = "https://files.pythonhosted.org/packages/b9/ca/bf1c68440d7a868180e11be653c85959502efd3a709323230314fda6e0b3/websockets-16.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:19c4dc84098e523fd63711e563077d39e90ec6702aff4b5d9e344a60cb3c0cb1", size = 185711 }, + { url = "https://files.pythonhosted.org/packages/c4/f8/fdc34643a989561f217bb477cbc47a3a07212cbda91c0e4389c43c296ebf/websockets-16.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:a5e18a238a2b2249c9a9235466b90e96ae4795672598a58772dd806edc7ac6d3", size = 184982 }, + { url = "https://files.pythonhosted.org/packages/dd/d1/574fa27e233764dbac9c52730d63fcf2823b16f0856b3329fc6268d6ae4f/websockets-16.0-cp314-cp314-win32.whl", hash = "sha256:a069d734c4a043182729edd3e9f247c3b2a4035415a9172fd0f1b71658a320a8", size = 177915 }, + { url = "https://files.pythonhosted.org/packages/8a/f1/ae6b937bf3126b5134ce1f482365fde31a357c784ac51852978768b5eff4/websockets-16.0-cp314-cp314-win_amd64.whl", hash = "sha256:c0ee0e63f23914732c6d7e0cce24915c48f3f1512ec1d079ed01fc629dab269d", size = 178381 }, + { url = "https://files.pythonhosted.org/packages/06/9b/f791d1db48403e1f0a27577a6beb37afae94254a8c6f08be4a23e4930bc0/websockets-16.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:a35539cacc3febb22b8f4d4a99cc79b104226a756aa7400adc722e83b0d03244", size = 177737 }, + { url = "https://files.pythonhosted.org/packages/bd/40/53ad02341fa33b3ce489023f635367a4ac98b73570102ad2cdd770dacc9a/websockets-16.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:b784ca5de850f4ce93ec85d3269d24d4c82f22b7212023c974c401d4980ebc5e", size = 175268 }, + { url = "https://files.pythonhosted.org/packages/74/9b/6158d4e459b984f949dcbbb0c5d270154c7618e11c01029b9bbd1bb4c4f9/websockets-16.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:569d01a4e7fba956c5ae4fc988f0d4e187900f5497ce46339c996dbf24f17641", size = 175486 }, + { url = "https://files.pythonhosted.org/packages/e5/2d/7583b30208b639c8090206f95073646c2c9ffd66f44df967981a64f849ad/websockets-16.0-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:50f23cdd8343b984957e4077839841146f67a3d31ab0d00e6b824e74c5b2f6e8", size = 185331 }, + { url = "https://files.pythonhosted.org/packages/45/b0/cce3784eb519b7b5ad680d14b9673a31ab8dcb7aad8b64d81709d2430aa8/websockets-16.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:152284a83a00c59b759697b7f9e9cddf4e3c7861dd0d964b472b70f78f89e80e", size = 186501 }, + { url = "https://files.pythonhosted.org/packages/19/60/b8ebe4c7e89fb5f6cdf080623c9d92789a53636950f7abacfc33fe2b3135/websockets-16.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bc59589ab64b0022385f429b94697348a6a234e8ce22544e3681b2e9331b5944", size = 186062 }, + { url = "https://files.pythonhosted.org/packages/88/a8/a080593f89b0138b6cba1b28f8df5673b5506f72879322288b031337c0b8/websockets-16.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:32da954ffa2814258030e5a57bc73a3635463238e797c7375dc8091327434206", size = 185356 }, + { url = "https://files.pythonhosted.org/packages/c2/b6/b9afed2afadddaf5ebb2afa801abf4b0868f42f8539bfe4b071b5266c9fe/websockets-16.0-cp314-cp314t-win32.whl", hash = "sha256:5a4b4cc550cb665dd8a47f868c8d04c8230f857363ad3c9caf7a0c3bf8c61ca6", size = 178085 }, + { url = "https://files.pythonhosted.org/packages/9f/3e/28135a24e384493fa804216b79a6a6759a38cc4ff59118787b9fb693df93/websockets-16.0-cp314-cp314t-win_amd64.whl", hash = "sha256:b14dc141ed6d2dde437cddb216004bcac6a1df0935d79656387bd41632ba0bbd", size = 178531 }, + { url = "https://files.pythonhosted.org/packages/6f/28/258ebab549c2bf3e64d2b0217b973467394a9cea8c42f70418ca2c5d0d2e/websockets-16.0-py3-none-any.whl", hash = "sha256:1637db62fad1dc833276dded54215f2c7fa46912301a24bd94d45d46a011ceec", size = 171598 }, +]