first commit
This commit is contained in:
commit
b47f825d54
83 changed files with 10016 additions and 0 deletions
1
ref/.github/FUNDING.yml
vendored
Normal file
1
ref/.github/FUNDING.yml
vendored
Normal file
|
|
@ -0,0 +1 @@
|
|||
buy_me_a_coffee: justinzeus
|
||||
14
ref/.github/dependabot.yml
vendored
Normal file
14
ref/.github/dependabot.yml
vendored
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
version: 2
|
||||
updates:
|
||||
- package-ecosystem: pip
|
||||
directory: /
|
||||
schedule:
|
||||
interval: weekly
|
||||
- package-ecosystem: npm
|
||||
directory: /frontend
|
||||
schedule:
|
||||
interval: weekly
|
||||
- package-ecosystem: github-actions
|
||||
directory: /
|
||||
schedule:
|
||||
interval: weekly
|
||||
BIN
ref/.github/logo-dark.png
vendored
Normal file
BIN
ref/.github/logo-dark.png
vendored
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 87 KiB |
197
ref/.github/workflows/ci.yml
vendored
Normal file
197
ref/.github/workflows/ci.yml
vendored
Normal file
|
|
@ -0,0 +1,197 @@
|
|||
name: CI
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
|
||||
jobs:
|
||||
repo-hygiene:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Enforce no generated artifacts
|
||||
run: ./scripts/check_no_generated_artifacts.sh
|
||||
|
||||
- name: Enforce env contract parity
|
||||
run: python3 scripts/check_env_contract.py
|
||||
|
||||
lint:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [repo-hygiene]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.12"
|
||||
- uses: astral-sh/setup-uv@v4
|
||||
- run: uv sync --extra dev
|
||||
- name: Ruff check
|
||||
run: uv run ruff check .
|
||||
- name: Ruff format check
|
||||
run: uv run ruff format --check .
|
||||
- name: Mypy
|
||||
run: uv run mypy app/ --ignore-missing-imports
|
||||
|
||||
test:
|
||||
runs-on: ubuntu-latest
|
||||
needs:
|
||||
- repo-hygiene
|
||||
- lint
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:15
|
||||
env:
|
||||
POSTGRES_DB: scholar
|
||||
POSTGRES_USER: scholar
|
||||
POSTGRES_PASSWORD: scholar
|
||||
ports:
|
||||
- 5432:5432
|
||||
options: >-
|
||||
--health-cmd "pg_isready -U scholar -d scholar"
|
||||
--health-interval 5s
|
||||
--health-timeout 5s
|
||||
--health-retries 20
|
||||
|
||||
env:
|
||||
DATABASE_URL: postgresql+asyncpg://scholar:scholar@localhost:5432/scholar
|
||||
SCHOLAR_IMAGE_UPLOAD_DIR: /tmp/scholarr_uploads/scholar_images
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Setup Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- name: Setup uv
|
||||
uses: astral-sh/setup-uv@v4
|
||||
|
||||
- name: Install dependencies
|
||||
run: uv sync --extra dev
|
||||
|
||||
- name: Migration smoke
|
||||
run: uv run alembic upgrade head
|
||||
|
||||
- name: Unit tests
|
||||
run: uv run pytest tests/unit
|
||||
|
||||
- name: Integration tests
|
||||
run: uv run pytest -m integration
|
||||
|
||||
frontend-quality:
|
||||
runs-on: ubuntu-latest
|
||||
needs:
|
||||
- repo-hygiene
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Setup Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- name: API contract drift check
|
||||
run: python3 scripts/check_frontend_api_contract.py
|
||||
|
||||
- name: Setup Node
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: "20"
|
||||
|
||||
- name: Install frontend dependencies
|
||||
working-directory: frontend
|
||||
run: npm ci
|
||||
|
||||
- name: Frontend theme token policy
|
||||
working-directory: frontend
|
||||
run: npm run check:theme-tokens
|
||||
|
||||
- name: Frontend typecheck
|
||||
working-directory: frontend
|
||||
run: npm run typecheck
|
||||
|
||||
- name: Frontend unit tests
|
||||
working-directory: frontend
|
||||
run: npm run test:run
|
||||
|
||||
- name: Frontend build
|
||||
working-directory: frontend
|
||||
run: npm run build
|
||||
|
||||
docs-quality:
|
||||
runs-on: ubuntu-latest
|
||||
needs:
|
||||
- repo-hygiene
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Setup Node
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: "20"
|
||||
|
||||
- name: Install docs dependencies
|
||||
run: npm ci --prefix docs/website
|
||||
|
||||
- name: Docs build
|
||||
run: npm --prefix docs/website run build
|
||||
|
||||
docker-publish:
|
||||
runs-on: ubuntu-latest
|
||||
needs:
|
||||
- repo-hygiene
|
||||
- test
|
||||
- frontend-quality
|
||||
- docs-quality
|
||||
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_PASSWORD }}
|
||||
|
||||
- name: Derive image metadata
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: justinzeus/scholarr
|
||||
tags: |
|
||||
type=raw,value=latest
|
||||
type=sha,format=short,prefix=sha-
|
||||
|
||||
- name: Build and push multi-arch image
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: .
|
||||
file: ./Dockerfile
|
||||
target: prod
|
||||
push: true
|
||||
platforms: linux/amd64,linux/arm64
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
26
ref/.github/workflows/codeql.yml
vendored
Normal file
26
ref/.github/workflows/codeql.yml
vendored
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
name: CodeQL
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
pull_request:
|
||||
branches: [main]
|
||||
schedule:
|
||||
- cron: "30 5 * * 1"
|
||||
|
||||
jobs:
|
||||
analyze:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
security-events: write
|
||||
contents: read
|
||||
strategy:
|
||||
matrix:
|
||||
language: [python, javascript]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: github/codeql-action/init@v3
|
||||
with:
|
||||
languages: ${{ matrix.language }}
|
||||
- uses: github/codeql-action/autobuild@v3
|
||||
- uses: github/codeql-action/analyze@v3
|
||||
57
ref/.github/workflows/docs-pages.yml
vendored
Normal file
57
ref/.github/workflows/docs-pages.yml
vendored
Normal file
|
|
@ -0,0 +1,57 @@
|
|||
name: Docs Pages
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
paths:
|
||||
- docs/**
|
||||
- .github/workflows/docs-pages.yml
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: docs-pages
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Setup Pages
|
||||
uses: actions/configure-pages@v5
|
||||
|
||||
- name: Setup Node
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: "20"
|
||||
|
||||
- name: Install docs dependencies
|
||||
run: npm ci --prefix docs/website
|
||||
|
||||
- name: Build docs site
|
||||
run: npm --prefix docs/website run build
|
||||
|
||||
- name: Upload pages artifact
|
||||
uses: actions/upload-pages-artifact@v3
|
||||
with:
|
||||
path: docs/website/build
|
||||
|
||||
deploy:
|
||||
needs: build
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
pages: write
|
||||
id-token: write
|
||||
environment:
|
||||
name: github-pages
|
||||
url: ${{ steps.deployment.outputs.page_url }}
|
||||
steps:
|
||||
- name: Deploy to GitHub Pages
|
||||
id: deployment
|
||||
uses: actions/deploy-pages@v4
|
||||
32
ref/.github/workflows/release.yml
vendored
Normal file
32
ref/.github/workflows/release.yml
vendored
Normal file
|
|
@ -0,0 +1,32 @@
|
|||
name: Release
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
|
||||
jobs:
|
||||
release:
|
||||
runs-on: ubuntu-latest
|
||||
if: github.repository == 'JustinZeus/scholarr'
|
||||
permissions:
|
||||
contents: write
|
||||
id-token: write
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.12"
|
||||
- uses: astral-sh/setup-uv@v4
|
||||
- run: uv sync --extra dev
|
||||
- name: Lint
|
||||
run: uv run ruff check
|
||||
- name: Test
|
||||
run: uv run pytest
|
||||
- name: Semantic Release
|
||||
id: release
|
||||
run: uv run semantic-release publish
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
54
ref/.github/workflows/scheduled-probes.yml
vendored
Normal file
54
ref/.github/workflows/scheduled-probes.yml
vendored
Normal file
|
|
@ -0,0 +1,54 @@
|
|||
name: Scheduled Probes
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
schedule:
|
||||
- cron: "20 6 * * *"
|
||||
|
||||
jobs:
|
||||
fixture-probes:
|
||||
runs-on: ubuntu-latest
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:15
|
||||
env:
|
||||
POSTGRES_DB: scholar
|
||||
POSTGRES_USER: scholar
|
||||
POSTGRES_PASSWORD: scholar
|
||||
ports:
|
||||
- 5432:5432
|
||||
options: >-
|
||||
--health-cmd "pg_isready -U scholar -d scholar"
|
||||
--health-interval 5s
|
||||
--health-timeout 5s
|
||||
--health-retries 20
|
||||
|
||||
env:
|
||||
DATABASE_URL: postgresql+asyncpg://scholar:scholar@localhost:5432/scholar
|
||||
SCHOLAR_IMAGE_UPLOAD_DIR: /tmp/scholarr_uploads/scholar_images
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Setup Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- name: Setup uv
|
||||
uses: astral-sh/setup-uv@v4
|
||||
|
||||
- name: Install dependencies
|
||||
run: uv sync --extra dev
|
||||
|
||||
- name: Fixture-backed parser unit probes
|
||||
run: >-
|
||||
uv run pytest
|
||||
tests/unit/test_scholar_parser.py
|
||||
tests/unit/test_scholar_search_safety.py
|
||||
|
||||
- name: Fixture-backed integration probe
|
||||
run: >-
|
||||
uv run pytest -m integration
|
||||
tests/integration/test_fixture_probe_runs.py
|
||||
67
ref/Dockerfile
Normal file
67
ref/Dockerfile
Normal file
|
|
@ -0,0 +1,67 @@
|
|||
FROM node:20-alpine AS frontend-builder
|
||||
|
||||
WORKDIR /frontend
|
||||
|
||||
COPY frontend/package.json frontend/package-lock.json ./
|
||||
RUN npm ci
|
||||
|
||||
COPY frontend/ ./
|
||||
RUN npm run build
|
||||
|
||||
FROM python:3.12-slim AS base
|
||||
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
UV_LINK_MODE=copy \
|
||||
UV_PROJECT_ENVIRONMENT=/opt/venv \
|
||||
PATH="/opt/venv/bin:$PATH"
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends build-essential curl \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
COPY --from=ghcr.io/astral-sh/uv:0.6.5 /uv /uvx /bin/
|
||||
|
||||
COPY pyproject.toml uv.lock README.md ./
|
||||
COPY app ./app
|
||||
COPY alembic.ini ./alembic.ini
|
||||
COPY alembic ./alembic
|
||||
COPY scripts ./scripts
|
||||
|
||||
RUN uv sync --frozen --extra dev
|
||||
|
||||
FROM base AS dev
|
||||
ENTRYPOINT ["/bin/sh", "/app/scripts/entrypoint.sh"]
|
||||
|
||||
FROM python:3.12-slim AS prod
|
||||
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
UV_LINK_MODE=copy \
|
||||
UV_PROJECT_ENVIRONMENT=/opt/venv \
|
||||
PATH="/opt/venv/bin:$PATH" \
|
||||
APP_RELOAD=0 \
|
||||
UVICORN_WORKERS=1 \
|
||||
FRONTEND_ENABLED=1 \
|
||||
FRONTEND_DIST_DIR=/app/frontend/dist
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends curl \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
COPY --from=ghcr.io/astral-sh/uv:0.6.5 /uv /uvx /bin/
|
||||
|
||||
COPY pyproject.toml uv.lock README.md ./
|
||||
COPY app ./app
|
||||
COPY alembic.ini ./alembic.ini
|
||||
COPY alembic ./alembic
|
||||
|
||||
RUN uv sync --frozen
|
||||
|
||||
COPY scripts ./scripts
|
||||
COPY --from=frontend-builder /frontend/dist /app/frontend/dist
|
||||
ENTRYPOINT ["/bin/sh", "/app/scripts/entrypoint.sh"]
|
||||
149
ref/README.md
Normal file
149
ref/README.md
Normal file
|
|
@ -0,0 +1,149 @@
|
|||
<div align="center">
|
||||
|
||||
<picture>
|
||||
<source media="(prefers-color-scheme: dark)" srcset=".github/logo-dark.png" />
|
||||
<source media="(prefers-color-scheme: light)" srcset="frontend/public/scholar_logo.png" />
|
||||
<img src="frontend/public/scholar_logo.png" alt="Scholarr" width="120" />
|
||||
</picture>
|
||||
|
||||
# Scholarr
|
||||
|
||||
**Self-hosted academic publication tracker.**
|
||||
|
||||
Track Google Scholar profiles, discover new papers automatically,
|
||||
resolve open-access PDFs, and stay on top of the literature you care about.
|
||||
|
||||
[](https://github.com/JustinZeus/scholarr/actions/workflows/ci.yml)
|
||||
[](https://github.com/JustinZeus/scholarr/actions/workflows/codeql.yml)
|
||||
[](https://github.com/JustinZeus/scholarr/releases)
|
||||
[](https://hub.docker.com/r/justinzeus/scholarr)
|
||||
[](https://www.python.org/)
|
||||
[](LICENSE)
|
||||
[](https://justinzeus.github.io/scholarr/)
|
||||
|
||||
</div>
|
||||
|
||||
---
|
||||
|
||||
<!-- TODO: Replace these placeholders with actual screenshots.
|
||||
Drop PNG/JPEG files into docs/assets/ and update the paths below.
|
||||
|
||||
Recommended screenshots:
|
||||
1. Dashboard / publications list (light or dark mode)
|
||||
2. Scholar profile view
|
||||
3. Run detail with progress
|
||||
|
||||
Ideal dimensions: 1200-1400px wide, PNG or WebP.
|
||||
-->
|
||||
|
||||
<p align="center">
|
||||
<img src="docs/assets/screenshot-dashboard.png" alt="Publications dashboard" width="720" />
|
||||
</p>
|
||||
|
||||
<!--
|
||||
<p align="center">
|
||||
<img src="docs/assets/screenshot-scholars.png" alt="Scholar profiles" width="360" />
|
||||
<img src="docs/assets/screenshot-run.png" alt="Ingestion run" width="360" />
|
||||
</p>
|
||||
-->
|
||||
|
||||
## Why Scholarr?
|
||||
|
||||
Most researchers track new papers by manually checking Google Scholar, setting up email alerts, or juggling RSS feeds. Scholarr replaces all of that with a single self-hosted service:
|
||||
|
||||
- **Add scholars once** -- by profile URL, Scholar ID, or name search
|
||||
- **Publications appear automatically** -- a background scheduler scrapes profiles on a configurable interval
|
||||
- **Open-access PDFs are resolved for you** -- Unpaywall and arXiv are queried automatically when a DOI is found
|
||||
- **Everything is deduplicated** -- publications are global records; no duplicates across scholars
|
||||
- **Your data stays yours** -- fully self-hosted, export/import your entire library at any time
|
||||
|
||||
## Features
|
||||
|
||||
| | |
|
||||
|---|---|
|
||||
| **Automated Ingestion** | Background scheduler with configurable intervals, continuation queue, and multi-page pagination |
|
||||
| **Identifier Resolution** | Cross-references arXiv, Crossref, and OpenAlex to gather DOIs, arXiv IDs, PMIDs |
|
||||
| **PDF Discovery** | Resolves open-access PDFs via Unpaywall API and arXiv, with automatic retry queue |
|
||||
| **Scrape Safety** | Rate limiting, cooldowns, and backoff strategies that prevent IP bans -- these are safety floors, not optional |
|
||||
| **Multi-User** | Session-based auth, admin user management, user-scoped scholar tracking |
|
||||
| **Theming** | 7 color presets with light/dark mode, tokenized component system |
|
||||
| **Import / Export** | Portable scholar data with full publication and read-state preservation |
|
||||
| **Single Container** | FastAPI backend + Vue 3 frontend ship as one Docker image |
|
||||
|
||||
## Quick Start
|
||||
|
||||
```bash
|
||||
# 1. Clone and configure
|
||||
git clone https://github.com/JustinZeus/scholarr.git
|
||||
cd scholarr
|
||||
cp .env.example .env
|
||||
|
||||
# 2. Set required secrets in .env
|
||||
# POSTGRES_PASSWORD=<secure-password>
|
||||
# SESSION_SECRET_KEY=<random-32-char-string>
|
||||
|
||||
# 3. Start
|
||||
docker compose up -d
|
||||
|
||||
# 4. Open http://localhost:8000
|
||||
```
|
||||
|
||||
To bootstrap an admin account on first run, add to `.env`:
|
||||
|
||||
```bash
|
||||
BOOTSTRAP_ADMIN_ON_START=1
|
||||
BOOTSTRAP_ADMIN_EMAIL=admin@example.com
|
||||
BOOTSTRAP_ADMIN_PASSWORD=<secure-password>
|
||||
```
|
||||
|
||||
## How It Works
|
||||
|
||||
```mermaid
|
||||
graph LR
|
||||
UI[Vue 3 Dashboard] <-->|REST + SSE| API[FastAPI]
|
||||
API --> Scheduler[Scheduler]
|
||||
Scheduler -->|Scrape HTML| Scholar[Google Scholar]
|
||||
Scholar -->|Parse & Deduplicate| DB[(PostgreSQL)]
|
||||
Scholar -.->|Identify| Ext[arXiv / Crossref / OpenAlex]
|
||||
Ext --> DB
|
||||
DB -->|DOIs| PDF[PDF Resolution]
|
||||
PDF -->|Unpaywall / arXiv| DB
|
||||
API <--> DB
|
||||
```
|
||||
|
||||
## Tech Stack
|
||||
|
||||
| Layer | Technology |
|
||||
|-------|------------|
|
||||
| Backend | Python 3.12, FastAPI, SQLAlchemy 2.0 (async), Alembic |
|
||||
| Frontend | TypeScript, Vue 3, Vite, Tailwind CSS |
|
||||
| Database | PostgreSQL 15 |
|
||||
| Infrastructure | Multi-stage Docker, Docker Compose |
|
||||
|
||||
## Documentation
|
||||
|
||||
Full documentation: **[justinzeus.github.io/scholarr](https://justinzeus.github.io/scholarr/)**
|
||||
|
||||
| Section | Covers |
|
||||
|---------|--------|
|
||||
| [User Guide](docs/user/overview.md) | Installation, configuration, all environment variables |
|
||||
| [Developer Guide](docs/developer/overview.md) | Architecture, local dev, contributing, testing |
|
||||
| [Operations](docs/operations/overview.md) | Deployment, database runbook, scrape safety |
|
||||
| [API Reference](docs/reference/api.md) | Envelope spec, all endpoints, DTO contracts |
|
||||
|
||||
## Contributing
|
||||
|
||||
Scholarr uses [conventional commits](https://www.conventionalcommits.org/) and [semantic versioning](https://semver.org/). See the [contributing guide](docs/developer/contributing.md) for PR process and code standards.
|
||||
|
||||
```bash
|
||||
# Dev environment
|
||||
docker compose -f docker-compose.yml -f docker-compose.dev.yml up --build
|
||||
|
||||
# Run tests (always in containers)
|
||||
docker compose -f docker-compose.yml -f docker-compose.dev.yml run --rm app \
|
||||
python -m pytest
|
||||
```
|
||||
|
||||
## License
|
||||
|
||||
See [LICENSE](LICENSE) for details.
|
||||
Loading…
Add table
Add a link
Reference in a new issue