feat: add compose deliverable, scenario definitions, eval stub, and CI
This commit is contained in:
parent
ef9e102b05
commit
7c3934daee
8 changed files with 148 additions and 0 deletions
13
.env.example
Normal file
13
.env.example
Normal file
|
|
@ -0,0 +1,13 @@
|
||||||
|
# Without this file (or with it empty) the app starts in demo mode:
|
||||||
|
# recorded sessions, honestly labeled, no external calls.
|
||||||
|
|
||||||
|
APP_MODE=live
|
||||||
|
|
||||||
|
# Spotify application (developer dashboard). The redirect URI must be
|
||||||
|
# registered on the app and match the compose host port.
|
||||||
|
SPOTIFY_CLIENT_ID=
|
||||||
|
SPOTIFY_CLIENT_SECRET=
|
||||||
|
SPOTIFY_REDIRECT_URI=http://127.0.0.1:8888/callback
|
||||||
|
|
||||||
|
# Anthropic API key
|
||||||
|
ANTHROPIC_API_KEY=
|
||||||
38
.github/workflows/ci.yml
vendored
Normal file
38
.github/workflows/ci.yml
vendored
Normal file
|
|
@ -0,0 +1,38 @@
|
||||||
|
name: ci
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
branches: [main]
|
||||||
|
pull_request:
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
backend:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
defaults:
|
||||||
|
run:
|
||||||
|
working-directory: backend
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
- uses: astral-sh/setup-uv@v5
|
||||||
|
- run: uv sync --frozen
|
||||||
|
- run: uv run ruff check .
|
||||||
|
- run: uv run ruff format --check .
|
||||||
|
- run: uv run mypy app
|
||||||
|
- run: uv run pytest
|
||||||
|
|
||||||
|
frontend:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
defaults:
|
||||||
|
run:
|
||||||
|
working-directory: frontend
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
- uses: actions/setup-node@v4
|
||||||
|
with:
|
||||||
|
node-version: 20
|
||||||
|
cache: npm
|
||||||
|
cache-dependency-path: frontend/package-lock.json
|
||||||
|
- run: npm ci
|
||||||
|
- run: npm run typecheck
|
||||||
|
- run: npm run lint
|
||||||
|
- run: npm run build
|
||||||
17
backend/Dockerfile
Normal file
17
backend/Dockerfile
Normal file
|
|
@ -0,0 +1,17 @@
|
||||||
|
# Build context is the repository root: one image serves API + built SPA.
|
||||||
|
FROM node:20-slim AS frontend
|
||||||
|
WORKDIR /build
|
||||||
|
COPY frontend/package.json frontend/package-lock.json ./
|
||||||
|
RUN npm ci
|
||||||
|
COPY frontend/ ./
|
||||||
|
RUN npm run build
|
||||||
|
|
||||||
|
FROM ghcr.io/astral-sh/uv:python3.13-bookworm-slim
|
||||||
|
WORKDIR /srv
|
||||||
|
COPY backend/pyproject.toml backend/uv.lock ./
|
||||||
|
RUN uv sync --frozen --no-dev --no-install-project
|
||||||
|
COPY backend/app ./app
|
||||||
|
COPY eval/scenarios.yaml ./eval/scenarios.yaml
|
||||||
|
COPY --from=frontend /build/dist ./app/static
|
||||||
|
EXPOSE 8000
|
||||||
|
CMD ["uv", "run", "--no-sync", "uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"]
|
||||||
12
docker-compose.yml
Normal file
12
docker-compose.yml
Normal file
|
|
@ -0,0 +1,12 @@
|
||||||
|
services:
|
||||||
|
app:
|
||||||
|
build:
|
||||||
|
context: .
|
||||||
|
dockerfile: backend/Dockerfile
|
||||||
|
env_file:
|
||||||
|
- path: .env
|
||||||
|
required: false
|
||||||
|
ports:
|
||||||
|
# host port is fixed: the registered Spotify redirect URI
|
||||||
|
# (http://127.0.0.1:8888/callback) must match exactly
|
||||||
|
- "127.0.0.1:8888:8000"
|
||||||
0
eval/fixtures/.gitkeep
Normal file
0
eval/fixtures/.gitkeep
Normal file
23
eval/run_eval.py
Normal file
23
eval/run_eval.py
Normal file
|
|
@ -0,0 +1,23 @@
|
||||||
|
"""Eval harness entry point.
|
||||||
|
|
||||||
|
Fixture mode (default, key-free): byte-snapshots of deterministic pipeline
|
||||||
|
stages against recorded cassettes. Live mode: property assertions only —
|
||||||
|
LLM output is constrained, not pinned.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import yaml
|
||||||
|
|
||||||
|
SCENARIOS_FILE = Path(__file__).parent / "scenarios.yaml"
|
||||||
|
|
||||||
|
|
||||||
|
def load_scenarios() -> dict:
|
||||||
|
"""Load the shared scenario definitions."""
|
||||||
|
with SCENARIOS_FILE.open() as handle:
|
||||||
|
return yaml.safe_load(handle)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
scenarios = load_scenarios()
|
||||||
|
print(f"{len(scenarios['scenarios'])} scenarios, {len(scenarios['refinements'])} refinements")
|
||||||
45
eval/scenarios.yaml
Normal file
45
eval/scenarios.yaml
Normal file
|
|
@ -0,0 +1,45 @@
|
||||||
|
# One source of truth for three consumers: golden eval queries, demo fixture
|
||||||
|
# keys, and UI suggestion chips. Every chip therefore has a fixture, and every
|
||||||
|
# fixture has a golden query.
|
||||||
|
|
||||||
|
scenarios:
|
||||||
|
- key: focus-coding
|
||||||
|
chip: "Focus while coding"
|
||||||
|
query: "something calm for while I'm programming, but not boring"
|
||||||
|
facets: { activity: coding, familiarity: mixed }
|
||||||
|
|
||||||
|
- key: workout-energy
|
||||||
|
chip: "Energy for the gym"
|
||||||
|
query: "high-energy music for a heavy workout"
|
||||||
|
facets: { activity: workout, mood: energetic }
|
||||||
|
|
||||||
|
- key: dutch-chill
|
||||||
|
chip: "Dutch-language & relaxed"
|
||||||
|
query: "relaxed Dutch-language music for lounging on the couch"
|
||||||
|
facets: { language: nl, mood: relaxed }
|
||||||
|
|
||||||
|
- key: nineties-nostalgia
|
||||||
|
chip: "90s nostalgia"
|
||||||
|
query: "take me back to the nineties"
|
||||||
|
facets: { era: "1990s" }
|
||||||
|
|
||||||
|
- key: discover-new
|
||||||
|
chip: "Surprise me with something new"
|
||||||
|
query: "something I don't know yet but will probably like"
|
||||||
|
facets: { familiarity: new } # asserts known-track exclusion
|
||||||
|
|
||||||
|
- key: rainy-sunday
|
||||||
|
chip: "Rainy Sunday"
|
||||||
|
query: "melancholic but warm, for a rainy Sunday morning"
|
||||||
|
facets: { mood: melancholic }
|
||||||
|
|
||||||
|
- key: dinner-background
|
||||||
|
chip: "Background for dinner"
|
||||||
|
query: "something jazzy for during dinner, not too present"
|
||||||
|
facets: { mood: understated }
|
||||||
|
|
||||||
|
# Multi-turn refinement goldens: reuse the grounded pool, no re-grounding.
|
||||||
|
refinements:
|
||||||
|
- key: focus-coding-refine
|
||||||
|
after: focus-coding
|
||||||
|
query: "a bit more electronic, and drop number 3"
|
||||||
0
eval/snapshots/.gitkeep
Normal file
0
eval/snapshots/.gitkeep
Normal file
Loading…
Add table
Add a link
Reference in a new issue