First product

This commit is contained in:
Justin Visser 2026-02-17 14:51:25 +01:00
parent 778da9e2fc
commit 4433d7d2c4
157 changed files with 23975 additions and 0 deletions

204
tests/unit/test_auth.py Normal file
View file

@ -0,0 +1,204 @@
import re
from collections.abc import AsyncIterator
from types import SimpleNamespace
from unittest.mock import AsyncMock
from fastapi.testclient import TestClient
import pytest
from app.auth.deps import get_auth_service, get_login_rate_limiter
from app.auth.rate_limit import SlidingWindowRateLimiter
from app.db.session import get_db_session
from app.main import app
CSRF_TOKEN_PATTERN = re.compile(r'name="csrf_token" value="([^"]+)"')
class StubAuthService:
def __init__(self, *, user: object | None) -> None:
self._user = user
async def authenticate_user(self, _db_session, *, email: str, password: str):
if self._user is None:
return None
if email.strip().lower() != str(self._user.email):
return None
if password != "correct-password":
return None
return self._user
def _extract_csrf_token(html: str) -> str:
match = CSRF_TOKEN_PATTERN.search(html)
assert match is not None
return match.group(1)
async def _override_db_session() -> AsyncIterator[object]:
yield object()
@pytest.fixture(autouse=True)
def clear_dependency_overrides() -> AsyncIterator[None]:
app.dependency_overrides.clear()
yield
app.dependency_overrides.clear()
def test_login_requires_csrf_token() -> None:
client = TestClient(app)
response = client.post(
"/login",
data={"email": "user@example.com", "password": "correct-password"},
follow_redirects=False,
)
assert response.status_code == 403
assert response.text == "CSRF token missing."
def test_successful_login_creates_session_and_allows_dashboard(monkeypatch) -> None:
limiter = SlidingWindowRateLimiter(max_attempts=5, window_seconds=60)
app.dependency_overrides[get_db_session] = _override_db_session
app.dependency_overrides[get_auth_service] = lambda: StubAuthService(
user=SimpleNamespace(id=1, email="user@example.com", is_admin=False)
)
app.dependency_overrides[get_login_rate_limiter] = lambda: limiter
client = TestClient(app)
monkeypatch.setattr(
"app.web.common.get_authenticated_user",
AsyncMock(
return_value=SimpleNamespace(
id=1,
email="user@example.com",
is_admin=False,
)
),
)
monkeypatch.setattr(
"app.web.routers.dashboard.publication_service.list_new_for_latest_run_for_user",
AsyncMock(return_value=[]),
)
monkeypatch.setattr(
"app.web.routers.dashboard.run_service.list_recent_runs_for_user",
AsyncMock(return_value=[]),
)
monkeypatch.setattr(
"app.web.routers.dashboard.run_service.queue_status_counts_for_user",
AsyncMock(return_value={"queued": 0, "retrying": 0, "dropped": 0}),
)
monkeypatch.setattr(
"app.web.routers.dashboard.user_settings_service.get_or_create_settings",
AsyncMock(return_value=SimpleNamespace(request_delay_seconds=0)),
)
login_page = client.get("/login")
csrf_token = _extract_csrf_token(login_page.text)
login_response = client.post(
"/login",
data={
"email": "user@example.com",
"password": "correct-password",
"csrf_token": csrf_token,
},
follow_redirects=False,
)
assert login_response.status_code == 303
assert login_response.headers["location"] == "/"
dashboard_response = client.get("/")
assert dashboard_response.status_code == 200
assert 'data-test="home-hero"' in dashboard_response.text
assert 'data-test="session-user"' in dashboard_response.text
assert "user@example.com" in dashboard_response.text
def test_login_rate_limiting_returns_429_after_threshold() -> None:
limiter = SlidingWindowRateLimiter(max_attempts=2, window_seconds=60)
app.dependency_overrides[get_db_session] = _override_db_session
app.dependency_overrides[get_auth_service] = lambda: StubAuthService(user=None)
app.dependency_overrides[get_login_rate_limiter] = lambda: limiter
client = TestClient(app)
login_page = client.get("/login")
csrf_token = _extract_csrf_token(login_page.text)
payload = {
"email": "user@example.com",
"password": "wrong-password",
"csrf_token": csrf_token,
}
first = client.post("/login", data=payload, follow_redirects=False)
second = client.post("/login", data=payload, follow_redirects=False)
third = client.post("/login", data=payload, follow_redirects=False)
assert first.status_code == 401
assert second.status_code == 401
assert third.status_code == 429
assert third.headers["Retry-After"] == "60"
def test_logout_requires_csrf_token_and_clears_session(monkeypatch) -> None:
limiter = SlidingWindowRateLimiter(max_attempts=5, window_seconds=60)
app.dependency_overrides[get_db_session] = _override_db_session
app.dependency_overrides[get_auth_service] = lambda: StubAuthService(
user=SimpleNamespace(id=1, email="user@example.com", is_admin=False)
)
app.dependency_overrides[get_login_rate_limiter] = lambda: limiter
client = TestClient(app)
monkeypatch.setattr(
"app.web.common.get_authenticated_user",
AsyncMock(
side_effect=[
SimpleNamespace(id=1, email="user@example.com", is_admin=False),
None,
]
),
)
monkeypatch.setattr(
"app.web.routers.dashboard.publication_service.list_new_for_latest_run_for_user",
AsyncMock(return_value=[]),
)
monkeypatch.setattr(
"app.web.routers.dashboard.run_service.list_recent_runs_for_user",
AsyncMock(return_value=[]),
)
monkeypatch.setattr(
"app.web.routers.dashboard.run_service.queue_status_counts_for_user",
AsyncMock(return_value={"queued": 0, "retrying": 0, "dropped": 0}),
)
monkeypatch.setattr(
"app.web.routers.dashboard.user_settings_service.get_or_create_settings",
AsyncMock(return_value=SimpleNamespace(request_delay_seconds=0)),
)
login_page = client.get("/login")
csrf_token = _extract_csrf_token(login_page.text)
client.post(
"/login",
data={
"email": "user@example.com",
"password": "correct-password",
"csrf_token": csrf_token,
},
follow_redirects=False,
)
failed_logout = client.post("/logout", data={}, follow_redirects=False)
assert failed_logout.status_code == 403
assert failed_logout.text == "CSRF token invalid."
dashboard_page = client.get("/")
logout_token = _extract_csrf_token(dashboard_page.text)
successful_logout = client.post(
"/logout",
data={"csrf_token": logout_token},
follow_redirects=False,
)
assert successful_logout.status_code == 303
assert successful_logout.headers["location"] == "/login"
assert client.get("/", follow_redirects=False).headers["location"] == "/login"

View file

@ -0,0 +1,34 @@
from app.auth.rate_limit import SlidingWindowRateLimiter
from app.auth.security import PasswordService
def test_password_service_hash_and_verify_round_trip() -> None:
password_service = PasswordService()
password_hash = password_service.hash_password("example-password")
assert password_hash.startswith("$argon2id$")
assert password_service.verify_password(password_hash, "example-password")
assert not password_service.verify_password(password_hash, "wrong-password")
def test_rate_limiter_enforces_windowed_attempt_budget() -> None:
clock = {"now": 0.0}
limiter = SlidingWindowRateLimiter(
max_attempts=2,
window_seconds=10,
now=lambda: clock["now"],
)
key = "127.0.0.1:test@example.com"
assert limiter.check(key).allowed
limiter.record_failure(key)
assert limiter.check(key).allowed
limiter.record_failure(key)
decision = limiter.check(key)
assert not decision.allowed
assert decision.retry_after_seconds == 10
clock["now"] = 11.0
assert limiter.check(key).allowed

View file

@ -0,0 +1,88 @@
from __future__ import annotations
from datetime import datetime, timezone
from app.db.models import CrawlRun, RunStatus, RunTriggerType
from app.presentation.dashboard import SECTION_TEMPLATES, build_dashboard_view_model
from app.services.publications import UnreadPublicationItem
def test_build_dashboard_view_model_maps_sections_and_unread_items() -> None:
unread_items = [
UnreadPublicationItem(
publication_id=10,
scholar_profile_id=3,
scholar_label="Ada Lovelace",
title="Analytical Engine Notes",
year=None,
citation_count=42,
venue_text="Computing Letters",
pub_url="https://example.test/pub-10",
)
]
runs = [
CrawlRun(
id=77,
user_id=1,
trigger_type=RunTriggerType.MANUAL,
status=RunStatus.SUCCESS,
start_dt=datetime(2026, 2, 16, 18, 0, tzinfo=timezone.utc),
scholar_count=1,
new_pub_count=1,
error_log={},
)
]
vm = build_dashboard_view_model(
unread_publications=unread_items,
recent_runs=runs,
request_delay_seconds=7,
queue_counts={"queued": 2, "retrying": 1, "dropped": 3},
)
assert vm.section_templates == SECTION_TEMPLATES
assert vm.run_controls.request_delay_seconds == 7
assert vm.run_controls.queue_queued_count == 2
assert vm.run_controls.queue_retrying_count == 1
assert vm.run_controls.queue_dropped_count == 3
assert vm.unread_publications[0].title == "Analytical Engine Notes"
assert vm.unread_publications[0].year_display == "-"
assert vm.unread_publications[0].citation_count == 42
assert vm.run_history[0].status_label == "success"
assert vm.run_history[0].status_badge == "ok"
assert vm.run_history[0].started_at_display == "2026-02-16 18:00 UTC"
assert vm.run_history[0].detail_url == "/runs/77"
def test_build_dashboard_view_model_maps_failed_and_partial_statuses() -> None:
runs = [
CrawlRun(
id=11,
user_id=1,
trigger_type=RunTriggerType.MANUAL,
status=RunStatus.FAILED,
start_dt=datetime(2026, 2, 16, 19, 0, tzinfo=timezone.utc),
scholar_count=2,
new_pub_count=0,
error_log={},
),
CrawlRun(
id=12,
user_id=1,
trigger_type=RunTriggerType.MANUAL,
status=RunStatus.PARTIAL_FAILURE,
start_dt=datetime(2026, 2, 16, 20, 0, tzinfo=timezone.utc),
scholar_count=2,
new_pub_count=1,
error_log={},
),
]
vm = build_dashboard_view_model(
unread_publications=[],
recent_runs=runs,
request_delay_seconds=1,
)
assert [item.status_badge for item in vm.run_history] == ["danger", "warn"]
assert [item.status_label for item in vm.run_history] == ["failed", "partial_failure"]

View file

@ -0,0 +1,25 @@
from unittest.mock import AsyncMock
from fastapi.testclient import TestClient
from app.main import app
def test_healthz_returns_200_when_database_is_available(monkeypatch) -> None:
monkeypatch.setattr("app.web.routers.health.check_database", AsyncMock(return_value=True))
client = TestClient(app)
response = client.get("/healthz")
assert response.status_code == 200
assert response.json() == {"status": "ok"}
def test_healthz_returns_500_when_database_is_unavailable(monkeypatch) -> None:
monkeypatch.setattr("app.web.routers.health.check_database", AsyncMock(return_value=False))
client = TestClient(app)
response = client.get("/healthz")
assert response.status_code == 500
assert response.json()["detail"] == "database unavailable"

View file

@ -0,0 +1,52 @@
from __future__ import annotations
import json
import logging
import re
from fastapi.testclient import TestClient
from app.logging_config import JsonLogFormatter, parse_redact_fields
from app.main import app
from app.web.middleware import REQUEST_ID_HEADER, parse_skip_paths
def test_json_log_formatter_redacts_sensitive_fields() -> None:
formatter = JsonLogFormatter(redact_fields=parse_redact_fields("api_key"))
record = logging.makeLogRecord(
{
"name": "tests.logging",
"levelno": logging.INFO,
"levelname": "INFO",
"msg": "test.event",
"args": (),
"password": "very-secret",
"payload": {
"csrf_token": "token-value",
"safe": "ok",
},
"color_message": "ANSI-noise",
}
)
payload = json.loads(formatter.format(record))
assert payload["event"] == "test.event"
assert re.fullmatch(r"\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}Z", payload["timestamp"])
assert payload["password"] == "[REDACTED]"
assert payload["payload"]["csrf_token"] == "[REDACTED]"
assert payload["payload"]["safe"] == "ok"
assert "color_message" not in payload
def test_request_logging_middleware_sets_request_id_header() -> None:
client = TestClient(app)
response = client.get("/login", headers={REQUEST_ID_HEADER: "request-123"})
assert response.status_code == 200
assert response.headers[REQUEST_ID_HEADER] == "request-123"
def test_parse_skip_paths_trims_and_discards_empty_segments() -> None:
assert parse_skip_paths(" /healthz , , /static/ ") == ("/healthz", "/static/")

54
tests/unit/test_pages.py Normal file
View file

@ -0,0 +1,54 @@
from fastapi.testclient import TestClient
from app.main import app
def test_home_page_redirects_to_login_when_unauthenticated() -> None:
client = TestClient(app)
response = client.get("/", follow_redirects=False)
assert response.status_code == 303
assert response.headers["location"] == "/login"
def test_dev_ui_page_is_removed() -> None:
client = TestClient(app)
response = client.get("/dev/ui", follow_redirects=False)
assert response.status_code == 404
def test_login_page_renders_html() -> None:
client = TestClient(app)
response = client.get("/login")
assert response.status_code == 200
assert response.headers["content-type"].startswith("text/html")
assert "scholarr" in response.text
assert 'data-test="login-form"' in response.text
assert 'name="csrf_token"' in response.text
assert 'data-theme-control' in response.text
set_cookie = response.headers["set-cookie"].lower()
assert "httponly" in set_cookie
assert "samesite=lax" in set_cookie
def test_theme_query_parameter_accepts_supported_theme() -> None:
client = TestClient(app)
response = client.get("/login?theme=spruce")
assert response.status_code == 200
assert 'data-theme="spruce"' in response.text
def test_theme_query_parameter_falls_back_for_unknown_theme() -> None:
client = TestClient(app)
response = client.get("/login?theme=not-a-theme")
assert response.status_code == 200
assert 'data-theme="terracotta"' in response.text

View file

@ -0,0 +1,20 @@
from fastapi.testclient import TestClient
from app.main import app
def test_phase2_pages_redirect_to_login_when_unauthenticated() -> None:
client = TestClient(app)
for path in (
"/users",
"/runs",
"/runs/1",
"/publications",
"/scholars",
"/settings",
"/account/password",
):
response = client.get(path, follow_redirects=False)
assert response.status_code == 303
assert response.headers["location"] == "/login"

View file

@ -0,0 +1,240 @@
from __future__ import annotations
from pathlib import Path
from app.services.scholar_parser import ParseState, parse_profile_page
from app.services.scholar_source import FetchResult
def _fixture(name: str) -> str:
path = Path("tests/fixtures/scholar") / name
return path.read_text(encoding="utf-8")
def _regression_fixture(name: str) -> str:
path = Path("tests/fixtures/scholar/regression") / name
return path.read_text(encoding="utf-8")
def test_parse_profile_page_extracts_core_fields_from_fixture() -> None:
fetch_result = FetchResult(
requested_url="https://scholar.google.com/citations?hl=en&user=amIMrIEAAAAJ",
status_code=200,
final_url="https://scholar.google.com/citations?hl=en&user=amIMrIEAAAAJ",
body=_fixture("profile_ok_amIMrIEAAAAJ.html"),
error=None,
)
parsed = parse_profile_page(fetch_result)
assert parsed.state == ParseState.OK
assert parsed.state_reason == "publications_extracted"
assert parsed.profile_name == "Bangar Raju Cherukuri"
assert len(parsed.publications) >= 10
assert parsed.has_show_more_button is True
assert parsed.articles_range is not None
first = parsed.publications[0]
assert first.title
assert first.cluster_id
assert first.citation_count is not None
def test_parse_profile_page_classifies_accounts_redirect_as_blocked() -> None:
fetch_result = FetchResult(
requested_url="https://scholar.google.com/citations?hl=en&user=AAAAAAAAAAAA",
status_code=200,
final_url="https://accounts.google.com/v3/signin/identifier?continue=...",
body="<html><body>Sign in</body></html>",
error=None,
)
parsed = parse_profile_page(fetch_result)
assert parsed.state == ParseState.BLOCKED_OR_CAPTCHA
assert parsed.state_reason == "blocked_accounts_redirect"
assert len(parsed.publications) == 0
def test_parse_profile_page_handles_missing_optional_metadata() -> None:
html = """
<html>
<div id="gsc_prf_in">Test Author</div>
<span id="gsc_a_nn">Articles 1-1</span>
<table>
<tbody id="gsc_a_b">
<tr class="gsc_a_tr">
<td class="gsc_a_t">
<a class="gsc_a_at" href="/citations?view_op=view_citation&citation_for_view=abc:def123">A Test Paper</a>
<div class="gs_gray">A Person</div>
</td>
<td class="gsc_a_c"><a class="gsc_a_ac">7</a></td>
<td class="gsc_a_y"><span class="gsc_a_h"></span></td>
</tr>
</tbody>
</table>
</html>
"""
fetch_result = FetchResult(
requested_url="https://scholar.google.com/citations?hl=en&user=abcDEF123456",
status_code=200,
final_url="https://scholar.google.com/citations?hl=en&user=abcDEF123456",
body=html,
error=None,
)
parsed = parse_profile_page(fetch_result)
assert parsed.state == ParseState.OK
assert parsed.state_reason == "publications_extracted"
assert len(parsed.publications) == 1
publication = parsed.publications[0]
assert publication.year is None
assert publication.venue_text is None
def test_parse_profile_page_detects_layout_change_when_markers_absent() -> None:
fetch_result = FetchResult(
requested_url="https://scholar.google.com/citations?hl=en&user=abcDEF123456",
status_code=200,
final_url="https://scholar.google.com/citations?hl=en&user=abcDEF123456",
body="<html><body><h1>Unexpected page</h1></body></html>",
error=None,
)
parsed = parse_profile_page(fetch_result)
assert parsed.state == ParseState.LAYOUT_CHANGED
assert parsed.state_reason == "layout_markers_missing"
assert "no_rows_detected" in parsed.warnings
def test_parse_profile_page_reports_network_reason_when_status_missing() -> None:
fetch_result = FetchResult(
requested_url="https://scholar.google.com/citations?hl=en&user=abcDEF123456",
status_code=None,
final_url=None,
body="",
error="timed out",
)
parsed = parse_profile_page(fetch_result)
assert parsed.state == ParseState.NETWORK_ERROR
assert parsed.state_reason == "network_error_missing_status_code"
def test_parse_profile_page_ignores_no_results_keyword_inside_script_blocks() -> None:
html = """
<html>
<script>
const message = "didn't match any articles";
</script>
<div id="gsc_prf_in">Scripted Author</div>
<table><tbody id="gsc_a_b"></tbody></table>
</html>
"""
fetch_result = FetchResult(
requested_url="https://scholar.google.com/citations?hl=en&user=abcDEF123456",
status_code=200,
final_url="https://scholar.google.com/citations?hl=en&user=abcDEF123456",
body=html,
error=None,
)
parsed = parse_profile_page(fetch_result)
assert parsed.state == ParseState.OK
assert parsed.state_reason == "no_rows_with_known_markers"
def test_parse_profile_page_treats_disabled_show_more_button_as_absent() -> None:
html = """
<html>
<div id="gsc_prf_in">Disabled Show More</div>
<span id="gsc_a_nn">Articles 1-1</span>
<table><tbody id="gsc_a_b">
<tr class="gsc_a_tr">
<td class="gsc_a_t">
<a class="gsc_a_at" href="/citations?view_op=view_citation&citation_for_view=abc:def">Paper</a>
</td>
<td class="gsc_a_c"><a class="gsc_a_ac">1</a></td>
<td class="gsc_a_y"><span class="gsc_a_h">2024</span></td>
</tr>
</tbody></table>
<button id="gsc_bpf_more" disabled>Show more</button>
</html>
"""
fetch_result = FetchResult(
requested_url="https://scholar.google.com/citations?hl=en&user=abcDEF123456",
status_code=200,
final_url="https://scholar.google.com/citations?hl=en&user=abcDEF123456",
body=html,
error=None,
)
parsed = parse_profile_page(fetch_result)
assert parsed.state == ParseState.OK
assert parsed.has_show_more_button is False
def test_parse_profile_page_regression_fixture_profile_p1rwlvo() -> None:
fetch_result = FetchResult(
requested_url="https://scholar.google.com/citations?hl=en&user=P1RwlvoAAAAJ",
status_code=200,
final_url="https://scholar.google.com/citations?hl=en&user=P1RwlvoAAAAJ",
body=_regression_fixture("profile_P1RwlvoAAAAJ.html"),
error=None,
)
parsed = parse_profile_page(fetch_result)
assert parsed.state == ParseState.OK
assert parsed.state_reason == "publications_extracted"
assert parsed.profile_name == "WENRUI ZUO"
assert len(parsed.publications) == 5
assert parsed.has_show_more_button is False
assert parsed.articles_range in {"Articles 1-5", "Articles 15"}
assert "possible_partial_page_show_more_present" not in parsed.warnings
assert all(item.cluster_id for item in parsed.publications)
def test_parse_profile_page_regression_fixture_profile_lz5d() -> None:
fetch_result = FetchResult(
requested_url="https://scholar.google.com/citations?hl=en&user=LZ5D_p4AAAAJ",
status_code=200,
final_url="https://scholar.google.com/citations?hl=en&user=LZ5D_p4AAAAJ",
body=_regression_fixture("profile_LZ5D_p4AAAAJ.html"),
error=None,
)
parsed = parse_profile_page(fetch_result)
assert parsed.state == ParseState.OK
assert parsed.state_reason == "publications_extracted"
assert parsed.profile_name == "Doaa Elmatary"
assert len(parsed.publications) == 12
assert parsed.has_show_more_button is False
assert parsed.articles_range in {"Articles 1-12", "Articles 112"}
assert "possible_partial_page_show_more_present" not in parsed.warnings
assert any(item.venue_text is None for item in parsed.publications)
def test_parse_profile_page_regression_fixture_blocked_redirect() -> None:
fetch_result = FetchResult(
requested_url="https://scholar.google.com/citations?hl=en&user=AAAAAAAAAAAA",
status_code=200,
final_url=(
"https://accounts.google.com/v3/signin/identifier"
"?continue=https%3A%2F%2Fscholar.google.com%2Fcitations%3Fhl%3Den%26user%3DAAAAAAAAAAAA"
),
body=_regression_fixture("profile_AAAAAAAAAAAA.html"),
error=None,
)
parsed = parse_profile_page(fetch_result)
assert parsed.state == ParseState.BLOCKED_OR_CAPTCHA
assert parsed.state_reason == "blocked_accounts_redirect"
assert parsed.profile_name is None
assert len(parsed.publications) == 0