First product
This commit is contained in:
parent
778da9e2fc
commit
4433d7d2c4
157 changed files with 23975 additions and 0 deletions
204
tests/unit/test_auth.py
Normal file
204
tests/unit/test_auth.py
Normal file
|
|
@ -0,0 +1,204 @@
|
|||
import re
|
||||
from collections.abc import AsyncIterator
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
from fastapi.testclient import TestClient
|
||||
import pytest
|
||||
|
||||
from app.auth.deps import get_auth_service, get_login_rate_limiter
|
||||
from app.auth.rate_limit import SlidingWindowRateLimiter
|
||||
from app.db.session import get_db_session
|
||||
from app.main import app
|
||||
|
||||
CSRF_TOKEN_PATTERN = re.compile(r'name="csrf_token" value="([^"]+)"')
|
||||
|
||||
|
||||
class StubAuthService:
|
||||
def __init__(self, *, user: object | None) -> None:
|
||||
self._user = user
|
||||
|
||||
async def authenticate_user(self, _db_session, *, email: str, password: str):
|
||||
if self._user is None:
|
||||
return None
|
||||
if email.strip().lower() != str(self._user.email):
|
||||
return None
|
||||
if password != "correct-password":
|
||||
return None
|
||||
return self._user
|
||||
|
||||
|
||||
def _extract_csrf_token(html: str) -> str:
|
||||
match = CSRF_TOKEN_PATTERN.search(html)
|
||||
assert match is not None
|
||||
return match.group(1)
|
||||
|
||||
|
||||
async def _override_db_session() -> AsyncIterator[object]:
|
||||
yield object()
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def clear_dependency_overrides() -> AsyncIterator[None]:
|
||||
app.dependency_overrides.clear()
|
||||
yield
|
||||
app.dependency_overrides.clear()
|
||||
|
||||
|
||||
def test_login_requires_csrf_token() -> None:
|
||||
client = TestClient(app)
|
||||
|
||||
response = client.post(
|
||||
"/login",
|
||||
data={"email": "user@example.com", "password": "correct-password"},
|
||||
follow_redirects=False,
|
||||
)
|
||||
|
||||
assert response.status_code == 403
|
||||
assert response.text == "CSRF token missing."
|
||||
|
||||
|
||||
def test_successful_login_creates_session_and_allows_dashboard(monkeypatch) -> None:
|
||||
limiter = SlidingWindowRateLimiter(max_attempts=5, window_seconds=60)
|
||||
app.dependency_overrides[get_db_session] = _override_db_session
|
||||
app.dependency_overrides[get_auth_service] = lambda: StubAuthService(
|
||||
user=SimpleNamespace(id=1, email="user@example.com", is_admin=False)
|
||||
)
|
||||
app.dependency_overrides[get_login_rate_limiter] = lambda: limiter
|
||||
client = TestClient(app)
|
||||
monkeypatch.setattr(
|
||||
"app.web.common.get_authenticated_user",
|
||||
AsyncMock(
|
||||
return_value=SimpleNamespace(
|
||||
id=1,
|
||||
email="user@example.com",
|
||||
is_admin=False,
|
||||
)
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"app.web.routers.dashboard.publication_service.list_new_for_latest_run_for_user",
|
||||
AsyncMock(return_value=[]),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"app.web.routers.dashboard.run_service.list_recent_runs_for_user",
|
||||
AsyncMock(return_value=[]),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"app.web.routers.dashboard.run_service.queue_status_counts_for_user",
|
||||
AsyncMock(return_value={"queued": 0, "retrying": 0, "dropped": 0}),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"app.web.routers.dashboard.user_settings_service.get_or_create_settings",
|
||||
AsyncMock(return_value=SimpleNamespace(request_delay_seconds=0)),
|
||||
)
|
||||
|
||||
login_page = client.get("/login")
|
||||
csrf_token = _extract_csrf_token(login_page.text)
|
||||
|
||||
login_response = client.post(
|
||||
"/login",
|
||||
data={
|
||||
"email": "user@example.com",
|
||||
"password": "correct-password",
|
||||
"csrf_token": csrf_token,
|
||||
},
|
||||
follow_redirects=False,
|
||||
)
|
||||
|
||||
assert login_response.status_code == 303
|
||||
assert login_response.headers["location"] == "/"
|
||||
|
||||
dashboard_response = client.get("/")
|
||||
assert dashboard_response.status_code == 200
|
||||
assert 'data-test="home-hero"' in dashboard_response.text
|
||||
assert 'data-test="session-user"' in dashboard_response.text
|
||||
assert "user@example.com" in dashboard_response.text
|
||||
|
||||
|
||||
def test_login_rate_limiting_returns_429_after_threshold() -> None:
|
||||
limiter = SlidingWindowRateLimiter(max_attempts=2, window_seconds=60)
|
||||
app.dependency_overrides[get_db_session] = _override_db_session
|
||||
app.dependency_overrides[get_auth_service] = lambda: StubAuthService(user=None)
|
||||
app.dependency_overrides[get_login_rate_limiter] = lambda: limiter
|
||||
client = TestClient(app)
|
||||
|
||||
login_page = client.get("/login")
|
||||
csrf_token = _extract_csrf_token(login_page.text)
|
||||
payload = {
|
||||
"email": "user@example.com",
|
||||
"password": "wrong-password",
|
||||
"csrf_token": csrf_token,
|
||||
}
|
||||
|
||||
first = client.post("/login", data=payload, follow_redirects=False)
|
||||
second = client.post("/login", data=payload, follow_redirects=False)
|
||||
third = client.post("/login", data=payload, follow_redirects=False)
|
||||
|
||||
assert first.status_code == 401
|
||||
assert second.status_code == 401
|
||||
assert third.status_code == 429
|
||||
assert third.headers["Retry-After"] == "60"
|
||||
|
||||
|
||||
def test_logout_requires_csrf_token_and_clears_session(monkeypatch) -> None:
|
||||
limiter = SlidingWindowRateLimiter(max_attempts=5, window_seconds=60)
|
||||
app.dependency_overrides[get_db_session] = _override_db_session
|
||||
app.dependency_overrides[get_auth_service] = lambda: StubAuthService(
|
||||
user=SimpleNamespace(id=1, email="user@example.com", is_admin=False)
|
||||
)
|
||||
app.dependency_overrides[get_login_rate_limiter] = lambda: limiter
|
||||
client = TestClient(app)
|
||||
monkeypatch.setattr(
|
||||
"app.web.common.get_authenticated_user",
|
||||
AsyncMock(
|
||||
side_effect=[
|
||||
SimpleNamespace(id=1, email="user@example.com", is_admin=False),
|
||||
None,
|
||||
]
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"app.web.routers.dashboard.publication_service.list_new_for_latest_run_for_user",
|
||||
AsyncMock(return_value=[]),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"app.web.routers.dashboard.run_service.list_recent_runs_for_user",
|
||||
AsyncMock(return_value=[]),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"app.web.routers.dashboard.run_service.queue_status_counts_for_user",
|
||||
AsyncMock(return_value={"queued": 0, "retrying": 0, "dropped": 0}),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"app.web.routers.dashboard.user_settings_service.get_or_create_settings",
|
||||
AsyncMock(return_value=SimpleNamespace(request_delay_seconds=0)),
|
||||
)
|
||||
|
||||
login_page = client.get("/login")
|
||||
csrf_token = _extract_csrf_token(login_page.text)
|
||||
client.post(
|
||||
"/login",
|
||||
data={
|
||||
"email": "user@example.com",
|
||||
"password": "correct-password",
|
||||
"csrf_token": csrf_token,
|
||||
},
|
||||
follow_redirects=False,
|
||||
)
|
||||
|
||||
failed_logout = client.post("/logout", data={}, follow_redirects=False)
|
||||
assert failed_logout.status_code == 403
|
||||
assert failed_logout.text == "CSRF token invalid."
|
||||
|
||||
dashboard_page = client.get("/")
|
||||
logout_token = _extract_csrf_token(dashboard_page.text)
|
||||
successful_logout = client.post(
|
||||
"/logout",
|
||||
data={"csrf_token": logout_token},
|
||||
follow_redirects=False,
|
||||
)
|
||||
|
||||
assert successful_logout.status_code == 303
|
||||
assert successful_logout.headers["location"] == "/login"
|
||||
assert client.get("/", follow_redirects=False).headers["location"] == "/login"
|
||||
34
tests/unit/test_auth_primitives.py
Normal file
34
tests/unit/test_auth_primitives.py
Normal file
|
|
@ -0,0 +1,34 @@
|
|||
from app.auth.rate_limit import SlidingWindowRateLimiter
|
||||
from app.auth.security import PasswordService
|
||||
|
||||
|
||||
def test_password_service_hash_and_verify_round_trip() -> None:
|
||||
password_service = PasswordService()
|
||||
password_hash = password_service.hash_password("example-password")
|
||||
|
||||
assert password_hash.startswith("$argon2id$")
|
||||
assert password_service.verify_password(password_hash, "example-password")
|
||||
assert not password_service.verify_password(password_hash, "wrong-password")
|
||||
|
||||
|
||||
def test_rate_limiter_enforces_windowed_attempt_budget() -> None:
|
||||
clock = {"now": 0.0}
|
||||
limiter = SlidingWindowRateLimiter(
|
||||
max_attempts=2,
|
||||
window_seconds=10,
|
||||
now=lambda: clock["now"],
|
||||
)
|
||||
key = "127.0.0.1:test@example.com"
|
||||
|
||||
assert limiter.check(key).allowed
|
||||
limiter.record_failure(key)
|
||||
assert limiter.check(key).allowed
|
||||
limiter.record_failure(key)
|
||||
|
||||
decision = limiter.check(key)
|
||||
assert not decision.allowed
|
||||
assert decision.retry_after_seconds == 10
|
||||
|
||||
clock["now"] = 11.0
|
||||
assert limiter.check(key).allowed
|
||||
|
||||
88
tests/unit/test_dashboard_view_model.py
Normal file
88
tests/unit/test_dashboard_view_model.py
Normal file
|
|
@ -0,0 +1,88 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from app.db.models import CrawlRun, RunStatus, RunTriggerType
|
||||
from app.presentation.dashboard import SECTION_TEMPLATES, build_dashboard_view_model
|
||||
from app.services.publications import UnreadPublicationItem
|
||||
|
||||
|
||||
def test_build_dashboard_view_model_maps_sections_and_unread_items() -> None:
|
||||
unread_items = [
|
||||
UnreadPublicationItem(
|
||||
publication_id=10,
|
||||
scholar_profile_id=3,
|
||||
scholar_label="Ada Lovelace",
|
||||
title="Analytical Engine Notes",
|
||||
year=None,
|
||||
citation_count=42,
|
||||
venue_text="Computing Letters",
|
||||
pub_url="https://example.test/pub-10",
|
||||
)
|
||||
]
|
||||
runs = [
|
||||
CrawlRun(
|
||||
id=77,
|
||||
user_id=1,
|
||||
trigger_type=RunTriggerType.MANUAL,
|
||||
status=RunStatus.SUCCESS,
|
||||
start_dt=datetime(2026, 2, 16, 18, 0, tzinfo=timezone.utc),
|
||||
scholar_count=1,
|
||||
new_pub_count=1,
|
||||
error_log={},
|
||||
)
|
||||
]
|
||||
|
||||
vm = build_dashboard_view_model(
|
||||
unread_publications=unread_items,
|
||||
recent_runs=runs,
|
||||
request_delay_seconds=7,
|
||||
queue_counts={"queued": 2, "retrying": 1, "dropped": 3},
|
||||
)
|
||||
|
||||
assert vm.section_templates == SECTION_TEMPLATES
|
||||
assert vm.run_controls.request_delay_seconds == 7
|
||||
assert vm.run_controls.queue_queued_count == 2
|
||||
assert vm.run_controls.queue_retrying_count == 1
|
||||
assert vm.run_controls.queue_dropped_count == 3
|
||||
assert vm.unread_publications[0].title == "Analytical Engine Notes"
|
||||
assert vm.unread_publications[0].year_display == "-"
|
||||
assert vm.unread_publications[0].citation_count == 42
|
||||
assert vm.run_history[0].status_label == "success"
|
||||
assert vm.run_history[0].status_badge == "ok"
|
||||
assert vm.run_history[0].started_at_display == "2026-02-16 18:00 UTC"
|
||||
assert vm.run_history[0].detail_url == "/runs/77"
|
||||
|
||||
|
||||
def test_build_dashboard_view_model_maps_failed_and_partial_statuses() -> None:
|
||||
runs = [
|
||||
CrawlRun(
|
||||
id=11,
|
||||
user_id=1,
|
||||
trigger_type=RunTriggerType.MANUAL,
|
||||
status=RunStatus.FAILED,
|
||||
start_dt=datetime(2026, 2, 16, 19, 0, tzinfo=timezone.utc),
|
||||
scholar_count=2,
|
||||
new_pub_count=0,
|
||||
error_log={},
|
||||
),
|
||||
CrawlRun(
|
||||
id=12,
|
||||
user_id=1,
|
||||
trigger_type=RunTriggerType.MANUAL,
|
||||
status=RunStatus.PARTIAL_FAILURE,
|
||||
start_dt=datetime(2026, 2, 16, 20, 0, tzinfo=timezone.utc),
|
||||
scholar_count=2,
|
||||
new_pub_count=1,
|
||||
error_log={},
|
||||
),
|
||||
]
|
||||
|
||||
vm = build_dashboard_view_model(
|
||||
unread_publications=[],
|
||||
recent_runs=runs,
|
||||
request_delay_seconds=1,
|
||||
)
|
||||
|
||||
assert [item.status_badge for item in vm.run_history] == ["danger", "warn"]
|
||||
assert [item.status_label for item in vm.run_history] == ["failed", "partial_failure"]
|
||||
25
tests/unit/test_healthz.py
Normal file
25
tests/unit/test_healthz.py
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
from unittest.mock import AsyncMock
|
||||
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from app.main import app
|
||||
|
||||
|
||||
def test_healthz_returns_200_when_database_is_available(monkeypatch) -> None:
|
||||
monkeypatch.setattr("app.web.routers.health.check_database", AsyncMock(return_value=True))
|
||||
client = TestClient(app)
|
||||
|
||||
response = client.get("/healthz")
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.json() == {"status": "ok"}
|
||||
|
||||
|
||||
def test_healthz_returns_500_when_database_is_unavailable(monkeypatch) -> None:
|
||||
monkeypatch.setattr("app.web.routers.health.check_database", AsyncMock(return_value=False))
|
||||
client = TestClient(app)
|
||||
|
||||
response = client.get("/healthz")
|
||||
|
||||
assert response.status_code == 500
|
||||
assert response.json()["detail"] == "database unavailable"
|
||||
52
tests/unit/test_logging.py
Normal file
52
tests/unit/test_logging.py
Normal file
|
|
@ -0,0 +1,52 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from app.logging_config import JsonLogFormatter, parse_redact_fields
|
||||
from app.main import app
|
||||
from app.web.middleware import REQUEST_ID_HEADER, parse_skip_paths
|
||||
|
||||
|
||||
def test_json_log_formatter_redacts_sensitive_fields() -> None:
|
||||
formatter = JsonLogFormatter(redact_fields=parse_redact_fields("api_key"))
|
||||
record = logging.makeLogRecord(
|
||||
{
|
||||
"name": "tests.logging",
|
||||
"levelno": logging.INFO,
|
||||
"levelname": "INFO",
|
||||
"msg": "test.event",
|
||||
"args": (),
|
||||
"password": "very-secret",
|
||||
"payload": {
|
||||
"csrf_token": "token-value",
|
||||
"safe": "ok",
|
||||
},
|
||||
"color_message": "ANSI-noise",
|
||||
}
|
||||
)
|
||||
|
||||
payload = json.loads(formatter.format(record))
|
||||
|
||||
assert payload["event"] == "test.event"
|
||||
assert re.fullmatch(r"\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}Z", payload["timestamp"])
|
||||
assert payload["password"] == "[REDACTED]"
|
||||
assert payload["payload"]["csrf_token"] == "[REDACTED]"
|
||||
assert payload["payload"]["safe"] == "ok"
|
||||
assert "color_message" not in payload
|
||||
|
||||
|
||||
def test_request_logging_middleware_sets_request_id_header() -> None:
|
||||
client = TestClient(app)
|
||||
|
||||
response = client.get("/login", headers={REQUEST_ID_HEADER: "request-123"})
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.headers[REQUEST_ID_HEADER] == "request-123"
|
||||
|
||||
|
||||
def test_parse_skip_paths_trims_and_discards_empty_segments() -> None:
|
||||
assert parse_skip_paths(" /healthz , , /static/ ") == ("/healthz", "/static/")
|
||||
54
tests/unit/test_pages.py
Normal file
54
tests/unit/test_pages.py
Normal file
|
|
@ -0,0 +1,54 @@
|
|||
from fastapi.testclient import TestClient
|
||||
|
||||
from app.main import app
|
||||
|
||||
|
||||
def test_home_page_redirects_to_login_when_unauthenticated() -> None:
|
||||
client = TestClient(app)
|
||||
|
||||
response = client.get("/", follow_redirects=False)
|
||||
|
||||
assert response.status_code == 303
|
||||
assert response.headers["location"] == "/login"
|
||||
|
||||
|
||||
def test_dev_ui_page_is_removed() -> None:
|
||||
client = TestClient(app)
|
||||
|
||||
response = client.get("/dev/ui", follow_redirects=False)
|
||||
|
||||
assert response.status_code == 404
|
||||
|
||||
|
||||
def test_login_page_renders_html() -> None:
|
||||
client = TestClient(app)
|
||||
|
||||
response = client.get("/login")
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.headers["content-type"].startswith("text/html")
|
||||
assert "scholarr" in response.text
|
||||
assert 'data-test="login-form"' in response.text
|
||||
assert 'name="csrf_token"' in response.text
|
||||
assert 'data-theme-control' in response.text
|
||||
set_cookie = response.headers["set-cookie"].lower()
|
||||
assert "httponly" in set_cookie
|
||||
assert "samesite=lax" in set_cookie
|
||||
|
||||
|
||||
def test_theme_query_parameter_accepts_supported_theme() -> None:
|
||||
client = TestClient(app)
|
||||
|
||||
response = client.get("/login?theme=spruce")
|
||||
|
||||
assert response.status_code == 200
|
||||
assert 'data-theme="spruce"' in response.text
|
||||
|
||||
|
||||
def test_theme_query_parameter_falls_back_for_unknown_theme() -> None:
|
||||
client = TestClient(app)
|
||||
|
||||
response = client.get("/login?theme=not-a-theme")
|
||||
|
||||
assert response.status_code == 200
|
||||
assert 'data-theme="terracotta"' in response.text
|
||||
20
tests/unit/test_phase2_pages.py
Normal file
20
tests/unit/test_phase2_pages.py
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
from fastapi.testclient import TestClient
|
||||
|
||||
from app.main import app
|
||||
|
||||
|
||||
def test_phase2_pages_redirect_to_login_when_unauthenticated() -> None:
|
||||
client = TestClient(app)
|
||||
|
||||
for path in (
|
||||
"/users",
|
||||
"/runs",
|
||||
"/runs/1",
|
||||
"/publications",
|
||||
"/scholars",
|
||||
"/settings",
|
||||
"/account/password",
|
||||
):
|
||||
response = client.get(path, follow_redirects=False)
|
||||
assert response.status_code == 303
|
||||
assert response.headers["location"] == "/login"
|
||||
240
tests/unit/test_scholar_parser.py
Normal file
240
tests/unit/test_scholar_parser.py
Normal file
|
|
@ -0,0 +1,240 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from app.services.scholar_parser import ParseState, parse_profile_page
|
||||
from app.services.scholar_source import FetchResult
|
||||
|
||||
|
||||
def _fixture(name: str) -> str:
|
||||
path = Path("tests/fixtures/scholar") / name
|
||||
return path.read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def _regression_fixture(name: str) -> str:
|
||||
path = Path("tests/fixtures/scholar/regression") / name
|
||||
return path.read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def test_parse_profile_page_extracts_core_fields_from_fixture() -> None:
|
||||
fetch_result = FetchResult(
|
||||
requested_url="https://scholar.google.com/citations?hl=en&user=amIMrIEAAAAJ",
|
||||
status_code=200,
|
||||
final_url="https://scholar.google.com/citations?hl=en&user=amIMrIEAAAAJ",
|
||||
body=_fixture("profile_ok_amIMrIEAAAAJ.html"),
|
||||
error=None,
|
||||
)
|
||||
|
||||
parsed = parse_profile_page(fetch_result)
|
||||
|
||||
assert parsed.state == ParseState.OK
|
||||
assert parsed.state_reason == "publications_extracted"
|
||||
assert parsed.profile_name == "Bangar Raju Cherukuri"
|
||||
assert len(parsed.publications) >= 10
|
||||
assert parsed.has_show_more_button is True
|
||||
assert parsed.articles_range is not None
|
||||
first = parsed.publications[0]
|
||||
assert first.title
|
||||
assert first.cluster_id
|
||||
assert first.citation_count is not None
|
||||
|
||||
|
||||
def test_parse_profile_page_classifies_accounts_redirect_as_blocked() -> None:
|
||||
fetch_result = FetchResult(
|
||||
requested_url="https://scholar.google.com/citations?hl=en&user=AAAAAAAAAAAA",
|
||||
status_code=200,
|
||||
final_url="https://accounts.google.com/v3/signin/identifier?continue=...",
|
||||
body="<html><body>Sign in</body></html>",
|
||||
error=None,
|
||||
)
|
||||
|
||||
parsed = parse_profile_page(fetch_result)
|
||||
|
||||
assert parsed.state == ParseState.BLOCKED_OR_CAPTCHA
|
||||
assert parsed.state_reason == "blocked_accounts_redirect"
|
||||
assert len(parsed.publications) == 0
|
||||
|
||||
|
||||
def test_parse_profile_page_handles_missing_optional_metadata() -> None:
|
||||
html = """
|
||||
<html>
|
||||
<div id="gsc_prf_in">Test Author</div>
|
||||
<span id="gsc_a_nn">Articles 1-1</span>
|
||||
<table>
|
||||
<tbody id="gsc_a_b">
|
||||
<tr class="gsc_a_tr">
|
||||
<td class="gsc_a_t">
|
||||
<a class="gsc_a_at" href="/citations?view_op=view_citation&citation_for_view=abc:def123">A Test Paper</a>
|
||||
<div class="gs_gray">A Person</div>
|
||||
</td>
|
||||
<td class="gsc_a_c"><a class="gsc_a_ac">7</a></td>
|
||||
<td class="gsc_a_y"><span class="gsc_a_h"></span></td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</html>
|
||||
"""
|
||||
fetch_result = FetchResult(
|
||||
requested_url="https://scholar.google.com/citations?hl=en&user=abcDEF123456",
|
||||
status_code=200,
|
||||
final_url="https://scholar.google.com/citations?hl=en&user=abcDEF123456",
|
||||
body=html,
|
||||
error=None,
|
||||
)
|
||||
|
||||
parsed = parse_profile_page(fetch_result)
|
||||
|
||||
assert parsed.state == ParseState.OK
|
||||
assert parsed.state_reason == "publications_extracted"
|
||||
assert len(parsed.publications) == 1
|
||||
publication = parsed.publications[0]
|
||||
assert publication.year is None
|
||||
assert publication.venue_text is None
|
||||
|
||||
|
||||
def test_parse_profile_page_detects_layout_change_when_markers_absent() -> None:
|
||||
fetch_result = FetchResult(
|
||||
requested_url="https://scholar.google.com/citations?hl=en&user=abcDEF123456",
|
||||
status_code=200,
|
||||
final_url="https://scholar.google.com/citations?hl=en&user=abcDEF123456",
|
||||
body="<html><body><h1>Unexpected page</h1></body></html>",
|
||||
error=None,
|
||||
)
|
||||
|
||||
parsed = parse_profile_page(fetch_result)
|
||||
|
||||
assert parsed.state == ParseState.LAYOUT_CHANGED
|
||||
assert parsed.state_reason == "layout_markers_missing"
|
||||
assert "no_rows_detected" in parsed.warnings
|
||||
|
||||
|
||||
def test_parse_profile_page_reports_network_reason_when_status_missing() -> None:
|
||||
fetch_result = FetchResult(
|
||||
requested_url="https://scholar.google.com/citations?hl=en&user=abcDEF123456",
|
||||
status_code=None,
|
||||
final_url=None,
|
||||
body="",
|
||||
error="timed out",
|
||||
)
|
||||
|
||||
parsed = parse_profile_page(fetch_result)
|
||||
|
||||
assert parsed.state == ParseState.NETWORK_ERROR
|
||||
assert parsed.state_reason == "network_error_missing_status_code"
|
||||
|
||||
|
||||
def test_parse_profile_page_ignores_no_results_keyword_inside_script_blocks() -> None:
|
||||
html = """
|
||||
<html>
|
||||
<script>
|
||||
const message = "didn't match any articles";
|
||||
</script>
|
||||
<div id="gsc_prf_in">Scripted Author</div>
|
||||
<table><tbody id="gsc_a_b"></tbody></table>
|
||||
</html>
|
||||
"""
|
||||
fetch_result = FetchResult(
|
||||
requested_url="https://scholar.google.com/citations?hl=en&user=abcDEF123456",
|
||||
status_code=200,
|
||||
final_url="https://scholar.google.com/citations?hl=en&user=abcDEF123456",
|
||||
body=html,
|
||||
error=None,
|
||||
)
|
||||
|
||||
parsed = parse_profile_page(fetch_result)
|
||||
|
||||
assert parsed.state == ParseState.OK
|
||||
assert parsed.state_reason == "no_rows_with_known_markers"
|
||||
|
||||
|
||||
def test_parse_profile_page_treats_disabled_show_more_button_as_absent() -> None:
|
||||
html = """
|
||||
<html>
|
||||
<div id="gsc_prf_in">Disabled Show More</div>
|
||||
<span id="gsc_a_nn">Articles 1-1</span>
|
||||
<table><tbody id="gsc_a_b">
|
||||
<tr class="gsc_a_tr">
|
||||
<td class="gsc_a_t">
|
||||
<a class="gsc_a_at" href="/citations?view_op=view_citation&citation_for_view=abc:def">Paper</a>
|
||||
</td>
|
||||
<td class="gsc_a_c"><a class="gsc_a_ac">1</a></td>
|
||||
<td class="gsc_a_y"><span class="gsc_a_h">2024</span></td>
|
||||
</tr>
|
||||
</tbody></table>
|
||||
<button id="gsc_bpf_more" disabled>Show more</button>
|
||||
</html>
|
||||
"""
|
||||
fetch_result = FetchResult(
|
||||
requested_url="https://scholar.google.com/citations?hl=en&user=abcDEF123456",
|
||||
status_code=200,
|
||||
final_url="https://scholar.google.com/citations?hl=en&user=abcDEF123456",
|
||||
body=html,
|
||||
error=None,
|
||||
)
|
||||
|
||||
parsed = parse_profile_page(fetch_result)
|
||||
|
||||
assert parsed.state == ParseState.OK
|
||||
assert parsed.has_show_more_button is False
|
||||
|
||||
|
||||
def test_parse_profile_page_regression_fixture_profile_p1rwlvo() -> None:
|
||||
fetch_result = FetchResult(
|
||||
requested_url="https://scholar.google.com/citations?hl=en&user=P1RwlvoAAAAJ",
|
||||
status_code=200,
|
||||
final_url="https://scholar.google.com/citations?hl=en&user=P1RwlvoAAAAJ",
|
||||
body=_regression_fixture("profile_P1RwlvoAAAAJ.html"),
|
||||
error=None,
|
||||
)
|
||||
|
||||
parsed = parse_profile_page(fetch_result)
|
||||
|
||||
assert parsed.state == ParseState.OK
|
||||
assert parsed.state_reason == "publications_extracted"
|
||||
assert parsed.profile_name == "WENRUI ZUO"
|
||||
assert len(parsed.publications) == 5
|
||||
assert parsed.has_show_more_button is False
|
||||
assert parsed.articles_range in {"Articles 1-5", "Articles 1–5"}
|
||||
assert "possible_partial_page_show_more_present" not in parsed.warnings
|
||||
assert all(item.cluster_id for item in parsed.publications)
|
||||
|
||||
|
||||
def test_parse_profile_page_regression_fixture_profile_lz5d() -> None:
|
||||
fetch_result = FetchResult(
|
||||
requested_url="https://scholar.google.com/citations?hl=en&user=LZ5D_p4AAAAJ",
|
||||
status_code=200,
|
||||
final_url="https://scholar.google.com/citations?hl=en&user=LZ5D_p4AAAAJ",
|
||||
body=_regression_fixture("profile_LZ5D_p4AAAAJ.html"),
|
||||
error=None,
|
||||
)
|
||||
|
||||
parsed = parse_profile_page(fetch_result)
|
||||
|
||||
assert parsed.state == ParseState.OK
|
||||
assert parsed.state_reason == "publications_extracted"
|
||||
assert parsed.profile_name == "Doaa Elmatary"
|
||||
assert len(parsed.publications) == 12
|
||||
assert parsed.has_show_more_button is False
|
||||
assert parsed.articles_range in {"Articles 1-12", "Articles 1–12"}
|
||||
assert "possible_partial_page_show_more_present" not in parsed.warnings
|
||||
assert any(item.venue_text is None for item in parsed.publications)
|
||||
|
||||
|
||||
def test_parse_profile_page_regression_fixture_blocked_redirect() -> None:
|
||||
fetch_result = FetchResult(
|
||||
requested_url="https://scholar.google.com/citations?hl=en&user=AAAAAAAAAAAA",
|
||||
status_code=200,
|
||||
final_url=(
|
||||
"https://accounts.google.com/v3/signin/identifier"
|
||||
"?continue=https%3A%2F%2Fscholar.google.com%2Fcitations%3Fhl%3Den%26user%3DAAAAAAAAAAAA"
|
||||
),
|
||||
body=_regression_fixture("profile_AAAAAAAAAAAA.html"),
|
||||
error=None,
|
||||
)
|
||||
|
||||
parsed = parse_profile_page(fetch_result)
|
||||
|
||||
assert parsed.state == ParseState.BLOCKED_OR_CAPTCHA
|
||||
assert parsed.state_reason == "blocked_accounts_redirect"
|
||||
assert parsed.profile_name is None
|
||||
assert len(parsed.publications) == 0
|
||||
Loading…
Add table
Add a link
Reference in a new issue