justinzeus/scholarr
1
0
Fork 0
Code Issues 1 Pull requests 18 Projects Releases Packages Wiki Activity Actions

streamlined

Browse source
This commit is contained in:
Justin Visser 2026-02-17 21:59:41 +01:00
parent d1dd3213dc
commit ab1d29b203
23 changed files with 853 additions and 22 deletions
Download patch file Download diff file Expand all files Collapse all files

182
tests/integration/test_fixture_probe_runs.py Normal file
View file

@ -0,0 +1,182 @@
from __future__ import annotations
from collections import defaultdict
from pathlib import Path
import pytest
from sqlalchemy import text
from sqlalchemy.ext.asyncio import AsyncSession
from app.db.models import RunStatus, RunTriggerType
from app.services.ingestion import ScholarIngestionService
from app.services.scholar_source import FetchResult
from tests.integration.helpers import insert_user
REGRESSION_FIXTURE_DIR = Path("tests/fixtures/scholar/regression")
def _fixture(name: str) -> str:
return (REGRESSION_FIXTURE_DIR / name).read_text(encoding="utf-8")
def _profile_fetch(*, scholar_id: str, body: str) -> FetchResult:
return FetchResult(
requested_url=f"https://scholar.google.com/citations?hl=en&user={scholar_id}",
status_code=200,
final_url=f"https://scholar.google.com/citations?hl=en&user={scholar_id}",
body=body,
error=None,
)
def _blocked_fetch(*, scholar_id: str, body: str) -> FetchResult:
return FetchResult(
requested_url=f"https://scholar.google.com/citations?hl=en&user={scholar_id}",
status_code=200,
final_url=(
"https://accounts.google.com/v3/signin/identifier"
"?continue=https%3A%2F%2Fscholar.google.com%2Fcitations"
),
body=body,
error=None,
)
def _network_timeout_fetch(*, scholar_id: str) -> FetchResult:
return FetchResult(
requested_url=f"https://scholar.google.com/citations?hl=en&user={scholar_id}",
status_code=None,
final_url=None,
body="",
error="timed out",
)
class FixtureScholarSource:
def __init__(self, responses: dict[tuple[str, int], list[FetchResult]]) -> None:
self._responses = responses
self._calls: defaultdict[tuple[str, int], int] = defaultdict(int)
async def fetch_profile_page_html(
self,
scholar_id: str,
*,
cstart: int,
pagesize: int,
) -> FetchResult:
_ = pagesize
key = (scholar_id, int(cstart))
fetches = self._responses.get(key)
if not fetches:
return _network_timeout_fetch(scholar_id=scholar_id)
call_index = self._calls[key]
self._calls[key] += 1
return fetches[min(call_index, len(fetches) - 1)]
async def fetch_profile_html(self, scholar_id: str) -> FetchResult:
return await self.fetch_profile_page_html(scholar_id, cstart=0, pagesize=100)
@pytest.mark.integration
@pytest.mark.db
@pytest.mark.asyncio
async def test_fixture_probe_run_emits_failure_and_retry_summary_metrics(
db_session: AsyncSession,
) -> None:
user_id = await insert_user(
db_session,
email="fixture-probe@example.com",
password="fixture-probe-password",
)
scholar_ids = {
"ok": "abcDEF123456",
"blocked": "A1B2C3D4E5F6",
"retry": "RSTUVWX12345",
}
for label in ("ok", "blocked", "retry"):
await db_session.execute(
text(
"""
INSERT INTO scholar_profiles (user_id, scholar_id, display_name, is_enabled)
VALUES (:user_id, :scholar_id, :display_name, true)
"""
),
{
"user_id": user_id,
"scholar_id": scholar_ids[label],
"display_name": f"fixture-{label}",
},
)
await db_session.commit()
source = FixtureScholarSource(
{
(scholar_ids["ok"], 0): [
_profile_fetch(
scholar_id=scholar_ids["ok"],
body=_fixture("profile_P1RwlvoAAAAJ.html"),
)
],
(scholar_ids["blocked"], 0): [
_blocked_fetch(
scholar_id=scholar_ids["blocked"],
body=_fixture("profile_AAAAAAAAAAAA.html"),
)
],
(scholar_ids["retry"], 0): [
_network_timeout_fetch(scholar_id=scholar_ids["retry"]),
_profile_fetch(
scholar_id=scholar_ids["retry"],
body=_fixture("profile_LZ5D_p4AAAAJ.html"),
),
],
}
)
ingestion = ScholarIngestionService(source=source)
summary = await ingestion.run_for_user(
db_session,
user_id=user_id,
trigger_type=RunTriggerType.MANUAL,
request_delay_seconds=0,
network_error_retries=1,
retry_backoff_seconds=0.0,
max_pages_per_scholar=1,
page_size=100,
auto_queue_continuations=False,
alert_blocked_failure_threshold=1,
alert_network_failure_threshold=1,
alert_retry_scheduled_threshold=1,
)
assert summary.status == RunStatus.PARTIAL_FAILURE
assert summary.scholar_count == 3
assert summary.succeeded_count == 2
assert summary.failed_count == 1
assert summary.partial_count == 0
run_result = await db_session.execute(
text("SELECT error_log FROM crawl_runs WHERE id = :run_id"),
{"run_id": summary.crawl_run_id},
)
error_log = run_result.scalar_one()
run_summary = error_log["summary"]
assert run_summary["failed_state_counts"]["blocked_or_captcha"] == 1
assert run_summary["failed_reason_counts"]["blocked_accounts_redirect"] == 1
assert run_summary["scrape_failure_counts"]["blocked_or_captcha"] == 1
assert run_summary["retry_counts"]["retries_scheduled_count"] == 1
assert run_summary["retry_counts"]["scholars_with_retries_count"] == 1
assert run_summary["retry_counts"]["retry_exhausted_count"] == 0
assert run_summary["alert_thresholds"]["blocked_failure_threshold"] == 1
assert run_summary["alert_thresholds"]["network_failure_threshold"] == 1
assert run_summary["alert_thresholds"]["retry_scheduled_threshold"] == 1
assert run_summary["alert_flags"]["blocked_failure_threshold_exceeded"] is True
assert run_summary["alert_flags"]["network_failure_threshold_exceeded"] is False
assert run_summary["alert_flags"]["retry_scheduled_threshold_exceeded"] is True

58
tests/unit/test_runs_summary.py Normal file
View file

@ -0,0 +1,58 @@
from __future__ import annotations
from app.services.runs import extract_run_summary
def test_extract_run_summary_includes_extended_metrics() -> None:
error_log = {
"summary": {
"succeeded_count": 3,
"failed_count": 1,
"partial_count": 2,
"failed_state_counts": {"network_error": 1},
"failed_reason_counts": {"network_timeout": 1},
"scrape_failure_counts": {"network_error": 1},
"retry_counts": {
"retries_scheduled_count": 4,
"scholars_with_retries_count": 2,
"retry_exhausted_count": 1,
},
"alert_thresholds": {
"blocked_failure_threshold": 1,
"network_failure_threshold": 2,
"retry_scheduled_threshold": 3,
},
"alert_flags": {
"blocked_failure_threshold_exceeded": False,
"network_failure_threshold_exceeded": True,
"retry_scheduled_threshold_exceeded": True,
},
}
}
summary = extract_run_summary(error_log)
assert summary["succeeded_count"] == 3
assert summary["failed_count"] == 1
assert summary["partial_count"] == 2
assert summary["failed_state_counts"] == {"network_error": 1}
assert summary["failed_reason_counts"] == {"network_timeout": 1}
assert summary["scrape_failure_counts"] == {"network_error": 1}
assert summary["retry_counts"]["retries_scheduled_count"] == 4
assert summary["retry_counts"]["scholars_with_retries_count"] == 2
assert summary["retry_counts"]["retry_exhausted_count"] == 1
assert summary["alert_thresholds"]["retry_scheduled_threshold"] == 3
assert summary["alert_flags"]["network_failure_threshold_exceeded"] is True
def test_extract_run_summary_defaults_extended_metrics() -> None:
summary = extract_run_summary({})
assert summary["scrape_failure_counts"] == {}
assert summary["retry_counts"] == {
"retries_scheduled_count": 0,
"scholars_with_retries_count": 0,
"retry_exhausted_count": 0,
}
assert summary["alert_thresholds"] == {}
assert summary["alert_flags"] == {}

83
tests/unit/test_scholar_search_safety.py
View file

@ -63,6 +63,16 @@ def _blocked_author_search_fetch() -> FetchResult:
)
def _network_timeout_fetch() -> FetchResult:
return FetchResult(
requested_url="https://scholar.google.com/citations?hl=en&view_op=search_authors&mauthors=ada",
status_code=None,
final_url=None,
body="",
error="timed out",
)
@pytest.mark.asyncio
async def test_search_author_candidates_serves_cached_response_for_same_query() -> None:
source = StubScholarSource([_ok_author_search_fetch()])
@ -150,6 +160,79 @@ async def test_search_author_candidates_trips_cooldown_after_blocked_responses()
assert "author_search_cooldown_active" in second.warnings
@pytest.mark.asyncio
async def test_search_author_candidates_emits_cooldown_alert_warning_after_threshold() -> None:
source = StubScholarSource([_blocked_author_search_fetch()])
_ = await scholar_service.search_author_candidates(
source=source,
query="Blocked Query",
limit=10,
network_error_retries=0,
retry_backoff_seconds=0.0,
search_enabled=True,
cache_ttl_seconds=0,
blocked_cache_ttl_seconds=0,
cache_max_entries=64,
min_interval_seconds=0.0,
interval_jitter_seconds=0.0,
cooldown_block_threshold=1,
cooldown_seconds=300,
cooldown_rejection_alert_threshold=1,
)
second = await scholar_service.search_author_candidates(
source=source,
query="Blocked Query",
limit=10,
network_error_retries=0,
retry_backoff_seconds=0.0,
search_enabled=True,
cache_ttl_seconds=0,
blocked_cache_ttl_seconds=0,
cache_max_entries=64,
min_interval_seconds=0.0,
interval_jitter_seconds=0.0,
cooldown_block_threshold=1,
cooldown_seconds=300,
cooldown_rejection_alert_threshold=1,
)
assert second.state == ParseState.BLOCKED_OR_CAPTCHA
assert "author_search_cooldown_alert_threshold_exceeded" in second.warnings
@pytest.mark.asyncio
async def test_search_author_candidates_adds_retry_threshold_warning() -> None:
source = StubScholarSource(
[
_network_timeout_fetch(),
_network_timeout_fetch(),
_ok_author_search_fetch(),
]
)
result = await scholar_service.search_author_candidates(
source=source,
query="Ada Lovelace",
limit=10,
network_error_retries=2,
retry_backoff_seconds=0.0,
search_enabled=True,
cache_ttl_seconds=0,
blocked_cache_ttl_seconds=0,
cache_max_entries=64,
min_interval_seconds=0.0,
interval_jitter_seconds=0.0,
cooldown_block_threshold=3,
cooldown_seconds=300,
retry_alert_threshold=2,
)
assert result.state == ParseState.OK
assert "author_search_retry_threshold_exceeded_2" in result.warnings
assert source.calls == 3
@pytest.mark.asyncio
async def test_search_author_candidates_can_be_disabled_by_configuration() -> None:
source = StubScholarSource([_ok_author_search_fetch()])