after audit
This commit is contained in:
parent
e8e20637e6
commit
a5165dc3ba
80 changed files with 8489 additions and 7302 deletions
|
|
@ -6,6 +6,7 @@ import random
|
|||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy.exc import InterfaceError, OperationalError
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.models import (
|
||||
|
|
@ -21,7 +22,15 @@ from app.services.ingestion.constants import (
|
|||
)
|
||||
from app.services.ingestion.pagination import PaginationEngine
|
||||
from app.services.ingestion.publication_upsert import upsert_profile_publications
|
||||
from app.services.ingestion.run_completion import apply_outcome_to_progress, build_failure_debug_context
|
||||
from app.services.ingestion.run_completion import apply_outcome_to_progress
|
||||
from app.services.ingestion.scholar_outcomes import (
|
||||
apply_first_page_profile_metadata,
|
||||
assert_valid_paged_parse_result,
|
||||
build_result_entry,
|
||||
skipped_no_change_outcome,
|
||||
unexpected_scholar_exception_outcome,
|
||||
upsert_publications_outcome,
|
||||
)
|
||||
from app.services.ingestion.types import (
|
||||
PagedParseResult,
|
||||
RunProgress,
|
||||
|
|
@ -33,254 +42,6 @@ from app.services.scholar.state_detection import is_hard_challenge_reason
|
|||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def assert_valid_paged_parse_result(
|
||||
*,
|
||||
scholar_id: str,
|
||||
paged_parse_result: PagedParseResult,
|
||||
) -> None:
|
||||
parsed_page = paged_parse_result.parsed_page
|
||||
if parsed_page.state in {ParseState.OK, ParseState.NO_RESULTS} and any(
|
||||
code.startswith("layout_") for code in parsed_page.warnings
|
||||
):
|
||||
raise RuntimeError(f"Layout warning marked as terminal for scholar_id={scholar_id}.")
|
||||
for publication in paged_parse_result.publications:
|
||||
if not publication.title.strip():
|
||||
raise RuntimeError(f"Malformed publication title for scholar_id={scholar_id}.")
|
||||
if publication.citation_count is not None and int(publication.citation_count) < 0:
|
||||
raise RuntimeError(f"Negative citation count for scholar_id={scholar_id}.")
|
||||
|
||||
|
||||
def apply_first_page_profile_metadata(
|
||||
*,
|
||||
scholar: ScholarProfile,
|
||||
paged_parse_result: PagedParseResult,
|
||||
run_dt: datetime,
|
||||
) -> None:
|
||||
first_page = paged_parse_result.first_page_parsed_page
|
||||
if first_page.profile_name and not (scholar.display_name or "").strip():
|
||||
scholar.display_name = first_page.profile_name
|
||||
if first_page.profile_image_url:
|
||||
scholar.profile_image_url = first_page.profile_image_url
|
||||
scholar.last_initial_page_checked_at = run_dt
|
||||
|
||||
|
||||
def build_result_entry(
|
||||
*,
|
||||
scholar: ScholarProfile,
|
||||
start_cstart: int,
|
||||
paged_parse_result: PagedParseResult,
|
||||
) -> dict[str, Any]:
|
||||
parsed_page = paged_parse_result.parsed_page
|
||||
return {
|
||||
"scholar_profile_id": scholar.id,
|
||||
"scholar_id": scholar.scholar_id,
|
||||
"state": parsed_page.state.value,
|
||||
"state_reason": parsed_page.state_reason,
|
||||
"outcome": "failed",
|
||||
"attempt_count": len(paged_parse_result.attempt_log),
|
||||
"publication_count": len(paged_parse_result.publications),
|
||||
"start_cstart": start_cstart,
|
||||
"articles_range": parsed_page.articles_range,
|
||||
"warnings": parsed_page.warnings,
|
||||
"has_show_more_button": parsed_page.has_show_more_button,
|
||||
"pages_fetched": paged_parse_result.pages_fetched,
|
||||
"pages_attempted": paged_parse_result.pages_attempted,
|
||||
"has_more_remaining": paged_parse_result.has_more_remaining,
|
||||
"pagination_truncated_reason": paged_parse_result.pagination_truncated_reason,
|
||||
"continuation_cstart": paged_parse_result.continuation_cstart,
|
||||
"skipped_no_change": paged_parse_result.skipped_no_change,
|
||||
"initial_page_fingerprint_sha256": paged_parse_result.first_page_fingerprint_sha256,
|
||||
}
|
||||
|
||||
|
||||
def skipped_no_change_outcome(
|
||||
*,
|
||||
scholar: ScholarProfile,
|
||||
run_dt: datetime,
|
||||
paged_parse_result: PagedParseResult,
|
||||
result_entry: dict[str, Any],
|
||||
) -> ScholarProcessingOutcome:
|
||||
first_page = paged_parse_result.first_page_parsed_page
|
||||
scholar.last_run_status = RunStatus.SUCCESS
|
||||
scholar.last_run_dt = run_dt
|
||||
result_entry["state"] = first_page.state.value
|
||||
result_entry["state_reason"] = "no_change_initial_page_signature"
|
||||
result_entry["outcome"] = "success"
|
||||
result_entry["publication_count"] = 0
|
||||
result_entry["warnings"] = first_page.warnings
|
||||
result_entry["debug"] = {
|
||||
"state_reason": "no_change_initial_page_signature",
|
||||
"first_page_fingerprint_sha256": paged_parse_result.first_page_fingerprint_sha256,
|
||||
"attempt_log": paged_parse_result.attempt_log,
|
||||
"page_logs": paged_parse_result.page_logs,
|
||||
}
|
||||
return ScholarProcessingOutcome(
|
||||
result_entry=result_entry,
|
||||
succeeded_count_delta=1,
|
||||
failed_count_delta=0,
|
||||
partial_count_delta=0,
|
||||
discovered_publication_count=0,
|
||||
)
|
||||
|
||||
|
||||
async def upsert_publications_outcome(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
run: CrawlRun,
|
||||
scholar: ScholarProfile,
|
||||
run_dt: datetime,
|
||||
paged_parse_result: PagedParseResult,
|
||||
result_entry: dict[str, Any],
|
||||
) -> ScholarProcessingOutcome:
|
||||
parsed_page = paged_parse_result.parsed_page
|
||||
publications = paged_parse_result.publications
|
||||
had_page_failure = parsed_page.state not in {ParseState.OK, ParseState.NO_RESULTS}
|
||||
has_partial_set = len(publications) > 0 and had_page_failure
|
||||
if (not had_page_failure) or has_partial_set:
|
||||
return await _upsert_success_or_exception(
|
||||
db_session,
|
||||
run=run,
|
||||
scholar=scholar,
|
||||
run_dt=run_dt,
|
||||
paged_parse_result=paged_parse_result,
|
||||
result_entry=result_entry,
|
||||
has_partial_publication_set=has_partial_set,
|
||||
)
|
||||
return _parse_failure_outcome(
|
||||
scholar=scholar,
|
||||
run_dt=run_dt,
|
||||
paged_parse_result=paged_parse_result,
|
||||
result_entry=result_entry,
|
||||
)
|
||||
|
||||
|
||||
async def _upsert_success_or_exception(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
run: CrawlRun,
|
||||
scholar: ScholarProfile,
|
||||
run_dt: datetime,
|
||||
paged_parse_result: PagedParseResult,
|
||||
result_entry: dict[str, Any],
|
||||
has_partial_publication_set: bool,
|
||||
) -> ScholarProcessingOutcome:
|
||||
try:
|
||||
return _upsert_success(
|
||||
run=run,
|
||||
scholar=scholar,
|
||||
run_dt=run_dt,
|
||||
paged_parse_result=paged_parse_result,
|
||||
result_entry=result_entry,
|
||||
has_partial_publication_set=has_partial_publication_set,
|
||||
)
|
||||
except Exception as exc:
|
||||
return _upsert_exception_outcome(
|
||||
run=run,
|
||||
scholar=scholar,
|
||||
run_dt=run_dt,
|
||||
paged_parse_result=paged_parse_result,
|
||||
result_entry=result_entry,
|
||||
exc=exc,
|
||||
)
|
||||
|
||||
|
||||
def _upsert_success(
|
||||
*,
|
||||
run: CrawlRun,
|
||||
scholar: ScholarProfile,
|
||||
run_dt: datetime,
|
||||
paged_parse_result: PagedParseResult,
|
||||
result_entry: dict[str, Any],
|
||||
has_partial_publication_set: bool,
|
||||
) -> ScholarProcessingOutcome:
|
||||
discovered_count = paged_parse_result.discovered_publication_count
|
||||
is_partial = (
|
||||
paged_parse_result.has_more_remaining
|
||||
or paged_parse_result.pagination_truncated_reason is not None
|
||||
or has_partial_publication_set
|
||||
)
|
||||
scholar.last_run_status = RunStatus.PARTIAL_FAILURE if is_partial else RunStatus.SUCCESS
|
||||
scholar.last_run_dt = run_dt
|
||||
if not is_partial and paged_parse_result.first_page_fingerprint_sha256:
|
||||
scholar.last_initial_page_fingerprint_sha256 = paged_parse_result.first_page_fingerprint_sha256
|
||||
result_entry["outcome"] = "partial" if is_partial else "success"
|
||||
if is_partial:
|
||||
result_entry["debug"] = build_failure_debug_context(
|
||||
fetch_result=paged_parse_result.fetch_result,
|
||||
parsed_page=paged_parse_result.parsed_page,
|
||||
attempt_log=paged_parse_result.attempt_log,
|
||||
page_logs=paged_parse_result.page_logs,
|
||||
)
|
||||
return ScholarProcessingOutcome(
|
||||
result_entry=result_entry,
|
||||
succeeded_count_delta=1,
|
||||
failed_count_delta=0,
|
||||
partial_count_delta=1 if is_partial else 0,
|
||||
discovered_publication_count=discovered_count,
|
||||
)
|
||||
|
||||
|
||||
def _upsert_exception_outcome(
|
||||
*,
|
||||
run: CrawlRun,
|
||||
scholar: ScholarProfile,
|
||||
run_dt: datetime,
|
||||
paged_parse_result: PagedParseResult,
|
||||
result_entry: dict[str, Any],
|
||||
exc: Exception,
|
||||
) -> ScholarProcessingOutcome:
|
||||
scholar.last_run_status = RunStatus.FAILED
|
||||
scholar.last_run_dt = run_dt
|
||||
result_entry["state"] = "ingestion_error"
|
||||
result_entry["state_reason"] = "publication_upsert_exception"
|
||||
result_entry["outcome"] = "failed"
|
||||
result_entry["error"] = str(exc)
|
||||
result_entry["debug"] = build_failure_debug_context(
|
||||
fetch_result=paged_parse_result.fetch_result,
|
||||
parsed_page=paged_parse_result.parsed_page,
|
||||
attempt_log=paged_parse_result.attempt_log,
|
||||
page_logs=paged_parse_result.page_logs,
|
||||
exception=exc,
|
||||
)
|
||||
logger.exception(
|
||||
"ingestion.scholar_failed",
|
||||
extra={
|
||||
"crawl_run_id": run.id,
|
||||
"scholar_profile_id": scholar.id,
|
||||
"scholar_id": scholar.scholar_id,
|
||||
},
|
||||
)
|
||||
return ScholarProcessingOutcome(result_entry, 0, 1, 0, 0)
|
||||
|
||||
|
||||
def _parse_failure_outcome(
|
||||
*,
|
||||
scholar: ScholarProfile,
|
||||
run_dt: datetime,
|
||||
paged_parse_result: PagedParseResult,
|
||||
result_entry: dict[str, Any],
|
||||
) -> ScholarProcessingOutcome:
|
||||
scholar.last_run_status = RunStatus.FAILED
|
||||
scholar.last_run_dt = run_dt
|
||||
result_entry["debug"] = build_failure_debug_context(
|
||||
fetch_result=paged_parse_result.fetch_result,
|
||||
parsed_page=paged_parse_result.parsed_page,
|
||||
attempt_log=paged_parse_result.attempt_log,
|
||||
page_logs=paged_parse_result.page_logs,
|
||||
)
|
||||
structured_log(
|
||||
logger,
|
||||
"warning",
|
||||
"ingestion.scholar_parse_failed",
|
||||
scholar_profile_id=scholar.id,
|
||||
scholar_id=scholar.scholar_id,
|
||||
state=paged_parse_result.parsed_page.state.value,
|
||||
state_reason=paged_parse_result.parsed_page.state_reason,
|
||||
status_code=paged_parse_result.fetch_result.status_code,
|
||||
)
|
||||
return ScholarProcessingOutcome(result_entry, 0, 1, 0, 0)
|
||||
|
||||
|
||||
async def sync_continuation_queue(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
|
|
@ -396,6 +157,8 @@ async def process_scholar(
|
|||
queue_delay_seconds=queue_delay_seconds,
|
||||
)
|
||||
return outcome
|
||||
except (OperationalError, InterfaceError):
|
||||
raise
|
||||
except Exception as exc:
|
||||
return unexpected_scholar_exception_outcome(
|
||||
run=run,
|
||||
|
|
@ -492,44 +255,6 @@ async def _resolve_scholar_outcome(
|
|||
)
|
||||
|
||||
|
||||
def unexpected_scholar_exception_outcome(
|
||||
*,
|
||||
run: CrawlRun,
|
||||
scholar: ScholarProfile,
|
||||
start_cstart: int,
|
||||
exc: Exception,
|
||||
) -> ScholarProcessingOutcome:
|
||||
scholar.last_run_status = RunStatus.FAILED
|
||||
scholar.last_run_dt = datetime.now(UTC)
|
||||
logger.exception(
|
||||
"ingestion.scholar_unexpected_failure",
|
||||
extra={
|
||||
"crawl_run_id": run.id,
|
||||
"scholar_profile_id": scholar.id,
|
||||
"scholar_id": scholar.scholar_id,
|
||||
},
|
||||
)
|
||||
return ScholarProcessingOutcome(
|
||||
result_entry={
|
||||
"scholar_profile_id": scholar.id,
|
||||
"scholar_id": scholar.scholar_id,
|
||||
"state": "ingestion_error",
|
||||
"state_reason": "scholar_processing_exception",
|
||||
"outcome": "failed",
|
||||
"attempt_count": 0,
|
||||
"publication_count": 0,
|
||||
"start_cstart": start_cstart,
|
||||
"warnings": [],
|
||||
"error": str(exc),
|
||||
"debug": {"exception_type": type(exc).__name__, "exception_message": str(exc)},
|
||||
},
|
||||
succeeded_count_delta=0,
|
||||
failed_count_delta=1,
|
||||
partial_count_delta=0,
|
||||
discovered_publication_count=0,
|
||||
)
|
||||
|
||||
|
||||
def _is_hard_challenge_outcome(outcome: ScholarProcessingOutcome) -> bool:
|
||||
entry = outcome.result_entry
|
||||
return str(entry.get("state", "")) == ParseState.BLOCKED_OR_CAPTCHA.value and is_hard_challenge_reason(
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue