ci: add CodeQL security scanning and Dependabot
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
ac002131d6
commit
3866c6d6f0
90 changed files with 40 additions and 1 deletions
13
app/services/dbops/__init__.py
Normal file
13
app/services/dbops/__init__.py
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
from app.services.domains.dbops.application import run_publication_link_repair
|
||||
from app.services.domains.dbops.integrity import collect_integrity_report
|
||||
from app.services.domains.dbops.near_duplicate_repair import (
|
||||
run_publication_near_duplicate_repair,
|
||||
)
|
||||
from app.services.domains.dbops.query import list_repair_jobs
|
||||
|
||||
__all__ = [
|
||||
"collect_integrity_report",
|
||||
"list_repair_jobs",
|
||||
"run_publication_link_repair",
|
||||
"run_publication_near_duplicate_repair",
|
||||
]
|
||||
353
app/services/dbops/application.py
Normal file
353
app/services/dbops/application.py
Normal file
|
|
@ -0,0 +1,353 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import delete, exists, func, select, update
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.models import DataRepairJob, IngestionQueueItem, Publication, ScholarProfile, ScholarPublication
|
||||
|
||||
REPAIR_STATUS_PLANNED = "planned"
|
||||
REPAIR_STATUS_RUNNING = "running"
|
||||
REPAIR_STATUS_COMPLETED = "completed"
|
||||
REPAIR_STATUS_FAILED = "failed"
|
||||
SCOPE_MODE_SINGLE_USER = "single_user"
|
||||
SCOPE_MODE_ALL_USERS = "all_users"
|
||||
|
||||
|
||||
def _utcnow() -> datetime:
|
||||
return datetime.now(UTC)
|
||||
|
||||
|
||||
def _normalize_scope_mode(scope_mode: str) -> str:
|
||||
normalized = scope_mode.strip().lower()
|
||||
if normalized in {SCOPE_MODE_SINGLE_USER, SCOPE_MODE_ALL_USERS}:
|
||||
return normalized
|
||||
raise ValueError("Unknown scope mode.")
|
||||
|
||||
|
||||
def _scope_user_id(*, scope_mode: str, user_id: int | None) -> int | None:
|
||||
if scope_mode == SCOPE_MODE_SINGLE_USER:
|
||||
if user_id is None:
|
||||
raise ValueError("user_id is required when scope_mode=single_user.")
|
||||
return int(user_id)
|
||||
if user_id is not None:
|
||||
raise ValueError("user_id must be omitted when scope_mode=all_users.")
|
||||
return None
|
||||
|
||||
|
||||
def _scope_payload(
|
||||
*,
|
||||
scope_mode: str,
|
||||
user_id: int | None,
|
||||
target_scholar_profile_ids: list[int],
|
||||
orphan_gc: bool,
|
||||
) -> dict[str, Any]:
|
||||
payload: dict[str, Any] = {
|
||||
"scope_mode": scope_mode,
|
||||
"scholar_profile_ids": [int(value) for value in target_scholar_profile_ids],
|
||||
"gc_orphan_publications": bool(orphan_gc),
|
||||
}
|
||||
if user_id is not None:
|
||||
payload["user_id"] = int(user_id)
|
||||
return payload
|
||||
|
||||
|
||||
async def _target_scholar_profile_ids(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
scope_mode: str,
|
||||
user_id: int | None,
|
||||
scholar_profile_ids: list[int] | None,
|
||||
) -> list[int]:
|
||||
stmt = select(ScholarProfile.id)
|
||||
if scope_mode == SCOPE_MODE_SINGLE_USER:
|
||||
stmt = stmt.where(ScholarProfile.user_id == user_id)
|
||||
if scholar_profile_ids:
|
||||
normalized_ids = [int(value) for value in scholar_profile_ids]
|
||||
stmt = stmt.where(ScholarProfile.id.in_(normalized_ids))
|
||||
result = await db_session.execute(stmt.order_by(ScholarProfile.id.asc()))
|
||||
ids = [int(row[0]) for row in result.all()]
|
||||
if not ids:
|
||||
raise ValueError("No target scholar profiles found for the requested scope.")
|
||||
return ids
|
||||
|
||||
|
||||
async def _count_scope(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int | None,
|
||||
target_scholar_profile_ids: list[int],
|
||||
) -> dict[str, int]:
|
||||
links_result = await db_session.execute(
|
||||
select(func.count())
|
||||
.select_from(ScholarPublication)
|
||||
.where(ScholarPublication.scholar_profile_id.in_(target_scholar_profile_ids))
|
||||
)
|
||||
queue_stmt = (
|
||||
select(func.count())
|
||||
.select_from(IngestionQueueItem)
|
||||
.where(IngestionQueueItem.scholar_profile_id.in_(target_scholar_profile_ids))
|
||||
)
|
||||
if user_id is not None:
|
||||
queue_stmt = queue_stmt.where(IngestionQueueItem.user_id == user_id)
|
||||
queue_result = await db_session.execute(queue_stmt)
|
||||
return {
|
||||
"target_scholar_count": len(target_scholar_profile_ids),
|
||||
"links_in_scope": int(links_result.scalar_one() or 0),
|
||||
"queue_items_in_scope": int(queue_result.scalar_one() or 0),
|
||||
}
|
||||
|
||||
|
||||
async def _count_orphan_publications(db_session: AsyncSession) -> int:
|
||||
stmt = (
|
||||
select(func.count())
|
||||
.select_from(Publication)
|
||||
.where(~exists(select(1).where(ScholarPublication.publication_id == Publication.id)))
|
||||
)
|
||||
result = await db_session.execute(stmt)
|
||||
return int(result.scalar_one() or 0)
|
||||
|
||||
|
||||
async def _create_job(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
requested_by: str | None,
|
||||
scope: dict[str, Any],
|
||||
dry_run: bool,
|
||||
) -> DataRepairJob:
|
||||
job = DataRepairJob(
|
||||
job_name="repair_publication_links",
|
||||
requested_by=(requested_by or "").strip() or None,
|
||||
scope=scope,
|
||||
dry_run=dry_run,
|
||||
status=REPAIR_STATUS_PLANNED,
|
||||
summary={},
|
||||
)
|
||||
db_session.add(job)
|
||||
await db_session.flush()
|
||||
return job
|
||||
|
||||
|
||||
def _job_summary(
|
||||
*,
|
||||
counts: dict[str, int],
|
||||
dry_run: bool,
|
||||
links_deleted: int,
|
||||
queue_items_deleted: int,
|
||||
scholars_reset: int,
|
||||
orphan_publications_before: int,
|
||||
orphan_publications_deleted: int,
|
||||
) -> dict[str, Any]:
|
||||
return {
|
||||
**counts,
|
||||
"dry_run": bool(dry_run),
|
||||
"links_deleted": int(links_deleted),
|
||||
"queue_items_deleted": int(queue_items_deleted),
|
||||
"scholars_reset": int(scholars_reset),
|
||||
"orphan_publications_before": int(orphan_publications_before),
|
||||
"orphan_publications_deleted": int(orphan_publications_deleted),
|
||||
}
|
||||
|
||||
|
||||
async def _delete_links_for_targets(db_session: AsyncSession, *, target_scholar_profile_ids: list[int]) -> int:
|
||||
result = await db_session.execute(
|
||||
delete(ScholarPublication).where(ScholarPublication.scholar_profile_id.in_(target_scholar_profile_ids))
|
||||
)
|
||||
return int(result.rowcount or 0)
|
||||
|
||||
|
||||
async def _delete_queue_for_targets(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int | None,
|
||||
target_scholar_profile_ids: list[int],
|
||||
) -> int:
|
||||
stmt = delete(IngestionQueueItem).where(IngestionQueueItem.scholar_profile_id.in_(target_scholar_profile_ids))
|
||||
if user_id is not None:
|
||||
stmt = stmt.where(IngestionQueueItem.user_id == user_id)
|
||||
result = await db_session.execute(stmt)
|
||||
return int(result.rowcount or 0)
|
||||
|
||||
|
||||
async def _reset_scholar_tracking_state(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int | None,
|
||||
target_scholar_profile_ids: list[int],
|
||||
) -> int:
|
||||
stmt = update(ScholarProfile).where(ScholarProfile.id.in_(target_scholar_profile_ids))
|
||||
if user_id is not None:
|
||||
stmt = stmt.where(ScholarProfile.user_id == user_id)
|
||||
result = await db_session.execute(
|
||||
stmt.values(
|
||||
baseline_completed=False,
|
||||
last_initial_page_fingerprint_sha256=None,
|
||||
last_initial_page_checked_at=None,
|
||||
last_run_dt=None,
|
||||
last_run_status=None,
|
||||
)
|
||||
)
|
||||
return int(result.rowcount or 0)
|
||||
|
||||
|
||||
async def _delete_orphan_publications(db_session: AsyncSession) -> int:
|
||||
result = await db_session.execute(
|
||||
delete(Publication).where(~exists(select(1).where(ScholarPublication.publication_id == Publication.id)))
|
||||
)
|
||||
return int(result.rowcount or 0)
|
||||
|
||||
|
||||
async def _mutation_counts(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int | None,
|
||||
target_ids: list[int],
|
||||
dry_run: bool,
|
||||
gc_orphan_publications: bool,
|
||||
) -> tuple[int, int, int, int]:
|
||||
if dry_run:
|
||||
return 0, 0, 0, 0
|
||||
links_deleted = await _delete_links_for_targets(
|
||||
db_session,
|
||||
target_scholar_profile_ids=target_ids,
|
||||
)
|
||||
queue_deleted = await _delete_queue_for_targets(
|
||||
db_session,
|
||||
user_id=user_id,
|
||||
target_scholar_profile_ids=target_ids,
|
||||
)
|
||||
scholars_reset = await _reset_scholar_tracking_state(
|
||||
db_session,
|
||||
user_id=user_id,
|
||||
target_scholar_profile_ids=target_ids,
|
||||
)
|
||||
orphan_deleted = 0
|
||||
if gc_orphan_publications:
|
||||
orphan_deleted = await _delete_orphan_publications(db_session)
|
||||
return links_deleted, queue_deleted, scholars_reset, orphan_deleted
|
||||
|
||||
|
||||
def _result_payload(*, job: DataRepairJob, scope: dict[str, Any], summary: dict[str, Any]) -> dict[str, Any]:
|
||||
return {
|
||||
"job_id": int(job.id),
|
||||
"status": job.status,
|
||||
"scope": scope,
|
||||
"summary": summary,
|
||||
}
|
||||
|
||||
|
||||
async def _complete_job(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
job: DataRepairJob,
|
||||
summary: dict[str, Any],
|
||||
scope: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
job.summary = summary
|
||||
job.status = REPAIR_STATUS_COMPLETED
|
||||
job.finished_at = _utcnow()
|
||||
await db_session.commit()
|
||||
return _result_payload(job=job, scope=scope, summary=summary)
|
||||
|
||||
|
||||
async def _fail_job(db_session: AsyncSession, *, job: DataRepairJob, error: Exception) -> None:
|
||||
await db_session.rollback()
|
||||
job.status = REPAIR_STATUS_FAILED
|
||||
job.error_text = str(error)
|
||||
job.finished_at = _utcnow()
|
||||
db_session.add(job)
|
||||
await db_session.commit()
|
||||
|
||||
|
||||
async def _prepare_repair_job(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
scope_mode: str,
|
||||
user_id: int | None,
|
||||
scholar_profile_ids: list[int] | None,
|
||||
dry_run: bool,
|
||||
gc_orphan_publications: bool,
|
||||
requested_by: str | None,
|
||||
) -> tuple[int | None, list[int], dict[str, Any], DataRepairJob]:
|
||||
normalized_scope = _normalize_scope_mode(scope_mode)
|
||||
scope_user_id = _scope_user_id(scope_mode=normalized_scope, user_id=user_id)
|
||||
target_ids = await _target_scholar_profile_ids(
|
||||
db_session,
|
||||
scope_mode=normalized_scope,
|
||||
user_id=scope_user_id,
|
||||
scholar_profile_ids=scholar_profile_ids,
|
||||
)
|
||||
scope = _scope_payload(
|
||||
scope_mode=normalized_scope,
|
||||
user_id=scope_user_id,
|
||||
target_scholar_profile_ids=target_ids,
|
||||
orphan_gc=gc_orphan_publications,
|
||||
)
|
||||
job = await _create_job(db_session, requested_by=requested_by, scope=scope, dry_run=dry_run)
|
||||
job.status = REPAIR_STATUS_RUNNING
|
||||
job.started_at = _utcnow()
|
||||
return scope_user_id, target_ids, scope, job
|
||||
|
||||
|
||||
async def _build_repair_summary(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
scope_user_id: int | None,
|
||||
target_ids: list[int],
|
||||
dry_run: bool,
|
||||
gc_orphan_publications: bool,
|
||||
) -> dict[str, Any]:
|
||||
counts = await _count_scope(db_session, user_id=scope_user_id, target_scholar_profile_ids=target_ids)
|
||||
orphan_before = await _count_orphan_publications(db_session)
|
||||
links_deleted, queue_deleted, scholars_reset, orphan_deleted = await _mutation_counts(
|
||||
db_session,
|
||||
user_id=scope_user_id,
|
||||
target_ids=target_ids,
|
||||
dry_run=dry_run,
|
||||
gc_orphan_publications=gc_orphan_publications,
|
||||
)
|
||||
return _job_summary(
|
||||
counts=counts,
|
||||
dry_run=dry_run,
|
||||
links_deleted=links_deleted,
|
||||
queue_items_deleted=queue_deleted,
|
||||
scholars_reset=scholars_reset,
|
||||
orphan_publications_before=orphan_before,
|
||||
orphan_publications_deleted=orphan_deleted,
|
||||
)
|
||||
|
||||
|
||||
async def run_publication_link_repair(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
scope_mode: str = SCOPE_MODE_SINGLE_USER,
|
||||
user_id: int | None = None,
|
||||
scholar_profile_ids: list[int] | None = None,
|
||||
dry_run: bool = True,
|
||||
gc_orphan_publications: bool = False,
|
||||
requested_by: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
scope_user_id, target_ids, scope, job = await _prepare_repair_job(
|
||||
db_session,
|
||||
scope_mode=scope_mode,
|
||||
user_id=user_id,
|
||||
scholar_profile_ids=scholar_profile_ids,
|
||||
dry_run=dry_run,
|
||||
gc_orphan_publications=gc_orphan_publications,
|
||||
requested_by=requested_by,
|
||||
)
|
||||
|
||||
try:
|
||||
summary = await _build_repair_summary(
|
||||
db_session,
|
||||
scope_user_id=scope_user_id,
|
||||
target_ids=target_ids,
|
||||
dry_run=dry_run,
|
||||
gc_orphan_publications=gc_orphan_publications,
|
||||
)
|
||||
return await _complete_job(db_session, job=job, summary=summary, scope=scope)
|
||||
except Exception as exc:
|
||||
await _fail_job(db_session, job=job, error=exc)
|
||||
raise
|
||||
175
app/services/dbops/integrity.py
Normal file
175
app/services/dbops/integrity.py
Normal file
|
|
@ -0,0 +1,175 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import func, select, text
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.models import Publication
|
||||
|
||||
INTEGRITY_CHECK_DEFS = (
|
||||
(
|
||||
"legacy_cluster_id_format",
|
||||
"warning",
|
||||
"Publications with non-namespaced cluster IDs.",
|
||||
),
|
||||
(
|
||||
"negative_citation_count",
|
||||
"failure",
|
||||
"Publications with negative citation counts.",
|
||||
),
|
||||
(
|
||||
"orphan_publications",
|
||||
"warning",
|
||||
"Publications with no scholar links.",
|
||||
),
|
||||
(
|
||||
"orphan_scholar_publication_links",
|
||||
"failure",
|
||||
"Link rows missing parent scholar/publication.",
|
||||
),
|
||||
(
|
||||
"duplicate_fingerprint_keys",
|
||||
"failure",
|
||||
"Duplicate publication fingerprint keys.",
|
||||
),
|
||||
(
|
||||
"duplicate_cluster_ids",
|
||||
"failure",
|
||||
"Duplicate non-null publication cluster IDs.",
|
||||
),
|
||||
(
|
||||
"missing_pdf_url",
|
||||
"metric",
|
||||
"Publications without a resolved PDF URL.",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
async def _legacy_cluster_id_count(db_session: AsyncSession) -> int:
|
||||
stmt = (
|
||||
select(func.count())
|
||||
.select_from(Publication)
|
||||
.where(Publication.cluster_id.is_not(None))
|
||||
.where(~Publication.cluster_id.like("cfv:%"))
|
||||
.where(~Publication.cluster_id.like("cluster:%"))
|
||||
)
|
||||
result = await db_session.execute(stmt)
|
||||
return int(result.scalar_one() or 0)
|
||||
|
||||
|
||||
async def _negative_citation_count(db_session: AsyncSession) -> int:
|
||||
result = await db_session.execute(
|
||||
select(func.count()).select_from(Publication).where(Publication.citation_count < 0)
|
||||
)
|
||||
return int(result.scalar_one() or 0)
|
||||
|
||||
|
||||
async def _missing_pdf_url_count(db_session: AsyncSession) -> int:
|
||||
result = await db_session.execute(
|
||||
select(func.count()).select_from(Publication).where(Publication.pdf_url.is_(None))
|
||||
)
|
||||
return int(result.scalar_one() or 0)
|
||||
|
||||
|
||||
async def _count_from_sql(db_session: AsyncSession, *, sql: str) -> int:
|
||||
result = await db_session.execute(text(sql))
|
||||
return int(result.scalar_one() or 0)
|
||||
|
||||
|
||||
def _issues_for_severity(*, checks: list[dict[str, Any]], severity: str) -> list[str]:
|
||||
return [row["name"] for row in checks if row["severity"] == severity and row["count"] > 0]
|
||||
|
||||
|
||||
def _check_row(*, name: str, count: int, severity: str, message: str) -> dict[str, Any]:
|
||||
return {
|
||||
"name": name,
|
||||
"count": int(count),
|
||||
"severity": severity,
|
||||
"message": message,
|
||||
}
|
||||
|
||||
|
||||
async def _collect_counts(db_session: AsyncSession) -> dict[str, int]:
|
||||
orphan_publications = await _count_from_sql(
|
||||
db_session,
|
||||
sql=(
|
||||
"SELECT count(*) FROM publications p "
|
||||
"LEFT JOIN scholar_publications sp ON sp.publication_id = p.id "
|
||||
"WHERE sp.publication_id IS NULL"
|
||||
),
|
||||
)
|
||||
orphan_links = await _count_from_sql(
|
||||
db_session,
|
||||
sql=(
|
||||
"SELECT count(*) FROM scholar_publications sp "
|
||||
"LEFT JOIN publications p ON p.id = sp.publication_id "
|
||||
"LEFT JOIN scholar_profiles s ON s.id = sp.scholar_profile_id "
|
||||
"WHERE p.id IS NULL OR s.id IS NULL"
|
||||
),
|
||||
)
|
||||
duplicate_fingerprints = await _count_from_sql(
|
||||
db_session,
|
||||
sql=(
|
||||
"SELECT count(*) FROM ("
|
||||
"SELECT fingerprint_sha256 FROM publications "
|
||||
"GROUP BY fingerprint_sha256 HAVING count(*) > 1"
|
||||
") dup"
|
||||
),
|
||||
)
|
||||
duplicate_cluster_ids = await _count_from_sql(
|
||||
db_session,
|
||||
sql=(
|
||||
"SELECT count(*) FROM ("
|
||||
"SELECT cluster_id FROM publications "
|
||||
"WHERE cluster_id IS NOT NULL "
|
||||
"GROUP BY cluster_id HAVING count(*) > 1"
|
||||
") dup"
|
||||
),
|
||||
)
|
||||
return {
|
||||
"legacy_cluster_id_format": await _legacy_cluster_id_count(db_session),
|
||||
"negative_citation_count": await _negative_citation_count(db_session),
|
||||
"orphan_publications": orphan_publications,
|
||||
"orphan_scholar_publication_links": orphan_links,
|
||||
"duplicate_fingerprint_keys": duplicate_fingerprints,
|
||||
"duplicate_cluster_ids": duplicate_cluster_ids,
|
||||
"missing_pdf_url": await _missing_pdf_url_count(db_session),
|
||||
}
|
||||
|
||||
|
||||
def _build_checks(*, counts: dict[str, int]) -> list[dict[str, Any]]:
|
||||
return [
|
||||
_check_row(
|
||||
name=name,
|
||||
count=counts[name],
|
||||
severity=severity,
|
||||
message=message,
|
||||
)
|
||||
for name, severity, message in INTEGRITY_CHECK_DEFS
|
||||
]
|
||||
|
||||
|
||||
def _status_from_issues(*, failures: list[str], warnings: list[str]) -> str:
|
||||
status = "ok"
|
||||
if failures:
|
||||
status = "failed"
|
||||
elif warnings:
|
||||
status = "warning"
|
||||
return status
|
||||
|
||||
|
||||
async def collect_integrity_report(db_session: AsyncSession) -> dict[str, Any]:
|
||||
counts = await _collect_counts(db_session)
|
||||
checks = _build_checks(counts=counts)
|
||||
failures = _issues_for_severity(checks=checks, severity="failure")
|
||||
warnings = _issues_for_severity(checks=checks, severity="warning")
|
||||
|
||||
return {
|
||||
"status": _status_from_issues(failures=failures, warnings=warnings),
|
||||
"checked_at": datetime.now(UTC).isoformat(),
|
||||
"failures": failures,
|
||||
"warnings": warnings,
|
||||
"checks": checks,
|
||||
}
|
||||
223
app/services/dbops/near_duplicate_repair.py
Normal file
223
app/services/dbops/near_duplicate_repair.py
Normal file
|
|
@ -0,0 +1,223 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.models import DataRepairJob
|
||||
from app.services.domains.publications import dedup as dedup_service
|
||||
|
||||
REPAIR_STATUS_PLANNED = "planned"
|
||||
REPAIR_STATUS_RUNNING = "running"
|
||||
REPAIR_STATUS_COMPLETED = "completed"
|
||||
REPAIR_STATUS_FAILED = "failed"
|
||||
NEAR_DUP_JOB_NAME = "repair_publication_near_duplicates"
|
||||
NEAR_DUP_DEFAULT_MAX_CLUSTERS = 25
|
||||
|
||||
|
||||
def _utcnow() -> datetime:
|
||||
return datetime.now(UTC)
|
||||
|
||||
|
||||
def _normalized_cluster_keys(values: list[str] | None) -> list[str]:
|
||||
if not values:
|
||||
return []
|
||||
seen: set[str] = set()
|
||||
normalized: list[str] = []
|
||||
for value in values:
|
||||
key = str(value or "").strip().lower()
|
||||
if not key or key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
normalized.append(key)
|
||||
return normalized
|
||||
|
||||
|
||||
def _normalized_max_clusters(value: int) -> int:
|
||||
return max(1, min(int(value), 200))
|
||||
|
||||
|
||||
def _scope_payload(
|
||||
*,
|
||||
similarity_threshold: float,
|
||||
min_shared_tokens: int,
|
||||
max_year_delta: int,
|
||||
max_clusters: int,
|
||||
selected_cluster_keys: list[str],
|
||||
) -> dict[str, Any]:
|
||||
return {
|
||||
"similarity_threshold": float(similarity_threshold),
|
||||
"min_shared_tokens": int(min_shared_tokens),
|
||||
"max_year_delta": int(max_year_delta),
|
||||
"max_clusters": int(max_clusters),
|
||||
"selected_cluster_keys": selected_cluster_keys,
|
||||
}
|
||||
|
||||
|
||||
async def _create_job(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
requested_by: str | None,
|
||||
scope: dict[str, Any],
|
||||
dry_run: bool,
|
||||
) -> DataRepairJob:
|
||||
job = DataRepairJob(
|
||||
job_name=NEAR_DUP_JOB_NAME,
|
||||
requested_by=(requested_by or "").strip() or None,
|
||||
scope=scope,
|
||||
dry_run=bool(dry_run),
|
||||
status=REPAIR_STATUS_PLANNED,
|
||||
summary={},
|
||||
)
|
||||
db_session.add(job)
|
||||
await db_session.flush()
|
||||
job.status = REPAIR_STATUS_RUNNING
|
||||
job.started_at = _utcnow()
|
||||
return job
|
||||
|
||||
|
||||
def _selected_clusters(
|
||||
*,
|
||||
clusters: list[dedup_service.NearDuplicateCluster],
|
||||
selected_cluster_keys: list[str],
|
||||
) -> tuple[list[dedup_service.NearDuplicateCluster], list[str]]:
|
||||
if not selected_cluster_keys:
|
||||
return [], []
|
||||
by_key = {cluster.cluster_key.lower(): cluster for cluster in clusters}
|
||||
selected: list[dedup_service.NearDuplicateCluster] = []
|
||||
missing: list[str] = []
|
||||
for key in selected_cluster_keys:
|
||||
cluster = by_key.get(key)
|
||||
if cluster is None:
|
||||
missing.append(key)
|
||||
continue
|
||||
selected.append(cluster)
|
||||
return selected, missing
|
||||
|
||||
|
||||
async def _merge_selected_clusters(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
selected_clusters: list[dedup_service.NearDuplicateCluster],
|
||||
) -> int:
|
||||
merged_publications = 0
|
||||
for cluster in selected_clusters:
|
||||
merged_publications += await dedup_service.merge_near_duplicate_cluster(
|
||||
db_session,
|
||||
cluster=cluster,
|
||||
)
|
||||
return merged_publications
|
||||
|
||||
|
||||
def _clusters_payload(
|
||||
*,
|
||||
clusters: list[dedup_service.NearDuplicateCluster],
|
||||
max_clusters: int,
|
||||
) -> list[dict[str, object]]:
|
||||
preview = clusters[:max_clusters]
|
||||
return [dedup_service.near_duplicate_cluster_payload(cluster) for cluster in preview]
|
||||
|
||||
|
||||
def _summary_payload(
|
||||
*,
|
||||
dry_run: bool,
|
||||
cluster_count: int,
|
||||
selected_count: int,
|
||||
missing_count: int,
|
||||
merged_publications: int,
|
||||
max_clusters: int,
|
||||
) -> dict[str, Any]:
|
||||
return {
|
||||
"dry_run": bool(dry_run),
|
||||
"candidate_cluster_count": int(cluster_count),
|
||||
"selected_cluster_count": int(selected_count),
|
||||
"missing_selected_cluster_count": int(missing_count),
|
||||
"merged_publications": int(merged_publications),
|
||||
"preview_cluster_count": int(min(cluster_count, max_clusters)),
|
||||
}
|
||||
|
||||
|
||||
async def _complete_job(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
job: DataRepairJob,
|
||||
scope: dict[str, Any],
|
||||
summary: dict[str, Any],
|
||||
clusters: list[dict[str, object]],
|
||||
) -> dict[str, Any]:
|
||||
job.status = REPAIR_STATUS_COMPLETED
|
||||
job.finished_at = _utcnow()
|
||||
job.summary = summary
|
||||
await db_session.commit()
|
||||
return {
|
||||
"job_id": int(job.id),
|
||||
"status": job.status,
|
||||
"scope": scope,
|
||||
"summary": summary,
|
||||
"clusters": clusters,
|
||||
}
|
||||
|
||||
|
||||
async def _fail_job(db_session: AsyncSession, *, job: DataRepairJob, error: Exception) -> None:
|
||||
await db_session.rollback()
|
||||
job.status = REPAIR_STATUS_FAILED
|
||||
job.error_text = str(error)
|
||||
job.finished_at = _utcnow()
|
||||
db_session.add(job)
|
||||
await db_session.commit()
|
||||
|
||||
|
||||
async def run_publication_near_duplicate_repair(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
dry_run: bool = True,
|
||||
similarity_threshold: float = dedup_service.NEAR_DUP_DEFAULT_SIMILARITY_THRESHOLD,
|
||||
min_shared_tokens: int = dedup_service.NEAR_DUP_DEFAULT_MIN_SHARED_TOKENS,
|
||||
max_year_delta: int = dedup_service.NEAR_DUP_DEFAULT_MAX_YEAR_DELTA,
|
||||
max_clusters: int = NEAR_DUP_DEFAULT_MAX_CLUSTERS,
|
||||
selected_cluster_keys: list[str] | None = None,
|
||||
requested_by: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
normalized_keys = _normalized_cluster_keys(selected_cluster_keys)
|
||||
bounded_clusters = _normalized_max_clusters(max_clusters)
|
||||
scope = _scope_payload(
|
||||
similarity_threshold=similarity_threshold,
|
||||
min_shared_tokens=min_shared_tokens,
|
||||
max_year_delta=max_year_delta,
|
||||
max_clusters=bounded_clusters,
|
||||
selected_cluster_keys=normalized_keys,
|
||||
)
|
||||
job = await _create_job(db_session, requested_by=requested_by, scope=scope, dry_run=dry_run)
|
||||
try:
|
||||
clusters = await dedup_service.find_near_duplicate_clusters(
|
||||
db_session,
|
||||
similarity_threshold=similarity_threshold,
|
||||
min_shared_tokens=min_shared_tokens,
|
||||
max_year_delta=max_year_delta,
|
||||
)
|
||||
selected, missing = _selected_clusters(clusters=clusters, selected_cluster_keys=normalized_keys)
|
||||
merged_publications = 0
|
||||
if not dry_run:
|
||||
if not selected:
|
||||
raise ValueError("No selected near-duplicate clusters matched current data.")
|
||||
merged_publications = await _merge_selected_clusters(db_session, selected_clusters=selected)
|
||||
preview = _clusters_payload(clusters=clusters, max_clusters=bounded_clusters)
|
||||
summary = _summary_payload(
|
||||
dry_run=dry_run,
|
||||
cluster_count=len(clusters),
|
||||
selected_count=len(selected),
|
||||
missing_count=len(missing),
|
||||
merged_publications=merged_publications,
|
||||
max_clusters=bounded_clusters,
|
||||
)
|
||||
return await _complete_job(
|
||||
db_session,
|
||||
job=job,
|
||||
scope=scope,
|
||||
summary=summary,
|
||||
clusters=preview,
|
||||
)
|
||||
except Exception as exc:
|
||||
await _fail_job(db_session, job=job, error=exc)
|
||||
raise
|
||||
20
app/services/dbops/query.py
Normal file
20
app/services/dbops/query.py
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.models import DataRepairJob
|
||||
|
||||
|
||||
def _bounded_limit(limit: int) -> int:
|
||||
return max(1, min(int(limit), 200))
|
||||
|
||||
|
||||
async def list_repair_jobs(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
limit: int = 50,
|
||||
) -> list[DataRepairJob]:
|
||||
bounded = _bounded_limit(limit)
|
||||
result = await db_session.execute(select(DataRepairJob).order_by(DataRepairJob.created_at.desc()).limit(bounded))
|
||||
return list(result.scalars())
|
||||
Loading…
Add table
Add a link
Reference in a new issue