from __future__ import annotations from datetime import UTC, datetime from typing import Any from sqlalchemy.ext.asyncio import AsyncSession from app.db.models import DataRepairJob from app.services.domains.publications import dedup as dedup_service REPAIR_STATUS_PLANNED = "planned" REPAIR_STATUS_RUNNING = "running" REPAIR_STATUS_COMPLETED = "completed" REPAIR_STATUS_FAILED = "failed" NEAR_DUP_JOB_NAME = "repair_publication_near_duplicates" NEAR_DUP_DEFAULT_MAX_CLUSTERS = 25 def _utcnow() -> datetime: return datetime.now(UTC) def _normalized_cluster_keys(values: list[str] | None) -> list[str]: if not values: return [] seen: set[str] = set() normalized: list[str] = [] for value in values: key = str(value or "").strip().lower() if not key or key in seen: continue seen.add(key) normalized.append(key) return normalized def _normalized_max_clusters(value: int) -> int: return max(1, min(int(value), 200)) def _scope_payload( *, similarity_threshold: float, min_shared_tokens: int, max_year_delta: int, max_clusters: int, selected_cluster_keys: list[str], ) -> dict[str, Any]: return { "similarity_threshold": float(similarity_threshold), "min_shared_tokens": int(min_shared_tokens), "max_year_delta": int(max_year_delta), "max_clusters": int(max_clusters), "selected_cluster_keys": selected_cluster_keys, } async def _create_job( db_session: AsyncSession, *, requested_by: str | None, scope: dict[str, Any], dry_run: bool, ) -> DataRepairJob: job = DataRepairJob( job_name=NEAR_DUP_JOB_NAME, requested_by=(requested_by or "").strip() or None, scope=scope, dry_run=bool(dry_run), status=REPAIR_STATUS_PLANNED, summary={}, ) db_session.add(job) await db_session.flush() job.status = REPAIR_STATUS_RUNNING job.started_at = _utcnow() return job def _selected_clusters( *, clusters: list[dedup_service.NearDuplicateCluster], selected_cluster_keys: list[str], ) -> tuple[list[dedup_service.NearDuplicateCluster], list[str]]: if not selected_cluster_keys: return [], [] by_key = {cluster.cluster_key.lower(): cluster for cluster in clusters} selected: list[dedup_service.NearDuplicateCluster] = [] missing: list[str] = [] for key in selected_cluster_keys: cluster = by_key.get(key) if cluster is None: missing.append(key) continue selected.append(cluster) return selected, missing async def _merge_selected_clusters( db_session: AsyncSession, *, selected_clusters: list[dedup_service.NearDuplicateCluster], ) -> int: merged_publications = 0 for cluster in selected_clusters: merged_publications += await dedup_service.merge_near_duplicate_cluster( db_session, cluster=cluster, ) return merged_publications def _clusters_payload( *, clusters: list[dedup_service.NearDuplicateCluster], max_clusters: int, ) -> list[dict[str, object]]: preview = clusters[:max_clusters] return [dedup_service.near_duplicate_cluster_payload(cluster) for cluster in preview] def _summary_payload( *, dry_run: bool, cluster_count: int, selected_count: int, missing_count: int, merged_publications: int, max_clusters: int, ) -> dict[str, Any]: return { "dry_run": bool(dry_run), "candidate_cluster_count": int(cluster_count), "selected_cluster_count": int(selected_count), "missing_selected_cluster_count": int(missing_count), "merged_publications": int(merged_publications), "preview_cluster_count": int(min(cluster_count, max_clusters)), } async def _complete_job( db_session: AsyncSession, *, job: DataRepairJob, scope: dict[str, Any], summary: dict[str, Any], clusters: list[dict[str, object]], ) -> dict[str, Any]: job.status = REPAIR_STATUS_COMPLETED job.finished_at = _utcnow() job.summary = summary await db_session.commit() return { "job_id": int(job.id), "status": job.status, "scope": scope, "summary": summary, "clusters": clusters, } async def _fail_job(db_session: AsyncSession, *, job: DataRepairJob, error: Exception) -> None: await db_session.rollback() job.status = REPAIR_STATUS_FAILED job.error_text = str(error) job.finished_at = _utcnow() db_session.add(job) await db_session.commit() async def run_publication_near_duplicate_repair( db_session: AsyncSession, *, dry_run: bool = True, similarity_threshold: float = dedup_service.NEAR_DUP_DEFAULT_SIMILARITY_THRESHOLD, min_shared_tokens: int = dedup_service.NEAR_DUP_DEFAULT_MIN_SHARED_TOKENS, max_year_delta: int = dedup_service.NEAR_DUP_DEFAULT_MAX_YEAR_DELTA, max_clusters: int = NEAR_DUP_DEFAULT_MAX_CLUSTERS, selected_cluster_keys: list[str] | None = None, requested_by: str | None = None, ) -> dict[str, Any]: normalized_keys = _normalized_cluster_keys(selected_cluster_keys) bounded_clusters = _normalized_max_clusters(max_clusters) scope = _scope_payload( similarity_threshold=similarity_threshold, min_shared_tokens=min_shared_tokens, max_year_delta=max_year_delta, max_clusters=bounded_clusters, selected_cluster_keys=normalized_keys, ) job = await _create_job(db_session, requested_by=requested_by, scope=scope, dry_run=dry_run) try: clusters = await dedup_service.find_near_duplicate_clusters( db_session, similarity_threshold=similarity_threshold, min_shared_tokens=min_shared_tokens, max_year_delta=max_year_delta, ) selected, missing = _selected_clusters(clusters=clusters, selected_cluster_keys=normalized_keys) merged_publications = 0 if not dry_run: if not selected: raise ValueError("No selected near-duplicate clusters matched current data.") merged_publications = await _merge_selected_clusters(db_session, selected_clusters=selected) preview = _clusters_payload(clusters=clusters, max_clusters=bounded_clusters) summary = _summary_payload( dry_run=dry_run, cluster_count=len(clusters), selected_count=len(selected), missing_count=len(missing), merged_publications=merged_publications, max_clusters=bounded_clusters, ) return await _complete_job( db_session, job=job, scope=scope, summary=summary, clusters=preview, ) except Exception as exc: await _fail_job(db_session, job=job, error=exc) raise