Intermediate commit
This commit is contained in:
parent
0e9e49df16
commit
3d4cfeff1a
65 changed files with 5507 additions and 333 deletions
|
|
@ -1,29 +1,30 @@
|
|||
"""Unit tests for the identifier-based publication dedup sweep.
|
||||
"""Unit tests for publication dedup operations."""
|
||||
|
||||
DB operations are mocked via AsyncMock so no database is required.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from app.db.models import ScholarPublication
|
||||
from app.services.domains.publications import dedup as dedup_service
|
||||
from app.services.domains.publications.dedup import (
|
||||
NearDuplicateCluster,
|
||||
NearDuplicateMember,
|
||||
find_identifier_duplicate_pairs,
|
||||
find_near_duplicate_clusters,
|
||||
merge_duplicate_publication,
|
||||
merge_near_duplicate_cluster,
|
||||
sweep_identifier_duplicates,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _make_result(rows: list) -> MagicMock:
|
||||
mock_result = MagicMock()
|
||||
mock_result.scalars.return_value.all.return_value = rows
|
||||
mock_result.__iter__ = lambda self: iter(rows)
|
||||
mock_result.all.return_value = rows
|
||||
return mock_result
|
||||
|
||||
|
||||
|
|
@ -35,10 +36,6 @@ def _session_with_execute_sequence(results: list) -> AsyncMock:
|
|||
return session
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# find_identifier_duplicate_pairs
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_find_identifier_duplicate_pairs_returns_pairs() -> None:
|
||||
session = AsyncMock()
|
||||
|
|
@ -60,56 +57,104 @@ async def test_find_identifier_duplicate_pairs_returns_empty_when_no_duplicates(
|
|||
assert pairs == []
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# merge_duplicate_publication
|
||||
# ---------------------------------------------------------------------------
|
||||
@pytest.mark.asyncio
|
||||
async def test_merge_duplicate_publication_migrates_links_and_identifiers() -> None:
|
||||
session = AsyncMock()
|
||||
winner = SimpleNamespace(
|
||||
year=2014,
|
||||
citation_count=10,
|
||||
author_text=None,
|
||||
venue_text=None,
|
||||
pub_url=None,
|
||||
pdf_url=None,
|
||||
cluster_id=None,
|
||||
canonical_title_hash=None,
|
||||
title_raw="winner",
|
||||
title_normalized="winner",
|
||||
)
|
||||
dup = SimpleNamespace(
|
||||
year=2015,
|
||||
citation_count=11,
|
||||
author_text="a",
|
||||
venue_text="v",
|
||||
pub_url="https://example.org",
|
||||
pdf_url="https://example.org/a.pdf",
|
||||
cluster_id="x",
|
||||
canonical_title_hash="hash",
|
||||
title_raw="dup",
|
||||
title_normalized="dup",
|
||||
)
|
||||
|
||||
with (
|
||||
patch(
|
||||
"app.services.domains.publications.dedup._load_publication",
|
||||
new=AsyncMock(side_effect=[winner, dup]),
|
||||
),
|
||||
patch(
|
||||
"app.services.domains.publications.dedup._migrate_scholar_links",
|
||||
new=AsyncMock(),
|
||||
) as mock_links,
|
||||
patch(
|
||||
"app.services.domains.publications.dedup._migrate_identifiers",
|
||||
new=AsyncMock(),
|
||||
) as mock_identifiers,
|
||||
):
|
||||
await merge_duplicate_publication(session, winner_id=1, dup_id=2)
|
||||
|
||||
assert session.execute.await_count == 1
|
||||
mock_links.assert_awaited_once_with(session, winner_id=1, dup_id=2)
|
||||
mock_identifiers.assert_awaited_once_with(session, winner_id=1, dup_id=2)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_merge_duplicate_migrates_orphaned_scholar_links() -> None:
|
||||
"""Scholar link that only the dup has should be migrated to winner."""
|
||||
async def test_merge_duplicate_publication_rejects_missing_publications() -> None:
|
||||
session = AsyncMock()
|
||||
|
||||
with patch(
|
||||
"app.services.domains.publications.dedup._load_publication",
|
||||
new=AsyncMock(side_effect=[None, None]),
|
||||
):
|
||||
with pytest.raises(ValueError):
|
||||
await merge_duplicate_publication(session, winner_id=1, dup_id=2)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_migrate_scholar_links_moves_orphans() -> None:
|
||||
dup_link = MagicMock(spec=ScholarPublication)
|
||||
dup_link.scholar_profile_id = 99
|
||||
dup_link.publication_id = 2
|
||||
|
||||
session = _session_with_execute_sequence(
|
||||
results=[
|
||||
[dup_link], # dup links
|
||||
[], # winner profile ids (no conflict)
|
||||
[], # execute(delete(Publication)) result
|
||||
[dup_link],
|
||||
[],
|
||||
]
|
||||
)
|
||||
|
||||
await merge_duplicate_publication(session, winner_id=1, dup_id=2)
|
||||
await dedup_service._migrate_scholar_links(session, winner_id=1, dup_id=2)
|
||||
|
||||
assert dup_link.publication_id == 1
|
||||
session.delete.assert_not_awaited() # not deleted; migrated instead
|
||||
session.delete.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_merge_duplicate_drops_conflicting_scholar_link() -> None:
|
||||
"""When winner already has a link for the same scholar, dup's link is deleted."""
|
||||
async def test_migrate_scholar_links_drops_conflicts() -> None:
|
||||
dup_link = MagicMock(spec=ScholarPublication)
|
||||
dup_link.scholar_profile_id = 88
|
||||
dup_link.publication_id = 2
|
||||
|
||||
session = _session_with_execute_sequence(
|
||||
results=[
|
||||
[dup_link], # dup links
|
||||
[(88,)], # winner profiles (conflict: profile 88 already linked)
|
||||
[], # execute(delete(Publication)) result
|
||||
[dup_link],
|
||||
[(88,)],
|
||||
]
|
||||
)
|
||||
|
||||
await merge_duplicate_publication(session, winner_id=1, dup_id=2)
|
||||
await dedup_service._migrate_scholar_links(session, winner_id=1, dup_id=2)
|
||||
|
||||
session.delete.assert_awaited_once_with(dup_link)
|
||||
assert dup_link.publication_id == 2 # unchanged — link was deleted, not migrated
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# sweep_identifier_duplicates
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sweep_returns_zero_when_no_pairs() -> None:
|
||||
with patch(
|
||||
|
|
@ -145,7 +190,6 @@ async def test_sweep_returns_merge_count() -> None:
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sweep_merges_each_dup_only_once() -> None:
|
||||
"""A dup sharing two identifiers with the winner appears twice in pairs but merged once."""
|
||||
with (
|
||||
patch(
|
||||
"app.services.domains.publications.dedup.find_identifier_duplicate_pairs",
|
||||
|
|
@ -161,3 +205,80 @@ async def test_sweep_merges_each_dup_only_once() -> None:
|
|||
|
||||
assert count == 1
|
||||
assert mock_merge.await_count == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_find_near_duplicate_clusters_groups_similar_titles() -> None:
|
||||
first = dedup_service._candidate_from_row(
|
||||
publication_id=10,
|
||||
title="Adam: A method for stochastic optimization",
|
||||
year=2014,
|
||||
citation_count=100,
|
||||
)
|
||||
second = dedup_service._candidate_from_row(
|
||||
publication_id=11,
|
||||
title="†œAdam: A method for stochastic optimization, †3rd Int. Conf. Learn. Represent.",
|
||||
year=2015,
|
||||
citation_count=50,
|
||||
)
|
||||
assert first is not None
|
||||
assert second is not None
|
||||
|
||||
with patch(
|
||||
"app.services.domains.publications.dedup._load_near_duplicate_candidates",
|
||||
new=AsyncMock(return_value=[first, second]),
|
||||
):
|
||||
clusters = await find_near_duplicate_clusters(AsyncMock())
|
||||
|
||||
assert len(clusters) == 1
|
||||
assert len(clusters[0].members) == 2
|
||||
assert clusters[0].winner_publication_id == 10
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_find_near_duplicate_clusters_skips_unrelated_titles() -> None:
|
||||
first = dedup_service._candidate_from_row(
|
||||
publication_id=21,
|
||||
title="Adam optimizer",
|
||||
year=2014,
|
||||
citation_count=10,
|
||||
)
|
||||
second = dedup_service._candidate_from_row(
|
||||
publication_id=22,
|
||||
title="Diffusion models in vision",
|
||||
year=2022,
|
||||
citation_count=10,
|
||||
)
|
||||
assert first is not None
|
||||
assert second is not None
|
||||
|
||||
with patch(
|
||||
"app.services.domains.publications.dedup._load_near_duplicate_candidates",
|
||||
new=AsyncMock(return_value=[first, second]),
|
||||
):
|
||||
clusters = await find_near_duplicate_clusters(AsyncMock())
|
||||
|
||||
assert clusters == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_merge_near_duplicate_cluster_merges_non_winner_members() -> None:
|
||||
cluster = NearDuplicateCluster(
|
||||
cluster_key="abc",
|
||||
winner_publication_id=5,
|
||||
similarity_score=1.0,
|
||||
members=(
|
||||
NearDuplicateMember(publication_id=5, title="Winner", year=2014, citation_count=10),
|
||||
NearDuplicateMember(publication_id=6, title="Dup", year=2014, citation_count=3),
|
||||
NearDuplicateMember(publication_id=7, title="Dup2", year=2014, citation_count=1),
|
||||
),
|
||||
)
|
||||
|
||||
with patch(
|
||||
"app.services.domains.publications.dedup.merge_duplicate_publication",
|
||||
new=AsyncMock(),
|
||||
) as mock_merge:
|
||||
merged = await merge_near_duplicate_cluster(AsyncMock(), cluster=cluster)
|
||||
|
||||
assert merged == 2
|
||||
assert mock_merge.await_count == 2
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue