feat: refactor backend services and add direct PDF links, import/export, and dashboard sync
This commit is contained in:
parent
ba7976d935
commit
7f7a8ce2b0
26 changed files with 4170 additions and 2440 deletions
|
|
@ -499,6 +499,177 @@ async def test_api_scholars_search_and_profile_image_management(
|
|||
)
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
@pytest.mark.db
|
||||
@pytest.mark.asyncio
|
||||
async def test_api_scholar_import_export_round_trip(
|
||||
db_session: AsyncSession,
|
||||
) -> None:
|
||||
user_id = await insert_user(
|
||||
db_session,
|
||||
email="api-import-export@example.com",
|
||||
password="api-password",
|
||||
)
|
||||
scholar_result = await db_session.execute(
|
||||
text(
|
||||
"""
|
||||
INSERT INTO scholar_profiles (user_id, scholar_id, display_name, is_enabled)
|
||||
VALUES (:user_id, :scholar_id, :display_name, true)
|
||||
RETURNING id
|
||||
"""
|
||||
),
|
||||
{
|
||||
"user_id": user_id,
|
||||
"scholar_id": "abcDEF123456",
|
||||
"display_name": "Existing Scholar",
|
||||
},
|
||||
)
|
||||
scholar_profile_id = int(scholar_result.scalar_one())
|
||||
publication_result = await db_session.execute(
|
||||
text(
|
||||
"""
|
||||
INSERT INTO publications (fingerprint_sha256, title_raw, title_normalized, citation_count)
|
||||
VALUES (:fingerprint, :title_raw, :title_normalized, 1)
|
||||
RETURNING id
|
||||
"""
|
||||
),
|
||||
{
|
||||
"fingerprint": f"{(user_id + 500):064x}",
|
||||
"title_raw": "Existing Publication",
|
||||
"title_normalized": "existingpublication",
|
||||
},
|
||||
)
|
||||
publication_id = int(publication_result.scalar_one())
|
||||
await db_session.execute(
|
||||
text(
|
||||
"""
|
||||
INSERT INTO scholar_publications (scholar_profile_id, publication_id, is_read)
|
||||
VALUES (:scholar_profile_id, :publication_id, false)
|
||||
"""
|
||||
),
|
||||
{
|
||||
"scholar_profile_id": scholar_profile_id,
|
||||
"publication_id": publication_id,
|
||||
},
|
||||
)
|
||||
await db_session.commit()
|
||||
|
||||
client = TestClient(app)
|
||||
login_user(client, email="api-import-export@example.com", password="api-password")
|
||||
headers = _api_csrf_headers(client)
|
||||
|
||||
export_response = client.get("/api/v1/scholars/export")
|
||||
assert export_response.status_code == 200
|
||||
export_payload = export_response.json()["data"]
|
||||
assert export_payload["schema_version"] == 1
|
||||
assert len(export_payload["scholars"]) == 1
|
||||
assert len(export_payload["publications"]) == 1
|
||||
|
||||
import_response = client.post(
|
||||
"/api/v1/scholars/import",
|
||||
json={
|
||||
"schema_version": 1,
|
||||
"scholars": [
|
||||
{
|
||||
"scholar_id": "abcDEF123456",
|
||||
"display_name": "Updated Scholar",
|
||||
"is_enabled": False,
|
||||
"profile_image_override_url": "https://cdn.example.com/avatar.png",
|
||||
},
|
||||
{
|
||||
"scholar_id": "zzzYYY111222",
|
||||
"display_name": "Imported Scholar",
|
||||
"is_enabled": True,
|
||||
"profile_image_override_url": None,
|
||||
},
|
||||
],
|
||||
"publications": [
|
||||
{
|
||||
"scholar_id": "abcDEF123456",
|
||||
"title": "Existing Publication",
|
||||
"year": 2024,
|
||||
"citation_count": 8,
|
||||
"author_text": "A. Author",
|
||||
"venue_text": "Test Venue",
|
||||
"pub_url": "https://example.org/existing",
|
||||
"pdf_url": "https://example.org/existing.pdf",
|
||||
"is_read": True,
|
||||
},
|
||||
{
|
||||
"scholar_id": "zzzYYY111222",
|
||||
"title": "Imported Publication",
|
||||
"year": 2025,
|
||||
"citation_count": 2,
|
||||
"author_text": "B. Author",
|
||||
"venue_text": "Another Venue",
|
||||
"pub_url": "https://example.org/imported",
|
||||
"pdf_url": "https://example.org/imported.pdf",
|
||||
"is_read": False,
|
||||
},
|
||||
],
|
||||
},
|
||||
headers=headers,
|
||||
)
|
||||
assert import_response.status_code == 200
|
||||
import_data = import_response.json()["data"]
|
||||
assert int(import_data["scholars_created"]) == 1
|
||||
assert int(import_data["scholars_updated"]) >= 1
|
||||
assert int(import_data["publications_created"]) == 1
|
||||
assert int(import_data["links_created"]) == 1
|
||||
|
||||
updated_scholar_result = await db_session.execute(
|
||||
text(
|
||||
"""
|
||||
SELECT display_name, is_enabled, profile_image_override_url
|
||||
FROM scholar_profiles
|
||||
WHERE user_id = :user_id AND scholar_id = :scholar_id
|
||||
"""
|
||||
),
|
||||
{
|
||||
"user_id": user_id,
|
||||
"scholar_id": "abcDEF123456",
|
||||
},
|
||||
)
|
||||
updated_scholar = updated_scholar_result.one()
|
||||
assert updated_scholar[0] == "Updated Scholar"
|
||||
assert updated_scholar[1] is False
|
||||
assert updated_scholar[2] == "https://cdn.example.com/avatar.png"
|
||||
|
||||
imported_pub_result = await db_session.execute(
|
||||
text(
|
||||
"""
|
||||
SELECT p.title_raw, p.pdf_url
|
||||
FROM publications p
|
||||
WHERE p.title_raw = :title
|
||||
"""
|
||||
),
|
||||
{"title": "Imported Publication"},
|
||||
)
|
||||
imported_pub = imported_pub_result.one()
|
||||
assert imported_pub[0] == "Imported Publication"
|
||||
assert imported_pub[1] == "https://example.org/imported.pdf"
|
||||
|
||||
updated_link_result = await db_session.execute(
|
||||
text(
|
||||
"""
|
||||
SELECT sp.is_read
|
||||
FROM scholar_publications sp
|
||||
JOIN scholar_profiles s ON s.id = sp.scholar_profile_id
|
||||
JOIN publications p ON p.id = sp.publication_id
|
||||
WHERE s.user_id = :user_id
|
||||
AND s.scholar_id = :scholar_id
|
||||
AND p.title_raw = :title
|
||||
"""
|
||||
),
|
||||
{
|
||||
"user_id": user_id,
|
||||
"scholar_id": "abcDEF123456",
|
||||
"title": "Existing Publication",
|
||||
},
|
||||
)
|
||||
assert bool(updated_link_result.scalar_one()) is True
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
@pytest.mark.db
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -1168,8 +1339,14 @@ async def test_api_publications_list_and_mark_read(db_session: AsyncSession) ->
|
|||
publication_a = await db_session.execute(
|
||||
text(
|
||||
"""
|
||||
INSERT INTO publications (fingerprint_sha256, title_raw, title_normalized, citation_count)
|
||||
VALUES (:fingerprint, :title_raw, :title_normalized, 10)
|
||||
INSERT INTO publications (
|
||||
fingerprint_sha256,
|
||||
title_raw,
|
||||
title_normalized,
|
||||
citation_count,
|
||||
pdf_url
|
||||
)
|
||||
VALUES (:fingerprint, :title_raw, :title_normalized, 10, :pdf_url)
|
||||
RETURNING id
|
||||
"""
|
||||
),
|
||||
|
|
@ -1177,6 +1354,7 @@ async def test_api_publications_list_and_mark_read(db_session: AsyncSession) ->
|
|||
"fingerprint": f"{user_id:064x}",
|
||||
"title_raw": "Paper A",
|
||||
"title_normalized": "paper a",
|
||||
"pdf_url": "https://example.org/paper-a.pdf",
|
||||
},
|
||||
)
|
||||
publication_a_id = int(publication_a.scalar_one())
|
||||
|
|
@ -1228,6 +1406,9 @@ async def test_api_publications_list_and_mark_read(db_session: AsyncSession) ->
|
|||
assert data["new_count"] == data["latest_count"]
|
||||
assert isinstance(data["publications"], list)
|
||||
assert len(data["publications"]) == 2
|
||||
pdf_urls = {item["title"]: item["pdf_url"] for item in data["publications"]}
|
||||
assert pdf_urls["Paper A"] == "https://example.org/paper-a.pdf"
|
||||
assert pdf_urls["Paper B"] is None
|
||||
|
||||
latest_response = client.get("/api/v1/publications?mode=latest")
|
||||
assert latest_response.status_code == 200
|
||||
|
|
|
|||
|
|
@ -98,6 +98,74 @@ def test_parse_profile_page_handles_missing_optional_metadata() -> None:
|
|||
assert publication.venue_text is None
|
||||
|
||||
|
||||
def test_parse_profile_page_extracts_direct_pdf_link() -> None:
|
||||
html = """
|
||||
<html>
|
||||
<div id="gsc_prf_in">Direct PDF Test</div>
|
||||
<span id="gsc_a_nn">Articles 1-1</span>
|
||||
<table>
|
||||
<tbody id="gsc_a_b">
|
||||
<tr class="gsc_a_tr">
|
||||
<td class="gsc_a_t">
|
||||
<a class="gsc_a_at" href="/citations?view_op=view_citation&citation_for_view=abc:def999">Paper</a>
|
||||
<div class="gs_ggs gs_fl">
|
||||
<a href="https://example.org/paper.pdf">[PDF]</a>
|
||||
</div>
|
||||
</td>
|
||||
<td class="gsc_a_c"><a class="gsc_a_ac">3</a></td>
|
||||
<td class="gsc_a_y"><span class="gsc_a_h">2025</span></td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</html>
|
||||
"""
|
||||
fetch_result = FetchResult(
|
||||
requested_url="https://scholar.google.com/citations?hl=en&user=abcDEF123456",
|
||||
status_code=200,
|
||||
final_url="https://scholar.google.com/citations?hl=en&user=abcDEF123456",
|
||||
body=html,
|
||||
error=None,
|
||||
)
|
||||
|
||||
parsed = parse_profile_page(fetch_result)
|
||||
|
||||
assert parsed.state == ParseState.OK
|
||||
assert len(parsed.publications) == 1
|
||||
assert parsed.publications[0].pdf_url == "https://example.org/paper.pdf"
|
||||
|
||||
|
||||
def test_parse_profile_page_fails_fast_when_citation_markup_is_unparseable() -> None:
|
||||
html = """
|
||||
<html>
|
||||
<div id="gsc_prf_in">Citation Parse Drift</div>
|
||||
<span id="gsc_a_nn">Articles 1-1</span>
|
||||
<table>
|
||||
<tbody id="gsc_a_b">
|
||||
<tr class="gsc_a_tr">
|
||||
<td class="gsc_a_t">
|
||||
<a class="gsc_a_at" href="/citations?view_op=view_citation&citation_for_view=abc:def777">Paper</a>
|
||||
</td>
|
||||
<td class="gsc_a_c"><a class="gsc_a_ac">Cited by none</a></td>
|
||||
<td class="gsc_a_y"><span class="gsc_a_h">2025</span></td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</html>
|
||||
"""
|
||||
fetch_result = FetchResult(
|
||||
requested_url="https://scholar.google.com/citations?hl=en&user=abcDEF123456",
|
||||
status_code=200,
|
||||
final_url="https://scholar.google.com/citations?hl=en&user=abcDEF123456",
|
||||
body=html,
|
||||
error=None,
|
||||
)
|
||||
|
||||
parsed = parse_profile_page(fetch_result)
|
||||
|
||||
assert parsed.state == ParseState.LAYOUT_CHANGED
|
||||
assert parsed.state_reason == "layout_row_citation_unparseable"
|
||||
|
||||
|
||||
def test_parse_profile_page_detects_layout_change_when_markers_absent() -> None:
|
||||
fetch_result = FetchResult(
|
||||
requested_url="https://scholar.google.com/citations?hl=en&user=abcDEF123456",
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue