modularize service layer, add mobile nav drawer, and sync docs

app/services/domains/portability/__init__.py

@@ -0,0 +1 @@
+from app.services.domains.portability.application import *

app/services/domains/portability/application.py

@@ -0,0 +1,67 @@
+from __future__ import annotations
+
+from typing import Any
+
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.db.models import Publication
+from app.services.domains.portability.constants import (
+    EXPORT_SCHEMA_VERSION,
+    MAX_IMPORT_PUBLICATIONS,
+    MAX_IMPORT_SCHOLARS,
+)
+from app.services.domains.portability.exporting import export_user_data
+from app.services.domains.portability.normalize import _validate_import_sizes
+from app.services.domains.portability.publication_import import (
+    _build_imported_publication_input,
+    _initialize_import_counters,
+    _upsert_imported_publication,
+)
+from app.services.domains.portability.scholar_import import _upsert_imported_scholars
+from app.services.domains.portability.types import ImportExportError, ImportedPublicationInput
+
+
+async def import_user_data(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+    scholars: list[dict[str, Any]],
+    publications: list[dict[str, Any]],
+) -> dict[str, int]:
+    _validate_import_sizes(scholars=scholars, publications=publications)
+    scholar_map, counters = await _upsert_imported_scholars(
+        db_session,
+        user_id=user_id,
+        scholars=scholars,
+    )
+    cluster_cache: dict[str, Publication | None] = {}
+    fingerprint_cache: dict[str, Publication | None] = {}
+    _initialize_import_counters(counters)
+    for item in publications:
+        parsed_item = _build_imported_publication_input(
+            item=item,
+            scholar_map=scholar_map,
+        )
+        if parsed_item is None:
+            counters["skipped_records"] += 1
+            continue
+        await _upsert_imported_publication(
+            db_session,
+            payload=parsed_item,
+            cluster_cache=cluster_cache,
+            fingerprint_cache=fingerprint_cache,
+            counters=counters,
+        )
+    await db_session.commit()
+    return counters
+
+
+__all__ = [
+    "EXPORT_SCHEMA_VERSION",
+    "MAX_IMPORT_SCHOLARS",
+    "MAX_IMPORT_PUBLICATIONS",
+    "ImportExportError",
+    "ImportedPublicationInput",
+    "export_user_data",
+    "import_user_data",
+]

app/services/domains/portability/constants.py

@@ -0,0 +1,9 @@
+from __future__ import annotations
+
+import re
+
+EXPORT_SCHEMA_VERSION = 1
+MAX_IMPORT_SCHOLARS = 10_000
+MAX_IMPORT_PUBLICATIONS = 100_000
+WORD_RE = re.compile(r"[a-z0-9]+")
+SHA256_RE = re.compile(r"^[0-9a-f]{64}$")

app/services/domains/portability/exporting.py

@@ -0,0 +1,91 @@
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from typing import Any
+
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.db.models import Publication, ScholarProfile, ScholarPublication
+from app.services.domains.portability.constants import EXPORT_SCHEMA_VERSION
+
+
+def _exported_at_iso() -> str:
+    return datetime.now(timezone.utc).replace(microsecond=0).isoformat()
+
+
+def _serialize_export_scholar(profile: ScholarProfile) -> dict[str, Any]:
+    return {
+        "scholar_id": profile.scholar_id,
+        "display_name": profile.display_name,
+        "is_enabled": bool(profile.is_enabled),
+        "profile_image_override_url": profile.profile_image_override_url,
+    }
+
+
+def _serialize_export_publication(row: tuple[Any, ...]) -> dict[str, Any]:
+    (
+        scholar_id,
+        cluster_id,
+        fingerprint_sha256,
+        title_raw,
+        year,
+        citation_count,
+        author_text,
+        venue_text,
+        pub_url,
+        pdf_url,
+        is_read,
+    ) = row
+    return {
+        "scholar_id": scholar_id,
+        "cluster_id": cluster_id,
+        "fingerprint_sha256": fingerprint_sha256,
+        "title": title_raw,
+        "year": year,
+        "citation_count": int(citation_count or 0),
+        "author_text": author_text,
+        "venue_text": venue_text,
+        "pub_url": pub_url,
+        "pdf_url": pdf_url,
+        "is_read": bool(is_read),
+    }
+
+
+async def export_user_data(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+) -> dict[str, Any]:
+    scholars_result = await db_session.execute(
+        select(ScholarProfile)
+        .where(ScholarProfile.user_id == user_id)
+        .order_by(ScholarProfile.id.asc())
+    )
+    publication_result = await db_session.execute(
+        select(
+            ScholarProfile.scholar_id,
+            Publication.cluster_id,
+            Publication.fingerprint_sha256,
+            Publication.title_raw,
+            Publication.year,
+            Publication.citation_count,
+            Publication.author_text,
+            Publication.venue_text,
+            Publication.pub_url,
+            Publication.pdf_url,
+            ScholarPublication.is_read,
+        )
+        .join(ScholarPublication, ScholarPublication.scholar_profile_id == ScholarProfile.id)
+        .join(Publication, Publication.id == ScholarPublication.publication_id)
+        .where(ScholarProfile.user_id == user_id)
+        .order_by(ScholarPublication.created_at.desc(), Publication.id.desc())
+    )
+    scholars = [_serialize_export_scholar(profile) for profile in scholars_result.scalars().all()]
+    publications = [_serialize_export_publication(row) for row in publication_result.all()]
+    return {
+        "schema_version": EXPORT_SCHEMA_VERSION,
+        "exported_at": _exported_at_iso(),
+        "scholars": scholars,
+        "publications": publications,
+    }

app/services/domains/portability/normalize.py

@@ -0,0 +1,109 @@
+from __future__ import annotations
+
+import hashlib
+from typing import Any
+
+from app.services.domains.ingestion.application import normalize_title
+from app.services.domains.portability.constants import (
+    MAX_IMPORT_PUBLICATIONS,
+    MAX_IMPORT_SCHOLARS,
+    SHA256_RE,
+    WORD_RE,
+)
+from app.services.domains.portability.types import ImportExportError
+
+
+def _normalize_optional_text(value: Any) -> str | None:
+    if value is None:
+        return None
+    normalized = str(value).strip()
+    return normalized or None
+
+
+def _normalize_optional_year(value: Any) -> int | None:
+    if value is None:
+        return None
+    try:
+        year = int(value)
+    except (TypeError, ValueError):
+        return None
+    if year < 1500 or year > 3000:
+        return None
+    return year
+
+
+def _normalize_citation_count(value: Any) -> int:
+    try:
+        parsed = int(value)
+    except (TypeError, ValueError):
+        return 0
+    return max(0, parsed)
+
+
+def _first_author_last_name(authors_text: str | None) -> str:
+    if not authors_text:
+        return ""
+    first_author = authors_text.split(",", maxsplit=1)[0].strip().lower()
+    words = WORD_RE.findall(first_author)
+    if not words:
+        return ""
+    return words[-1]
+
+
+def _first_venue_word(venue_text: str | None) -> str:
+    if not venue_text:
+        return ""
+    words = WORD_RE.findall(venue_text.lower())
+    if not words:
+        return ""
+    return words[0]
+
+
+def _build_fingerprint(
+    *,
+    title: str,
+    year: int | None,
+    author_text: str | None,
+    venue_text: str | None,
+) -> str:
+    canonical = "|".join(
+        [
+            normalize_title(title),
+            str(year) if year is not None else "",
+            _first_author_last_name(author_text),
+            _first_venue_word(venue_text),
+        ]
+    )
+    return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
+
+
+def _resolve_fingerprint(
+    *,
+    title: str,
+    year: int | None,
+    author_text: str | None,
+    venue_text: str | None,
+    provided_fingerprint: Any,
+) -> str:
+    normalized = _normalize_optional_text(provided_fingerprint)
+    if normalized and SHA256_RE.fullmatch(normalized.lower()):
+        return normalized.lower()
+    return _build_fingerprint(
+        title=title,
+        year=year,
+        author_text=author_text,
+        venue_text=venue_text,
+    )
+
+
+def _validate_import_sizes(
+    *,
+    scholars: list[dict[str, Any]],
+    publications: list[dict[str, Any]],
+) -> None:
+    if len(scholars) > MAX_IMPORT_SCHOLARS:
+        raise ImportExportError(f"Import exceeds max scholars ({MAX_IMPORT_SCHOLARS}).")
+    if len(publications) > MAX_IMPORT_PUBLICATIONS:
+        raise ImportExportError(
+            f"Import exceeds max publications ({MAX_IMPORT_PUBLICATIONS})."
+        )

app/services/domains/portability/publication_import.py

@@ -0,0 +1,368 @@
+from __future__ import annotations
+
+from typing import Any
+
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.db.models import Publication, ScholarProfile, ScholarPublication
+from app.services.domains.ingestion.application import build_publication_url, normalize_title
+from app.services.domains.portability.normalize import (
+    _normalize_citation_count,
+    _normalize_optional_text,
+    _normalize_optional_year,
+    _resolve_fingerprint,
+)
+from app.services.domains.portability.types import ImportedPublicationInput
+
+
+async def _find_publication_by_cluster(
+    db_session: AsyncSession,
+    *,
+    cluster_id: str,
+) -> Publication | None:
+    result = await db_session.execute(
+        select(Publication).where(Publication.cluster_id == cluster_id)
+    )
+    return result.scalar_one_or_none()
+
+
+async def _find_publication_by_fingerprint(
+    db_session: AsyncSession,
+    *,
+    fingerprint_sha256: str,
+) -> Publication | None:
+    result = await db_session.execute(
+        select(Publication).where(Publication.fingerprint_sha256 == fingerprint_sha256)
+    )
+    return result.scalar_one_or_none()
+
+
+async def _find_linked_publication_by_title(
+    db_session: AsyncSession,
+    *,
+    scholar_profile_id: int,
+    title: str,
+) -> Publication | None:
+    normalized_title = normalize_title(title)
+    result = await db_session.execute(
+        select(Publication)
+        .join(
+            ScholarPublication,
+            ScholarPublication.publication_id == Publication.id,
+        )
+        .where(
+            ScholarPublication.scholar_profile_id == scholar_profile_id,
+            Publication.title_normalized == normalized_title,
+        )
+        .order_by(Publication.id.asc())
+        .limit(1)
+    )
+    return result.scalar_one_or_none()
+
+
+def _apply_imported_publication_values(
+    *,
+    publication: Publication,
+    title: str,
+    year: int | None,
+    citation_count: int,
+    author_text: str | None,
+    venue_text: str | None,
+    pub_url: str | None,
+    pdf_url: str | None,
+    cluster_id: str | None,
+) -> bool:
+    updated = False
+    if cluster_id and publication.cluster_id != cluster_id:
+        publication.cluster_id = cluster_id
+        updated = True
+    if publication.title_raw != title:
+        publication.title_raw = title
+        publication.title_normalized = normalize_title(title)
+        updated = True
+    if publication.year != year:
+        publication.year = year
+        updated = True
+    if int(publication.citation_count or 0) != citation_count:
+        publication.citation_count = citation_count
+        updated = True
+    if publication.author_text != author_text:
+        publication.author_text = author_text
+        updated = True
+    if publication.venue_text != venue_text:
+        publication.venue_text = venue_text
+        updated = True
+    if pub_url and publication.pub_url != pub_url:
+        publication.pub_url = pub_url
+        updated = True
+    if pdf_url and publication.pdf_url != pdf_url:
+        publication.pdf_url = pdf_url
+        updated = True
+    return updated
+
+
+def _new_publication(
+    *,
+    cluster_id: str | None,
+    fingerprint_sha256: str,
+    title: str,
+    year: int | None,
+    citation_count: int,
+    author_text: str | None,
+    venue_text: str | None,
+    pub_url: str | None,
+    pdf_url: str | None,
+) -> Publication:
+    return Publication(
+        cluster_id=cluster_id,
+        fingerprint_sha256=fingerprint_sha256,
+        title_raw=title,
+        title_normalized=normalize_title(title),
+        year=year,
+        citation_count=citation_count,
+        author_text=author_text,
+        venue_text=venue_text,
+        pub_url=pub_url,
+        pdf_url=pdf_url,
+    )
+
+
+async def _resolve_publication_for_import(
+    db_session: AsyncSession,
+    *,
+    scholar_profile_id: int,
+    title: str,
+    cluster_id: str | None,
+    fingerprint_sha256: str,
+    cluster_cache: dict[str, Publication | None],
+    fingerprint_cache: dict[str, Publication | None],
+) -> Publication | None:
+    if cluster_id:
+        if cluster_id not in cluster_cache:
+            cluster_cache[cluster_id] = await _find_publication_by_cluster(
+                db_session,
+                cluster_id=cluster_id,
+            )
+        if cluster_cache[cluster_id] is not None:
+            return cluster_cache[cluster_id]
+    if fingerprint_sha256 not in fingerprint_cache:
+        fingerprint_cache[fingerprint_sha256] = await _find_publication_by_fingerprint(
+            db_session,
+            fingerprint_sha256=fingerprint_sha256,
+        )
+    if fingerprint_cache[fingerprint_sha256] is not None:
+        return fingerprint_cache[fingerprint_sha256]
+    return await _find_linked_publication_by_title(
+        db_session,
+        scholar_profile_id=scholar_profile_id,
+        title=title,
+    )
+
+
+async def _upsert_scholar_publication_link(
+    db_session: AsyncSession,
+    *,
+    scholar_profile_id: int,
+    publication_id: int,
+    is_read: bool,
+) -> tuple[bool, bool]:
+    result = await db_session.execute(
+        select(ScholarPublication).where(
+            ScholarPublication.scholar_profile_id == scholar_profile_id,
+            ScholarPublication.publication_id == publication_id,
+        )
+    )
+    link = result.scalar_one_or_none()
+    if link is None:
+        db_session.add(
+            ScholarPublication(
+                scholar_profile_id=scholar_profile_id,
+                publication_id=publication_id,
+                is_read=bool(is_read),
+            )
+        )
+        return True, False
+    if bool(link.is_read) == bool(is_read):
+        return False, False
+    link.is_read = bool(is_read)
+    return False, True
+
+
+def _initialize_import_counters(counters: dict[str, int]) -> None:
+    counters.update(
+        {
+            "publications_created": 0,
+            "publications_updated": 0,
+            "links_created": 0,
+            "links_updated": 0,
+        }
+    )
+
+
+def _build_imported_publication_input(
+    *,
+    item: dict[str, Any],
+    scholar_map: dict[str, ScholarProfile],
+) -> ImportedPublicationInput | None:
+    scholar_id = _normalize_optional_text(item.get("scholar_id"))
+    title = _normalize_optional_text(item.get("title"))
+    if not scholar_id or not title:
+        return None
+
+    profile = scholar_map.get(scholar_id)
+    if profile is None:
+        return None
+
+    year = _normalize_optional_year(item.get("year"))
+    author_text = _normalize_optional_text(item.get("author_text"))
+    venue_text = _normalize_optional_text(item.get("venue_text"))
+    return ImportedPublicationInput(
+        profile=profile,
+        title=title,
+        year=year,
+        citation_count=_normalize_citation_count(item.get("citation_count")),
+        author_text=author_text,
+        venue_text=venue_text,
+        cluster_id=_normalize_optional_text(item.get("cluster_id")),
+        pub_url=build_publication_url(_normalize_optional_text(item.get("pub_url"))),
+        pdf_url=build_publication_url(_normalize_optional_text(item.get("pdf_url"))),
+        fingerprint=_resolve_fingerprint(
+            title=title,
+            year=year,
+            author_text=author_text,
+            venue_text=venue_text,
+            provided_fingerprint=item.get("fingerprint_sha256"),
+        ),
+        is_read=bool(item.get("is_read", False)),
+    )
+
+
+def _update_link_counters(
+    *,
+    counters: dict[str, int],
+    link_created: bool,
+    link_updated: bool,
+) -> None:
+    if link_created:
+        counters["links_created"] += 1
+    if link_updated:
+        counters["links_updated"] += 1
+
+
+def _cache_resolved_publication(
+    *,
+    publication: Publication,
+    cluster_id: str | None,
+    fingerprint_sha256: str,
+    cluster_cache: dict[str, Publication | None],
+    fingerprint_cache: dict[str, Publication | None],
+) -> None:
+    if cluster_id:
+        cluster_cache[cluster_id] = publication
+    fingerprint_cache[fingerprint_sha256] = publication
+
+
+async def _create_import_publication(
+    db_session: AsyncSession,
+    *,
+    payload: ImportedPublicationInput,
+) -> Publication:
+    publication = _new_publication(
+        cluster_id=payload.cluster_id,
+        fingerprint_sha256=payload.fingerprint,
+        title=payload.title,
+        year=payload.year,
+        citation_count=payload.citation_count,
+        author_text=payload.author_text,
+        venue_text=payload.venue_text,
+        pub_url=payload.pub_url,
+        pdf_url=payload.pdf_url,
+    )
+    db_session.add(publication)
+    await db_session.flush()
+    return publication
+
+
+def _update_import_publication(
+    *,
+    publication: Publication,
+    payload: ImportedPublicationInput,
+) -> bool:
+    return _apply_imported_publication_values(
+        publication=publication,
+        title=payload.title,
+        year=payload.year,
+        citation_count=payload.citation_count,
+        author_text=payload.author_text,
+        venue_text=payload.venue_text,
+        pub_url=payload.pub_url,
+        pdf_url=payload.pdf_url,
+        cluster_id=payload.cluster_id,
+    )
+
+
+async def _upsert_publication_entity(
+    db_session: AsyncSession,
+    *,
+    payload: ImportedPublicationInput,
+    cluster_cache: dict[str, Publication | None],
+    fingerprint_cache: dict[str, Publication | None],
+) -> tuple[Publication, bool, bool]:
+    publication = await _resolve_publication_for_import(
+        db_session,
+        scholar_profile_id=int(payload.profile.id),
+        title=payload.title,
+        cluster_id=payload.cluster_id,
+        fingerprint_sha256=payload.fingerprint,
+        cluster_cache=cluster_cache,
+        fingerprint_cache=fingerprint_cache,
+    )
+    created = False
+    updated = False
+    if publication is None:
+        publication = await _create_import_publication(db_session, payload=payload)
+        created = True
+    else:
+        updated = _update_import_publication(publication=publication, payload=payload)
+
+    _cache_resolved_publication(
+        publication=publication,
+        cluster_id=payload.cluster_id,
+        fingerprint_sha256=payload.fingerprint,
+        cluster_cache=cluster_cache,
+        fingerprint_cache=fingerprint_cache,
+    )
+    return publication, created, updated
+
+
+async def _upsert_imported_publication(
+    db_session: AsyncSession,
+    *,
+    payload: ImportedPublicationInput,
+    cluster_cache: dict[str, Publication | None],
+    fingerprint_cache: dict[str, Publication | None],
+    counters: dict[str, int],
+) -> None:
+    publication, created, updated = await _upsert_publication_entity(
+        db_session,
+        payload=payload,
+        cluster_cache=cluster_cache,
+        fingerprint_cache=fingerprint_cache,
+    )
+    if created:
+        counters["publications_created"] += 1
+    if updated:
+        counters["publications_updated"] += 1
+
+    link_created, link_updated = await _upsert_scholar_publication_link(
+        db_session,
+        scholar_profile_id=int(payload.profile.id),
+        publication_id=int(publication.id),
+        is_read=payload.is_read,
+    )
+    _update_link_counters(
+        counters=counters,
+        link_created=link_created,
+        link_updated=link_updated,
+    )

app/services/domains/portability/scholar_import.py

@@ -0,0 +1,107 @@
+from __future__ import annotations
+
+from typing import Any
+
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.db.models import ScholarProfile
+from app.services.domains.portability.normalize import _normalize_optional_text
+from app.services.domains.scholars import application as scholar_service
+
+
+async def _load_user_scholar_map(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+) -> dict[str, ScholarProfile]:
+    result = await db_session.execute(
+        select(ScholarProfile).where(ScholarProfile.user_id == user_id)
+    )
+    profiles = list(result.scalars().all())
+    return {profile.scholar_id: profile for profile in profiles}
+
+
+def _apply_imported_scholar_values(
+    *,
+    profile: ScholarProfile,
+    display_name: str | None,
+    profile_image_override_url: str | None,
+    is_enabled: bool,
+) -> bool:
+    updated = False
+    if display_name and profile.display_name != display_name:
+        profile.display_name = display_name
+        updated = True
+    if profile.profile_image_override_url != profile_image_override_url:
+        profile.profile_image_override_url = profile_image_override_url
+        updated = True
+    if bool(profile.is_enabled) != bool(is_enabled):
+        profile.is_enabled = bool(is_enabled)
+        updated = True
+    return updated
+
+
+def _new_scholar_profile(
+    *,
+    user_id: int,
+    scholar_id: str,
+    display_name: str | None,
+    profile_image_override_url: str | None,
+    is_enabled: bool,
+) -> ScholarProfile:
+    return ScholarProfile(
+        user_id=user_id,
+        scholar_id=scholar_id,
+        display_name=display_name,
+        profile_image_override_url=profile_image_override_url,
+        is_enabled=bool(is_enabled),
+    )
+
+
+async def _upsert_imported_scholars(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+    scholars: list[dict[str, Any]],
+) -> tuple[dict[str, ScholarProfile], dict[str, int]]:
+    scholar_map = await _load_user_scholar_map(db_session, user_id=user_id)
+    counters = {"scholars_created": 0, "scholars_updated": 0, "skipped_records": 0}
+    for item in scholars:
+        try:
+            scholar_id = scholar_service.validate_scholar_id(str(item["scholar_id"]))
+            display_name = scholar_service.normalize_display_name(
+                str(item.get("display_name") or "")
+            )
+            override_url = scholar_service.normalize_profile_image_url(
+                _normalize_optional_text(item.get("profile_image_override_url"))
+            )
+        except (KeyError, scholar_service.ScholarServiceError):
+            counters["skipped_records"] += 1
+            continue
+
+        is_enabled = bool(item.get("is_enabled", True))
+        existing = scholar_map.get(scholar_id)
+        if existing is None:
+            profile = _new_scholar_profile(
+                user_id=user_id,
+                scholar_id=scholar_id,
+                display_name=display_name,
+                profile_image_override_url=override_url,
+                is_enabled=is_enabled,
+            )
+            db_session.add(profile)
+            scholar_map[scholar_id] = profile
+            counters["scholars_created"] += 1
+            continue
+
+        if _apply_imported_scholar_values(
+            profile=existing,
+            display_name=display_name,
+            profile_image_override_url=override_url,
+            is_enabled=is_enabled,
+        ):
+            counters["scholars_updated"] += 1
+
+    await db_session.flush()
+    return scholar_map, counters

app/services/domains/portability/types.py

@@ -0,0 +1,24 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from app.db.models import ScholarProfile
+
+
+class ImportExportError(ValueError):
+    """Raised when import/export payload constraints are violated."""
+
+
+@dataclass(frozen=True)
+class ImportedPublicationInput:
+    profile: ScholarProfile
+    title: str
+    year: int | None
+    citation_count: int
+    author_text: str | None
+    venue_text: str | None
+    cluster_id: str | None
+    pub_url: str | None
+    pdf_url: str | None
+    fingerprint: str
+    is_read: bool