temp commit

This commit is contained in:
Justin Visser 2026-02-26 12:54:19 +01:00
parent 8760f27b51
commit 0e9e49df16
193 changed files with 23228 additions and 935 deletions

View file

@ -0,0 +1 @@

27
build/lib/app/db/base.py Normal file
View file

@ -0,0 +1,27 @@
from datetime import datetime, timezone
from sqlalchemy import MetaData, event
from sqlalchemy.orm import DeclarativeBase
NAMING_CONVENTION = {
"ix": "ix_%(column_0_label)s",
"uq": "uq_%(table_name)s_%(column_0_name)s",
"ck": "ck_%(table_name)s_%(constraint_name)s",
"fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s",
"pk": "pk_%(table_name)s",
}
class Base(DeclarativeBase):
metadata = MetaData(naming_convention=NAMING_CONVENTION)
@event.listens_for(Base, "before_update", propagate=True)
def _set_updated_at_before_update(_mapper, _connection, target) -> None:
# Keep audit timestamps current for ORM-managed updates.
if hasattr(target, "updated_at"):
target.updated_at = datetime.now(timezone.utc)
metadata = Base.metadata

546
build/lib/app/db/models.py Normal file
View file

@ -0,0 +1,546 @@
from __future__ import annotations
from datetime import datetime
from enum import StrEnum
from sqlalchemy import (
Boolean,
CheckConstraint,
DateTime,
Enum,
Float,
ForeignKey,
Index,
Integer,
String,
Text,
UniqueConstraint,
func,
text,
)
from sqlalchemy.dialects.postgresql import JSONB
from sqlalchemy.orm import Mapped, mapped_column
from app.db.base import Base
class RunTriggerType(StrEnum):
MANUAL = "manual"
SCHEDULED = "scheduled"
class RunStatus(StrEnum):
RUNNING = "running"
SUCCESS = "success"
PARTIAL_FAILURE = "partial_failure"
FAILED = "failed"
CANCELED = "canceled"
class QueueItemStatus(StrEnum):
QUEUED = "queued"
RETRYING = "retrying"
DROPPED = "dropped"
RUN_STATUS_DB_ENUM = Enum(
RunStatus,
name="run_status",
values_callable=lambda members: [member.value for member in members],
)
RUN_TRIGGER_TYPE_DB_ENUM = Enum(
RunTriggerType,
name="run_trigger_type",
values_callable=lambda members: [member.value for member in members],
)
class User(Base):
__tablename__ = "users"
id: Mapped[int] = mapped_column(primary_key=True)
email: Mapped[str] = mapped_column(String(255), nullable=False, unique=True)
password_hash: Mapped[str] = mapped_column(String(255), nullable=False)
is_active: Mapped[bool] = mapped_column(
Boolean, nullable=False, server_default=text("true")
)
is_admin: Mapped[bool] = mapped_column(
Boolean, nullable=False, server_default=text("false")
)
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now()
)
updated_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now()
)
class UserSetting(Base):
__tablename__ = "user_settings"
__table_args__ = (
CheckConstraint(
"run_interval_minutes >= 15",
name="run_interval_minutes_min_15",
),
CheckConstraint(
"request_delay_seconds >= 2",
name="request_delay_seconds_min_2",
),
)
user_id: Mapped[int] = mapped_column(
ForeignKey("users.id", ondelete="CASCADE"), primary_key=True
)
auto_run_enabled: Mapped[bool] = mapped_column(
Boolean, nullable=False, server_default=text("false")
)
run_interval_minutes: Mapped[int] = mapped_column(
Integer, nullable=False, server_default=text("1440")
)
request_delay_seconds: Mapped[int] = mapped_column(
Integer, nullable=False, server_default=text("10")
)
nav_visible_pages: Mapped[list[str]] = mapped_column(
JSONB,
nullable=False,
server_default=text(
'\'["dashboard","scholars","publications","settings","style-guide","runs","users"]\'::jsonb'
),
)
scrape_safety_state: Mapped[dict] = mapped_column(
JSONB,
nullable=False,
server_default=text("'{}'::jsonb"),
)
scrape_cooldown_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
scrape_cooldown_reason: Mapped[str | None] = mapped_column(String(64))
openalex_api_key: Mapped[str | None] = mapped_column(String(255))
crossref_api_token: Mapped[str | None] = mapped_column(String(255))
crossref_api_mailto: Mapped[str | None] = mapped_column(String(255))
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now()
)
updated_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now()
)
class ScholarProfile(Base):
__tablename__ = "scholar_profiles"
__table_args__ = (
UniqueConstraint("user_id", "scholar_id", name="uq_scholar_profiles_user_scholar"),
Index("ix_scholar_profiles_user_enabled", "user_id", "is_enabled"),
)
id: Mapped[int] = mapped_column(primary_key=True)
user_id: Mapped[int] = mapped_column(
ForeignKey("users.id", ondelete="CASCADE"), nullable=False
)
scholar_id: Mapped[str] = mapped_column(String(64), nullable=False)
display_name: Mapped[str | None] = mapped_column(String(255))
profile_image_url: Mapped[str | None] = mapped_column(Text)
profile_image_override_url: Mapped[str | None] = mapped_column(Text)
profile_image_upload_path: Mapped[str | None] = mapped_column(Text)
last_initial_page_fingerprint_sha256: Mapped[str | None] = mapped_column(String(64))
last_initial_page_checked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
is_enabled: Mapped[bool] = mapped_column(
Boolean, nullable=False, server_default=text("true")
)
baseline_completed: Mapped[bool] = mapped_column(
Boolean, nullable=False, server_default=text("false")
)
last_run_dt: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
last_run_status: Mapped[RunStatus | None] = mapped_column(
RUN_STATUS_DB_ENUM,
)
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now()
)
updated_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now()
)
class CrawlRun(Base):
__tablename__ = "crawl_runs"
__table_args__ = (
Index("ix_crawl_runs_user_start", "user_id", "start_dt"),
Index(
"uq_crawl_runs_user_manual_idempotency_key",
"user_id",
"idempotency_key",
unique=True,
postgresql_where=text(
"idempotency_key IS NOT NULL AND trigger_type = 'manual'::run_trigger_type"
),
),
)
id: Mapped[int] = mapped_column(primary_key=True)
user_id: Mapped[int] = mapped_column(
ForeignKey("users.id", ondelete="CASCADE"), nullable=False
)
trigger_type: Mapped[RunTriggerType] = mapped_column(
RUN_TRIGGER_TYPE_DB_ENUM, nullable=False
)
status: Mapped[RunStatus] = mapped_column(
RUN_STATUS_DB_ENUM, nullable=False
)
start_dt: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now()
)
end_dt: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
scholar_count: Mapped[int] = mapped_column(
Integer, nullable=False, server_default=text("0")
)
new_pub_count: Mapped[int] = mapped_column(
Integer, nullable=False, server_default=text("0")
)
idempotency_key: Mapped[str | None] = mapped_column(String(128))
error_log: Mapped[dict] = mapped_column(
JSONB, nullable=False, server_default=text("'{}'::jsonb")
)
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now()
)
updated_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now()
)
class Publication(Base):
__tablename__ = "publications"
__table_args__ = (
UniqueConstraint("fingerprint_sha256", name="uq_publications_fingerprint"),
Index(
"uq_publications_cluster_id_not_null",
"cluster_id",
unique=True,
postgresql_where=text("cluster_id IS NOT NULL"),
),
)
id: Mapped[int] = mapped_column(primary_key=True)
cluster_id: Mapped[str | None] = mapped_column(String(64))
fingerprint_sha256: Mapped[str] = mapped_column(String(64), nullable=False)
title_raw: Mapped[str] = mapped_column(Text, nullable=False)
title_normalized: Mapped[str] = mapped_column(Text, nullable=False)
year: Mapped[int | None] = mapped_column(Integer)
citation_count: Mapped[int] = mapped_column(
Integer, nullable=False, server_default=text("0")
)
author_text: Mapped[str | None] = mapped_column(Text)
venue_text: Mapped[str | None] = mapped_column(Text)
pub_url: Mapped[str | None] = mapped_column(Text)
pdf_url: Mapped[str | None] = mapped_column(Text)
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now()
)
updated_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now()
)
class PublicationIdentifier(Base):
__tablename__ = "publication_identifiers"
__table_args__ = (
UniqueConstraint(
"publication_id",
"kind",
"value_normalized",
name="uq_publication_identifiers_publication_kind_value",
),
CheckConstraint(
"confidence_score >= 0 AND confidence_score <= 1",
name="publication_identifiers_confidence_score_range",
),
Index(
"ix_publication_identifiers_kind_value",
"kind",
"value_normalized",
),
Index(
"ix_publication_identifiers_publication_id",
"publication_id",
),
)
id: Mapped[int] = mapped_column(primary_key=True)
publication_id: Mapped[int] = mapped_column(
ForeignKey("publications.id", ondelete="CASCADE"),
nullable=False,
)
kind: Mapped[str] = mapped_column(String(32), nullable=False)
value_raw: Mapped[str] = mapped_column(Text, nullable=False)
value_normalized: Mapped[str] = mapped_column(String(255), nullable=False)
source: Mapped[str] = mapped_column(String(64), nullable=False)
confidence_score: Mapped[float] = mapped_column(
Float,
nullable=False,
server_default=text("0"),
)
evidence_url: Mapped[str | None] = mapped_column(Text)
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now()
)
updated_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now()
)
class PublicationPdfJob(Base):
__tablename__ = "publication_pdf_jobs"
__table_args__ = (
CheckConstraint(
"status IN ('queued', 'running', 'resolved', 'failed')",
name="publication_pdf_jobs_status_valid",
),
Index("ix_publication_pdf_jobs_status", "status"),
Index("ix_publication_pdf_jobs_updated_at", "updated_at"),
Index("ix_publication_pdf_jobs_queued_at", "queued_at"),
)
publication_id: Mapped[int] = mapped_column(
ForeignKey("publications.id", ondelete="CASCADE"),
primary_key=True,
)
status: Mapped[str] = mapped_column(
String(16),
nullable=False,
server_default=text("'queued'"),
)
attempt_count: Mapped[int] = mapped_column(
Integer,
nullable=False,
server_default=text("0"),
)
queued_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
last_attempt_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
resolved_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
last_failure_reason: Mapped[str | None] = mapped_column(String(64))
last_failure_detail: Mapped[str | None] = mapped_column(Text)
last_source: Mapped[str | None] = mapped_column(String(32))
last_requested_by_user_id: Mapped[int | None] = mapped_column(
ForeignKey("users.id", ondelete="SET NULL"),
)
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now()
)
updated_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now()
)
class PublicationPdfJobEvent(Base):
__tablename__ = "publication_pdf_job_events"
__table_args__ = (
Index("ix_publication_pdf_job_events_publication_created", "publication_id", "created_at"),
Index("ix_publication_pdf_job_events_created_at", "created_at"),
Index("ix_publication_pdf_job_events_event_type", "event_type"),
)
id: Mapped[int] = mapped_column(primary_key=True)
publication_id: Mapped[int] = mapped_column(
ForeignKey("publications.id", ondelete="CASCADE"),
nullable=False,
)
user_id: Mapped[int | None] = mapped_column(
ForeignKey("users.id", ondelete="SET NULL"),
)
event_type: Mapped[str] = mapped_column(String(32), nullable=False)
status: Mapped[str | None] = mapped_column(String(16))
source: Mapped[str | None] = mapped_column(String(32))
failure_reason: Mapped[str | None] = mapped_column(String(64))
message: Mapped[str | None] = mapped_column(Text)
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now()
)
class ScholarPublication(Base):
__tablename__ = "scholar_publications"
__table_args__ = (
Index("ix_scholar_publications_is_read", "is_read"),
Index("ix_scholar_publications_is_favorite", "is_favorite"),
)
scholar_profile_id: Mapped[int] = mapped_column(
ForeignKey("scholar_profiles.id", ondelete="CASCADE"),
primary_key=True,
)
publication_id: Mapped[int] = mapped_column(
ForeignKey("publications.id", ondelete="CASCADE"),
primary_key=True,
)
is_read: Mapped[bool] = mapped_column(
Boolean, nullable=False, server_default=text("false")
)
is_favorite: Mapped[bool] = mapped_column(
Boolean, nullable=False, server_default=text("false")
)
first_seen_run_id: Mapped[int | None] = mapped_column(
ForeignKey("crawl_runs.id", ondelete="SET NULL")
)
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now()
)
class IngestionQueueItem(Base):
__tablename__ = "ingestion_queue_items"
__table_args__ = (
UniqueConstraint(
"user_id",
"scholar_profile_id",
name="uq_ingestion_queue_user_scholar",
),
CheckConstraint(
"status IN ('queued', 'retrying', 'dropped')",
name="ingestion_queue_status_valid",
),
Index("ix_ingestion_queue_next_attempt", "next_attempt_dt"),
Index("ix_ingestion_queue_status_next_attempt", "status", "next_attempt_dt"),
)
id: Mapped[int] = mapped_column(primary_key=True)
user_id: Mapped[int] = mapped_column(
ForeignKey("users.id", ondelete="CASCADE"),
nullable=False,
)
scholar_profile_id: Mapped[int] = mapped_column(
ForeignKey("scholar_profiles.id", ondelete="CASCADE"),
nullable=False,
)
resume_cstart: Mapped[int] = mapped_column(
Integer,
nullable=False,
server_default=text("0"),
)
reason: Mapped[str] = mapped_column(String(64), nullable=False)
status: Mapped[str] = mapped_column(
String(16),
nullable=False,
server_default=text("'queued'"),
)
attempt_count: Mapped[int] = mapped_column(
Integer,
nullable=False,
server_default=text("0"),
)
next_attempt_dt: Mapped[datetime] = mapped_column(
DateTime(timezone=True),
nullable=False,
server_default=func.now(),
)
last_run_id: Mapped[int | None] = mapped_column(
ForeignKey("crawl_runs.id", ondelete="SET NULL"),
)
last_error: Mapped[str | None] = mapped_column(Text)
dropped_reason: Mapped[str | None] = mapped_column(String(128))
dropped_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now()
)
updated_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now()
)
class DataRepairJob(Base):
__tablename__ = "data_repair_jobs"
__table_args__ = (
CheckConstraint(
"status IN ('planned', 'running', 'completed', 'failed')",
name="data_repair_jobs_status_valid",
),
Index("ix_data_repair_jobs_created_at", "created_at"),
Index("ix_data_repair_jobs_job_name_created_at", "job_name", "created_at"),
)
id: Mapped[int] = mapped_column(primary_key=True)
job_name: Mapped[str] = mapped_column(String(64), nullable=False)
requested_by: Mapped[str | None] = mapped_column(String(255))
scope: Mapped[dict] = mapped_column(
JSONB,
nullable=False,
server_default=text("'{}'::jsonb"),
)
dry_run: Mapped[bool] = mapped_column(
Boolean,
nullable=False,
server_default=text("true"),
)
status: Mapped[str] = mapped_column(
String(16),
nullable=False,
server_default=text("'planned'"),
)
summary: Mapped[dict] = mapped_column(
JSONB,
nullable=False,
server_default=text("'{}'::jsonb"),
)
error_text: Mapped[str | None] = mapped_column(Text)
started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now()
)
updated_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now()
)
class AuthorSearchRuntimeState(Base):
__tablename__ = "author_search_runtime_state"
state_key: Mapped[str] = mapped_column(String(64), primary_key=True)
last_live_request_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
cooldown_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
consecutive_blocked_count: Mapped[int] = mapped_column(
Integer,
nullable=False,
server_default=text("0"),
)
cooldown_rejection_count: Mapped[int] = mapped_column(
Integer,
nullable=False,
server_default=text("0"),
)
cooldown_alert_emitted: Mapped[bool] = mapped_column(
Boolean,
nullable=False,
server_default=text("false"),
)
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now()
)
updated_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now()
)
class AuthorSearchCacheEntry(Base):
__tablename__ = "author_search_cache_entries"
__table_args__ = (
Index("ix_author_search_cache_expires_at", "expires_at"),
Index("ix_author_search_cache_cached_at", "cached_at"),
)
query_key: Mapped[str] = mapped_column(String(256), primary_key=True)
payload: Mapped[dict] = mapped_column(
JSONB,
nullable=False,
)
expires_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True),
nullable=False,
)
cached_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now()
)
updated_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now()
)

View file

@ -0,0 +1,99 @@
from collections.abc import AsyncIterator
import logging
import os
from sqlalchemy import text
from sqlalchemy.ext.asyncio import (
AsyncEngine,
AsyncSession,
async_sessionmaker,
create_async_engine,
)
from sqlalchemy.pool import NullPool
from app.settings import settings
logger = logging.getLogger(__name__)
_engine: AsyncEngine | None = None
_session_factory: async_sessionmaker[AsyncSession] | None = None
def _normalized_pool_mode(raw_mode: str) -> str:
mode = (raw_mode or "").strip().lower()
if mode == "auto":
if os.getenv("PYTEST_CURRENT_TEST"):
return "null"
app_env = (os.getenv("APP_ENV") or "").strip().lower()
if app_env in {"test", "development", "dev", "local"}:
return "null"
return "queue"
if mode in {"null", "queue"}:
return mode
logger.warning(
"db.invalid_pool_mode_fallback",
extra={
"event": "db.invalid_pool_mode_fallback",
"database_pool_mode": raw_mode,
"fallback_mode": "queue",
},
)
return "queue"
def get_engine() -> AsyncEngine:
global _engine
if _engine is None:
pool_mode = _normalized_pool_mode(settings.database_pool_mode)
engine_kwargs: dict[str, object] = {
"pool_pre_ping": True,
}
if pool_mode == "null":
engine_kwargs["poolclass"] = NullPool
else:
engine_kwargs["pool_size"] = max(1, int(settings.database_pool_size))
engine_kwargs["max_overflow"] = max(0, int(settings.database_pool_max_overflow))
engine_kwargs["pool_timeout"] = max(1, int(settings.database_pool_timeout_seconds))
_engine = create_async_engine(settings.database_url, **engine_kwargs)
logger.info(
"db.engine_initialized",
extra={
"event": "db.engine_initialized",
"pool_mode": pool_mode,
},
)
return _engine
def get_session_factory() -> async_sessionmaker[AsyncSession]:
global _session_factory
if _session_factory is None:
_session_factory = async_sessionmaker(get_engine(), expire_on_commit=False)
return _session_factory
async def get_db_session() -> AsyncIterator[AsyncSession]:
session_factory = get_session_factory()
async with session_factory() as session:
yield session
async def check_database() -> bool:
engine = get_engine()
try:
async with engine.connect() as conn:
result = await conn.execute(text("SELECT 1"))
return result.scalar_one() == 1
except Exception:
logger.exception("db.healthcheck_failed", extra={"event": "db.healthcheck_failed"})
return False
async def close_engine() -> None:
global _engine, _session_factory
if _engine is not None:
await _engine.dispose()
logger.info("db.engine_disposed", extra={"event": "db.engine_disposed"})
_engine = None
_session_factory = None