First product
This commit is contained in:
parent
778da9e2fc
commit
4433d7d2c4
157 changed files with 23975 additions and 0 deletions
1
app/db/__init__.py
Normal file
1
app/db/__init__.py
Normal file
|
|
@ -0,0 +1 @@
|
|||
|
||||
18
app/db/base.py
Normal file
18
app/db/base.py
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
from sqlalchemy import MetaData
|
||||
from sqlalchemy.orm import DeclarativeBase
|
||||
|
||||
|
||||
NAMING_CONVENTION = {
|
||||
"ix": "ix_%(column_0_label)s",
|
||||
"uq": "uq_%(table_name)s_%(column_0_name)s",
|
||||
"ck": "ck_%(table_name)s_%(constraint_name)s",
|
||||
"fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s",
|
||||
"pk": "pk_%(table_name)s",
|
||||
}
|
||||
|
||||
|
||||
class Base(DeclarativeBase):
|
||||
metadata = MetaData(naming_convention=NAMING_CONVENTION)
|
||||
|
||||
|
||||
metadata = Base.metadata
|
||||
292
app/db/models.py
Normal file
292
app/db/models.py
Normal file
|
|
@ -0,0 +1,292 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from enum import StrEnum
|
||||
|
||||
from sqlalchemy import (
|
||||
Boolean,
|
||||
CheckConstraint,
|
||||
DateTime,
|
||||
Enum,
|
||||
ForeignKey,
|
||||
Index,
|
||||
Integer,
|
||||
String,
|
||||
Text,
|
||||
UniqueConstraint,
|
||||
func,
|
||||
text,
|
||||
)
|
||||
from sqlalchemy.dialects.postgresql import JSONB
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from app.db.base import Base
|
||||
|
||||
|
||||
class RunTriggerType(StrEnum):
|
||||
MANUAL = "manual"
|
||||
SCHEDULED = "scheduled"
|
||||
|
||||
|
||||
class RunStatus(StrEnum):
|
||||
RUNNING = "running"
|
||||
SUCCESS = "success"
|
||||
PARTIAL_FAILURE = "partial_failure"
|
||||
FAILED = "failed"
|
||||
|
||||
|
||||
class QueueItemStatus(StrEnum):
|
||||
QUEUED = "queued"
|
||||
RETRYING = "retrying"
|
||||
DROPPED = "dropped"
|
||||
|
||||
|
||||
RUN_STATUS_DB_ENUM = Enum(
|
||||
RunStatus,
|
||||
name="run_status",
|
||||
values_callable=lambda members: [member.value for member in members],
|
||||
)
|
||||
RUN_TRIGGER_TYPE_DB_ENUM = Enum(
|
||||
RunTriggerType,
|
||||
name="run_trigger_type",
|
||||
values_callable=lambda members: [member.value for member in members],
|
||||
)
|
||||
|
||||
|
||||
class User(Base):
|
||||
__tablename__ = "users"
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
email: Mapped[str] = mapped_column(String(255), nullable=False, unique=True)
|
||||
password_hash: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
is_active: Mapped[bool] = mapped_column(
|
||||
Boolean, nullable=False, server_default=text("true")
|
||||
)
|
||||
is_admin: Mapped[bool] = mapped_column(
|
||||
Boolean, nullable=False, server_default=text("false")
|
||||
)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
|
||||
|
||||
class UserSetting(Base):
|
||||
__tablename__ = "user_settings"
|
||||
__table_args__ = (
|
||||
CheckConstraint(
|
||||
"run_interval_minutes >= 15",
|
||||
name="run_interval_minutes_min_15",
|
||||
),
|
||||
CheckConstraint(
|
||||
"request_delay_seconds >= 1",
|
||||
name="request_delay_seconds_min_1",
|
||||
),
|
||||
)
|
||||
|
||||
user_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("users.id", ondelete="CASCADE"), primary_key=True
|
||||
)
|
||||
auto_run_enabled: Mapped[bool] = mapped_column(
|
||||
Boolean, nullable=False, server_default=text("false")
|
||||
)
|
||||
run_interval_minutes: Mapped[int] = mapped_column(
|
||||
Integer, nullable=False, server_default=text("1440")
|
||||
)
|
||||
request_delay_seconds: Mapped[int] = mapped_column(
|
||||
Integer, nullable=False, server_default=text("10")
|
||||
)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
|
||||
|
||||
class ScholarProfile(Base):
|
||||
__tablename__ = "scholar_profiles"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("user_id", "scholar_id", name="uq_scholar_profiles_user_scholar"),
|
||||
Index("ix_scholar_profiles_user_enabled", "user_id", "is_enabled"),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
user_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("users.id", ondelete="CASCADE"), nullable=False
|
||||
)
|
||||
scholar_id: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
display_name: Mapped[str | None] = mapped_column(String(255))
|
||||
is_enabled: Mapped[bool] = mapped_column(
|
||||
Boolean, nullable=False, server_default=text("true")
|
||||
)
|
||||
baseline_completed: Mapped[bool] = mapped_column(
|
||||
Boolean, nullable=False, server_default=text("false")
|
||||
)
|
||||
last_run_dt: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
last_run_status: Mapped[RunStatus | None] = mapped_column(
|
||||
RUN_STATUS_DB_ENUM,
|
||||
)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
|
||||
|
||||
class CrawlRun(Base):
|
||||
__tablename__ = "crawl_runs"
|
||||
__table_args__ = (
|
||||
Index("ix_crawl_runs_user_start", "user_id", "start_dt"),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
user_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("users.id", ondelete="CASCADE"), nullable=False
|
||||
)
|
||||
trigger_type: Mapped[RunTriggerType] = mapped_column(
|
||||
RUN_TRIGGER_TYPE_DB_ENUM, nullable=False
|
||||
)
|
||||
status: Mapped[RunStatus] = mapped_column(
|
||||
RUN_STATUS_DB_ENUM, nullable=False
|
||||
)
|
||||
start_dt: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
end_dt: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
scholar_count: Mapped[int] = mapped_column(
|
||||
Integer, nullable=False, server_default=text("0")
|
||||
)
|
||||
new_pub_count: Mapped[int] = mapped_column(
|
||||
Integer, nullable=False, server_default=text("0")
|
||||
)
|
||||
error_log: Mapped[dict] = mapped_column(
|
||||
JSONB, nullable=False, server_default=text("'{}'::jsonb")
|
||||
)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
|
||||
|
||||
class Publication(Base):
|
||||
__tablename__ = "publications"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("fingerprint_sha256", name="uq_publications_fingerprint"),
|
||||
Index(
|
||||
"uq_publications_cluster_id_not_null",
|
||||
"cluster_id",
|
||||
unique=True,
|
||||
postgresql_where=text("cluster_id IS NOT NULL"),
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
cluster_id: Mapped[str | None] = mapped_column(String(64))
|
||||
fingerprint_sha256: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
title_raw: Mapped[str] = mapped_column(Text, nullable=False)
|
||||
title_normalized: Mapped[str] = mapped_column(Text, nullable=False)
|
||||
year: Mapped[int | None] = mapped_column(Integer)
|
||||
citation_count: Mapped[int] = mapped_column(
|
||||
Integer, nullable=False, server_default=text("0")
|
||||
)
|
||||
author_text: Mapped[str | None] = mapped_column(Text)
|
||||
venue_text: Mapped[str | None] = mapped_column(Text)
|
||||
pub_url: Mapped[str | None] = mapped_column(Text)
|
||||
pdf_url: Mapped[str | None] = mapped_column(Text)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
|
||||
|
||||
class ScholarPublication(Base):
|
||||
__tablename__ = "scholar_publications"
|
||||
__table_args__ = (
|
||||
Index("ix_scholar_publications_is_read", "is_read"),
|
||||
)
|
||||
|
||||
scholar_profile_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("scholar_profiles.id", ondelete="CASCADE"),
|
||||
primary_key=True,
|
||||
)
|
||||
publication_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("publications.id", ondelete="CASCADE"),
|
||||
primary_key=True,
|
||||
)
|
||||
is_read: Mapped[bool] = mapped_column(
|
||||
Boolean, nullable=False, server_default=text("false")
|
||||
)
|
||||
first_seen_run_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("crawl_runs.id", ondelete="SET NULL")
|
||||
)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
|
||||
|
||||
class IngestionQueueItem(Base):
|
||||
__tablename__ = "ingestion_queue_items"
|
||||
__table_args__ = (
|
||||
UniqueConstraint(
|
||||
"user_id",
|
||||
"scholar_profile_id",
|
||||
name="uq_ingestion_queue_user_scholar",
|
||||
),
|
||||
CheckConstraint(
|
||||
"status IN ('queued', 'retrying', 'dropped')",
|
||||
name="ingestion_queue_status_valid",
|
||||
),
|
||||
Index("ix_ingestion_queue_next_attempt", "next_attempt_dt"),
|
||||
Index("ix_ingestion_queue_status_next_attempt", "status", "next_attempt_dt"),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
user_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("users.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
)
|
||||
scholar_profile_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("scholar_profiles.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
)
|
||||
resume_cstart: Mapped[int] = mapped_column(
|
||||
Integer,
|
||||
nullable=False,
|
||||
server_default=text("0"),
|
||||
)
|
||||
reason: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
status: Mapped[str] = mapped_column(
|
||||
String(16),
|
||||
nullable=False,
|
||||
server_default=text("'queued'"),
|
||||
)
|
||||
attempt_count: Mapped[int] = mapped_column(
|
||||
Integer,
|
||||
nullable=False,
|
||||
server_default=text("0"),
|
||||
)
|
||||
next_attempt_dt: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True),
|
||||
nullable=False,
|
||||
server_default=func.now(),
|
||||
)
|
||||
last_run_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("crawl_runs.id", ondelete="SET NULL"),
|
||||
)
|
||||
last_error: Mapped[str | None] = mapped_column(Text)
|
||||
dropped_reason: Mapped[str | None] = mapped_column(String(128))
|
||||
dropped_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
64
app/db/session.py
Normal file
64
app/db/session.py
Normal file
|
|
@ -0,0 +1,64 @@
|
|||
from collections.abc import AsyncIterator
|
||||
import logging
|
||||
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.ext.asyncio import (
|
||||
AsyncEngine,
|
||||
AsyncSession,
|
||||
async_sessionmaker,
|
||||
create_async_engine,
|
||||
)
|
||||
from sqlalchemy.pool import NullPool
|
||||
|
||||
from app.settings import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_engine: AsyncEngine | None = None
|
||||
_session_factory: async_sessionmaker[AsyncSession] | None = None
|
||||
|
||||
|
||||
def get_engine() -> AsyncEngine:
|
||||
global _engine
|
||||
if _engine is None:
|
||||
# NullPool avoids cross-event-loop connection reuse in tests and dev tools.
|
||||
_engine = create_async_engine(
|
||||
settings.database_url,
|
||||
pool_pre_ping=True,
|
||||
poolclass=NullPool,
|
||||
)
|
||||
logger.info("db.engine_initialized", extra={"event": "db.engine_initialized"})
|
||||
return _engine
|
||||
|
||||
|
||||
def get_session_factory() -> async_sessionmaker[AsyncSession]:
|
||||
global _session_factory
|
||||
if _session_factory is None:
|
||||
_session_factory = async_sessionmaker(get_engine(), expire_on_commit=False)
|
||||
return _session_factory
|
||||
|
||||
|
||||
async def get_db_session() -> AsyncIterator[AsyncSession]:
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as session:
|
||||
yield session
|
||||
|
||||
|
||||
async def check_database() -> bool:
|
||||
engine = get_engine()
|
||||
try:
|
||||
async with engine.connect() as conn:
|
||||
result = await conn.execute(text("SELECT 1"))
|
||||
return result.scalar_one() == 1
|
||||
except Exception:
|
||||
logger.exception("db.healthcheck_failed", extra={"event": "db.healthcheck_failed"})
|
||||
return False
|
||||
|
||||
|
||||
async def close_engine() -> None:
|
||||
global _engine, _session_factory
|
||||
if _engine is not None:
|
||||
await _engine.dispose()
|
||||
logger.info("db.engine_disposed", extra={"event": "db.engine_disposed"})
|
||||
_engine = None
|
||||
_session_factory = None
|
||||
Loading…
Add table
Add a link
Reference in a new issue