ci: add CodeQL security scanning and Dependabot

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Justin Visser 2026-02-27 00:05:17 +01:00
parent ac002131d6
commit 3866c6d6f0
90 changed files with 40 additions and 1 deletions

View file

@ -0,0 +1,51 @@
from app.services.domains.runs.application import (
QUEUE_STATUS_DROPPED as QUEUE_STATUS_DROPPED,
)
from app.services.domains.runs.application import (
QUEUE_STATUS_QUEUED as QUEUE_STATUS_QUEUED,
)
from app.services.domains.runs.application import (
QUEUE_STATUS_RETRYING as QUEUE_STATUS_RETRYING,
)
from app.services.domains.runs.application import (
QueueClearResult as QueueClearResult,
)
from app.services.domains.runs.application import (
QueueListItem as QueueListItem,
)
from app.services.domains.runs.application import (
QueueTransitionError as QueueTransitionError,
)
from app.services.domains.runs.application import (
clear_queue_item_for_user as clear_queue_item_for_user,
)
from app.services.domains.runs.application import (
drop_queue_item_for_user as drop_queue_item_for_user,
)
from app.services.domains.runs.application import (
extract_run_summary as extract_run_summary,
)
from app.services.domains.runs.application import (
get_manual_run_by_idempotency_key as get_manual_run_by_idempotency_key,
)
from app.services.domains.runs.application import (
get_queue_item_for_user as get_queue_item_for_user,
)
from app.services.domains.runs.application import (
get_run_for_user as get_run_for_user,
)
from app.services.domains.runs.application import (
list_queue_items_for_user as list_queue_items_for_user,
)
from app.services.domains.runs.application import (
list_recent_runs_for_user as list_recent_runs_for_user,
)
from app.services.domains.runs.application import (
list_runs_for_user as list_runs_for_user,
)
from app.services.domains.runs.application import (
queue_status_counts_for_user as queue_status_counts_for_user,
)
from app.services.domains.runs.application import (
retry_queue_item_for_user as retry_queue_item_for_user,
)

View file

@ -0,0 +1,45 @@
from __future__ import annotations
from app.services.domains.runs.queue_service import (
clear_queue_item_for_user,
drop_queue_item_for_user,
get_queue_item_for_user,
list_queue_items_for_user,
queue_status_counts_for_user,
retry_queue_item_for_user,
)
from app.services.domains.runs.runs_service import (
get_manual_run_by_idempotency_key,
get_run_for_user,
list_recent_runs_for_user,
list_runs_for_user,
)
from app.services.domains.runs.summary import extract_run_summary
from app.services.domains.runs.types import (
QUEUE_STATUS_DROPPED,
QUEUE_STATUS_QUEUED,
QUEUE_STATUS_RETRYING,
QueueClearResult,
QueueListItem,
QueueTransitionError,
)
__all__ = [
"QUEUE_STATUS_DROPPED",
"QUEUE_STATUS_QUEUED",
"QUEUE_STATUS_RETRYING",
"QueueClearResult",
"QueueListItem",
"QueueTransitionError",
"clear_queue_item_for_user",
"drop_queue_item_for_user",
"extract_run_summary",
"get_manual_run_by_idempotency_key",
"get_queue_item_for_user",
"get_run_for_user",
"list_queue_items_for_user",
"list_recent_runs_for_user",
"list_runs_for_user",
"queue_status_counts_for_user",
"retry_queue_item_for_user",
]

View file

@ -0,0 +1,60 @@
import asyncio
import json
import logging
from collections.abc import AsyncGenerator
from typing import Any
logger = logging.getLogger(__name__)
class RunEventPublisher:
def __init__(self) -> None:
# Maps run_id to a set of subscriber queues
self._subscribers: dict[int, set[asyncio.Queue]] = {}
def subscribe(self, run_id: int) -> asyncio.Queue:
if run_id not in self._subscribers:
self._subscribers[run_id] = set()
queue: asyncio.Queue[Any] = asyncio.Queue()
self._subscribers[run_id].add(queue)
logger.debug(f"New subscriber for run {run_id}. Total: {len(self._subscribers[run_id])}")
return queue
def unsubscribe(self, run_id: int, queue: asyncio.Queue) -> None:
if run_id in self._subscribers:
self._subscribers[run_id].discard(queue)
if not self._subscribers[run_id]:
self._subscribers.pop(run_id, None)
async def publish(self, run_id: int, event_type: str, data: dict[str, Any]) -> None:
if run_id not in self._subscribers:
return
message = {"type": event_type, "data": data}
# Fan-out to all active subscribers for this run
for queue in list(self._subscribers[run_id]):
try:
queue.put_nowait(message)
except asyncio.QueueFull:
logger.warning(f"Subscriber queue full for run {run_id}, dropping message")
run_events = RunEventPublisher()
async def event_generator(run_id: int) -> AsyncGenerator[str, None]:
queue = run_events.subscribe(run_id)
try:
while True:
# Wait for a new event
message = await queue.get()
# Server-Sent Events format: "event: <type>\ndata: <json>\n\n"
event_type = message["type"]
data_str = json.dumps(message["data"])
yield f"event: {event_type}\ndata: {data_str}\n\n"
except asyncio.CancelledError:
logger.debug(f"Client disconnected from SSE stream for run {run_id}")
raise
finally:
run_events.unsubscribe(run_id, queue)

View file

@ -0,0 +1,70 @@
from __future__ import annotations
from sqlalchemy import and_, select
from app.db.models import IngestionQueueItem, ScholarProfile
from app.services.domains.runs.types import QueueListItem
def queue_item_columns() -> tuple:
return (
IngestionQueueItem.id,
IngestionQueueItem.scholar_profile_id,
ScholarProfile.display_name,
ScholarProfile.scholar_id,
IngestionQueueItem.status,
IngestionQueueItem.reason,
IngestionQueueItem.dropped_reason,
IngestionQueueItem.attempt_count,
IngestionQueueItem.resume_cstart,
IngestionQueueItem.next_attempt_dt,
IngestionQueueItem.updated_at,
IngestionQueueItem.last_error,
IngestionQueueItem.last_run_id,
)
def queue_item_select(*, user_id: int):
return (
select(*queue_item_columns())
.join(
ScholarProfile,
and_(
ScholarProfile.id == IngestionQueueItem.scholar_profile_id,
ScholarProfile.user_id == IngestionQueueItem.user_id,
),
)
.where(IngestionQueueItem.user_id == user_id)
)
def queue_list_item_from_row(row: tuple) -> QueueListItem:
(
item_id,
scholar_profile_id,
display_name,
scholar_id,
status,
reason,
dropped_reason,
attempt_count,
resume_cstart,
next_attempt_dt,
updated_at,
last_error,
last_run_id,
) = row
return QueueListItem(
id=int(item_id),
scholar_profile_id=int(scholar_profile_id),
scholar_label=(display_name or scholar_id),
status=str(status),
reason=str(reason),
dropped_reason=dropped_reason,
attempt_count=int(attempt_count or 0),
resume_cstart=int(resume_cstart or 0),
next_attempt_dt=next_attempt_dt,
updated_at=updated_at,
last_error=last_error,
last_run_id=int(last_run_id) if last_run_id is not None else None,
)

View file

@ -0,0 +1,180 @@
from __future__ import annotations
from sqlalchemy import case, func, select
from sqlalchemy.ext.asyncio import AsyncSession
from app.db.models import IngestionQueueItem
from app.services.domains.ingestion import queue as queue_mutations
from app.services.domains.runs.queue_queries import queue_item_select, queue_list_item_from_row
from app.services.domains.runs.types import (
QUEUE_STATUS_DROPPED,
QUEUE_STATUS_QUEUED,
QUEUE_STATUS_RETRYING,
QueueClearResult,
QueueListItem,
QueueTransitionError,
)
async def list_queue_items_for_user(
db_session: AsyncSession,
*,
user_id: int,
limit: int = 200,
) -> list[QueueListItem]:
result = await db_session.execute(
queue_item_select(user_id=user_id)
.order_by(
case((IngestionQueueItem.status == QUEUE_STATUS_DROPPED, 1), else_=0).asc(),
IngestionQueueItem.next_attempt_dt.asc(),
IngestionQueueItem.id.asc(),
)
.limit(limit)
)
return [queue_list_item_from_row(row) for row in result.all()]
async def get_queue_item_for_user(
db_session: AsyncSession,
*,
user_id: int,
queue_item_id: int,
) -> QueueListItem | None:
result = await db_session.execute(
queue_item_select(user_id=user_id).where(IngestionQueueItem.id == queue_item_id).limit(1)
)
row = result.one_or_none()
if row is None:
return None
return queue_list_item_from_row(row)
async def retry_queue_item_for_user(
db_session: AsyncSession,
*,
user_id: int,
queue_item_id: int,
) -> QueueListItem | None:
result = await db_session.execute(
select(IngestionQueueItem).where(
IngestionQueueItem.id == queue_item_id,
IngestionQueueItem.user_id == user_id,
)
)
item = result.scalar_one_or_none()
if item is None:
return None
if item.status == QUEUE_STATUS_QUEUED:
raise QueueTransitionError(
code="queue_item_already_queued",
message="Queue item is already queued.",
current_status=item.status,
)
if item.status == QUEUE_STATUS_RETRYING:
raise QueueTransitionError(
code="queue_item_retrying",
message="Queue item is currently retrying.",
current_status=item.status,
)
await queue_mutations.mark_queued_now(
db_session,
job_id=item.id,
reason="manual_retry",
reset_attempt_count=(item.status == QUEUE_STATUS_DROPPED),
)
await db_session.commit()
return await get_queue_item_for_user(
db_session,
user_id=user_id,
queue_item_id=queue_item_id,
)
async def drop_queue_item_for_user(
db_session: AsyncSession,
*,
user_id: int,
queue_item_id: int,
) -> QueueListItem | None:
result = await db_session.execute(
select(IngestionQueueItem).where(
IngestionQueueItem.id == queue_item_id,
IngestionQueueItem.user_id == user_id,
)
)
item = result.scalar_one_or_none()
if item is None:
return None
if item.status == QUEUE_STATUS_DROPPED:
raise QueueTransitionError(
code="queue_item_already_dropped",
message="Queue item is already dropped.",
current_status=item.status,
)
await queue_mutations.mark_dropped(
db_session,
job_id=int(item.id),
reason="manual_drop",
)
await db_session.commit()
return await get_queue_item_for_user(
db_session,
user_id=user_id,
queue_item_id=queue_item_id,
)
async def clear_queue_item_for_user(
db_session: AsyncSession,
*,
user_id: int,
queue_item_id: int,
) -> QueueClearResult | None:
result = await db_session.execute(
select(IngestionQueueItem).where(
IngestionQueueItem.id == queue_item_id,
IngestionQueueItem.user_id == user_id,
)
)
item = result.scalar_one_or_none()
if item is None:
return None
if item.status != QUEUE_STATUS_DROPPED:
raise QueueTransitionError(
code="queue_item_not_dropped",
message="Queue item can only be cleared after it is dropped.",
current_status=item.status,
)
item_id = int(item.id)
previous_status = str(item.status)
deleted = await queue_mutations.delete_job_by_id(db_session, job_id=item_id)
await db_session.commit()
if not deleted:
return None
return QueueClearResult(queue_item_id=item_id, previous_status=previous_status)
async def queue_status_counts_for_user(
db_session: AsyncSession,
*,
user_id: int,
) -> dict[str, int]:
result = await db_session.execute(
select(
IngestionQueueItem.status,
func.count(IngestionQueueItem.id),
)
.where(IngestionQueueItem.user_id == user_id)
.group_by(IngestionQueueItem.status)
)
counts: dict[str, int] = {
QUEUE_STATUS_QUEUED: 0,
QUEUE_STATUS_RETRYING: 0,
QUEUE_STATUS_DROPPED: 0,
}
for status, count in result.all():
counts[str(status)] = int(count or 0)
return counts

View file

@ -0,0 +1,77 @@
from __future__ import annotations
from sqlalchemy import or_, select
from sqlalchemy.ext.asyncio import AsyncSession
from app.db.models import CrawlRun, RunStatus, RunTriggerType
async def list_recent_runs_for_user(
db_session: AsyncSession,
*,
user_id: int,
limit: int = 20,
) -> list[CrawlRun]:
result = await db_session.execute(
select(CrawlRun)
.where(CrawlRun.user_id == user_id)
.order_by(CrawlRun.start_dt.desc(), CrawlRun.id.desc())
.limit(limit)
)
return list(result.scalars().all())
async def list_runs_for_user(
db_session: AsyncSession,
*,
user_id: int,
limit: int = 100,
failed_only: bool = False,
) -> list[CrawlRun]:
stmt = (
select(CrawlRun)
.where(CrawlRun.user_id == user_id)
.order_by(CrawlRun.start_dt.desc(), CrawlRun.id.desc())
.limit(limit)
)
if failed_only:
stmt = stmt.where(CrawlRun.status.in_([RunStatus.FAILED, RunStatus.PARTIAL_FAILURE]))
result = await db_session.execute(stmt)
return list(result.scalars().all())
async def get_run_for_user(
db_session: AsyncSession,
*,
user_id: int,
run_id: int,
) -> CrawlRun | None:
result = await db_session.execute(
select(CrawlRun).where(
CrawlRun.user_id == user_id,
CrawlRun.id == run_id,
)
)
return result.scalar_one_or_none()
async def get_manual_run_by_idempotency_key(
db_session: AsyncSession,
*,
user_id: int,
idempotency_key: str,
) -> CrawlRun | None:
result = await db_session.execute(
select(CrawlRun)
.where(
CrawlRun.user_id == user_id,
CrawlRun.trigger_type == RunTriggerType.MANUAL,
or_(
CrawlRun.idempotency_key == idempotency_key,
CrawlRun.error_log["meta"]["idempotency_key"].astext == idempotency_key,
),
)
.order_by(CrawlRun.start_dt.desc(), CrawlRun.id.desc())
.limit(1)
)
return result.scalar_one_or_none()

View file

@ -0,0 +1,70 @@
from __future__ import annotations
from typing import Any, cast
def _safe_int(value: object, default: int = 0) -> int:
try:
return int(cast(Any, value))
except (TypeError, ValueError):
return default
def _summary_dict(error_log: object) -> dict[str, Any]:
if not isinstance(error_log, dict):
return {}
summary = error_log.get("summary")
if not isinstance(summary, dict):
return {}
return summary
def _summary_int_dict(summary: dict[str, Any], key: str) -> dict[str, int]:
value = summary.get(key)
if not isinstance(value, dict):
return {}
return {
str(item_key): _safe_int(item_value, 0) for item_key, item_value in value.items() if isinstance(item_key, str)
}
def _summary_bool_dict(summary: dict[str, Any], key: str) -> dict[str, bool]:
value = summary.get(key)
if not isinstance(value, dict):
return {}
return {str(item_key): bool(item_value) for item_key, item_value in value.items() if isinstance(item_key, str)}
def _retry_counts(summary: dict[str, Any]) -> dict[str, int]:
retry_counts = summary.get("retry_counts")
if not isinstance(retry_counts, dict):
retry_counts = {}
return {
"retries_scheduled_count": _safe_int(
retry_counts.get("retries_scheduled_count", 0),
0,
),
"scholars_with_retries_count": _safe_int(
retry_counts.get("scholars_with_retries_count", 0),
0,
),
"retry_exhausted_count": _safe_int(
retry_counts.get("retry_exhausted_count", 0),
0,
),
}
def extract_run_summary(error_log: object) -> dict[str, Any]:
summary = _summary_dict(error_log)
return {
"succeeded_count": _safe_int(summary.get("succeeded_count", 0)),
"failed_count": _safe_int(summary.get("failed_count", 0)),
"partial_count": _safe_int(summary.get("partial_count", 0)),
"failed_state_counts": _summary_int_dict(summary, "failed_state_counts"),
"failed_reason_counts": _summary_int_dict(summary, "failed_reason_counts"),
"scrape_failure_counts": _summary_int_dict(summary, "scrape_failure_counts"),
"retry_counts": _retry_counts(summary),
"alert_thresholds": _summary_int_dict(summary, "alert_thresholds"),
"alert_flags": _summary_bool_dict(summary, "alert_flags"),
}

View file

@ -0,0 +1,38 @@
from __future__ import annotations
from dataclasses import dataclass
from datetime import datetime
QUEUE_STATUS_QUEUED = "queued"
QUEUE_STATUS_RETRYING = "retrying"
QUEUE_STATUS_DROPPED = "dropped"
@dataclass(frozen=True)
class QueueListItem:
id: int
scholar_profile_id: int
scholar_label: str
status: str
reason: str
dropped_reason: str | None
attempt_count: int
resume_cstart: int
next_attempt_dt: datetime | None
updated_at: datetime
last_error: str | None
last_run_id: int | None
@dataclass(frozen=True)
class QueueClearResult:
queue_item_id: int
previous_status: str
class QueueTransitionError(RuntimeError):
def __init__(self, *, code: str, message: str, current_status: str) -> None:
super().__init__(message)
self.code = code
self.message = message
self.current_status = current_status