justinzeus/scholarr
1
0
Fork 0
Code Issues 1 Pull requests 18 Projects Releases Packages Wiki Activity Actions

Bugfixeds

Browse source
This commit is contained in:
Justin Visser 2026-03-01 22:54:23 +01:00
parent a5165dc3ba
commit 4403c9ebde
20 changed files with 199 additions and 85 deletions
Download patch file Download diff file Expand all files Collapse all files

4
.github/workflows/release.yml vendored
View file

@ -21,6 +21,10 @@ jobs:
python-version: "3.12"
- uses: astral-sh/setup-uv@v4
- run: uv sync --extra dev
- name: Lint
run: uv run ruff check
- name: Test
run: uv run pytest
- name: Semantic Release
id: release
run: uv run semantic-release publish

6
app/api/routers/run_manual.py
View file

@ -1,7 +1,7 @@
from __future__ import annotations
import logging
from typing import Any
from typing import Any, NoReturn
from fastapi import Request
from sqlalchemy.exc import IntegrityError
@ -215,7 +215,7 @@ async def execute_manual_run(
raise_manual_failed(exc=exc, user_id=user_id)
def raise_manual_blocked_safety(*, exc, user_id: int) -> None:
def raise_manual_blocked_safety(*, exc, user_id: int) -> NoReturn:
structured_log(
logger,
"info",
@ -233,7 +233,7 @@ def raise_manual_blocked_safety(*, exc, user_id: int) -> None:
) from exc
def raise_manual_failed(*, exc: Exception, user_id: int) -> None:
def raise_manual_failed(*, exc: Exception, user_id: int) -> NoReturn:
structured_log(
logger,
"exception",

8
app/api/schemas/admin.py
View file

@ -35,7 +35,7 @@ class AdminUsersListEnvelope(BaseModel):
class AdminUserCreateRequest(BaseModel):
email: str = Field(max_length=254)
password: str = Field(max_length=128)
password: str = Field(min_length=8, max_length=128)
is_admin: bool = False
model_config = ConfigDict(extra="forbid")
@ -55,7 +55,7 @@ class AdminUserEnvelope(BaseModel):
class AdminResetPasswordRequest(BaseModel):
new_password: str = Field(max_length=128)
new_password: str = Field(min_length=8, max_length=128)
model_config = ConfigDict(extra="forbid")
@ -64,7 +64,7 @@ class AdminScholarHttpSettingsData(BaseModel):
user_agent: str
rotate_user_agent: bool
accept_language: str
cookie: str
cookie: str | None = None
model_config = ConfigDict(extra="forbid")
@ -80,7 +80,7 @@ class AdminScholarHttpSettingsUpdateRequest(BaseModel):
user_agent: str
rotate_user_agent: bool
accept_language: str
cookie: str
cookie: str | None = None
model_config = ConfigDict(extra="forbid")

8
app/api/schemas/auth.py
View file

@ -45,7 +45,7 @@ class CsrfBootstrapEnvelope(BaseModel):
class LoginRequest(BaseModel):
email: str = Field(max_length=254)
password: str = Field(max_length=128)
password: str = Field(min_length=8, max_length=128)
model_config = ConfigDict(extra="forbid")
@ -66,8 +66,8 @@ class LoginEnvelope(BaseModel):
class ChangePasswordRequest(BaseModel):
current_password: str = Field(max_length=128)
new_password: str = Field(max_length=128)
confirm_password: str = Field(max_length=128)
current_password: str = Field(min_length=8, max_length=128)
new_password: str = Field(min_length=8, max_length=128)
confirm_password: str = Field(min_length=8, max_length=128)
model_config = ConfigDict(extra="forbid")

20
app/main.py
View file

@ -81,6 +81,26 @@ async def lifespan(_: FastAPI):
error=str(exc),
)
try:
session_factory = get_session_factory()
async with session_factory() as session:
await session.execute(
text(
"UPDATE publication_pdf_jobs SET status = 'queued'"
" WHERE status = 'running'"
" AND (last_attempt_at IS NULL OR last_attempt_at < NOW() - INTERVAL '10 minutes')"
)
)
await session.commit()
structured_log(logger, "info", "app.startup_stuck_pdf_jobs_recovered")
except Exception as exc:
structured_log(
logger,
"error",
"app.startup_stuck_pdf_jobs_recovery_failed",
error=str(exc),
)
await scheduler_service.start()
yield
await scheduler_service.stop()

21
app/services/arxiv/rate_limit.py
View file

@ -68,8 +68,10 @@ async def _run_serialized_fetch(
source_path: str,
) -> tuple[httpx.Response, bool]:
session_factory = get_session_factory()
async with session_factory() as lock_session:
await _acquire_arxiv_lock(lock_session)
try:
async with session_factory() as db_session, db_session.begin():
await _acquire_arxiv_lock(db_session)
runtime_state = await _load_runtime_state_for_update(db_session)
wait_seconds = await _wait_for_allowed_slot_or_raise(
runtime_state,
@ -88,6 +90,9 @@ async def _run_serialized_fetch(
response_status=int(response.status_code),
source_path=source_path,
)
cooldown_until_value = runtime_state.cooldown_until
finally:
await _release_arxiv_lock(lock_session)
structured_log(
logger,
"info",
@ -95,7 +100,7 @@ async def _run_serialized_fetch(
status_code=int(response.status_code),
wait_seconds=wait_seconds,
cooldown_remaining_seconds=_cooldown_remaining_seconds(
runtime_state.cooldown_until, now_utc=datetime.now(UTC)
cooldown_until_value, now_utc=datetime.now(UTC)
),
source_path=source_path,
)
@ -104,7 +109,17 @@ async def _run_serialized_fetch(
async def _acquire_arxiv_lock(db_session: AsyncSession) -> None:
await db_session.execute(
text("SELECT pg_advisory_xact_lock(:namespace, :lock_key)"),
text("SELECT pg_advisory_lock(:namespace, :lock_key)"),
{
"namespace": ARXIV_RATE_LIMIT_LOCK_NAMESPACE,
"lock_key": ARXIV_RATE_LIMIT_LOCK_KEY,
},
)
async def _release_arxiv_lock(db_session: AsyncSession) -> None:
await db_session.execute(
text("SELECT pg_advisory_unlock(:namespace, :lock_key)"),
{
"namespace": ARXIV_RATE_LIMIT_LOCK_NAMESPACE,
"lock_key": ARXIV_RATE_LIMIT_LOCK_KEY,

3
app/services/crossref/application.py
View file

@ -20,7 +20,7 @@ if TYPE_CHECKING:
from app.services.publications.types import PublicationListItem, UnreadPublicationItem
TOKEN_RE = re.compile(r"[a-z0-9]+")
NON_ALNUM_RE = re.compile(r"[^a-z0-9\\s]+")
NON_ALNUM_RE = re.compile(r"[^a-z0-9\s]+")
STOP_WORDS = {"the", "and", "for", "with", "from", "method", "study", "analysis"}
_RATE_LOCK = threading.Lock()
_LAST_REQUEST_AT = 0.0
@ -37,6 +37,7 @@ def _rate_limit_wait(min_interval_seconds: float) -> None:
remaining = interval - elapsed
if remaining > 0:
time.sleep(remaining)
with _RATE_LOCK:
_LAST_REQUEST_AT = time.monotonic()

3
app/services/dbops/near_duplicate_repair.py
View file

@ -162,7 +162,10 @@ async def _complete_job(
async def _fail_job(db_session: AsyncSession, *, job: DataRepairJob, error: Exception) -> None:
from sqlalchemy.orm import make_transient
await db_session.rollback()
make_transient(job)
job.status = REPAIR_STATUS_FAILED
job.error_text = str(error)
job.finished_at = _utcnow()

1
app/services/ingestion/application.py
View file

@ -401,6 +401,7 @@ class ScholarIngestionService:
if run_to_fail:
run_to_fail.status = RunStatus.FAILED
run_to_fail.end_dt = datetime.now(UTC)
run_to_fail.error_log = run_to_fail.error_log or {}
run_to_fail.error_log["terminal_exception"] = str(exc)
await cleanup_session.commit()
except Exception:

12
app/services/ingestion/publication_upsert.py
View file

@ -176,6 +176,7 @@ async def upsert_profile_publications(
seen_publication_ids: set[int] = set()
discovered_count = 0
try:
for candidate in publications:
publication = await resolve_publication(db_session, candidate)
if publication.id in seen_publication_ids:
@ -201,7 +202,7 @@ async def upsert_profile_publications(
db_session.add(link)
discovered_count += 1
await commit_discovered_publication(
await flush_discovered_publication(
db_session,
run=run,
scholar=scholar,
@ -211,10 +212,15 @@ async def upsert_profile_publications(
if not scholar.baseline_completed:
scholar.baseline_completed = True
await db_session.commit()
except Exception:
await db_session.rollback()
raise
return discovered_count
async def commit_discovered_publication(
async def flush_discovered_publication(
db_session: AsyncSession,
*,
run: CrawlRun,
@ -222,7 +228,7 @@ async def commit_discovered_publication(
publication: Publication,
) -> None:
run.new_pub_count = int(run.new_pub_count or 0) + 1
await db_session.commit()
await db_session.flush()
await run_events.publish(
run_id=run.id,
event_type="publication_discovered",

1
app/services/ingestion/queue.py
View file

@ -196,6 +196,7 @@ async def list_due_jobs(
IngestionQueueItem.id.asc(),
)
.limit(limit)
.with_for_update(skip_locked=True)
)
rows = list(result.scalars().all())
jobs: list[ContinuationQueueJob] = []

1
app/services/publications/pdf_queue_common.py
View file

@ -47,4 +47,5 @@ def queued_job(
status=PDF_STATUS_QUEUED,
queued_at=now,
last_requested_by_user_id=user_id,
attempt_count=0,
)

22
app/services/publications/pdf_queue_resolution.py
View file

@ -46,7 +46,7 @@ async def _mark_attempt_started(
db_session.add(job)
job.status = PDF_STATUS_RUNNING
job.last_attempt_at = utcnow()
job.attempt_count = int(job.attempt_count) + 1
job.attempt_count = int(job.attempt_count or 0) + 1
db_session.add(
event_row(
publication_id=publication_id,
@ -240,6 +240,26 @@ async def _run_resolution_task(
detail="Stopping PDF resolution batch — OpenAlex daily budget exhausted",
)
break
except Exception:
structured_log(
logger,
"exception",
"pdf_queue.row_failed",
publication_id=row.publication_id,
)
try:
await _persist_outcome(
publication_id=row.publication_id,
user_id=user_id,
outcome=_failed_outcome(row=row),
)
except Exception:
structured_log(
logger,
"exception",
"pdf_queue.row_fail_persist_error",
publication_id=row.publication_id,
)
def _register_task(task: asyncio.Task[None]) -> None:

13
app/services/runs/events.py
View file

@ -8,6 +8,8 @@ from app.logging_utils import structured_log
logger = logging.getLogger(__name__)
_SUBSCRIBER_QUEUE_MAXSIZE = 256
class RunEventPublisher:
def __init__(self) -> None:
@ -17,7 +19,7 @@ class RunEventPublisher:
def subscribe(self, run_id: int) -> asyncio.Queue:
if run_id not in self._subscribers:
self._subscribers[run_id] = set()
queue: asyncio.Queue[Any] = asyncio.Queue()
queue: asyncio.Queue[Any] = asyncio.Queue(maxsize=_SUBSCRIBER_QUEUE_MAXSIZE)
self._subscribers[run_id].add(queue)
structured_log(
logger,
@ -51,6 +53,10 @@ class RunEventPublisher:
"runs.event_subscriber_queue_full",
run_id=run_id,
)
self._subscribers[run_id].discard(queue)
async def publish_run_complete(self, run_id: int) -> None:
await self.publish(run_id, "run_complete", {})
run_events = RunEventPublisher()
@ -62,8 +68,11 @@ async def event_generator(run_id: int) -> AsyncGenerator[str, None]:
while True:
# Wait for a new event
message = await queue.get()
# Server-Sent Events format: "event: <type>\ndata: <json>\n\n"
event_type = message["type"]
if event_type == "run_complete":
yield f"event: {event_type}\ndata: {{}}\n\n"
break
# Server-Sent Events format: "event: <type>\ndata: <json>\n\n"
data_str = json.dumps(message["data"])
yield f"event: {event_type}\ndata: {data_str}\n\n"
except asyncio.CancelledError:

2
frontend/src/features/publications/components/PublicationTableRow.test.ts
View file

@ -13,7 +13,7 @@ function buildItem(overrides: Partial<PublicationItem> = {}): PublicationItem {
citation_count: 42,
pub_url: "https://example.com/pub",
pdf_url: null,
pdf_status: null,
pdf_status: "untracked",
is_read: false,
is_favorite: false,
is_new_in_latest_run: true,

6
frontend/src/features/publications/composables/usePublicationData.ts
View file

@ -1,4 +1,4 @@
import { computed, ref, watch } from "vue";
import { computed, onScopeDispose, ref, watch } from "vue";
import { useRoute, useRouter } from "vue-router";
import {
listPublications,
@ -271,6 +271,10 @@ export function usePublicationData() {
}, 300);
});
onScopeDispose(() => {
if (searchDebounceTimer !== null) clearTimeout(searchDebounceTimer);
});
// --- Run-triggered refresh watcher ---
watch(

6
frontend/src/features/settings/SettingsAdminPanel.vue
View file

@ -36,10 +36,8 @@ async function refreshForSection(): Promise<void> {
return;
}
if (props.section === SECTION_REPAIRS) {
await Promise.all([
usersRef.value?.load(),
repairsRef.value?.load(),
]);
await usersRef.value?.load();
await repairsRef.value?.load();
return;
}
if (props.section === SECTION_PDF && pdfQueueRef.value) {

14
frontend/src/features/settings/components/AdminPdfQueueSection.vue
View file

@ -7,6 +7,7 @@ import AppHelpHint from "@/components/ui/AppHelpHint.vue";
import AppRefreshButton from "@/components/ui/AppRefreshButton.vue";
import AppSelect from "@/components/ui/AppSelect.vue";
import AppTable from "@/components/ui/AppTable.vue";
import RequestStateAlerts from "@/components/patterns/RequestStateAlerts.vue";
import {
listAdminPdfQueue,
requeueAdminPdfLookup,
@ -15,7 +16,7 @@ import {
} from "@/features/admin_dbops";
import { useRequestState } from "@/composables/useRequestState";
const { clearAlerts, assignError, setSuccess } = useRequestState();
const { errorMessage, errorRequestId, successMessage, clearAlerts, assignError, setSuccess } = useRequestState();
const refreshingPdfQueue = ref(false);
const requeueingPublicationId = ref<number | null>(null);
@ -131,6 +132,16 @@ defineExpose({ load });
</script>
<template>
<section class="grid gap-4">
<RequestStateAlerts
:success-message="successMessage"
:error-message="errorMessage"
:error-request-id="errorRequestId"
success-title="PDF queue operation complete"
error-title="PDF queue operation failed"
@dismiss-success="successMessage = null"
/>
<AppCard class="space-y-3">
<div class="flex flex-wrap items-center justify-between gap-2">
<div class="flex items-center gap-1">
@ -200,4 +211,5 @@ defineExpose({ load });
</div>
</div>
</AppCard>
</section>
</template>

12
frontend/src/features/settings/components/AdminRepairsSection.vue
View file

@ -9,6 +9,7 @@ import AppInput from "@/components/ui/AppInput.vue";
import AppRefreshButton from "@/components/ui/AppRefreshButton.vue";
import AppSelect from "@/components/ui/AppSelect.vue";
import AppTable from "@/components/ui/AppTable.vue";
import RequestStateAlerts from "@/components/patterns/RequestStateAlerts.vue";
import {
dropAllPublications,
listAdminDbRepairJobs,
@ -34,7 +35,7 @@ const DROP_PUBLICATIONS_CONFIRM_TEXT = "DROP ALL PUBLICATIONS";
type RepairScopeMode = typeof SCOPE_SINGLE_USER | typeof SCOPE_ALL_USERS;
const { clearAlerts, assignError, setSuccess } = useRequestState();
const { errorMessage, errorRequestId, successMessage, clearAlerts, assignError, setSuccess } = useRequestState();
const refreshingJobs = ref(false);
const runningRepair = ref(false);
@ -246,6 +247,15 @@ defineExpose({ load });
<template>
<section class="grid gap-4">
<RequestStateAlerts
:success-message="successMessage"
:error-message="errorMessage"
:error-request-id="errorRequestId"
success-title="Repair operation complete"
error-title="Repair operation failed"
@dismiss-success="successMessage = null"
/>
<AppCard class="space-y-3">
<div class="flex items-center gap-1">
<h2 class="text-lg font-semibold text-ink-primary">Publication Link Repair</h2>

9
frontend/src/pages/PublicationsPage.vue
View file

@ -335,6 +335,7 @@ async function onStartRun(): Promise<void> {
pub.successMessage.value = null;
pub.errorMessage.value = null;
pub.errorRequestId.value = null;
try {
const result = await runStatus.startManualCheck();
if (result.kind === "started") { pub.successMessage.value = `Run #${result.runId} started.`; return; }
if (result.kind === "already_running") {
@ -343,6 +344,14 @@ async function onStartRun(): Promise<void> {
}
pub.errorMessage.value = result.message;
pub.errorRequestId.value = result.requestId;
} catch (error) {
if (error instanceof ApiRequestError) {
pub.errorMessage.value = error.message;
pub.errorRequestId.value = error.requestId;
} else {
pub.errorMessage.value = "Unable to start manual run.";
}
}
}
// --- Lifecycle ---