(
+ "/admin/db/drop-all-publications",
+ {
+ method: "POST",
+ body: { confirmation_text: confirmationText },
+ },
+ );
+ return response.data;
+}
diff --git a/frontend/src/features/publications/index.ts b/frontend/src/features/publications/index.ts
index f238a0e..3f80806 100644
--- a/frontend/src/features/publications/index.ts
+++ b/frontend/src/features/publications/index.ts
@@ -19,7 +19,6 @@ export interface PublicationItem {
citation_count: number;
venue_text: string | null;
pub_url: string | null;
- doi: string | null;
display_identifier: DisplayIdentifier | null;
pdf_url: string | null;
pdf_status: "untracked" | "queued" | "running" | "resolved" | "failed";
@@ -44,6 +43,7 @@ export interface PublicationsResult {
total_count: number;
page: number;
page_size: number;
+ snapshot: string;
has_next: boolean;
has_prev: boolean;
publications: PublicationItem[];
@@ -53,8 +53,12 @@ export interface PublicationsQuery {
mode?: PublicationMode;
favoriteOnly?: boolean;
scholarProfileId?: number;
+ search?: string;
+ sortBy?: string;
+ sortDir?: "asc" | "desc";
page?: number;
pageSize?: number;
+ snapshot?: string;
}
export interface PublicationSelection {
@@ -74,12 +78,24 @@ export async function listPublications(query: PublicationsQuery = {}): Promise 0) {
+ params.set("search", query.search.trim());
+ }
+ if (query.sortBy) {
+ params.set("sort_by", query.sortBy);
+ }
+ if (query.sortDir) {
+ params.set("sort_dir", query.sortDir);
+ }
const parsedPage = Number.isFinite(query.page) ? Math.max(1, Math.trunc(Number(query.page))) : 1;
const parsedPageSize = Number.isFinite(query.pageSize)
? Math.max(1, Math.min(500, Math.trunc(Number(query.pageSize))))
: 100;
params.set("page", String(parsedPage));
params.set("page_size", String(parsedPageSize));
+ if (query.snapshot && query.snapshot.trim().length > 0) {
+ params.set("snapshot", query.snapshot.trim());
+ }
const suffix = params.toString();
const response = await apiRequest(
diff --git a/frontend/src/features/runs/index.ts b/frontend/src/features/runs/index.ts
index 97d49c6..735776d 100644
--- a/frontend/src/features/runs/index.ts
+++ b/frontend/src/features/runs/index.ts
@@ -145,6 +145,13 @@ export async function triggerManualRun(): Promise<{
return response.data;
}
+export async function cancelRun(runId: number): Promise {
+ const response = await apiRequest(`/runs/${runId}/cancel`, {
+ method: "POST",
+ });
+ return response.data;
+}
+
export async function listQueueItems(limit = 200): Promise {
const response = await apiRequest(`/runs/queue/items?limit=${limit}`, {
method: "GET",
diff --git a/frontend/src/features/scholars/index.ts b/frontend/src/features/scholars/index.ts
index db19e76..ef3c8c4 100644
--- a/frontend/src/features/scholars/index.ts
+++ b/frontend/src/features/scholars/index.ts
@@ -54,7 +54,6 @@ export interface PublicationExportItem {
author_text: string | null;
venue_text: string | null;
pub_url: string | null;
- doi: string | null;
pdf_url: string | null;
is_read: boolean;
}
@@ -86,7 +85,7 @@ interface ScholarsListData {
scholars: ScholarProfile[];
}
-interface ScholarSearchData extends ScholarSearchResult {}
+interface ScholarSearchData extends ScholarSearchResult { }
export async function listScholars(): Promise {
const response = await apiRequest("/scholars", { method: "GET" });
diff --git a/frontend/src/features/settings/SettingsAdminPanel.vue b/frontend/src/features/settings/SettingsAdminPanel.vue
index 7edf67b..e5edfb4 100644
--- a/frontend/src/features/settings/SettingsAdminPanel.vue
+++ b/frontend/src/features/settings/SettingsAdminPanel.vue
@@ -14,6 +14,7 @@ import AppRefreshButton from "@/components/ui/AppRefreshButton.vue";
import AppSelect from "@/components/ui/AppSelect.vue";
import AppTable from "@/components/ui/AppTable.vue";
import {
+ dropAllPublications,
getAdminDbIntegrityReport,
listAdminDbRepairJobs,
listAdminPdfQueue,
@@ -42,6 +43,7 @@ const SECTION_PDF = "pdf";
const SCOPE_SINGLE_USER = "single_user";
const SCOPE_ALL_USERS = "all_users";
const APPLY_ALL_USERS_CONFIRM_TEXT = "REPAIR ALL USERS";
+const DROP_PUBLICATIONS_CONFIRM_TEXT = "DROP ALL PUBLICATIONS";
type RepairScopeMode = typeof SCOPE_SINGLE_USER | typeof SCOPE_ALL_USERS;
@@ -81,6 +83,13 @@ const repairDryRun = ref(true);
const repairGcOrphans = ref(false);
const repairConfirmationText = ref("");
+const droppingPublications = ref(false);
+const dropConfirmationText = ref("");
+const dropResult = ref<{ deleted_count: number; message: string } | null>(null);
+const dropConfirmationValid = computed(
+ () => dropConfirmationText.value.trim() === DROP_PUBLICATIONS_CONFIRM_TEXT,
+);
+
const refreshingPdfQueue = ref(false);
const requeueingPublicationId = ref(null);
const requeueingAllPdfs = ref(false);
@@ -433,6 +442,22 @@ async function onRequeueAllPdfs(): Promise {
}
}
+async function onDropAllPublications(): Promise {
+ droppingPublications.value = true;
+ clearAlerts();
+ dropResult.value = null;
+ try {
+ const result = await dropAllPublications(dropConfirmationText.value.trim());
+ dropResult.value = result;
+ successMessage.value = result.message;
+ dropConfirmationText.value = "";
+ } catch (error) {
+ assignError(error, "Unable to drop publications.");
+ } finally {
+ droppingPublications.value = false;
+ }
+}
+
onMounted(async () => {
loading.value = true;
clearAlerts();
@@ -686,6 +711,45 @@ watch(
+
+
+
+
Drop All Publications
+
+
+ This action is irreversible . It deletes every publication record across all users.
+ The next ingestion run will re-populate all data from scratch.
+
+
+
+
+
+
+
Deleted: {{ dropResult.deleted_count }}
+
+
{{ dropResult.message }}
+
diff --git a/frontend/src/features/settings/index.ts b/frontend/src/features/settings/index.ts
index e8ef325..6ff6c94 100644
--- a/frontend/src/features/settings/index.ts
+++ b/frontend/src/features/settings/index.ts
@@ -19,6 +19,9 @@ export interface UserSettings {
nav_visible_pages: string[];
policy: UserSettingsPolicy;
safety_state: ScrapeSafetyState;
+ openalex_api_key: string | null;
+ crossref_api_token: string | null;
+ crossref_api_mailto: string | null;
}
export interface UserSettingsUpdate {
@@ -26,6 +29,9 @@ export interface UserSettingsUpdate {
run_interval_minutes: number;
request_delay_seconds: number;
nav_visible_pages: string[];
+ openalex_api_key: string | null;
+ crossref_api_token: string | null;
+ crossref_api_mailto: string | null;
}
export interface ChangePasswordPayload {
diff --git a/frontend/src/pages/DashboardPage.vue b/frontend/src/pages/DashboardPage.vue
index e61df8d..90d8513 100644
--- a/frontend/src/pages/DashboardPage.vue
+++ b/frontend/src/pages/DashboardPage.vue
@@ -64,6 +64,7 @@ const startCheckLabel = computed(() => {
const isStartCheckAnimating = computed(
() => runStatus.isSubmitting || runStatus.isLikelyRunning,
);
+const isCancelAnimating = ref(false);
const displayedLatestRun = computed(() => {
const snapshotLatest = snapshot.value?.latestRun ?? null;
@@ -102,6 +103,25 @@ const recentRuns = computed(() => {
mergedRuns.splice(existingIndex, 1, latest);
return mergedRuns;
});
+
+const recentPublications = computed(() => {
+ const stream = runStatus.livePublications;
+ // DashboardSnapshot's recentPublications is compatible with PublicationItem,
+ // but let's cast it slightly to guarantee iteration.
+ const base = snapshot.value?.recentPublications ?? [];
+ const merged = [...stream, ...base];
+
+ const seenIds = new Set();
+ const deduped: any[] = [];
+ for (const item of merged) {
+ if (!seenIds.has(item.publication_id)) {
+ seenIds.add(item.publication_id);
+ deduped.push(item);
+ }
+ }
+ return deduped;
+});
+
const activeRunId = computed(() => runStatus.latestRun?.status === "running" ? runStatus.latestRun.id : null);
const showRunningHint = computed(
() => runStatus.isLikelyRunning && displayedLatestRun.value?.status !== "running",
@@ -178,7 +198,6 @@ async function onTriggerRun(): Promise {
return;
}
- errorMessage.value = result.message;
errorRequestId.value = result.requestId;
} catch {
errorMessage.value = "Unable to start an update check.";
@@ -186,6 +205,28 @@ async function onTriggerRun(): Promise {
}
}
+async function onCancelRun(): Promise {
+ if (!activeRunId.value) return;
+ errorMessage.value = null;
+ errorRequestId.value = null;
+ successMessage.value = null;
+ isCancelAnimating.value = true;
+
+ try {
+ const result = await runStatus.cancelActiveCheck();
+ if (result.kind === "success") {
+ successMessage.value = "Update check canceled successfully.";
+ await loadSnapshot();
+ } else {
+ errorMessage.value = result.message;
+ }
+ } catch {
+ errorMessage.value = "Unable to cancel the update check.";
+ } finally {
+ isCancelAnimating.value = false;
+ }
+}
+
onMounted(() => {
void loadSnapshot();
void runStatus.syncLatest();
@@ -252,14 +293,14 @@ watch(
@@ -328,6 +369,22 @@ watch(
/>
+
+
+
+
+ Cancel check
+
+
("first_seen");
const sortDirection = ref<"asc" | "desc">("desc");
const currentPage = ref(1);
const pageSize = ref("50");
+const publicationSnapshot = ref(null);
const scholars = ref([]);
const listState = ref(null);
@@ -190,27 +189,42 @@ const selectedScholarName = computed(() => {
});
const filteredPublications = computed(() => {
+ let stream = [...runStatus.livePublications];
+ if (favoriteOnly.value) {
+ stream = stream.filter((p) => p.is_favorite);
+ }
+ if (mode.value === "unread") {
+ stream = stream.filter((p) => !p.is_read);
+ }
+ const selectedScholarId = Number(selectedScholarFilter.value);
+ if (Number.isInteger(selectedScholarId) && selectedScholarId > 0) {
+ stream = stream.filter((p) => p.scholar_profile_id === selectedScholarId);
+ }
+
const base = listState.value?.publications ?? [];
+ const merged = [...stream, ...base];
+ const seenIds = new Set();
+ const deduped: typeof base = [];
+ for (const item of merged) {
+ if (!seenIds.has(item.publication_id)) {
+ seenIds.add(item.publication_id);
+ deduped.push(item);
+ }
+ }
+
const normalized = searchQuery.value.trim().toLowerCase();
if (!normalized) {
- return base;
+ return deduped;
}
- return base.filter((item) => {
- const year = item.year === null ? "" : String(item.year);
- return [item.title, item.scholar_label, item.venue_text || "", year]
- .join(" ")
- .toLowerCase()
- .includes(normalized);
- });
+ // Client-side fallback: filter live-discovered publications that haven't been
+ // server-round-tripped yet. The main server query already filters by search.
+ return deduped;
});
function publicationSortValue(item: PublicationItem, key: PublicationSortKey): number | string {
if (key === "title") {
return item.title;
}
- if (key === "favorite") {
- return item.is_favorite ? 1 : 0;
- }
if (key === "scholar") {
return item.scholar_label;
}
@@ -220,38 +234,14 @@ function publicationSortValue(item: PublicationItem, key: PublicationSortKey): n
if (key === "citations") {
return item.citation_count;
}
- if (key === "status") {
- if (item.is_read) {
- return 2;
- }
- if (item.is_new_in_latest_run) {
- return 0;
- }
- return 1;
- }
const timestamp = Date.parse(item.first_seen_at);
return Number.isNaN(timestamp) ? 0 : timestamp;
}
const sortedPublications = computed(() => {
- const sorted = [...filteredPublications.value];
- sorted.sort((a, b) => {
- const left = publicationSortValue(a, sortKey.value);
- const right = publicationSortValue(b, sortKey.value);
-
- let comparison: number;
- if (typeof left === "string" && typeof right === "string") {
- comparison = textCollator.compare(left, right);
- } else {
- comparison = Number(left) - Number(right);
- }
-
- if (comparison === 0) {
- comparison = textCollator.compare(a.title, b.title);
- }
- return sortDirection.value === "asc" ? comparison : comparison * -1;
- });
- return sorted;
+ // Server already returns data in the correct sort order.
+ // We just pass through the filtered/merged list without client-side re-sorting.
+ return filteredPublications.value;
});
const visibleUnreadKeys = computed(() => {
@@ -437,13 +427,16 @@ watch(hasSelection, (nextHasSelection) => {
bulkAction.value = nextHasSelection ? "mark_selected_read" : "mark_all_unread_read";
});
-function toggleSort(nextKey: PublicationSortKey): void {
+async function toggleSort(nextKey: PublicationSortKey): Promise {
if (sortKey.value === nextKey) {
sortDirection.value = sortDirection.value === "asc" ? "desc" : "asc";
- return;
+ } else {
+ sortKey.value = nextKey;
+ sortDirection.value = nextKey === "first_seen" ? "desc" : "asc";
}
- sortKey.value = nextKey;
- sortDirection.value = nextKey === "first_seen" ? "desc" : "asc";
+ currentPage.value = 1;
+ publicationSnapshot.value = null;
+ await loadPublications();
}
function sortMarker(key: PublicationSortKey): string {
@@ -476,9 +469,14 @@ async function loadPublications(): Promise {
mode: mode.value,
favoriteOnly: favoriteOnly.value,
scholarProfileId: selectedScholarId(),
+ search: searchQuery.value.trim() || undefined,
+ sortBy: sortKey.value,
+ sortDir: sortDirection.value,
page: currentPage.value,
pageSize: pageSizeValue.value,
+ snapshot: publicationSnapshot.value ?? undefined,
});
+ publicationSnapshot.value = listState.value.snapshot;
currentPage.value = listState.value.page;
pageSize.value = String(listState.value.page_size);
selectedPublicationKeys.value = new Set();
@@ -498,12 +496,14 @@ async function loadPublications(): Promise {
async function onModeChanged(): Promise {
currentPage.value = 1;
+ publicationSnapshot.value = null;
await syncFiltersToRoute();
await loadPublications();
}
async function onScholarFilterChanged(): Promise {
currentPage.value = 1;
+ publicationSnapshot.value = null;
await syncFiltersToRoute();
await loadPublications();
}
@@ -511,12 +511,14 @@ async function onScholarFilterChanged(): Promise {
async function onFavoriteOnlyChanged(): Promise {
favoriteOnly.value = !favoriteOnly.value;
currentPage.value = 1;
+ publicationSnapshot.value = null;
await syncFiltersToRoute();
await loadPublications();
}
async function onPageSizeChanged(): Promise {
currentPage.value = 1;
+ publicationSnapshot.value = null;
await syncFiltersToRoute();
await loadPublications();
}
@@ -605,15 +607,15 @@ function canRetryPublicationPdf(item: PublicationItem): boolean {
function pdfPendingLabel(item: PublicationItem): string {
if (item.pdf_status === "queued") {
- return "queued";
+ return "Queued";
}
if (item.pdf_status === "running") {
- return "resolving";
+ return "Resolving...";
}
if (item.pdf_status === "failed") {
- return "failed";
+ return "Missing";
}
- return "untracked";
+ return "Untracked";
}
function replacePublication(updated: PublicationItem): void {
@@ -784,6 +786,18 @@ function resetSearchQuery(): void {
searchQuery.value = "";
}
+let searchDebounceTimer: ReturnType | null = null;
+watch(searchQuery, () => {
+ if (searchDebounceTimer !== null) {
+ clearTimeout(searchDebounceTimer);
+ }
+ searchDebounceTimer = setTimeout(() => {
+ currentPage.value = 1;
+ publicationSnapshot.value = null;
+ void loadPublications();
+ }, 300);
+});
+
onMounted(() => {
syncFiltersFromRoute();
void Promise.all([loadScholarFilters(), loadPublications(), runStatus.syncLatest()]);
@@ -792,10 +806,18 @@ onMounted(() => {
watch(
() => [route.query.scholar, route.query.favorite, route.query.page],
async () => {
+ const previousScholar = selectedScholarFilter.value;
+ const previousFavorite = favoriteOnly.value;
const changed = syncFiltersFromRoute();
if (!changed) {
return;
}
+ if (
+ selectedScholarFilter.value !== previousScholar
+ || favoriteOnly.value !== previousFavorite
+ ) {
+ publicationSnapshot.value = null;
+ }
await loadPublications();
},
);
@@ -843,7 +865,7 @@ watch(
Search
-
-
-
- ★ {{ sortMarker('favorite') }}
-
-
+
★
Title {{ sortMarker('title') }}
@@ -990,7 +1008,7 @@ watch(
Scholar {{ sortMarker('scholar') }}
-
PDF
+
PDF
Year {{ sortMarker('year') }}
@@ -1001,11 +1019,7 @@ watch(
Citations {{ sortMarker('citations') }}
-
-
- Read status {{ sortMarker('status') }}
-
-
+
Read status
First seen {{ sortMarker('first_seen') }}
@@ -1073,8 +1087,12 @@ watch(
target="_blank"
rel="noreferrer"
class="pdf-link-button"
+ title="Open PDF"
>
- PDF
+
+
+ Available
- {{ isRetryingPublication(item) ? "..." : "retry" }}
+
+
+ {{ isRetryingPublication(item) ? "Retrying..." : "Missing (Retry)" }}
- {{ pdfPendingLabel(item) }}
+
+
+
+ {{ pdfPendingLabel(item) }}
+
{{ item.year ?? "n/a" }}
{{ item.citation_count }}
diff --git a/frontend/src/pages/RunsPage.vue b/frontend/src/pages/RunsPage.vue
index 711158c..d28a3dc 100644
--- a/frontend/src/pages/RunsPage.vue
+++ b/frontend/src/pages/RunsPage.vue
@@ -23,6 +23,7 @@ import {
type RunListItem,
} from "@/features/runs";
import { ApiRequestError } from "@/lib/api/errors";
+import { useAuthStore } from "@/stores/auth";
import { useRunStatusStore } from "@/stores/run_status";
import { useUserSettingsStore } from "@/stores/user_settings";
@@ -33,8 +34,10 @@ const errorMessage = ref(null);
const errorRequestId = ref(null);
const successMessage = ref(null);
const activeQueueItemId = ref(null);
+const auth = useAuthStore();
const runStatus = useRunStatusStore();
const userSettings = useUserSettingsStore();
+const isCancelAnimating = ref(false);
function formatDate(value: string | null): string {
if (!value) {
@@ -62,7 +65,7 @@ function queueHealth() {
}
const queueCounts = computed(() => queueHealth());
-const activeRunId = computed(() => runStatus.latestRun?.status === "running" ? runStatus.latestRun.id : null);
+const activeRunId = computed(() => runStatus.isRunActive && runStatus.latestRun ? runStatus.latestRun.id : null);
const isStartBlocked = computed(
() =>
runStatus.isRunActive ||
@@ -151,6 +154,28 @@ async function onTriggerManualRun(): Promise {
}
}
+async function onCancelRun(): Promise {
+ if (!activeRunId.value) return;
+ errorMessage.value = null;
+ errorRequestId.value = null;
+ successMessage.value = null;
+ isCancelAnimating.value = true;
+
+ try {
+ const result = await runStatus.cancelActiveCheck();
+ if (result.kind === "success") {
+ successMessage.value = "Update check canceled successfully.";
+ await loadData();
+ } else {
+ errorMessage.value = result.message;
+ }
+ } catch {
+ errorMessage.value = "Unable to cancel the update check.";
+ } finally {
+ isCancelAnimating.value = false;
+ }
+}
+
async function runQueueAction(itemId: number, action: "retry" | "drop" | "clear"): Promise {
activeQueueItemId.value = itemId;
successMessage.value = null;
@@ -208,6 +233,22 @@ onMounted(() => {
:manual-run-allowed="userSettings.manualRunAllowed"
/>
+
+
+
+
+ Cancel check
+
+
{{ runButtonLabel }}
diff --git a/frontend/src/pages/SettingsPage.vue b/frontend/src/pages/SettingsPage.vue
index a7b00b1..66f12eb 100644
--- a/frontend/src/pages/SettingsPage.vue
+++ b/frontend/src/pages/SettingsPage.vue
@@ -30,6 +30,7 @@ const TAB_ADMIN_USERS = "admin-users";
const TAB_ADMIN_INTEGRITY = "admin-integrity";
const TAB_ADMIN_REPAIRS = "admin-repairs";
const TAB_ADMIN_PDF = "admin-pdf";
+const TAB_ADMIN_INTEGRATIONS = "admin-integrations";
const auth = useAuthStore();
const userSettings = useUserSettingsStore();
@@ -45,6 +46,9 @@ const autoRunEnabled = ref(false);
const runIntervalMinutes = ref("60");
const requestDelaySeconds = ref("2");
const navVisiblePages = ref
([]);
+const openalexApiKey = ref("");
+const crossrefApiToken = ref("");
+const crossrefApiMailto = ref("");
const currentPassword = ref("");
const newPassword = ref("");
@@ -72,6 +76,7 @@ const tabItems = computed(() => {
{ id: TAB_ADMIN_INTEGRITY, label: "Integrity" },
{ id: TAB_ADMIN_REPAIRS, label: "Repairs" },
{ id: TAB_ADMIN_PDF, label: "PDF Queue" },
+ { id: TAB_ADMIN_INTEGRATIONS, label: "Integrations" },
);
}
return tabs;
@@ -129,6 +134,10 @@ function hydrateSettings(settings: UserSettings): void {
requestDelaySeconds.value = String(settings.request_delay_seconds);
navVisiblePages.value = normalizeUserNavVisiblePages(settings.nav_visible_pages);
+ openalexApiKey.value = settings.openalex_api_key ?? "";
+ crossrefApiToken.value = settings.crossref_api_token ?? "";
+ crossrefApiMailto.value = settings.crossref_api_mailto ?? "";
+
userSettings.applySettings(settings);
runStatus.setSafetyState(settings.safety_state);
}
@@ -183,6 +192,9 @@ async function onSaveSettings(): Promise {
minRequestDelaySeconds.value,
),
nav_visible_pages: normalizeUserNavVisiblePages(navVisiblePages.value),
+ openalex_api_key: openalexApiKey.value.trim() || null,
+ crossref_api_token: crossrefApiToken.value.trim() || null,
+ crossref_api_mailto: crossrefApiMailto.value.trim() || null,
};
const saved = await updateSettings(payload);
@@ -357,6 +369,50 @@ onMounted(async () => {
+
+
+ If no keys are provided, the system will gracefully fall back to free unauthenticated tiers where available.
+
+
+
+
OpenAlex
+
+ OpenAlex is a free index of the world's research. Providing a key unlocks a higher rate limit.
+ Values returned from the backend might be 'SET' to hide the raw key for security.
+
+
+ API Key
+
+
+
+
+
Crossref
+
+ Crossref metadata search for DOIs. A "mailto" address puts you in their "Polite Pool" for better performance.
+ A Plus API token unlocks the fastest tier.
+
+
+ API Token (Plus)
+
+
+ Mailto (Polite Pool)
+
+
+
+
+ {{ saving ? "Saving..." : "Save integrations" }}
+
+
+
[]) {
describe("run status store", () => {
const mockedListRuns = vi.mocked(listRuns);
const mockedTriggerManualRun = vi.mocked(triggerManualRun);
+ const mockedCancelRun = vi.mocked(cancelRun);
beforeEach(() => {
setActivePinia(createPinia());
mockedListRuns.mockReset();
mockedTriggerManualRun.mockReset();
+ mockedCancelRun.mockReset();
vi.useRealTimers();
});
@@ -223,4 +226,87 @@ describe("run status store", () => {
await startPromise;
expect(store.isRunActive).toBe(true);
});
+
+ it("cancels an active check and transitions to canceled state", async () => {
+ mockedCancelRun.mockResolvedValueOnce({
+ run: buildRun({ id: 50, status: "canceled" }),
+ summary: {
+ succeeded_count: 0,
+ failed_count: 0,
+ partial_count: 0,
+ failed_state_counts: {},
+ failed_reason_counts: {},
+ scrape_failure_counts: {},
+ retry_counts: {
+ retries_scheduled_count: 0,
+ scholars_with_retries_count: 0,
+ retry_exhausted_count: 0,
+ },
+ alert_thresholds: {},
+ alert_flags: {},
+ },
+ scholar_results: [],
+ safety_state: createDefaultSafetyState(),
+ } as any);
+
+ const store = useRunStatusStore();
+ store.setLatestRun(buildRun({ id: 50, status: "running", end_dt: null }));
+ expect(store.isRunActive).toBe(true);
+
+ const result = await store.cancelActiveCheck();
+
+ expect(result.kind).toBe("success");
+ expect(store.latestRun?.status).toBe("canceled");
+ expect(store.isRunActive).toBe(false);
+ expect(store.isPolling).toBe(false);
+ });
+
+ it("cancels a resolving run using server status as source of truth", async () => {
+ mockedCancelRun.mockResolvedValueOnce({
+ run: buildRun({ id: 72, status: "failed" }),
+ summary: {
+ succeeded_count: 0,
+ failed_count: 1,
+ partial_count: 0,
+ failed_state_counts: {},
+ failed_reason_counts: {},
+ scrape_failure_counts: {},
+ retry_counts: {
+ retries_scheduled_count: 0,
+ scholars_with_retries_count: 0,
+ retry_exhausted_count: 0,
+ },
+ alert_thresholds: {},
+ alert_flags: {},
+ },
+ scholar_results: [],
+ safety_state: createDefaultSafetyState(),
+ } as any);
+
+ const store = useRunStatusStore();
+ store.setLatestRun(buildRun({ id: 72, status: "resolving", end_dt: null }));
+
+ const result = await store.cancelActiveCheck();
+
+ expect(result.kind).toBe("success");
+ expect(store.latestRun?.status).toBe("failed");
+ expect(store.isRunActive).toBe(false);
+ });
+
+ it("reconciles poll responses without regressing publication counters", async () => {
+ mockedListRuns.mockResolvedValueOnce(
+ buildRunsPayload([buildRun({ id: 99, status: "running", new_publication_count: 1, end_dt: null })]),
+ );
+
+ const store = useRunStatusStore();
+ await store.syncLatest();
+ store.latestRun = buildRun({ id: 99, status: "running", new_publication_count: 5, end_dt: null });
+
+ mockedListRuns.mockResolvedValueOnce(
+ buildRunsPayload([buildRun({ id: 99, status: "running", new_publication_count: 3, end_dt: null })]),
+ );
+ await store.syncLatest();
+
+ expect(store.latestRun?.new_publication_count).toBe(5);
+ });
});
diff --git a/frontend/src/stores/run_status.ts b/frontend/src/stores/run_status.ts
index 9c12540..e5b5cd7 100644
--- a/frontend/src/stores/run_status.ts
+++ b/frontend/src/stores/run_status.ts
@@ -1,11 +1,12 @@
import { defineStore } from "pinia";
-import { listRuns, triggerManualRun, type RunListItem } from "@/features/runs";
+import { listRuns, triggerManualRun, cancelRun, type RunListItem } from "@/features/runs";
import {
createDefaultSafetyState,
normalizeSafetyState,
type ScrapeSafetyState,
} from "@/features/safety";
+import { type PublicationItem } from "@/features/publications";
import { ApiRequestError } from "@/lib/api/errors";
export const RUN_STATUS_POLL_INTERVAL_MS = 5000;
@@ -13,24 +14,31 @@ export const RUN_STATUS_STARTING_PHASE_MS = 1500;
export type StartManualCheckResult =
| {
- kind: "started";
- runId: number;
- reusedExistingRun: boolean;
- }
+ kind: "started";
+ runId: number;
+ reusedExistingRun: boolean;
+ }
| {
- kind: "already_running";
- runId: number | null;
- requestId: string | null;
- }
+ kind: "already_running";
+ runId: number | null;
+ requestId: string | null;
+ }
| {
- kind: "error";
- message: string;
- requestId: string | null;
- };
+ kind: "error";
+ message: string;
+ requestId: string | null;
+ };
+
+export type CancelCheckResult =
+ | { kind: "success" }
+ | { kind: "error"; message: string };
let pollTimer: ReturnType | null = null;
let syncPromise: Promise | null = null;
let submittingPhaseTimer: ReturnType | null = null;
+let eventSource: EventSource | null = null;
+let activeStreamRunId: number | null = null;
+const ACTIVE_STATUSES = new Set(["running", "resolving"]);
function parseRunId(value: unknown): number | null {
if (typeof value === "number" && Number.isFinite(value)) {
@@ -67,6 +75,32 @@ function extractSafetyStateFromDetails(details: unknown): ScrapeSafetyState | nu
return normalizeSafetyState(candidate);
}
+function isActiveStatus(value: string | null | undefined): boolean {
+ return value !== undefined && value !== null && ACTIVE_STATUSES.has(value);
+}
+
+function parsePublicationCount(value: unknown, fallback: number): number {
+ if (typeof value === "number" && Number.isFinite(value)) {
+ return Math.max(Math.trunc(value), 0);
+ }
+ return fallback;
+}
+
+function reconcileRunCounters(previous: RunListItem | null, next: RunListItem | null): RunListItem | null {
+ if (previous === null || next === null) {
+ return next;
+ }
+ if (previous.id !== next.id) {
+ return next;
+ }
+ const previousCount = parsePublicationCount(previous.new_publication_count, 0);
+ const nextCount = parsePublicationCount(next.new_publication_count, previousCount);
+ return {
+ ...next,
+ new_publication_count: Math.max(previousCount, nextCount),
+ };
+}
+
function buildPlaceholderRunningRun(runId: number): RunListItem {
return {
id: runId,
@@ -91,13 +125,16 @@ export const useRunStatusStore = defineStore("runStatus", {
lastErrorMessage: null as string | null,
lastErrorRequestId: null as string | null,
lastSyncAt: null as number | null,
+ livePublications: [] as Array,
}),
getters: {
isRunActive(state): boolean {
- return state.isSubmitting || state.latestRun?.status === "running";
+ const s = state.latestRun?.status;
+ return state.isSubmitting || s === "running" || s === "resolving";
},
isLikelyRunning(state): boolean {
- return state.latestRun?.status === "running" || state.assumeRunningFromSubmission;
+ const s = state.latestRun?.status;
+ return s === "running" || s === "resolving" || state.assumeRunningFromSubmission;
},
canStart(): boolean {
return !this.isRunActive && !this.safetyState.cooldown_active;
@@ -125,7 +162,7 @@ export const useRunStatusStore = defineStore("runStatus", {
this.lastSyncAt = Date.now();
this.lastErrorMessage = null;
this.lastErrorRequestId = null;
- if (run?.status === "running") {
+ if (isActiveStatus(run?.status)) {
this.assumeRunningFromSubmission = true;
} else if (!this.isSubmitting) {
this.assumeRunningFromSubmission = false;
@@ -156,9 +193,69 @@ export const useRunStatusStore = defineStore("runStatus", {
updatePolling(): void {
if (this.isRunActive) {
this.startPolling();
+ this.updateEventSource();
return;
}
this.stopPolling();
+ this.updateEventSource();
+ },
+ updateEventSource(): void {
+ const targetRunId = this.latestRun?.status === "running" ? this.latestRun.id : null;
+ if (activeStreamRunId === targetRunId) {
+ return;
+ }
+ if (eventSource !== null) {
+ eventSource.close();
+ eventSource = null;
+ activeStreamRunId = null;
+ }
+ if (targetRunId !== null) {
+ if (typeof EventSource === "undefined") {
+ return;
+ }
+ activeStreamRunId = targetRunId;
+ this.livePublications = [];
+ eventSource = new EventSource(`/api/v1/runs/${targetRunId}/stream`);
+ eventSource.addEventListener("publication_discovered", (e) => {
+ try {
+ const data = JSON.parse(e.data);
+ if (this.latestRun && this.latestRun.id === targetRunId) {
+ const baseline = parsePublicationCount(this.latestRun.new_publication_count, 0);
+ const payloadCount = parsePublicationCount(data?.new_publication_count, baseline + 1);
+ this.latestRun.new_publication_count = Math.max(baseline, payloadCount);
+ }
+ this.livePublications.unshift({
+ publication_id: data.publication_id,
+ scholar_profile_id: data.scholar_profile_id,
+ scholar_label: data.scholar_label,
+ title: data.title,
+ pub_url: data.pub_url,
+ first_seen_at: data.first_seen_at,
+ year: null,
+ citation_count: 0,
+ venue_text: null,
+ display_identifier: null,
+ pdf_url: null,
+ pdf_status: "untracked",
+ pdf_attempt_count: 0,
+ pdf_failure_reason: null,
+ pdf_failure_detail: null,
+ is_read: false,
+ is_favorite: false,
+ is_new_in_latest_run: true,
+ });
+ if (this.livePublications.length > 50) {
+ this.livePublications.pop();
+ }
+ } catch (err) {
+ console.error("Failed to parse SSE event", err);
+ }
+ });
+ eventSource.onerror = () => {
+ // Reconnecting is handled automatically by EventSource,
+ // but if it's permanently closed, we could do something here.
+ };
+ }
},
async syncLatest(): Promise {
if (syncPromise) {
@@ -169,13 +266,13 @@ export const useRunStatusStore = defineStore("runStatus", {
syncPromise = (async () => {
try {
const payload = await listRuns({ limit: 1 });
- const latest = payload.runs[0] ?? null;
+ const latest = reconcileRunCounters(this.latestRun, payload.runs[0] ?? null);
this.latestRun = latest;
this.safetyState = normalizeSafetyState(payload.safety_state);
this.lastSyncAt = Date.now();
this.lastErrorMessage = null;
this.lastErrorRequestId = null;
- if (latest?.status === "running") {
+ if (isActiveStatus(latest?.status)) {
this.assumeRunningFromSubmission = true;
} else if (!this.isSubmitting) {
this.assumeRunningFromSubmission = false;
@@ -294,14 +391,36 @@ export const useRunStatusStore = defineStore("runStatus", {
} finally {
this.isSubmitting = false;
this.clearSubmittingPhaseTimer();
- if (this.latestRun?.status !== "running") {
+ if (!isActiveStatus(this.latestRun?.status)) {
this.assumeRunningFromSubmission = false;
}
this.updatePolling();
}
},
+ async cancelActiveCheck(): Promise<{ kind: "success" } | { kind: "error"; message: string }> {
+ if (!this.latestRun || !isActiveStatus(this.latestRun.status)) {
+ return { kind: "error", message: "No active run to cancel." };
+ }
+ try {
+ const response = await cancelRun(this.latestRun.id);
+ this.setLatestRun(reconcileRunCounters(this.latestRun, response.run));
+ this.setSafetyState(response.safety_state);
+ return { kind: "success" };
+ } catch (error) {
+ let errMessage = "Failed to cancel check.";
+ if (error instanceof ApiRequestError) {
+ errMessage = error.message;
+ }
+ return { kind: "error", message: errMessage };
+ }
+ },
reset(): void {
this.stopPolling();
+ if (eventSource !== null) {
+ eventSource.close();
+ eventSource = null;
+ activeStreamRunId = null;
+ }
this.clearSubmittingPhaseTimer();
this.latestRun = null;
this.isSubmitting = false;
diff --git a/frontend/src/stores/user_settings.test.ts b/frontend/src/stores/user_settings.test.ts
index 72cbb81..46b25f5 100644
--- a/frontend/src/stores/user_settings.test.ts
+++ b/frontend/src/stores/user_settings.test.ts
@@ -43,6 +43,9 @@ describe("user settings store", () => {
last_evaluated_run_id: null,
},
},
+ openalex_api_key: null,
+ crossref_api_token: null,
+ crossref_api_mailto: null,
});
expect(store.networkFailureThreshold).toBe(2);
diff --git a/pyproject.toml b/pyproject.toml
index e469967..27426bc 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -17,7 +17,9 @@ dependencies = [
"httpx>=0.28,<0.29",
"itsdangerous>=2.2,<3.0",
"python-multipart>=0.0.9,<0.1",
+ "rapidfuzz>=3.14.3",
"sqlalchemy>=2.0,<2.1",
+ "tenacity>=9.1.4",
"uvicorn[standard]>=0.34,<0.35",
]
diff --git a/test_enrich.py b/test_enrich.py
new file mode 100644
index 0000000..9a76001
--- /dev/null
+++ b/test_enrich.py
@@ -0,0 +1,42 @@
+import asyncio
+from sqlalchemy.ext.asyncio import create_async_engine, async_sessionmaker
+from app.settings import settings
+from app.db.models import ScholarProfile
+from app.services.domains.ingestion.application import ScholarIngestionService
+from app.services.domains.scholar.parser_types import PublicationCandidate
+from app.services.domains.openalex.client import OpenAlexClient
+
+async def main():
+ service = ScholarIngestionService()
+ scholar = ScholarProfile(
+ id=1,
+ scholar_id="SaiiI5MAAAAJ",
+ name="Test Scholar"
+ )
+ pubs = [
+ PublicationCandidate(
+ title="A fast quantum mechanical algorithm for database search",
+ year=1996,
+ citation_count=1000,
+ authors_text="LK Grover",
+ venue_text="Proceedings of the 28th Annual ACM Symposium on the Theory of Computing",
+ cluster_id=None,
+ title_url=None,
+ pdf_url=None
+ )
+ ]
+
+ print("Enriching...")
+ try:
+ enriched = await service._enrich_publications_with_openalex(scholar, pubs)
+ for p in enriched:
+ print(f"Title: {p.title}")
+ print(f"Year: {p.year}")
+ print(f"Citations: {p.citation_count}")
+ print(f"PDF URL: {p.pdf_url}")
+ except Exception as e:
+ import traceback
+ traceback.print_exc()
+
+if __name__ == "__main__":
+ asyncio.run(main())
diff --git a/tests/integration/test_api_v1.py b/tests/integration/test_api_v1.py
index bb21f34..e6a5422 100644
--- a/tests/integration/test_api_v1.py
+++ b/tests/integration/test_api_v1.py
@@ -1,5 +1,6 @@
from __future__ import annotations
+import asyncio
from datetime import datetime, timedelta, timezone
from pathlib import Path
@@ -66,6 +67,31 @@ def _assert_safety_state_contract(payload: dict[str, object]) -> None:
assert set(counters.keys()) == SAFETY_COUNTER_KEYS
+_ACTIVE_RUN_STATUSES = {"running", "resolving"}
+
+
+async def _wait_for_run_complete(
+ client: TestClient,
+ run_id: int,
+ *,
+ max_retries: int = 300,
+ poll_interval: float = 0.2,
+) -> dict:
+ """Poll GET /api/v1/runs/{run_id} until the run is in a terminal state.
+
+ Returns the final run detail data dict.
+ """
+ for _ in range(max_retries):
+ await asyncio.sleep(poll_interval)
+ r = client.get(f"/api/v1/runs/{run_id}")
+ assert r.status_code == 200
+ data = r.json()["data"]
+ if data["run"]["status"] not in _ACTIVE_RUN_STATUSES:
+ return data
+ r = client.get(f"/api/v1/runs/{run_id}")
+ return r.json()["data"]
+
+
async def _seed_publication_link_for_user(
db_session: AsyncSession,
*,
@@ -1041,44 +1067,42 @@ async def test_api_manual_run_skips_unchanged_initial_page_for_scholar(
app.dependency_overrides[get_scholar_source] = lambda: StubScholarSource()
try:
- client = TestClient(app)
- login_user(client, email="api-skip-unchanged@example.com", password="api-password")
- headers = _api_csrf_headers(client)
+ with TestClient(app) as client:
+ login_user(client, email="api-skip-unchanged@example.com", password="api-password")
+ headers = _api_csrf_headers(client)
- create_response = client.post(
- "/api/v1/scholars",
- json={"scholar_id": "abcDEF123456"},
- headers=headers,
- )
- assert create_response.status_code == 201
+ create_response = client.post(
+ "/api/v1/scholars",
+ json={"scholar_id": "abcDEF123456"},
+ headers=headers,
+ )
+ assert create_response.status_code == 201
- first_run_response = client.post(
- "/api/v1/runs/manual",
- headers={**headers, "Idempotency-Key": "skip-unchanged-run-001"},
- )
- assert first_run_response.status_code == 200
- first_run_id = int(first_run_response.json()["data"]["run_id"])
+ first_run_response = client.post(
+ "/api/v1/runs/manual",
+ headers={**headers, "Idempotency-Key": "skip-unchanged-run-001"},
+ )
+ assert first_run_response.status_code == 200
+ first_run_id = int(first_run_response.json()["data"]["run_id"])
- first_run_detail = client.get(f"/api/v1/runs/{first_run_id}")
- assert first_run_detail.status_code == 200
- first_results = first_run_detail.json()["data"]["scholar_results"]
- assert len(first_results) == 1
- assert first_results[0]["state_reason"] != "no_change_initial_page_signature"
+ first_detail_data = await _wait_for_run_complete(client, first_run_id)
+ first_results = first_detail_data["scholar_results"]
+ assert len(first_results) == 1
+ assert first_results[0]["state_reason"] != "no_change_initial_page_signature"
- second_run_response = client.post(
- "/api/v1/runs/manual",
- headers={**headers, "Idempotency-Key": "skip-unchanged-run-002"},
- )
- assert second_run_response.status_code == 200
- second_run_id = int(second_run_response.json()["data"]["run_id"])
+ second_run_response = client.post(
+ "/api/v1/runs/manual",
+ headers={**headers, "Idempotency-Key": "skip-unchanged-run-002"},
+ )
+ assert second_run_response.status_code == 200
+ second_run_id = int(second_run_response.json()["data"]["run_id"])
- second_run_detail = client.get(f"/api/v1/runs/{second_run_id}")
- assert second_run_detail.status_code == 200
- second_results = second_run_detail.json()["data"]["scholar_results"]
- assert len(second_results) == 1
- assert second_results[0]["state_reason"] == "no_change_initial_page_signature"
- assert second_results[0]["publication_count"] == 0
- assert second_results[0]["outcome"] == "success"
+ second_detail_data = await _wait_for_run_complete(client, second_run_id)
+ second_results = second_detail_data["scholar_results"]
+ assert len(second_results) == 1
+ assert second_results[0]["state_reason"] == "no_change_initial_page_signature"
+ assert second_results[0]["publication_count"] == 0
+ assert second_results[0]["outcome"] == "success"
finally:
app.dependency_overrides.pop(get_scholar_source, None)
@@ -1227,7 +1251,7 @@ async def test_api_runs_manual_and_queue_actions(db_session: AsyncSession) -> No
assert run_response.status_code == 200
run_payload = run_response.json()["data"]
assert "run_id" in run_payload
- assert run_payload["status"] in {"success", "partial_failure", "failed"}
+ assert run_payload["status"] in {"running", "resolving", "success", "partial_failure", "failed"}
assert run_payload["reused_existing_run"] is False
assert run_payload["idempotency_key"] == "manual-run-0001"
_assert_safety_state_contract(run_payload["safety_state"])
@@ -1243,11 +1267,15 @@ async def test_api_runs_manual_and_queue_actions(db_session: AsyncSession) -> No
"/api/v1/runs/manual",
headers={**headers, "Idempotency-Key": "manual-run-0001"},
)
- assert replay_response.status_code == 200
- replay_payload = replay_response.json()["data"]
- assert replay_payload["run_id"] == run_payload["run_id"]
- assert replay_payload["reused_existing_run"] is True
- _assert_safety_state_contract(replay_payload["safety_state"])
+ assert replay_response.status_code in {200, 409}
+ if replay_response.status_code == 200:
+ replay_payload = replay_response.json()["data"]
+ assert replay_payload["run_id"] == run_payload["run_id"]
+ assert replay_payload["reused_existing_run"] is True
+ _assert_safety_state_contract(replay_payload["safety_state"])
+ else:
+ replay_error = replay_response.json()["error"]
+ assert replay_error["code"] == "run_in_progress"
runs_response = client.get("/api/v1/runs")
assert runs_response.status_code == 200
@@ -1429,43 +1457,45 @@ async def test_api_manual_run_enforces_scrape_safety_cooldown(db_session: AsyncS
object.__setattr__(settings, "ingestion_safety_cooldown_blocked_seconds", 600)
try:
- client = TestClient(app)
- login_user(client, email="api-runs-safety@example.com", password="api-password")
- headers = _api_csrf_headers(client)
+ with TestClient(app) as client:
+ login_user(client, email="api-runs-safety@example.com", password="api-password")
+ headers = _api_csrf_headers(client)
- first_run_response = client.post(
- "/api/v1/runs/manual",
- headers={**headers, "Idempotency-Key": "safety-cooldown-run-1"},
- )
- assert first_run_response.status_code == 200
+ first_run_response = client.post(
+ "/api/v1/runs/manual",
+ headers={**headers, "Idempotency-Key": "safety-cooldown-run-1"},
+ )
+ assert first_run_response.status_code == 200
+ first_run_id = int(first_run_response.json()["data"]["run_id"])
+ await _wait_for_run_complete(client, first_run_id)
- settings_response = client.get("/api/v1/settings")
- assert settings_response.status_code == 200
- safety_state = settings_response.json()["data"]["safety_state"]
- _assert_safety_state_contract(safety_state)
- assert safety_state["cooldown_active"] is True
- assert safety_state["cooldown_reason"] == "blocked_failure_threshold_exceeded"
- assert int(safety_state["cooldown_remaining_seconds"]) > 0
- assert int(safety_state["counters"]["cooldown_entry_count"]) >= 1
- assert int(safety_state["counters"]["last_blocked_failure_count"]) >= 1
+ settings_response = client.get("/api/v1/settings")
+ assert settings_response.status_code == 200
+ safety_state = settings_response.json()["data"]["safety_state"]
+ _assert_safety_state_contract(safety_state)
+ assert safety_state["cooldown_active"] is True
+ assert safety_state["cooldown_reason"] == "blocked_failure_threshold_exceeded"
+ assert int(safety_state["cooldown_remaining_seconds"]) > 0
+ assert int(safety_state["counters"]["cooldown_entry_count"]) >= 1
+ assert int(safety_state["counters"]["last_blocked_failure_count"]) >= 1
- blocked_start_response = client.post(
- "/api/v1/runs/manual",
- headers={**headers, "Idempotency-Key": "safety-cooldown-run-2"},
- )
- assert blocked_start_response.status_code == 429
- blocked_payload = blocked_start_response.json()
- assert blocked_payload["error"]["code"] == "scrape_cooldown_active"
- blocked_state = blocked_payload["error"]["details"]["safety_state"]
- _assert_safety_state_contract(blocked_state)
- assert blocked_state["cooldown_active"] is True
- assert blocked_state["cooldown_reason"] == "blocked_failure_threshold_exceeded"
- assert int(blocked_state["counters"]["blocked_start_count"]) >= 1
+ blocked_start_response = client.post(
+ "/api/v1/runs/manual",
+ headers={**headers, "Idempotency-Key": "safety-cooldown-run-2"},
+ )
+ assert blocked_start_response.status_code == 429
+ blocked_payload = blocked_start_response.json()
+ assert blocked_payload["error"]["code"] == "scrape_cooldown_active"
+ blocked_state = blocked_payload["error"]["details"]["safety_state"]
+ _assert_safety_state_contract(blocked_state)
+ assert blocked_state["cooldown_active"] is True
+ assert blocked_state["cooldown_reason"] == "blocked_failure_threshold_exceeded"
+ assert int(blocked_state["counters"]["blocked_start_count"]) >= 1
- runs_response = client.get("/api/v1/runs")
- assert runs_response.status_code == 200
- _assert_safety_state_contract(runs_response.json()["data"]["safety_state"])
- assert runs_response.json()["data"]["safety_state"]["cooldown_active"] is True
+ runs_response = client.get("/api/v1/runs")
+ assert runs_response.status_code == 200
+ _assert_safety_state_contract(runs_response.json()["data"]["safety_state"])
+ assert runs_response.json()["data"]["safety_state"]["cooldown_active"] is True
finally:
object.__setattr__(settings, "ingestion_alert_blocked_failure_threshold", previous_blocked_threshold)
object.__setattr__(settings, "ingestion_safety_cooldown_blocked_seconds", previous_blocked_cooldown_seconds)
@@ -1535,37 +1565,39 @@ async def test_api_manual_run_enforces_network_failure_safety_cooldown(
object.__setattr__(settings, "ingestion_retry_backoff_seconds", 0.0)
try:
- client = TestClient(app)
- login_user(client, email="api-runs-safety-network@example.com", password="api-password")
- headers = _api_csrf_headers(client)
+ with TestClient(app) as client:
+ login_user(client, email="api-runs-safety-network@example.com", password="api-password")
+ headers = _api_csrf_headers(client)
- first_run_response = client.post(
- "/api/v1/runs/manual",
- headers={**headers, "Idempotency-Key": "safety-network-cooldown-run-1"},
- )
- assert first_run_response.status_code == 200
+ first_run_response = client.post(
+ "/api/v1/runs/manual",
+ headers={**headers, "Idempotency-Key": "safety-network-cooldown-run-1"},
+ )
+ assert first_run_response.status_code == 200
+ first_run_id = int(first_run_response.json()["data"]["run_id"])
+ await _wait_for_run_complete(client, first_run_id)
- settings_response = client.get("/api/v1/settings")
- assert settings_response.status_code == 200
- safety_state = settings_response.json()["data"]["safety_state"]
- _assert_safety_state_contract(safety_state)
- assert safety_state["cooldown_active"] is True
- assert safety_state["cooldown_reason"] == "network_failure_threshold_exceeded"
- assert int(safety_state["cooldown_remaining_seconds"]) > 0
- assert int(safety_state["counters"]["last_network_failure_count"]) >= 1
+ settings_response = client.get("/api/v1/settings")
+ assert settings_response.status_code == 200
+ safety_state = settings_response.json()["data"]["safety_state"]
+ _assert_safety_state_contract(safety_state)
+ assert safety_state["cooldown_active"] is True
+ assert safety_state["cooldown_reason"] == "network_failure_threshold_exceeded"
+ assert int(safety_state["cooldown_remaining_seconds"]) > 0
+ assert int(safety_state["counters"]["last_network_failure_count"]) >= 1
- blocked_start_response = client.post(
- "/api/v1/runs/manual",
- headers={**headers, "Idempotency-Key": "safety-network-cooldown-run-2"},
- )
- assert blocked_start_response.status_code == 429
- blocked_payload = blocked_start_response.json()
- assert blocked_payload["error"]["code"] == "scrape_cooldown_active"
- blocked_state = blocked_payload["error"]["details"]["safety_state"]
- _assert_safety_state_contract(blocked_state)
- assert blocked_state["cooldown_active"] is True
- assert blocked_state["cooldown_reason"] == "network_failure_threshold_exceeded"
- assert int(blocked_state["counters"]["blocked_start_count"]) >= 1
+ blocked_start_response = client.post(
+ "/api/v1/runs/manual",
+ headers={**headers, "Idempotency-Key": "safety-network-cooldown-run-2"},
+ )
+ assert blocked_start_response.status_code == 429
+ blocked_payload = blocked_start_response.json()
+ assert blocked_payload["error"]["code"] == "scrape_cooldown_active"
+ blocked_state = blocked_payload["error"]["details"]["safety_state"]
+ _assert_safety_state_contract(blocked_state)
+ assert blocked_state["cooldown_active"] is True
+ assert blocked_state["cooldown_reason"] == "network_failure_threshold_exceeded"
+ assert int(blocked_state["counters"]["blocked_start_count"]) >= 1
finally:
object.__setattr__(settings, "ingestion_alert_network_failure_threshold", previous_network_threshold)
object.__setattr__(settings, "ingestion_safety_cooldown_network_seconds", previous_network_cooldown_seconds)
@@ -2152,7 +2184,6 @@ async def test_api_publication_retry_pdf_queues_resolution_job(
citation_count=item.citation_count,
venue_text=item.venue_text,
pub_url=item.pub_url,
- doi=item.doi,
pdf_url=item.pdf_url,
is_read=item.is_read,
first_seen_at=item.first_seen_at,
@@ -2189,11 +2220,10 @@ async def test_api_publication_retry_pdf_queues_resolution_job(
assert payload["publication"]["pdf_failure_reason"] == "no_pdf_found"
stored = await db_session.execute(
- text("SELECT doi, pdf_url FROM publications WHERE id = :publication_id"),
+ text("SELECT pdf_url FROM publications WHERE id = :publication_id"),
{"publication_id": publication_id},
)
- stored_doi, stored_pdf_url = stored.one()
- assert stored_doi is None
+ stored_pdf_url = stored.scalar_one()
assert stored_pdf_url is None
diff --git a/tests/integration/test_deferred_enrichment.py b/tests/integration/test_deferred_enrichment.py
new file mode 100644
index 0000000..c51861e
--- /dev/null
+++ b/tests/integration/test_deferred_enrichment.py
@@ -0,0 +1,103 @@
+from __future__ import annotations
+
+import pytest
+from unittest.mock import patch, AsyncMock, MagicMock
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+from datetime import datetime, timezone
+
+from app.db.models import CrawlRun, Publication, ScholarProfile, ScholarPublication, RunStatus, RunTriggerType
+from app.services.domains.ingestion.application import ScholarIngestionService
+from app.services.domains.openalex.types import OpenAlexWork
+from tests.integration.helpers import insert_user
+
+@pytest.mark.integration
+@pytest.mark.asyncio
+async def test_deferred_enrichment_sweeps_previous_runs(db_session: AsyncSession) -> None:
+ # 1. Setup: Create user and scholar
+ user_id = await insert_user(db_session, email="test@example.com", password="password123")
+
+ scholar = ScholarProfile(
+ user_id=user_id,
+ scholar_id="SaiiI5MAAAAJ",
+ display_name="Test Scholar",
+ is_enabled=True,
+ )
+ db_session.add(scholar)
+ await db_session.flush()
+
+ # 2. Simulate a previous FAILED run that left an un-enriched publication
+ failed_run = CrawlRun(
+ user_id=user_id,
+ status=RunStatus.FAILED,
+ trigger_type=RunTriggerType.MANUAL,
+ start_dt=datetime.now(timezone.utc),
+ )
+ db_session.add(failed_run)
+ await db_session.flush()
+
+ pub = Publication(
+ title_raw="A fast quantum mechanical algorithm for database search",
+ title_normalized="a fast quantum mechanical algorithm for database search",
+ fingerprint_sha256="dummy_fingerprint_for_test",
+ author_text="LK Grover",
+ openalex_enriched=False,
+ )
+ db_session.add(pub)
+ await db_session.flush()
+
+ link = ScholarPublication(
+ scholar_profile_id=scholar.id,
+ publication_id=pub.id,
+ first_seen_run_id=failed_run.id,
+ )
+ db_session.add(link)
+ await db_session.commit()
+
+ # 3. Create a NEW run
+ new_run = CrawlRun(
+ user_id=user_id,
+ status=RunStatus.RUNNING,
+ trigger_type=RunTriggerType.MANUAL,
+ start_dt=datetime.now(timezone.utc),
+ )
+ db_session.add(new_run)
+ await db_session.commit()
+
+ # 4. Mock OpenAlex client to return enrichment data
+ mock_work = OpenAlexWork(
+ openalex_id="W1234567",
+ doi="10.1145/237814.237866",
+ pmid=None,
+ pmcid=None,
+ title="A fast quantum mechanical algorithm for database search",
+ publication_year=1996,
+ cited_by_count=1000,
+ is_oa=True,
+ oa_url="http://example.com/grover.pdf",
+ )
+
+ mock_source = MagicMock()
+ service = ScholarIngestionService(source=mock_source)
+
+ # We patch the client at its source, and also mock arXiv to avoid real HTTP calls
+ with patch("app.services.domains.openalex.client.OpenAlexClient") as MockClient, \
+ patch("app.services.domains.arxiv.application.discover_arxiv_id_for_publication", new=AsyncMock(return_value=None)):
+ mock_instance = MockClient.return_value
+ mock_instance.get_works_by_filter = AsyncMock(return_value=[mock_work])
+
+ # 5. Execute the enrichment pass for the NEW run
+ await service._enrich_pending_publications(db_session, run_id=new_run.id)
+ await db_session.commit()
+
+ # 6. Verification: The publication from the FAILED run should now be enriched
+ await db_session.refresh(pub)
+ assert pub.openalex_enriched is True
+ assert pub.citation_count == 1000
+ assert pub.pdf_url == "http://example.com/grover.pdf"
+
+ # Double check it was indeed processed
+ stmt = select(Publication).where(Publication.id == pub.id)
+ result = await db_session.execute(stmt)
+ enriched_pub = result.scalar_one()
+ assert enriched_pub.openalex_enriched is True
diff --git a/tests/integration/test_fixture_probe_runs.py b/tests/integration/test_fixture_probe_runs.py
index eb3762a..65361f2 100644
--- a/tests/integration/test_fixture_probe_runs.py
+++ b/tests/integration/test_fixture_probe_runs.py
@@ -144,7 +144,9 @@ async def test_fixture_probe_run_emits_failure_and_retry_summary_metrics(
request_delay_seconds=0,
network_error_retries=1,
retry_backoff_seconds=0.0,
- max_pages_per_scholar=1,
+ rate_limit_retries=0,
+ rate_limit_backoff_seconds=0.0,
+ max_pages_per_scholar=10,
page_size=100,
auto_queue_continuations=False,
alert_blocked_failure_threshold=1,
diff --git a/tests/integration/test_migrations.py b/tests/integration/test_migrations.py
index 7b4295a..f8c3e0a 100644
--- a/tests/integration/test_migrations.py
+++ b/tests/integration/test_migrations.py
@@ -19,7 +19,7 @@ EXPECTED_TABLES = {
}
EXPECTED_ENUMS = {"run_status", "run_trigger_type"}
-EXPECTED_REVISION = "20260221_0015"
+EXPECTED_REVISION = "20260225_0022"
@pytest.mark.integration
@@ -137,6 +137,28 @@ async def test_crawl_runs_has_manual_idempotency_unique_index(db_session: AsyncS
assert "WHERE" in indexdef
+@pytest.mark.integration
+@pytest.mark.db
+@pytest.mark.migrations
+@pytest.mark.asyncio
+async def test_crawl_runs_has_single_active_run_unique_index(db_session: AsyncSession) -> None:
+ result = await db_session.execute(
+ text(
+ """
+ SELECT indexdef
+ FROM pg_indexes
+ WHERE schemaname = 'public'
+ AND tablename = 'crawl_runs'
+ AND indexname = 'uq_crawl_runs_user_active'
+ """
+ )
+ )
+ indexdef = result.scalar_one()
+ assert "UNIQUE INDEX" in indexdef
+ assert "(user_id)" in indexdef
+ assert "WHERE" in indexdef
+
+
@pytest.mark.integration
@pytest.mark.db
@pytest.mark.migrations
diff --git a/tests/integration/test_run_lifecycle_consistency.py b/tests/integration/test_run_lifecycle_consistency.py
new file mode 100644
index 0000000..28020e6
--- /dev/null
+++ b/tests/integration/test_run_lifecycle_consistency.py
@@ -0,0 +1,514 @@
+from __future__ import annotations
+
+import asyncio
+from typing import Any
+
+import pytest
+from fastapi.testclient import TestClient
+from sqlalchemy import text
+from sqlalchemy.exc import IntegrityError
+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
+
+from app.api.runtime_deps import get_ingestion_service
+from app.db.models import CrawlRun, Publication, RunStatus, RunTriggerType
+from app.main import app
+from app.services.domains.ingestion.application import ScholarIngestionService
+from tests.integration.helpers import insert_user, login_user
+
+
+def _csrf_headers(client: TestClient) -> dict[str, str]:
+ response = client.get("/api/v1/auth/me")
+ assert response.status_code == 200
+ csrf_token = response.json()["data"]["csrf_token"]
+ assert isinstance(csrf_token, str) and csrf_token
+ return {"X-CSRF-Token": csrf_token}
+
+
+@pytest.mark.integration
+@pytest.mark.db
+@pytest.mark.asyncio
+async def test_crawl_runs_enforce_single_active_run_per_user(db_session: AsyncSession) -> None:
+ user_id = await insert_user(
+ db_session,
+ email="api-single-active@example.com",
+ password="api-password",
+ )
+ await db_session.execute(
+ text(
+ """
+ INSERT INTO crawl_runs (user_id, trigger_type, status, scholar_count, new_pub_count)
+ VALUES (:user_id, 'manual', 'running', 1, 0)
+ """
+ ),
+ {"user_id": user_id},
+ )
+ await db_session.commit()
+
+ with pytest.raises(IntegrityError):
+ await db_session.execute(
+ text(
+ """
+ INSERT INTO crawl_runs (user_id, trigger_type, status, scholar_count, new_pub_count)
+ VALUES (:user_id, 'scheduled', 'resolving', 1, 0)
+ """
+ ),
+ {"user_id": user_id},
+ )
+ await db_session.commit()
+ await db_session.rollback()
+
+
+@pytest.mark.integration
+@pytest.mark.db
+@pytest.mark.asyncio
+async def test_api_manual_run_conflicts_when_an_active_run_exists(
+ db_session: AsyncSession,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ user_id = await insert_user(
+ db_session,
+ email="api-run-conflict@example.com",
+ password="api-password",
+ )
+ await db_session.execute(
+ text(
+ """
+ INSERT INTO scholar_profiles (user_id, scholar_id, display_name, is_enabled)
+ VALUES (:user_id, :scholar_id, :display_name, true)
+ """
+ ),
+ {"user_id": user_id, "scholar_id": "runConflict001", "display_name": "Run Conflict"},
+ )
+ await db_session.commit()
+
+ service = ScholarIngestionService(source=object())
+
+ async def _stall_execute_run(**_kwargs: Any) -> None:
+ await asyncio.sleep(0.4)
+
+ monkeypatch.setattr(service, "execute_run", _stall_execute_run)
+ app.dependency_overrides[get_ingestion_service] = lambda: service
+ try:
+ client = TestClient(app)
+ login_user(client, email="api-run-conflict@example.com", password="api-password")
+ headers = _csrf_headers(client)
+
+ first_response = client.post("/api/v1/runs/manual", headers=headers)
+ assert first_response.status_code == 200
+ assert first_response.json()["data"]["status"] == "running"
+
+ second_response = client.post("/api/v1/runs/manual", headers=headers)
+ assert second_response.status_code == 409
+ payload = second_response.json()
+ assert payload["error"]["code"] == "run_in_progress"
+ await asyncio.sleep(0.45)
+ finally:
+ app.dependency_overrides.pop(get_ingestion_service, None)
+
+
+@pytest.mark.integration
+@pytest.mark.db
+@pytest.mark.asyncio
+async def test_api_cancel_allows_resolving_and_rejects_terminal_run(db_session: AsyncSession) -> None:
+ user_id = await insert_user(
+ db_session,
+ email="api-cancel-resolving@example.com",
+ password="api-password",
+ )
+ run_result = await db_session.execute(
+ text(
+ """
+ INSERT INTO crawl_runs (user_id, trigger_type, status, scholar_count, new_pub_count)
+ VALUES (:user_id, 'manual', 'resolving', 1, 2)
+ RETURNING id
+ """
+ ),
+ {"user_id": user_id},
+ )
+ run_id = int(run_result.scalar_one())
+ await db_session.commit()
+
+ client = TestClient(app)
+ login_user(client, email="api-cancel-resolving@example.com", password="api-password")
+ headers = _csrf_headers(client)
+
+ cancel_response = client.post(f"/api/v1/runs/{run_id}/cancel", headers=headers)
+ assert cancel_response.status_code == 200
+ assert cancel_response.json()["data"]["run"]["status"] == "canceled"
+
+ cancel_again_response = client.post(f"/api/v1/runs/{run_id}/cancel", headers=headers)
+ assert cancel_again_response.status_code == 409
+ assert cancel_again_response.json()["error"]["code"] == "run_not_cancelable"
+
+
+@pytest.mark.integration
+@pytest.mark.db
+@pytest.mark.asyncio
+async def test_background_enrichment_preserves_canceled_status(
+ db_session: AsyncSession,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ user_id = await insert_user(
+ db_session,
+ email="api-cancel-enrichment@example.com",
+ password="api-password",
+ )
+ scholar_result = await db_session.execute(
+ text(
+ """
+ INSERT INTO scholar_profiles (user_id, scholar_id, display_name, is_enabled)
+ VALUES (:user_id, 'cancelResolve001', 'Cancel Resolve', true)
+ RETURNING id
+ """
+ ),
+ {"user_id": user_id},
+ )
+ scholar_profile_id = int(scholar_result.scalar_one())
+ run = CrawlRun(
+ user_id=user_id,
+ trigger_type=RunTriggerType.MANUAL,
+ status=RunStatus.RESOLVING,
+ scholar_count=1,
+ )
+ publication = Publication(
+ fingerprint_sha256=f"{(user_id + 9000):064x}",
+ title_raw="Cancel During Resolving",
+ title_normalized="cancel during resolving",
+ citation_count=0,
+ openalex_enriched=False,
+ )
+ db_session.add_all([run, publication])
+ await db_session.flush()
+ await db_session.execute(
+ text(
+ """
+ INSERT INTO scholar_publications (
+ scholar_profile_id,
+ publication_id,
+ first_seen_run_id,
+ is_read
+ )
+ VALUES (:scholar_profile_id, :publication_id, :run_id, false)
+ """
+ ),
+ {
+ "scholar_profile_id": scholar_profile_id,
+ "publication_id": int(publication.id),
+ "run_id": int(run.id),
+ },
+ )
+ await db_session.commit()
+
+ session_factory = async_sessionmaker(db_session.bind, expire_on_commit=False)
+
+ class _OpenAlexClientStub:
+ def __init__(self, *args: Any, **kwargs: Any) -> None:
+ _ = (args, kwargs)
+
+ async def get_works_by_filter(self, *_args: Any, **_kwargs: Any) -> list[Any]:
+ async with session_factory() as cancel_session:
+ run_to_cancel = await cancel_session.get(CrawlRun, int(run.id))
+ assert run_to_cancel is not None
+ run_to_cancel.status = RunStatus.CANCELED
+ await cancel_session.commit()
+ return []
+
+ monkeypatch.setattr(
+ "app.services.domains.openalex.client.OpenAlexClient",
+ _OpenAlexClientStub,
+ )
+
+ service = ScholarIngestionService(source=object())
+ await service._background_enrich(
+ session_factory,
+ run_id=int(run.id),
+ intended_final_status=RunStatus.SUCCESS,
+ )
+
+ await db_session.refresh(run)
+ await db_session.refresh(publication)
+ assert run.status == RunStatus.CANCELED
+ assert publication.openalex_enriched is False
+
+
+@pytest.mark.integration
+@pytest.mark.db
+@pytest.mark.asyncio
+async def test_publications_list_endpoint_does_not_raise_name_error(
+ db_session: AsyncSession,
+) -> None:
+ user_id = await insert_user(
+ db_session,
+ email="api-publications-nameerror@example.com",
+ password="api-password",
+ )
+ scholar_result = await db_session.execute(
+ text(
+ """
+ INSERT INTO scholar_profiles (user_id, scholar_id, display_name, is_enabled)
+ VALUES (:user_id, 'pubNameError001', 'Pub NameError', true)
+ RETURNING id
+ """
+ ),
+ {"user_id": user_id},
+ )
+ scholar_profile_id = int(scholar_result.scalar_one())
+ publication_result = await db_session.execute(
+ text(
+ """
+ INSERT INTO publications (fingerprint_sha256, title_raw, title_normalized, citation_count)
+ VALUES (:fingerprint, :title_raw, :title_normalized, 1)
+ RETURNING id
+ """
+ ),
+ {
+ "fingerprint": f"{(user_id + 2222):064x}",
+ "title_raw": "NameError Regression",
+ "title_normalized": "nameerror regression",
+ },
+ )
+ publication_id = int(publication_result.scalar_one())
+ await db_session.execute(
+ text(
+ """
+ INSERT INTO scholar_publications (scholar_profile_id, publication_id, is_read, is_favorite)
+ VALUES (:scholar_profile_id, :publication_id, false, false)
+ """
+ ),
+ {"scholar_profile_id": scholar_profile_id, "publication_id": publication_id},
+ )
+ await db_session.commit()
+
+ client = TestClient(app)
+ login_user(client, email="api-publications-nameerror@example.com", password="api-password")
+ response = client.get("/api/v1/publications?mode=all&limit=10&offset=0")
+ assert response.status_code == 200
+ assert len(response.json()["data"]["publications"]) == 1
+
+
+@pytest.mark.integration
+@pytest.mark.db
+@pytest.mark.asyncio
+async def test_partial_discovery_exception_keeps_new_pub_count_consistent(
+ db_session: AsyncSession,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ user_id = await insert_user(
+ db_session,
+ email="api-pub-count-consistency@example.com",
+ password="api-password",
+ )
+ scholar_result = await db_session.execute(
+ text(
+ """
+ INSERT INTO scholar_profiles (user_id, scholar_id, display_name, is_enabled)
+ VALUES (:user_id, 'countConsistency001', 'Count Consistency', true)
+ RETURNING id
+ """
+ ),
+ {"user_id": user_id},
+ )
+ scholar_profile_id = int(scholar_result.scalar_one())
+
+ run = CrawlRun(
+ user_id=user_id,
+ trigger_type=RunTriggerType.MANUAL,
+ status=RunStatus.RUNNING,
+ scholar_count=1,
+ new_pub_count=0,
+ )
+ publication = Publication(
+ fingerprint_sha256=f"{(user_id + 777):064x}",
+ title_raw="Persisted Discovery",
+ title_normalized="persisted discovery",
+ citation_count=0,
+ )
+ db_session.add_all([run, publication])
+ await db_session.commit()
+
+ service = ScholarIngestionService(source=object())
+ call_count = 0
+
+ async def _resolve_publication_stub(*_args: Any, **_kwargs: Any) -> Publication:
+ nonlocal call_count
+ call_count += 1
+ if call_count == 1:
+ return publication
+ raise RuntimeError("mid_page_failure")
+
+ monkeypatch.setattr(service, "_resolve_publication", _resolve_publication_stub)
+
+ from app.services.domains.scholar.parser_types import PublicationCandidate
+ from app.db.models import ScholarProfile
+
+ scholar = await db_session.get(ScholarProfile, scholar_profile_id)
+ assert scholar is not None
+
+ publications = [
+ PublicationCandidate(
+ title="Persisted Discovery",
+ title_url=None,
+ cluster_id=None,
+ year=None,
+ citation_count=0,
+ authors_text=None,
+ venue_text=None,
+ pdf_url=None,
+ ),
+ PublicationCandidate(
+ title="Will Fail",
+ title_url=None,
+ cluster_id=None,
+ year=None,
+ citation_count=0,
+ authors_text=None,
+ venue_text=None,
+ pdf_url=None,
+ ),
+ ]
+
+ with pytest.raises(RuntimeError, match="mid_page_failure"):
+ await service._upsert_profile_publications(
+ db_session,
+ run=run,
+ scholar=scholar,
+ publications=publications,
+ )
+
+ refreshed_run = await db_session.get(CrawlRun, int(run.id))
+ assert refreshed_run is not None
+ assert int(refreshed_run.new_pub_count) == 1
+ link_count_result = await db_session.execute(
+ text(
+ """
+ SELECT count(*)
+ FROM scholar_publications
+ WHERE scholar_profile_id = :scholar_profile_id
+ AND first_seen_run_id = :run_id
+ """
+ ),
+ {"scholar_profile_id": scholar_profile_id, "run_id": int(run.id)},
+ )
+ assert int(link_count_result.scalar_one()) == 1
+
+
+@pytest.mark.integration
+@pytest.mark.db
+@pytest.mark.asyncio
+async def test_publications_pagination_snapshot_stays_stable_across_inserts(
+ db_session: AsyncSession,
+) -> None:
+ user_id = await insert_user(
+ db_session,
+ email="api-publications-snapshot@example.com",
+ password="api-password",
+ )
+ scholar_result = await db_session.execute(
+ text(
+ """
+ INSERT INTO scholar_profiles (user_id, scholar_id, display_name, is_enabled)
+ VALUES (:user_id, 'pagingSnapshot001', 'Paging Snapshot', true)
+ RETURNING id
+ """
+ ),
+ {"user_id": user_id},
+ )
+ scholar_profile_id = int(scholar_result.scalar_one())
+
+ publication_ids: list[int] = []
+ for index in range(4):
+ created = await db_session.execute(
+ text(
+ """
+ INSERT INTO publications (fingerprint_sha256, title_raw, title_normalized, citation_count)
+ VALUES (:fingerprint, :title_raw, :title_normalized, 1)
+ RETURNING id
+ """
+ ),
+ {
+ "fingerprint": f"{(user_id + 1000 + index):064x}",
+ "title_raw": f"Stable Page {index}",
+ "title_normalized": f"stable page {index}",
+ },
+ )
+ publication_ids.append(int(created.scalar_one()))
+
+ for publication_id in publication_ids:
+ await db_session.execute(
+ text(
+ """
+ INSERT INTO scholar_publications (scholar_profile_id, publication_id, is_read, is_favorite)
+ VALUES (:scholar_profile_id, :publication_id, false, false)
+ """
+ ),
+ {"scholar_profile_id": scholar_profile_id, "publication_id": publication_id},
+ )
+ await db_session.commit()
+
+ client = TestClient(app)
+ login_user(client, email="api-publications-snapshot@example.com", password="api-password")
+
+ first_page_response = client.get("/api/v1/publications?mode=all&page=1&page_size=2")
+ assert first_page_response.status_code == 200
+ first_page = first_page_response.json()["data"]
+ first_page_ids = [int(item["publication_id"]) for item in first_page["publications"]]
+ snapshot = first_page["snapshot"]
+ assert isinstance(snapshot, str) and snapshot
+
+ inserted = await db_session.execute(
+ text(
+ """
+ INSERT INTO publications (fingerprint_sha256, title_raw, title_normalized, citation_count)
+ VALUES (:fingerprint, :title_raw, :title_normalized, 1)
+ RETURNING id
+ """
+ ),
+ {
+ "fingerprint": f"{(user_id + 5000):064x}",
+ "title_raw": "Inserted After Snapshot",
+ "title_normalized": "inserted after snapshot",
+ },
+ )
+ inserted_publication_id = int(inserted.scalar_one())
+ await db_session.execute(
+ text(
+ """
+ INSERT INTO scholar_publications (scholar_profile_id, publication_id, is_read, is_favorite)
+ VALUES (:scholar_profile_id, :publication_id, false, false)
+ """
+ ),
+ {"scholar_profile_id": scholar_profile_id, "publication_id": inserted_publication_id},
+ )
+ await db_session.commit()
+
+ second_page_response = client.get(
+ "/api/v1/publications",
+ params={
+ "mode": "all",
+ "page": 2,
+ "page_size": 2,
+ "snapshot": snapshot,
+ },
+ )
+ assert second_page_response.status_code == 200
+ second_page = second_page_response.json()["data"]
+ second_page_ids = [int(item["publication_id"]) for item in second_page["publications"]]
+
+ first_page_again_response = client.get(
+ "/api/v1/publications",
+ params={
+ "mode": "all",
+ "page": 1,
+ "page_size": 2,
+ "snapshot": snapshot,
+ },
+ )
+ assert first_page_again_response.status_code == 200
+ first_page_again = first_page_again_response.json()["data"]
+ first_page_again_ids = [
+ int(item["publication_id"]) for item in first_page_again["publications"]
+ ]
+
+ assert first_page_again_ids == first_page_ids
+ assert not (set(first_page_ids) & set(second_page_ids))
+ assert inserted_publication_id not in set(first_page_ids + second_page_ids)
diff --git a/tests/unit/services/domains/openalex/test_client.py b/tests/unit/services/domains/openalex/test_client.py
new file mode 100644
index 0000000..4b31cb3
--- /dev/null
+++ b/tests/unit/services/domains/openalex/test_client.py
@@ -0,0 +1,64 @@
+import pytest
+from app.services.domains.openalex.types import OpenAlexWork
+
+def test_parse_openalex_work_from_api_dict() -> None:
+ raw_api_response = {
+ "id": "https://openalex.org/W2741809807",
+ "doi": "https://doi.org/10.1038/s41586-020-0315-z",
+ "title": "Machine learning and the physical sciences",
+ "publication_year": 2019,
+ "cited_by_count": 1420,
+ "ids": {
+ "openalex": "https://openalex.org/W2741809807",
+ "doi": "https://doi.org/10.1038/s41586-020-0315-z",
+ "mag": "2741809807",
+ "pmid": "https://pubmed.ncbi.nlm.nih.gov/32040050",
+ "pmcid": "https://www.ncbi.nlm.nih.gov/pmc/articles/PMC7325852"
+ },
+ "open_access": {
+ "is_oa": True,
+ "oa_url": "https://example.com/pdf"
+ },
+ "authorships": [
+ {
+ "author_position": "first",
+ "author": {
+ "id": "https://openalex.org/A1969205032",
+ "display_name": "Giuseppe Carleo"
+ }
+ },
+ {
+ "author_position": "middle",
+ "author": {
+ "id": "https://openalex.org/A4356881717",
+ "display_name": "Ignacio Cirac"
+ }
+ }
+ ]
+ }
+
+ work = OpenAlexWork.from_api_dict(raw_api_response)
+
+ assert work.openalex_id == "https://openalex.org/W2741809807"
+ assert work.title == "Machine learning and the physical sciences"
+ assert work.doi == "10.1038/s41586-020-0315-z"
+ assert work.pmid == "32040050"
+ assert work.pmcid == "PMC7325852"
+ assert work.publication_year == 2019
+ assert work.cited_by_count == 1420
+ assert work.is_oa is True
+ assert work.oa_url == "https://example.com/pdf"
+ assert len(work.authors) == 2
+ assert work.authors[0].display_name == "Giuseppe Carleo"
+ assert work.authors[1].display_name == "Ignacio Cirac"
+
+def test_parse_openalex_work_empty() -> None:
+ work = OpenAlexWork.from_api_dict({"id": "W123"})
+ assert work.openalex_id == "W123"
+ assert work.doi is None
+ assert work.pmid is None
+ assert work.title is None
+ assert work.publication_year is None
+ assert work.cited_by_count == 0
+ assert work.is_oa is False
+ assert len(work.authors) == 0
diff --git a/tests/unit/services/domains/openalex/test_matching.py b/tests/unit/services/domains/openalex/test_matching.py
new file mode 100644
index 0000000..b8583a0
--- /dev/null
+++ b/tests/unit/services/domains/openalex/test_matching.py
@@ -0,0 +1,55 @@
+import pytest
+from app.services.domains.openalex.types import OpenAlexWork
+from app.services.domains.openalex.matching import find_best_match
+
+def test_find_best_match_exact_title():
+ cand1 = OpenAlexWork.from_api_dict({"id": "W1", "title": "Exact Title of the Paper"})
+ cand2 = OpenAlexWork.from_api_dict({"id": "W2", "title": "Totally Different Paper"})
+
+ match = find_best_match("Exact Title of the Paper", 2020, "Author A", [cand1, cand2])
+ assert match is not None
+ assert match.openalex_id == "W1"
+
+def test_find_best_match_fuzzy_title():
+ # Only differences are punctuation or minor phrasing (e.g., matching a preprint title vs published)
+ cand1 = OpenAlexWork.from_api_dict({"id": "W1", "title": "Fuzzier Title: A Study on OpenAlex"})
+ cand2 = OpenAlexWork.from_api_dict({"id": "W2", "title": "Some completely unrelated work"})
+
+ match = find_best_match("Fuzzier Title A Study on OpenAlex", 2021, "Author B", [cand1, cand2])
+ assert match is not None
+ assert match.openalex_id == "W1"
+
+def test_find_best_match_rejects_low_score():
+ cand1 = OpenAlexWork.from_api_dict({"id": "W1", "title": "Cats in hats"})
+
+ match = find_best_match("Dogs with logs", 2020, "Author A", [cand1])
+ assert match is None
+
+def test_find_best_match_year_tiebreaker():
+ # Both titles are very similar, one has exact year.
+ cand1 = OpenAlexWork.from_api_dict({"id": "W1", "title": "The exact same title", "publication_year": 2018})
+ cand2 = OpenAlexWork.from_api_dict({"id": "W2", "title": "The exact same title", "publication_year": 2020})
+
+ match = find_best_match("The exact same title", 2020, "Author A", [cand1, cand2])
+ assert match is not None
+ assert match.openalex_id == "W2"
+
+def test_find_best_match_author_tiebreaker():
+ # Titles and years match exactly. Author overlap decides it.
+ cand1 = OpenAlexWork.from_api_dict({
+ "id": "W1",
+ "title": "A popular title",
+ "publication_year": 2020,
+ "authorships": [{"author": {"display_name": "Smith, J"}}]
+ })
+ cand2 = OpenAlexWork.from_api_dict({
+ "id": "W2",
+ "title": "A popular title",
+ "publication_year": 2020,
+ "authorships": [{"author": {"display_name": "Doe, J"}}]
+ })
+
+ # Target authors contains "Doe"
+ match = find_best_match("A popular title", 2020, "A Einstein, J Doe", [cand1, cand2])
+ assert match is not None
+ assert match.openalex_id == "W2"
diff --git a/tests/unit/services/domains/publications/test_dedup.py b/tests/unit/services/domains/publications/test_dedup.py
new file mode 100644
index 0000000..71542f0
--- /dev/null
+++ b/tests/unit/services/domains/publications/test_dedup.py
@@ -0,0 +1,163 @@
+"""Unit tests for the identifier-based publication dedup sweep.
+
+DB operations are mocked via AsyncMock so no database is required.
+"""
+from __future__ import annotations
+
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+
+from app.db.models import ScholarPublication
+from app.services.domains.publications.dedup import (
+ find_identifier_duplicate_pairs,
+ merge_duplicate_publication,
+ sweep_identifier_duplicates,
+)
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+def _make_result(rows: list) -> MagicMock:
+ mock_result = MagicMock()
+ mock_result.scalars.return_value.all.return_value = rows
+ mock_result.__iter__ = lambda self: iter(rows)
+ return mock_result
+
+
+def _session_with_execute_sequence(results: list) -> AsyncMock:
+ session = AsyncMock()
+ session.execute = AsyncMock(side_effect=[_make_result(r) for r in results])
+ session.delete = AsyncMock()
+ session.flush = AsyncMock()
+ return session
+
+
+# ---------------------------------------------------------------------------
+# find_identifier_duplicate_pairs
+# ---------------------------------------------------------------------------
+
+@pytest.mark.asyncio
+async def test_find_identifier_duplicate_pairs_returns_pairs() -> None:
+ session = AsyncMock()
+ session.execute = AsyncMock(return_value=_make_result([(1, 2)]))
+
+ pairs = await find_identifier_duplicate_pairs(session)
+
+ assert pairs == [(1, 2)]
+ session.execute.assert_awaited_once()
+
+
+@pytest.mark.asyncio
+async def test_find_identifier_duplicate_pairs_returns_empty_when_no_duplicates() -> None:
+ session = AsyncMock()
+ session.execute = AsyncMock(return_value=_make_result([]))
+
+ pairs = await find_identifier_duplicate_pairs(session)
+
+ assert pairs == []
+
+
+# ---------------------------------------------------------------------------
+# merge_duplicate_publication
+# ---------------------------------------------------------------------------
+
+@pytest.mark.asyncio
+async def test_merge_duplicate_migrates_orphaned_scholar_links() -> None:
+ """Scholar link that only the dup has should be migrated to winner."""
+ dup_link = MagicMock(spec=ScholarPublication)
+ dup_link.scholar_profile_id = 99
+ dup_link.publication_id = 2
+
+ session = _session_with_execute_sequence(
+ results=[
+ [dup_link], # dup links
+ [], # winner profile ids (no conflict)
+ [], # execute(delete(Publication)) result
+ ]
+ )
+
+ await merge_duplicate_publication(session, winner_id=1, dup_id=2)
+
+ assert dup_link.publication_id == 1
+ session.delete.assert_not_awaited() # not deleted; migrated instead
+
+
+@pytest.mark.asyncio
+async def test_merge_duplicate_drops_conflicting_scholar_link() -> None:
+ """When winner already has a link for the same scholar, dup's link is deleted."""
+ dup_link = MagicMock(spec=ScholarPublication)
+ dup_link.scholar_profile_id = 88
+ dup_link.publication_id = 2
+
+ session = _session_with_execute_sequence(
+ results=[
+ [dup_link], # dup links
+ [(88,)], # winner profiles (conflict: profile 88 already linked)
+ [], # execute(delete(Publication)) result
+ ]
+ )
+
+ await merge_duplicate_publication(session, winner_id=1, dup_id=2)
+
+ session.delete.assert_awaited_once_with(dup_link)
+ assert dup_link.publication_id == 2 # unchanged — link was deleted, not migrated
+
+
+# ---------------------------------------------------------------------------
+# sweep_identifier_duplicates
+# ---------------------------------------------------------------------------
+
+@pytest.mark.asyncio
+async def test_sweep_returns_zero_when_no_pairs() -> None:
+ with patch(
+ "app.services.domains.publications.dedup.find_identifier_duplicate_pairs",
+ new=AsyncMock(return_value=[]),
+ ):
+ session = AsyncMock()
+ count = await sweep_identifier_duplicates(session)
+
+ assert count == 0
+ session.flush.assert_not_awaited()
+
+
+@pytest.mark.asyncio
+async def test_sweep_returns_merge_count() -> None:
+ with (
+ patch(
+ "app.services.domains.publications.dedup.find_identifier_duplicate_pairs",
+ new=AsyncMock(return_value=[(1, 2), (3, 4)]),
+ ),
+ patch(
+ "app.services.domains.publications.dedup.merge_duplicate_publication",
+ new=AsyncMock(),
+ ) as mock_merge,
+ ):
+ session = AsyncMock()
+ count = await sweep_identifier_duplicates(session)
+
+ assert count == 2
+ assert mock_merge.await_count == 2
+ session.flush.assert_awaited_once()
+
+
+@pytest.mark.asyncio
+async def test_sweep_merges_each_dup_only_once() -> None:
+ """A dup sharing two identifiers with the winner appears twice in pairs but merged once."""
+ with (
+ patch(
+ "app.services.domains.publications.dedup.find_identifier_duplicate_pairs",
+ new=AsyncMock(return_value=[(1, 2), (1, 2)]),
+ ),
+ patch(
+ "app.services.domains.publications.dedup.merge_duplicate_publication",
+ new=AsyncMock(),
+ ) as mock_merge,
+ ):
+ session = AsyncMock()
+ count = await sweep_identifier_duplicates(session)
+
+ assert count == 1
+ assert mock_merge.await_count == 1
diff --git a/tests/unit/test_fingerprints.py b/tests/unit/test_fingerprints.py
new file mode 100644
index 0000000..2065713
--- /dev/null
+++ b/tests/unit/test_fingerprints.py
@@ -0,0 +1,242 @@
+from __future__ import annotations
+
+import pytest
+
+from app.services.domains.ingestion.fingerprints import (
+ _dedupe_publication_candidates,
+ canonical_title_for_dedup,
+ fuzzy_titles_match,
+ normalize_title,
+)
+from app.services.domains.scholar.parser import PublicationCandidate
+
+
+def _candidate(
+ title: str,
+ *,
+ cluster_id: str | None = None,
+ year: int | None = 2024,
+ authors_text: str | None = "Smith, J",
+ venue_text: str | None = "ICML",
+) -> PublicationCandidate:
+ return PublicationCandidate(
+ title=title,
+ title_url=None,
+ cluster_id=cluster_id,
+ year=year,
+ citation_count=None,
+ authors_text=authors_text,
+ venue_text=venue_text,
+ pdf_url=None,
+ )
+
+
+class TestFuzzyTitlesMatch:
+ def test_identical_titles(self) -> None:
+ assert fuzzy_titles_match("Deep Learning for NLP", "deep learning for nlp") is True
+
+ def test_minor_word_difference(self) -> None:
+ assert fuzzy_titles_match(
+ "A Survey on Deep Learning Methods for NLP",
+ "Survey on Deep Learning Methods for NLP",
+ ) is True
+
+ def test_punctuation_difference(self) -> None:
+ assert fuzzy_titles_match(
+ "Attention Is All You Need",
+ "Attention Is All You Need.",
+ ) is True
+
+ def test_colon_vs_dash_subtitle(self) -> None:
+ assert fuzzy_titles_match(
+ "Deep Learning: A Comprehensive Survey",
+ "Deep Learning - A Comprehensive Survey",
+ ) is True
+
+ def test_completely_different_titles(self) -> None:
+ assert fuzzy_titles_match(
+ "Deep Learning for NLP",
+ "Climate Change Impact on Agriculture",
+ ) is False
+
+ def test_short_title_no_false_positive(self) -> None:
+ assert fuzzy_titles_match("On Trees", "On Forests") is False
+
+ def test_empty_title(self) -> None:
+ assert fuzzy_titles_match("", "Deep Learning") is False
+
+ def test_custom_threshold(self) -> None:
+ # Lower threshold catches more distant matches
+ assert fuzzy_titles_match(
+ "A Survey on Deep Learning",
+ "Survey on Machine Learning Approaches",
+ threshold=0.3,
+ ) is True
+ # Default threshold rejects them
+ assert fuzzy_titles_match(
+ "A Survey on Deep Learning",
+ "Survey on Machine Learning Approaches",
+ ) is False
+
+
+class TestDedupePublicationCandidates:
+ def test_exact_duplicates_by_cluster_id(self) -> None:
+ pubs = [
+ _candidate("Title A", cluster_id="c1"),
+ _candidate("Title A Copy", cluster_id="c1"),
+ ]
+ result = _dedupe_publication_candidates(pubs)
+ assert len(result) == 1
+ assert result[0].title == "Title A"
+
+ def test_fuzzy_duplicates_without_cluster_id(self) -> None:
+ pubs = [
+ _candidate("Attention Is All You Need"),
+ _candidate("Attention Is All You Need."),
+ ]
+ result = _dedupe_publication_candidates(pubs)
+ assert len(result) == 1
+
+ def test_distinct_titles_preserved(self) -> None:
+ pubs = [
+ _candidate("Deep Learning for NLP"),
+ _candidate("Reinforcement Learning for Robotics"),
+ ]
+ result = _dedupe_publication_candidates(pubs)
+ assert len(result) == 2
+
+ def test_fallback_aligned_with_db_fingerprint(self) -> None:
+ """Same title/year/first_author/first_venue should deduplicate even with
+ different full authors_text or venue_text."""
+ pubs = [
+ _candidate(
+ "My Paper",
+ authors_text="Smith, J; Jones, A",
+ venue_text="International Conference on ML",
+ ),
+ _candidate(
+ "My Paper",
+ authors_text="Smith, J; Baker, B",
+ venue_text="International Conference for ML",
+ ),
+ ]
+ result = _dedupe_publication_candidates(pubs)
+ # Both share first_author_last_name="smith" and first_venue_word="international"
+ assert len(result) == 1
+
+ def test_mixed_cluster_and_fuzzy(self) -> None:
+ pubs = [
+ _candidate("A Comprehensive Survey on Deep Learning Methods", cluster_id="c1"),
+ _candidate("Comprehensive Survey on Deep Learning Methods"), # fuzzy match (subtitle stripped)
+ _candidate("Completely Different Study"),
+ ]
+ result = _dedupe_publication_candidates(pubs)
+ assert len(result) == 2
+ titles = [p.title for p in result]
+ assert "A Comprehensive Survey on Deep Learning Methods" in titles
+ assert "Completely Different Study" in titles
+
+ def test_scholar_noise_variants_collapse_to_one(self) -> None:
+ """The motivating case: three Scholar display variants of the Adam paper."""
+ pubs = [
+ _candidate(
+ "Adam: A method for stochastic optimization, preprint (2014)",
+ year=2014,
+ venue_text="",
+ ),
+ _candidate(
+ "Adam: A Method for Stochastic Optimization. arXiv, Jan 29, 2017. doi: 10.48550/arxiv.1412.6980",
+ year=2017,
+ venue_text="arXiv",
+ ),
+ _candidate(
+ "Adam a method for stochastic optimization. Comput. Sci",
+ year=2015,
+ venue_text="Comput. Sci",
+ ),
+ ]
+ result = _dedupe_publication_candidates(pubs)
+ assert len(result) == 1
+ assert result[0].title == pubs[0].title
+
+ def test_distinct_papers_not_merged(self) -> None:
+ """Papers with different core titles must not be collapsed."""
+ pubs = [
+ _candidate("Adam: A Method for Stochastic Optimization"),
+ _candidate("SGD: Stochastic Gradient Descent Revisited"),
+ _candidate("Attention Is All You Need"),
+ ]
+ result = _dedupe_publication_candidates(pubs)
+ assert len(result) == 3
+
+ def test_cross_page_dedup_via_seen_canonical(self) -> None:
+ """seen_canonical threads state across two separate calls (simulating two pages)."""
+ seen: set[str] = set()
+ page1 = [_candidate("Adam: A Method for Stochastic Optimization")]
+ page2 = [
+ _candidate(
+ "Adam: A method for stochastic optimization, preprint (2014)",
+ year=2014,
+ ),
+ _candidate("An Entirely Different Paper"),
+ ]
+
+ result1 = _dedupe_publication_candidates(page1, seen_canonical=seen)
+ result2 = _dedupe_publication_candidates(page2, seen_canonical=seen)
+
+ assert len(result1) == 1
+ # Noisy Adam variant from page 2 is suppressed; distinct paper survives
+ assert len(result2) == 1
+ assert result2[0].title == "An Entirely Different Paper"
+
+ def test_first_seen_wins_in_noise_collapse(self) -> None:
+ """First occurrence in page order is the kept candidate."""
+ pubs = [
+ _candidate("Adam: A Method for Stochastic Optimization", year=2015),
+ _candidate("Adam: A method for stochastic optimization, preprint (2014)", year=2014),
+ ]
+ result = _dedupe_publication_candidates(pubs)
+ assert len(result) == 1
+ assert result[0].year == 2015 # first wins
+
+
+class TestCanonicalTitleForDedup:
+ def test_strips_doi_suffix(self) -> None:
+ title = "Adam: A Method for Stochastic Optimization. doi: 10.48550/arxiv.1412.6980"
+ assert canonical_title_for_dedup(title) == normalize_title(
+ "Adam: A Method for Stochastic Optimization"
+ )
+
+ def test_strips_arxiv_metadata_suffix(self) -> None:
+ title = "Adam: A Method for Stochastic Optimization. arXiv, Jan 29, 2017"
+ assert canonical_title_for_dedup(title) == normalize_title(
+ "Adam: A Method for Stochastic Optimization"
+ )
+
+ def test_strips_preprint_parenthetical(self) -> None:
+ title = "Adam: A method for stochastic optimization, preprint (2014)"
+ assert canonical_title_for_dedup(title) == normalize_title(
+ "Adam: A method for stochastic optimization"
+ )
+
+ def test_strips_venue_sentence_suffix(self) -> None:
+ title = "Adam a method for stochastic optimization. Comput. Sci"
+ assert canonical_title_for_dedup(title) == normalize_title(
+ "Adam a method for stochastic optimization"
+ )
+
+ def test_strips_trailing_year_in_parens(self) -> None:
+ assert canonical_title_for_dedup("Deep Learning (2018)") == normalize_title("Deep Learning")
+
+ def test_preserves_clean_title(self) -> None:
+ title = "Attention Is All You Need"
+ assert canonical_title_for_dedup(title) == normalize_title(title)
+
+ def test_adam_variants_produce_identical_canonical(self) -> None:
+ variants = [
+ "Adam: A method for stochastic optimization, preprint (2014)",
+ "Adam: A Method for Stochastic Optimization. arXiv, Jan 29, 2017. doi: 10.48550/arxiv.1412.6980",
+ "Adam a method for stochastic optimization. Comput. Sci",
+ ]
+ canonicals = [canonical_title_for_dedup(v) for v in variants]
+ assert len(set(canonicals)) == 1, f"Expected one canonical, got: {canonicals}"
diff --git a/tests/unit/test_publication_identifiers.py b/tests/unit/test_publication_identifiers.py
index feab56b..88f5fcf 100644
--- a/tests/unit/test_publication_identifiers.py
+++ b/tests/unit/test_publication_identifiers.py
@@ -36,3 +36,72 @@ def test_derive_display_identifier_uses_pmcid_when_present() -> None:
assert display is not None
assert display.kind == "pmcid"
assert display.value == "PMC2175868"
+
+
+def test_normalize_arxiv_id_handles_urls() -> None:
+ from app.services.domains.publication_identifiers.normalize import normalize_arxiv_id
+
+ assert normalize_arxiv_id("https://arxiv.org/abs/1504.08025") == "1504.08025"
+ assert normalize_arxiv_id("http://arxiv.org/pdf/1504.08025v2.pdf") == "1504.08025v2"
+ assert normalize_arxiv_id("https://arxiv.org/html/1504.08025v2") == "1504.08025v2"
+ # Modern arxiv format
+ assert normalize_arxiv_id("https://arxiv.org/abs/2012.00001") == "2012.00001"
+ # Old arxiv format
+ assert normalize_arxiv_id("https://arxiv.org/abs/math/9901123") == "math/9901123"
+
+
+def test_normalize_arxiv_id_handles_raw_text() -> None:
+ from app.services.domains.publication_identifiers.normalize import normalize_arxiv_id
+
+ assert normalize_arxiv_id("arXiv:1504.08025") == "1504.08025"
+ assert normalize_arxiv_id("arxiv: 1504.08025v1") == "1504.08025v1"
+ assert normalize_arxiv_id("Preprint at arXiv:math/9901123v2") == "math/9901123v2"
+ assert normalize_arxiv_id("Not an arxiv: 123") is None
+
+
+def test_normalize_pmcid_handles_urls_and_text() -> None:
+ from app.services.domains.publication_identifiers.normalize import normalize_pmcid
+
+ assert normalize_pmcid("https://pmc.ncbi.nlm.nih.gov/articles/PMC2175868/") == "PMC2175868"
+ assert normalize_pmcid("http://www.ncbi.nlm.nih.gov/pmc/articles/PMC1234567") == "PMC1234567"
+ assert normalize_pmcid("PMCID: PMC1234567") == "PMC1234567"
+ assert normalize_pmcid("pmc1234567") == "PMC1234567"
+ assert normalize_pmcid("Not a PMCID 1234567") is None
+
+
+def test_normalize_pmid_handles_urls() -> None:
+ from app.services.domains.publication_identifiers.normalize import normalize_pmid
+
+ assert normalize_pmid("https://pubmed.ncbi.nlm.nih.gov/12345678/") == "12345678"
+ assert normalize_pmid("http://pubmed.ncbi.nlm.nih.gov/12345678") == "12345678"
+ assert normalize_pmid("https://pubmed.ncbi.nlm.nih.gov/not-a-pmid/") is None
+
+
+import pytest
+from app.services.domains.arxiv import application as arxiv_service
+from app.services.domains.publications.types import UnreadPublicationItem
+
+@pytest.mark.asyncio
+async def test_discover_arxiv_id_returns_none_if_no_title() -> None:
+ item = UnreadPublicationItem(
+ publication_id=1,
+ scholar_profile_id=1,
+ scholar_label="First Last",
+ title="",
+ year=2023,
+ citation_count=0,
+ venue_text=None,
+ pub_url=None,
+ pdf_url=None,
+ )
+ result = await arxiv_service.discover_arxiv_id_for_publication(item=item)
+ assert result is None
+
+
+@pytest.mark.asyncio
+async def test_build_arxiv_query() -> None:
+ query = arxiv_service._build_arxiv_query("Super AI Model", "Smith")
+ assert query == 'ti:"Super AI Model" AND au:"Smith"'
+
+ query2 = arxiv_service._build_arxiv_query("Only Title Here", None)
+ assert query2 == 'ti:"Only Title Here"'
diff --git a/tests/unit/test_publication_pdf_queue_policy.py b/tests/unit/test_publication_pdf_queue_policy.py
index 0cc3a7c..494d752 100644
--- a/tests/unit/test_publication_pdf_queue_policy.py
+++ b/tests/unit/test_publication_pdf_queue_policy.py
@@ -6,6 +6,7 @@ from types import SimpleNamespace
import pytest
from app.db.models import PublicationPdfJob
+from app.services.domains.arxiv.application import ArxivRateLimitError
from app.services.domains.publications import pdf_queue
from app.services.domains.publications.pdf_resolution_pipeline import PipelineOutcome
from app.services.domains.unpaywall.application import OaResolutionOutcome
@@ -133,7 +134,7 @@ def test_pdf_queue_manual_requeue_still_blocks_when_inflight() -> None:
@pytest.mark.asyncio
async def test_fetch_outcome_for_row_uses_pipeline_outcome(monkeypatch: pytest.MonkeyPatch) -> None:
- async def _fake_pipeline(*, row, request_email=None):
+ async def _fake_pipeline(*, row, request_email=None, openalex_api_key=None):
assert request_email == "user@example.com"
return PipelineOutcome(
outcome=OaResolutionOutcome(
@@ -141,7 +142,7 @@ async def test_fetch_outcome_for_row_uses_pipeline_outcome(monkeypatch: pytest.M
doi=None,
pdf_url="https://arxiv.org/pdf/1703.06103",
failure_reason=None,
- source=pdf_queue.PDF_SOURCE_SCHOLAR_PUBLICATION_PAGE,
+ source="openalex",
used_crossref=False,
),
scholar_candidates=None,
@@ -152,7 +153,7 @@ async def test_fetch_outcome_for_row_uses_pipeline_outcome(monkeypatch: pytest.M
outcome = await pdf_queue._fetch_outcome_for_row(row=_row(), request_email="user@example.com")
assert outcome.pdf_url == "https://arxiv.org/pdf/1703.06103"
- assert outcome.source == pdf_queue.PDF_SOURCE_SCHOLAR_PUBLICATION_PAGE
+ assert outcome.source == "openalex"
assert outcome.used_crossref is False
@@ -160,7 +161,7 @@ async def test_fetch_outcome_for_row_uses_pipeline_outcome(monkeypatch: pytest.M
async def test_fetch_outcome_for_row_returns_failed_outcome_when_pipeline_returns_none(
monkeypatch: pytest.MonkeyPatch,
) -> None:
- async def _fake_pipeline(*, row, request_email=None):
+ async def _fake_pipeline(*, row, request_email=None, openalex_api_key=None):
assert request_email == "user@example.com"
return PipelineOutcome(outcome=None, scholar_candidates=None)
@@ -170,3 +171,66 @@ async def test_fetch_outcome_for_row_returns_failed_outcome_when_pipeline_return
assert outcome.pdf_url is None
assert outcome.failure_reason == pdf_queue.FAILURE_RESOLUTION_EXCEPTION
+
+
+@pytest.mark.asyncio
+async def test_resolve_publication_row_persists_failed_outcome_before_reraising_arxiv_rate_limit(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ captured: list[tuple[int, int, OaResolutionOutcome]] = []
+
+ async def _noop_mark_attempt_started(*, publication_id: int, user_id: int) -> None:
+ return None
+
+ async def _raise_rate_limit(*, row, request_email=None, openalex_api_key=None):
+ raise ArxivRateLimitError("arXiv rate limit hit (429) — stopping batch")
+
+ async def _capture_persist_outcome(*, publication_id: int, user_id: int, outcome: OaResolutionOutcome) -> None:
+ captured.append((publication_id, user_id, outcome))
+
+ monkeypatch.setattr(pdf_queue, "_mark_attempt_started", _noop_mark_attempt_started)
+ monkeypatch.setattr(pdf_queue, "_fetch_outcome_for_row", _raise_rate_limit)
+ monkeypatch.setattr(pdf_queue, "_persist_outcome", _capture_persist_outcome)
+
+ with pytest.raises(ArxivRateLimitError):
+ await pdf_queue._resolve_publication_row(
+ user_id=42,
+ request_email="user@example.com",
+ row=_row(),
+ openalex_api_key="key",
+ )
+
+ assert len(captured) == 1
+ publication_id, user_id, outcome = captured[0]
+ assert publication_id == 1
+ assert user_id == 42
+ assert outcome.pdf_url is None
+ assert outcome.failure_reason == pdf_queue.FAILURE_RESOLUTION_EXCEPTION
+
+
+@pytest.mark.asyncio
+async def test_run_resolution_task_stops_batch_on_arxiv_rate_limit(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ calls: list[int] = []
+ first = _row()
+ second = SimpleNamespace(**{**first.__dict__, "publication_id": 2})
+
+ def _raise_session_factory_error():
+ raise RuntimeError("skip user settings lookup in test")
+
+ async def _fake_resolve_publication_row(*, user_id: int, request_email: str | None, row, openalex_api_key=None):
+ calls.append(int(row.publication_id))
+ if row.publication_id == 1:
+ raise ArxivRateLimitError("arXiv rate limit hit (429) — stopping batch")
+
+ monkeypatch.setattr(pdf_queue, "get_session_factory", _raise_session_factory_error)
+ monkeypatch.setattr(pdf_queue, "_resolve_publication_row", _fake_resolve_publication_row)
+
+ await pdf_queue._run_resolution_task(
+ user_id=42,
+ request_email="user@example.com",
+ rows=[first, second],
+ )
+
+ assert calls == [1]
diff --git a/tests/unit/test_publication_pdf_resolution_pipeline.py b/tests/unit/test_publication_pdf_resolution_pipeline.py
index bc7bb5c..74768c2 100644
--- a/tests/unit/test_publication_pdf_resolution_pipeline.py
+++ b/tests/unit/test_publication_pdf_resolution_pipeline.py
@@ -1,32 +1,16 @@
from __future__ import annotations
-from dataclasses import dataclass
from datetime import datetime, timezone
from types import SimpleNamespace
import pytest
from app.services.domains.publications import pdf_resolution_pipeline as pipeline
+from app.services.domains.publication_identifiers.types import DisplayIdentifier
from app.services.domains.unpaywall.application import OaResolutionOutcome
-@dataclass(frozen=True)
-class _Candidate:
- url: str
- confidence_score: float
- label_present: bool
- reason: str
-
-
-@dataclass(frozen=True)
-class _Candidates:
- container_seen: bool
- labeled_candidate: _Candidate | None
- fallback_candidate: _Candidate | None
- warnings: tuple[str, ...] = ()
-
-
-def _row(*, doi: str | None = None) -> SimpleNamespace:
+def _row(*, display_identifier: DisplayIdentifier | None = None) -> SimpleNamespace:
return SimpleNamespace(
publication_id=1,
scholar_profile_id=1,
@@ -36,7 +20,7 @@ def _row(*, doi: str | None = None) -> SimpleNamespace:
citation_count=0,
venue_text=None,
pub_url="https://scholar.google.com/citations?view_op=view_citation&citation_for_view=abc:xyz",
- doi=doi,
+ display_identifier=display_identifier,
pdf_url=None,
is_read=False,
is_favorite=False,
@@ -45,7 +29,20 @@ def _row(*, doi: str | None = None) -> SimpleNamespace:
)
-def _oa_outcome(*, pdf_url: str | None, source: str = "unpaywall") -> OaResolutionOutcome:
+def _api_outcome(*, pdf_url: str | None, source: str = "unpaywall") -> OaResolutionOutcome | None:
+ if not pdf_url:
+ return None
+ return OaResolutionOutcome(
+ publication_id=1,
+ doi="10.1000/example",
+ pdf_url=pdf_url,
+ failure_reason=None if pdf_url else "no_pdf_found",
+ source=source,
+ used_crossref=False,
+ )
+
+
+def _oa_fallback_outcome(*, pdf_url: str | None, source: str = "unpaywall") -> OaResolutionOutcome:
return OaResolutionOutcome(
publication_id=1,
doi="10.1000/example",
@@ -57,97 +54,63 @@ def _oa_outcome(*, pdf_url: str | None, source: str = "unpaywall") -> OaResoluti
@pytest.mark.asyncio
-async def test_pipeline_prefers_labeled_scholar_candidate_before_oa(monkeypatch: pytest.MonkeyPatch) -> None:
- async def _fake_candidates(_url):
- return _Candidates(
- container_seen=True,
- labeled_candidate=_Candidate(
- url="https://arxiv.org/pdf/1703.06103",
- confidence_score=0.98,
- label_present=True,
- reason="scholar_link_labeled_pdf",
- ),
- fallback_candidate=None,
- )
+async def test_pipeline_prefers_openalex_before_arxiv(monkeypatch: pytest.MonkeyPatch) -> None:
+ async def _fake_openalex(row, request_email: str | None = None, openalex_api_key: str | None = None):
+ return _api_outcome(pdf_url="https://oa.example.org/found.pdf", source="openalex")
- async def _fail_oa(*, row, request_email):
- raise AssertionError(f"OA should not run when labeled Scholar candidate exists: {row.publication_id} {request_email}")
+ async def _fail_arxiv(row):
+ raise AssertionError(f"arXiv should not run when OpenAlex candidate exists.")
- monkeypatch.setattr(pipeline, "fetch_link_candidates_from_scholar_publication_page", _fake_candidates)
- monkeypatch.setattr(pipeline, "_oa_outcome", _fail_oa)
-
- result = await pipeline.resolve_publication_pdf_outcome_for_row(row=_row(), request_email="user@example.com")
-
- assert result.outcome is not None
- assert result.outcome.pdf_url == "https://arxiv.org/pdf/1703.06103"
- assert result.outcome.source == pipeline.PDF_SOURCE_SCHOLAR_PUBLICATION_PAGE
-
-
-@pytest.mark.asyncio
-async def test_pipeline_uses_oa_result_before_unlabeled_fallback(monkeypatch: pytest.MonkeyPatch) -> None:
- async def _fake_candidates(_url):
- return _Candidates(
- container_seen=True,
- labeled_candidate=None,
- fallback_candidate=_Candidate(
- url="https://example.org/download/42",
- confidence_score=0.2,
- label_present=False,
- reason="scholar_link_unlabeled_fallback",
- ),
- )
-
- async def _fake_oa(*, row, request_email):
- assert request_email == "user@example.com"
- return _oa_outcome(pdf_url="https://oa.example.org/found.pdf")
-
- async def _fail_fallback(*, row, candidate):
- raise AssertionError(f"Unlabeled fallback should not run when OA returns PDF: {row.publication_id} {candidate.url}")
-
- monkeypatch.setattr(pipeline, "fetch_link_candidates_from_scholar_publication_page", _fake_candidates)
- monkeypatch.setattr(pipeline, "_oa_outcome", _fake_oa)
- monkeypatch.setattr(pipeline, "_unlabeled_fallback_outcome", _fail_fallback)
+ monkeypatch.setattr(pipeline, "_openalex_outcome", _fake_openalex)
+ monkeypatch.setattr(pipeline, "_arxiv_outcome", _fail_arxiv)
result = await pipeline.resolve_publication_pdf_outcome_for_row(row=_row(), request_email="user@example.com")
assert result.outcome is not None
assert result.outcome.pdf_url == "https://oa.example.org/found.pdf"
- assert result.outcome.source == "unpaywall"
+ assert result.outcome.source == "openalex"
@pytest.mark.asyncio
-async def test_pipeline_uses_unlabeled_fallback_after_oa_failure(monkeypatch: pytest.MonkeyPatch) -> None:
- fallback_candidate = _Candidate(
- url="https://example.org/download/42",
- confidence_score=0.2,
- label_present=False,
- reason="scholar_link_unlabeled_fallback",
- )
+async def test_pipeline_uses_arxiv_after_openalex_failure(monkeypatch: pytest.MonkeyPatch) -> None:
+ async def _fake_openalex(row, request_email: str | None = None, openalex_api_key: str | None = None):
+ return None
- async def _fake_candidates(_url):
- return _Candidates(container_seen=True, labeled_candidate=None, fallback_candidate=fallback_candidate)
+ async def _fake_arxiv(row, request_email: str | None = None):
+ return _api_outcome(pdf_url="https://arxiv.org/pdf/1234.5678.pdf", source="arxiv")
+
+ async def _fail_oa(*, row, request_email):
+ raise AssertionError("Unpaywall should not run when arXiv returns PDF.")
+
+ monkeypatch.setattr(pipeline, "_openalex_outcome", _fake_openalex)
+ monkeypatch.setattr(pipeline, "_arxiv_outcome", _fake_arxiv)
+ monkeypatch.setattr(pipeline, "_oa_outcome", _fail_oa)
+
+ result = await pipeline.resolve_publication_pdf_outcome_for_row(row=_row(), request_email="user@example.com")
+
+ assert result.outcome is not None
+ assert result.outcome.pdf_url == "https://arxiv.org/pdf/1234.5678.pdf"
+ assert result.outcome.source == "arxiv"
+
+
+@pytest.mark.asyncio
+async def test_pipeline_uses_unpaywall_after_arxiv_failure(monkeypatch: pytest.MonkeyPatch) -> None:
+ async def _fake_openalex(row, request_email: str | None = None, openalex_api_key: str | None = None):
+ return None
+
+ async def _fake_arxiv(row, request_email: str | None = None):
+ return None
async def _fake_oa(*, row, request_email):
assert request_email == "user@example.com"
- return _oa_outcome(pdf_url=None)
+ return _oa_fallback_outcome(pdf_url="https://example.org/fallback.pdf", source="unpaywall")
- async def _fake_fallback(*, row, candidate):
- assert candidate == fallback_candidate
- return OaResolutionOutcome(
- publication_id=row.publication_id,
- doi=row.doi,
- pdf_url="https://example.org/fallback.pdf",
- failure_reason=None,
- source=pipeline.PDF_SOURCE_SCHOLAR_PUBLICATION_PAGE_UNLABELED,
- used_crossref=False,
- )
-
- monkeypatch.setattr(pipeline, "fetch_link_candidates_from_scholar_publication_page", _fake_candidates)
+ monkeypatch.setattr(pipeline, "_openalex_outcome", _fake_openalex)
+ monkeypatch.setattr(pipeline, "_arxiv_outcome", _fake_arxiv)
monkeypatch.setattr(pipeline, "_oa_outcome", _fake_oa)
- monkeypatch.setattr(pipeline, "_unlabeled_fallback_outcome", _fake_fallback)
result = await pipeline.resolve_publication_pdf_outcome_for_row(row=_row(), request_email="user@example.com")
assert result.outcome is not None
assert result.outcome.pdf_url == "https://example.org/fallback.pdf"
- assert result.outcome.source == pipeline.PDF_SOURCE_SCHOLAR_PUBLICATION_PAGE_UNLABELED
+ assert result.outcome.source == "unpaywall"
diff --git a/tests/unit/test_scholar_publication_pdf.py b/tests/unit/test_scholar_publication_pdf.py
deleted file mode 100644
index 7382b77..0000000
--- a/tests/unit/test_scholar_publication_pdf.py
+++ /dev/null
@@ -1,76 +0,0 @@
-from __future__ import annotations
-
-import pytest
-
-from app.services.domains.scholar.parser_types import ScholarDomInvariantError
-from app.services.domains.scholar.publication_pdf import (
- extract_link_candidates_from_publication_detail_html,
- is_scholar_publication_detail_url,
-)
-
-
-def test_extract_link_candidates_from_publication_detail_html_reads_gsc_oci_pdf_link() -> None:
- html = """
-
-
-
- """
- candidates = extract_link_candidates_from_publication_detail_html(html)
- assert candidates.labeled_candidate is not None
- assert candidates.labeled_candidate.url == "https://arxiv.org/pdf/1703.06103"
-
-
-def test_extract_link_candidates_from_publication_detail_html_returns_no_candidates_when_container_missing() -> None:
- html = "No PDF section
"
- candidates = extract_link_candidates_from_publication_detail_html(html)
- assert candidates.container_seen is False
- assert candidates.labeled_candidate is None
- assert candidates.fallback_candidate is None
-
-
-def test_extract_pdf_url_from_publication_detail_html_fails_fast_on_malformed_pdf_container() -> None:
- html = """
-
-
-
- """
- with pytest.raises(ScholarDomInvariantError) as exc:
- extract_link_candidates_from_publication_detail_html(html)
- assert exc.value.code == "layout_publication_link_missing_href"
-
-
-def test_extract_link_candidates_from_publication_detail_html_keeps_unlabeled_fallback() -> None:
- html = """
-
-
-
- """
- candidates = extract_link_candidates_from_publication_detail_html(html)
- assert candidates.container_seen is True
- assert candidates.labeled_candidate is None
- assert candidates.fallback_candidate is not None
- assert candidates.fallback_candidate.url == "https://example.org/download?id=42"
- assert candidates.fallback_candidate.label_present is False
- assert "scholar_publication_link_unlabeled_only" in candidates.warnings
- assert candidates.labeled_candidate is None
-
-
-def test_is_scholar_publication_detail_url_matches_view_citation_links() -> None:
- assert is_scholar_publication_detail_url(
- "https://scholar.google.com/citations?view_op=view_citation&hl=en&user=8200InoAAAAJ&citation_for_view=8200InoAAAAJ:gsN89kCJA0AC"
- ) is True
- assert is_scholar_publication_detail_url("https://example.org/paper") is False
diff --git a/tests/unit/test_unpaywall_resolution.py b/tests/unit/test_unpaywall_resolution.py
index 651b33d..b29cc2c 100644
--- a/tests/unit/test_unpaywall_resolution.py
+++ b/tests/unit/test_unpaywall_resolution.py
@@ -6,6 +6,7 @@ from datetime import datetime, timezone
import pytest
from app.services.domains.publications.types import PublicationListItem
+from app.services.domains.publication_identifiers.types import DisplayIdentifier
from app.services.domains.unpaywall import application as unpaywall_app
@@ -31,7 +32,7 @@ def _item(publication_id: int) -> PublicationListItem:
citation_count=1000,
venue_text="Cell",
pub_url="https://doi.org/10.1016/j.cell.2007.11.019",
- doi=None,
+ display_identifier=None,
pdf_url=None,
is_read=False,
first_seen_at=datetime.now(timezone.utc),
@@ -44,7 +45,13 @@ def test_publication_doi_uses_stored_value_when_metadata_has_no_doi() -> None:
_item(99),
pub_url="https://scholar.google.com/citations?view_op=view_citation&citation_for_view=abc:123",
venue_text="Cell 130 (5), 2007",
- doi="10.1016/j.cell.2007.11.019",
+ display_identifier=DisplayIdentifier(
+ kind="doi",
+ value="10.1016/j.cell.2007.11.019",
+ label="DOI",
+ url=None,
+ confidence_score=1.0,
+ ),
)
assert unpaywall_app._publication_doi(item) == "10.1016/j.cell.2007.11.019"
@@ -54,7 +61,13 @@ def test_publication_doi_prefers_explicit_metadata_doi_over_stored_value() -> No
_item(100),
pub_url="https://doi.org/10.2000/fresh-value",
venue_text="Cell",
- doi="10.1000/stale-value",
+ display_identifier=DisplayIdentifier(
+ kind="doi",
+ value="10.1000/stale-value",
+ label="DOI",
+ url=None,
+ confidence_score=1.0,
+ ),
)
assert unpaywall_app._publication_doi(item) == "10.2000/fresh-value"
diff --git a/uv.lock b/uv.lock
index ac9d550..f4f139b 100644
--- a/uv.lock
+++ b/uv.lock
@@ -824,6 +824,69 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341 },
]
+[[package]]
+name = "rapidfuzz"
+version = "3.14.3"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/d3/28/9d808fe62375b9aab5ba92fa9b29371297b067c2790b2d7cda648b1e2f8d/rapidfuzz-3.14.3.tar.gz", hash = "sha256:2491937177868bc4b1e469087601d53f925e8d270ccc21e07404b4b5814b7b5f", size = 57863900 }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/fa/8e/3c215e860b458cfbedb3ed73bc72e98eb7e0ed72f6b48099604a7a3260c2/rapidfuzz-3.14.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:685c93ea961d135893b5984a5a9851637d23767feabe414ec974f43babbd8226", size = 1945306 },
+ { url = "https://files.pythonhosted.org/packages/36/d9/31b33512015c899f4a6e6af64df8dfe8acddf4c8b40a4b3e0e6e1bcd00e5/rapidfuzz-3.14.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fa7c8f26f009f8c673fbfb443792f0cf8cf50c4e18121ff1e285b5e08a94fbdb", size = 1390788 },
+ { url = "https://files.pythonhosted.org/packages/a9/67/2ee6f8de6e2081ccd560a571d9c9063184fe467f484a17fa90311a7f4a2e/rapidfuzz-3.14.3-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:57f878330c8d361b2ce76cebb8e3e1dc827293b6abf404e67d53260d27b5d941", size = 1374580 },
+ { url = "https://files.pythonhosted.org/packages/30/83/80d22997acd928eda7deadc19ccd15883904622396d6571e935993e0453a/rapidfuzz-3.14.3-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6c5f545f454871e6af05753a0172849c82feaf0f521c5ca62ba09e1b382d6382", size = 3154947 },
+ { url = "https://files.pythonhosted.org/packages/5b/cf/9f49831085a16384695f9fb096b99662f589e30b89b4a589a1ebc1a19d34/rapidfuzz-3.14.3-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:07aa0b5d8863e3151e05026a28e0d924accf0a7a3b605da978f0359bb804df43", size = 1223872 },
+ { url = "https://files.pythonhosted.org/packages/c8/0f/41ee8034e744b871c2e071ef0d360686f5ccfe5659f4fd96c3ec406b3c8b/rapidfuzz-3.14.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:73b07566bc7e010e7b5bd490fb04bb312e820970180df6b5655e9e6224c137db", size = 2392512 },
+ { url = "https://files.pythonhosted.org/packages/da/86/280038b6b0c2ccec54fb957c732ad6b41cc1fd03b288d76545b9cf98343f/rapidfuzz-3.14.3-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:6de00eb84c71476af7d3110cf25d8fe7c792d7f5fa86764ef0b4ca97e78ca3ed", size = 2521398 },
+ { url = "https://files.pythonhosted.org/packages/fa/7b/05c26f939607dca0006505e3216248ae2de631e39ef94dd63dbbf0860021/rapidfuzz-3.14.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:d7843a1abf0091773a530636fdd2a49a41bcae22f9910b86b4f903e76ddc82dc", size = 4259416 },
+ { url = "https://files.pythonhosted.org/packages/40/eb/9e3af4103d91788f81111af1b54a28de347cdbed8eaa6c91d5e98a889aab/rapidfuzz-3.14.3-cp312-cp312-win32.whl", hash = "sha256:dea97ac3ca18cd3ba8f3d04b5c1fe4aa60e58e8d9b7793d3bd595fdb04128d7a", size = 1709527 },
+ { url = "https://files.pythonhosted.org/packages/b8/63/d06ecce90e2cf1747e29aeab9f823d21e5877a4c51b79720b2d3be7848f8/rapidfuzz-3.14.3-cp312-cp312-win_amd64.whl", hash = "sha256:b5100fd6bcee4d27f28f4e0a1c6b5127bc8ba7c2a9959cad9eab0bf4a7ab3329", size = 1538989 },
+ { url = "https://files.pythonhosted.org/packages/fc/6d/beee32dcda64af8128aab3ace2ccb33d797ed58c434c6419eea015fec779/rapidfuzz-3.14.3-cp312-cp312-win_arm64.whl", hash = "sha256:4e49c9e992bc5fc873bd0fff7ef16a4405130ec42f2ce3d2b735ba5d3d4eb70f", size = 811161 },
+ { url = "https://files.pythonhosted.org/packages/e4/4f/0d94d09646853bd26978cb3a7541b6233c5760687777fa97da8de0d9a6ac/rapidfuzz-3.14.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:dbcb726064b12f356bf10fffdb6db4b6dce5390b23627c08652b3f6e49aa56ae", size = 1939646 },
+ { url = "https://files.pythonhosted.org/packages/b6/eb/f96aefc00f3bbdbab9c0657363ea8437a207d7545ac1c3789673e05d80bd/rapidfuzz-3.14.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:1704fc70d214294e554a2421b473779bcdeef715881c5e927dc0f11e1692a0ff", size = 1385512 },
+ { url = "https://files.pythonhosted.org/packages/26/34/71c4f7749c12ee223dba90017a5947e8f03731a7cc9f489b662a8e9e643d/rapidfuzz-3.14.3-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cc65e72790ddfd310c2c8912b45106e3800fefe160b0c2ef4d6b6fec4e826457", size = 1373571 },
+ { url = "https://files.pythonhosted.org/packages/32/00/ec8597a64f2be301ce1ee3290d067f49f6a7afb226b67d5f15b56d772ba5/rapidfuzz-3.14.3-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:43e38c1305cffae8472572a0584d4ffc2f130865586a81038ca3965301f7c97c", size = 3156759 },
+ { url = "https://files.pythonhosted.org/packages/61/d5/b41eeb4930501cc899d5a9a7b5c9a33d85a670200d7e81658626dcc0ecc0/rapidfuzz-3.14.3-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:e195a77d06c03c98b3fc06b8a28576ba824392ce40de8c708f96ce04849a052e", size = 1222067 },
+ { url = "https://files.pythonhosted.org/packages/2a/7d/6d9abb4ffd1027c6ed837b425834f3bed8344472eb3a503ab55b3407c721/rapidfuzz-3.14.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:1b7ef2f4b8583a744338a18f12c69693c194fb6777c0e9ada98cd4d9e8f09d10", size = 2394775 },
+ { url = "https://files.pythonhosted.org/packages/15/ce/4f3ab4c401c5a55364da1ffff8cc879fc97b4e5f4fa96033827da491a973/rapidfuzz-3.14.3-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:a2135b138bcdcb4c3742d417f215ac2d8c2b87bde15b0feede231ae95f09ec41", size = 2526123 },
+ { url = "https://files.pythonhosted.org/packages/c1/4b/54f804975376a328f57293bd817c12c9036171d15cf7292032e3f5820b2d/rapidfuzz-3.14.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:33a325ed0e8e1aa20c3e75f8ab057a7b248fdea7843c2a19ade0008906c14af0", size = 4262874 },
+ { url = "https://files.pythonhosted.org/packages/e9/b6/958db27d8a29a50ee6edd45d33debd3ce732e7209183a72f57544cd5fe22/rapidfuzz-3.14.3-cp313-cp313-win32.whl", hash = "sha256:8383b6d0d92f6cd008f3c9216535be215a064b2cc890398a678b56e6d280cb63", size = 1707972 },
+ { url = "https://files.pythonhosted.org/packages/07/75/fde1f334b0cec15b5946d9f84d73250fbfcc73c236b4bc1b25129d90876b/rapidfuzz-3.14.3-cp313-cp313-win_amd64.whl", hash = "sha256:e6b5e3036976f0fde888687d91be86d81f9ac5f7b02e218913c38285b756be6c", size = 1537011 },
+ { url = "https://files.pythonhosted.org/packages/2e/d7/d83fe001ce599dc7ead57ba1debf923dc961b6bdce522b741e6b8c82f55c/rapidfuzz-3.14.3-cp313-cp313-win_arm64.whl", hash = "sha256:7ba009977601d8b0828bfac9a110b195b3e4e79b350dcfa48c11269a9f1918a0", size = 810744 },
+ { url = "https://files.pythonhosted.org/packages/92/13/a486369e63ff3c1a58444d16b15c5feb943edd0e6c28a1d7d67cb8946b8f/rapidfuzz-3.14.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:a0a28add871425c2fe94358c6300bbeb0bc2ed828ca003420ac6825408f5a424", size = 1967702 },
+ { url = "https://files.pythonhosted.org/packages/f1/82/efad25e260b7810f01d6b69122685e355bed78c94a12784bac4e0beb2afb/rapidfuzz-3.14.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:010e12e2411a4854b0434f920e72b717c43f8ec48d57e7affe5c42ecfa05dd0e", size = 1410702 },
+ { url = "https://files.pythonhosted.org/packages/ba/1a/34c977b860cde91082eae4a97ae503f43e0d84d4af301d857679b66f9869/rapidfuzz-3.14.3-cp313-cp313t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5cfc3d57abd83c734d1714ec39c88a34dd69c85474918ebc21296f1e61eb5ca8", size = 1382337 },
+ { url = "https://files.pythonhosted.org/packages/88/74/f50ea0e24a5880a9159e8fd256b84d8f4634c2f6b4f98028bdd31891d907/rapidfuzz-3.14.3-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:89acb8cbb52904f763e5ac238083b9fc193bed8d1f03c80568b20e4cef43a519", size = 3165563 },
+ { url = "https://files.pythonhosted.org/packages/e8/7a/e744359404d7737049c26099423fc54bcbf303de5d870d07d2fb1410f567/rapidfuzz-3.14.3-cp313-cp313t-manylinux_2_31_armv7l.whl", hash = "sha256:7d9af908c2f371bfb9c985bd134e295038e3031e666e4b2ade1e7cb7f5af2f1a", size = 1214727 },
+ { url = "https://files.pythonhosted.org/packages/d3/2e/87adfe14ce75768ec6c2b8acd0e05e85e84be4be5e3d283cdae360afc4fe/rapidfuzz-3.14.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:1f1925619627f8798f8c3a391d81071336942e5fe8467bc3c567f982e7ce2897", size = 2403349 },
+ { url = "https://files.pythonhosted.org/packages/70/17/6c0b2b2bff9c8b12e12624c07aa22e922b0c72a490f180fa9183d1ef2c75/rapidfuzz-3.14.3-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:152555187360978119e98ce3e8263d70dd0c40c7541193fc302e9b7125cf8f58", size = 2507596 },
+ { url = "https://files.pythonhosted.org/packages/c3/d1/87852a7cbe4da7b962174c749a47433881a63a817d04f3e385ea9babcd9e/rapidfuzz-3.14.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:52619d25a09546b8db078981ca88939d72caa6b8701edd8b22e16482a38e799f", size = 4273595 },
+ { url = "https://files.pythonhosted.org/packages/c1/ab/1d0354b7d1771a28fa7fe089bc23acec2bdd3756efa2419f463e3ed80e16/rapidfuzz-3.14.3-cp313-cp313t-win32.whl", hash = "sha256:489ce98a895c98cad284f0a47960c3e264c724cb4cfd47a1430fa091c0c25204", size = 1757773 },
+ { url = "https://files.pythonhosted.org/packages/0b/0c/71ef356adc29e2bdf74cd284317b34a16b80258fa0e7e242dd92cc1e6d10/rapidfuzz-3.14.3-cp313-cp313t-win_amd64.whl", hash = "sha256:656e52b054d5b5c2524169240e50cfa080b04b1c613c5f90a2465e84888d6f15", size = 1576797 },
+ { url = "https://files.pythonhosted.org/packages/fe/d2/0e64fc27bb08d4304aa3d11154eb5480bcf5d62d60140a7ee984dc07468a/rapidfuzz-3.14.3-cp313-cp313t-win_arm64.whl", hash = "sha256:c7e40c0a0af02ad6e57e89f62bef8604f55a04ecae90b0ceeda591bbf5923317", size = 829940 },
+ { url = "https://files.pythonhosted.org/packages/32/6f/1b88aaeade83abc5418788f9e6b01efefcd1a69d65ded37d89cd1662be41/rapidfuzz-3.14.3-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:442125473b247227d3f2de807a11da6c08ccf536572d1be943f8e262bae7e4ea", size = 1942086 },
+ { url = "https://files.pythonhosted.org/packages/a0/2c/b23861347436cb10f46c2bd425489ec462790faaa360a54a7ede5f78de88/rapidfuzz-3.14.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1ec0c8c0c3d4f97ced46b2e191e883f8c82dbbf6d5ebc1842366d7eff13cd5a6", size = 1386993 },
+ { url = "https://files.pythonhosted.org/packages/83/86/5d72e2c060aa1fbdc1f7362d938f6b237dff91f5b9fc5dd7cc297e112250/rapidfuzz-3.14.3-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2dc37bc20272f388b8c3a4eba4febc6e77e50a8f450c472def4751e7678f55e4", size = 1379126 },
+ { url = "https://files.pythonhosted.org/packages/c9/bc/ef2cee3e4d8b3fc22705ff519f0d487eecc756abdc7c25d53686689d6cf2/rapidfuzz-3.14.3-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dee362e7e79bae940a5e2b3f6d09c6554db6a4e301cc68343886c08be99844f1", size = 3159304 },
+ { url = "https://files.pythonhosted.org/packages/a0/36/dc5f2f62bbc7bc90be1f75eeaf49ed9502094bb19290dfb4747317b17f12/rapidfuzz-3.14.3-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:4b39921df948388a863f0e267edf2c36302983459b021ab928d4b801cbe6a421", size = 1218207 },
+ { url = "https://files.pythonhosted.org/packages/df/7e/8f4be75c1bc62f47edf2bbbe2370ee482fae655ebcc4718ac3827ead3904/rapidfuzz-3.14.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:beda6aa9bc44d1d81242e7b291b446be352d3451f8217fcb068fc2933927d53b", size = 2401245 },
+ { url = "https://files.pythonhosted.org/packages/05/38/f7c92759e1bb188dd05b80d11c630ba59b8d7856657baf454ff56059c2ab/rapidfuzz-3.14.3-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:6a014ba09657abfcfeed64b7d09407acb29af436d7fc075b23a298a7e4a6b41c", size = 2518308 },
+ { url = "https://files.pythonhosted.org/packages/c7/ac/85820f70fed5ecb5f1d9a55f1e1e2090ef62985ef41db289b5ac5ec56e28/rapidfuzz-3.14.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:32eeafa3abce138bb725550c0e228fc7eaeec7059aa8093d9cbbec2b58c2371a", size = 4265011 },
+ { url = "https://files.pythonhosted.org/packages/46/a9/616930721ea9835c918af7cde22bff17f9db3639b0c1a7f96684be7f5630/rapidfuzz-3.14.3-cp314-cp314-win32.whl", hash = "sha256:adb44d996fc610c7da8c5048775b21db60dd63b1548f078e95858c05c86876a3", size = 1742245 },
+ { url = "https://files.pythonhosted.org/packages/06/8a/f2fa5e9635b1ccafda4accf0e38246003f69982d7c81f2faa150014525a4/rapidfuzz-3.14.3-cp314-cp314-win_amd64.whl", hash = "sha256:f3d15d8527e2b293e38ce6e437631af0708df29eafd7c9fc48210854c94472f9", size = 1584856 },
+ { url = "https://files.pythonhosted.org/packages/ef/97/09e20663917678a6d60d8e0e29796db175b1165e2079830430342d5298be/rapidfuzz-3.14.3-cp314-cp314-win_arm64.whl", hash = "sha256:576e4b9012a67e0bf54fccb69a7b6c94d4e86a9540a62f1a5144977359133583", size = 833490 },
+ { url = "https://files.pythonhosted.org/packages/03/1b/6b6084576ba87bf21877c77218a0c97ba98cb285b0c02eaaee3acd7c4513/rapidfuzz-3.14.3-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:cec3c0da88562727dd5a5a364bd9efeb535400ff0bfb1443156dd139a1dd7b50", size = 1968658 },
+ { url = "https://files.pythonhosted.org/packages/38/c0/fb02a0db80d95704b0a6469cc394e8c38501abf7e1c0b2afe3261d1510c2/rapidfuzz-3.14.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:d1fa009f8b1100e4880868137e7bf0501422898f7674f2adcd85d5a67f041296", size = 1410742 },
+ { url = "https://files.pythonhosted.org/packages/a4/72/3fbf12819fc6afc8ec75a45204013b40979d068971e535a7f3512b05e765/rapidfuzz-3.14.3-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1b86daa7419b5e8b180690efd1fdbac43ff19230803282521c5b5a9c83977655", size = 1382810 },
+ { url = "https://files.pythonhosted.org/packages/0f/18/0f1991d59bb7eee28922a00f79d83eafa8c7bfb4e8edebf4af2a160e7196/rapidfuzz-3.14.3-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c7bd1816db05d6c5ffb3a4df0a2b7b56fb8c81ef584d08e37058afa217da91b1", size = 3166349 },
+ { url = "https://files.pythonhosted.org/packages/0d/f0/baa958b1989c8f88c78bbb329e969440cf330b5a01a982669986495bb980/rapidfuzz-3.14.3-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:33da4bbaf44e9755b0ce192597f3bde7372fe2e381ab305f41b707a95ac57aa7", size = 1214994 },
+ { url = "https://files.pythonhosted.org/packages/e4/a0/cd12ec71f9b2519a3954febc5740291cceabc64c87bc6433afcb36259f3b/rapidfuzz-3.14.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:3fecce764cf5a991ee2195a844196da840aba72029b2612f95ac68a8b74946bf", size = 2403919 },
+ { url = "https://files.pythonhosted.org/packages/0b/ce/019bd2176c1644098eced4f0595cb4b3ef52e4941ac9a5854f209d0a6e16/rapidfuzz-3.14.3-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:ecd7453e02cf072258c3a6b8e930230d789d5d46cc849503729f9ce475d0e785", size = 2508346 },
+ { url = "https://files.pythonhosted.org/packages/23/f8/be16c68e2c9e6c4f23e8f4adbb7bccc9483200087ed28ff76c5312da9b14/rapidfuzz-3.14.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ea188aa00e9bcae8c8411f006a5f2f06c4607a02f24eab0d8dc58566aa911f35", size = 4274105 },
+ { url = "https://files.pythonhosted.org/packages/a1/d1/5ab148e03f7e6ec8cd220ccf7af74d3aaa4de26dd96df58936beb7cba820/rapidfuzz-3.14.3-cp314-cp314t-win32.whl", hash = "sha256:7ccbf68100c170e9a0581accbe9291850936711548c6688ce3bfb897b8c589ad", size = 1793465 },
+ { url = "https://files.pythonhosted.org/packages/cd/97/433b2d98e97abd9fff1c470a109b311669f44cdec8d0d5aa250aceaed1fb/rapidfuzz-3.14.3-cp314-cp314t-win_amd64.whl", hash = "sha256:9ec02e62ae765a318d6de38df609c57fc6dacc65c0ed1fd489036834fd8a620c", size = 1623491 },
+ { url = "https://files.pythonhosted.org/packages/e2/f6/e2176eb94f94892441bce3ddc514c179facb65db245e7ce3356965595b19/rapidfuzz-3.14.3-cp314-cp314t-win_arm64.whl", hash = "sha256:e805e52322ae29aa945baf7168b6c898120fbc16d2b8f940b658a5e9e3999253", size = 851487 },
+]
+
[[package]]
name = "requests"
version = "2.32.5"
@@ -852,7 +915,9 @@ dependencies = [
{ name = "httpx" },
{ name = "itsdangerous" },
{ name = "python-multipart" },
+ { name = "rapidfuzz" },
{ name = "sqlalchemy" },
+ { name = "tenacity" },
{ name = "uvicorn", extra = ["standard"] },
]
@@ -874,7 +939,9 @@ requires-dist = [
{ name = "pytest", marker = "extra == 'dev'", specifier = ">=8.3,<9.0" },
{ name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=0.25,<0.26" },
{ name = "python-multipart", specifier = ">=0.0.9,<0.1" },
+ { name = "rapidfuzz", specifier = ">=3.14.3" },
{ name = "sqlalchemy", specifier = ">=2.0,<2.1" },
+ { name = "tenacity", specifier = ">=9.1.4" },
{ name = "uvicorn", extras = ["standard"], specifier = ">=0.34,<0.35" },
]
provides-extras = ["dev"]
@@ -948,6 +1015,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/be/72/2db2f49247d0a18b4f1bb9a5a39a0162869acf235f3a96418363947b3d46/starlette-0.48.0-py3-none-any.whl", hash = "sha256:0764ca97b097582558ecb498132ed0c7d942f233f365b86ba37770e026510659", size = 73736 },
]
+[[package]]
+name = "tenacity"
+version = "9.1.4"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/47/c6/ee486fd809e357697ee8a44d3d69222b344920433d3b6666ccd9b374630c/tenacity-9.1.4.tar.gz", hash = "sha256:adb31d4c263f2bd041081ab33b498309a57c77f9acf2db65aadf0898179cf93a", size = 49413 }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/d7/c1/eb8f9debc45d3b7918a32ab756658a0904732f75e555402972246b0b8e71/tenacity-9.1.4-py3-none-any.whl", hash = "sha256:6095a360c919085f28c6527de529e76a06ad89b23659fa881ae0649b867a9d55", size = 28926 },
+]
+
[[package]]
name = "traitlets"
version = "5.14.3"