Initial release prep
This commit is contained in:
parent
c9b9d892f4
commit
92395b2b4b
100 changed files with 405 additions and 401 deletions
|
|
@ -80,17 +80,17 @@ Commit message: "refactor: add structured_log utility to eliminate logging boile
|
||||||
|
|
||||||
## Slice 3: Migrate Ingestion Logging to `structured_log()`
|
## Slice 3: Migrate Ingestion Logging to `structured_log()`
|
||||||
|
|
||||||
**Why:** `app/services/domains/ingestion/application.py` is 3,089 lines. 8 `_log_*` helpers consume ~239 lines. 34 inline logger calls duplicate event names and include dead metric fields. This slice removes ~300 lines.
|
**Why:** `app/services/ingestion/application.py` is 3,089 lines. 8 `_log_*` helpers consume ~239 lines. 34 inline logger calls duplicate event names and include dead metric fields. This slice removes ~300 lines.
|
||||||
|
|
||||||
**Files to modify:**
|
**Files to modify:**
|
||||||
- `app/services/domains/ingestion/application.py`
|
- `app/services/ingestion/application.py`
|
||||||
|
|
||||||
**Context files:**
|
**Context files:**
|
||||||
- `app/logging_utils.py` (created in Slice 2)
|
- `app/logging_utils.py` (created in Slice 2)
|
||||||
|
|
||||||
**Prompt:**
|
**Prompt:**
|
||||||
```
|
```
|
||||||
You are working on the scholarr repository. The file `app/services/domains/ingestion/application.py` (3,089 lines) has 8 `_log_*` helper functions consuming ~239 lines, plus 34 inline logger calls with duplicated `"event":` keys and mixed `metric_name`/`metric_value` fields.
|
You are working on the scholarr repository. The file `app/services/ingestion/application.py` (3,089 lines) has 8 `_log_*` helper functions consuming ~239 lines, plus 34 inline logger calls with duplicated `"event":` keys and mixed `metric_name`/`metric_value` fields.
|
||||||
|
|
||||||
Migrate ALL logging in this file to use `structured_log()` from `app.logging_utils`.
|
Migrate ALL logging in this file to use `structured_log()` from `app.logging_utils`.
|
||||||
|
|
||||||
|
|
@ -148,11 +148,11 @@ Commit message: "refactor: migrate ingestion service logging to structured_log"
|
||||||
**Why:** 11 more `_log_*` helpers remain across 8 files (183 lines total), plus inline logger calls with the same duplication pattern in ~20 files.
|
**Why:** 11 more `_log_*` helpers remain across 8 files (183 lines total), plus inline logger calls with the same duplication pattern in ~20 files.
|
||||||
|
|
||||||
**Files to modify (all have `_log_*` helpers to delete):**
|
**Files to modify (all have `_log_*` helpers to delete):**
|
||||||
- `app/services/domains/ingestion/scheduler.py` — `_log_queue_item_resolved` (line ~518, 21 lines)
|
- `app/services/ingestion/scheduler.py` — `_log_queue_item_resolved` (line ~518, 21 lines)
|
||||||
- `app/services/domains/arxiv/rate_limit.py` — `_log_request_scheduled` (199), `_log_request_completed` (216), `_log_cooldown_activated` (235)
|
- `app/services/arxiv/rate_limit.py` — `_log_request_scheduled` (199), `_log_request_completed` (216), `_log_cooldown_activated` (235)
|
||||||
- `app/services/domains/arxiv/client.py` — `_log_cache_event` (283), `_log_request_skipped_for_cooldown` (299)
|
- `app/services/arxiv/client.py` — `_log_cache_event` (283), `_log_request_skipped_for_cooldown` (299)
|
||||||
- `app/services/domains/publications/pdf_resolution_pipeline.py` — `_log_arxiv_skip` (148)
|
- `app/services/publications/pdf_resolution_pipeline.py` — `_log_arxiv_skip` (148)
|
||||||
- `app/services/domains/unpaywall/application.py` — `_log_resolution_summary` (194)
|
- `app/services/unpaywall/application.py` — `_log_resolution_summary` (194)
|
||||||
- `app/api/routers/publications.py` — `_log_retry_pdf_result` (343)
|
- `app/api/routers/publications.py` — `_log_retry_pdf_result` (343)
|
||||||
- `app/api/routers/settings.py` — `_log_settings_update` (103)
|
- `app/api/routers/settings.py` — `_log_settings_update` (103)
|
||||||
- `app/main.py` — `_log_startup_build_marker` (53)
|
- `app/main.py` — `_log_startup_build_marker` (53)
|
||||||
|
|
@ -168,16 +168,16 @@ Commit message: "refactor: migrate ingestion service logging to structured_log"
|
||||||
- `app/api/routers/admin.py`
|
- `app/api/routers/admin.py`
|
||||||
- `app/api/routers/auth.py`
|
- `app/api/routers/auth.py`
|
||||||
- `app/api/routers/publications.py`
|
- `app/api/routers/publications.py`
|
||||||
- `app/services/domains/scholars/application.py`
|
- `app/services/scholars/application.py`
|
||||||
- `app/services/domains/runs/events.py`
|
- `app/services/runs/events.py`
|
||||||
- `app/services/domains/openalex/client.py`
|
- `app/services/openalex/client.py`
|
||||||
- `app/services/domains/openalex/matching.py`
|
- `app/services/openalex/matching.py`
|
||||||
- `app/services/domains/crossref/application.py`
|
- `app/services/crossref/application.py`
|
||||||
- `app/services/domains/publications/dedup.py`
|
- `app/services/publications/dedup.py`
|
||||||
- `app/services/domains/publications/enrichment.py`
|
- `app/services/publications/enrichment.py`
|
||||||
- `app/services/domains/publications/pdf_queue.py`
|
- `app/services/publications/pdf_queue.py`
|
||||||
- `app/services/domains/scholar/source.py`
|
- `app/services/scholar/source.py`
|
||||||
- `app/services/domains/arxiv/gateway.py`
|
- `app/services/arxiv/gateway.py`
|
||||||
|
|
||||||
**Prompt:**
|
**Prompt:**
|
||||||
```
|
```
|
||||||
|
|
@ -185,14 +185,14 @@ You are working on the scholarr repository. Slice 3 migrated `ingestion/applicat
|
||||||
|
|
||||||
1. Delete these `_log_*` helper functions and replace their call sites with inline `structured_log()`:
|
1. Delete these `_log_*` helper functions and replace their call sites with inline `structured_log()`:
|
||||||
|
|
||||||
- `app/services/domains/ingestion/scheduler.py:~518` — `_log_queue_item_resolved` (21 lines)
|
- `app/services/ingestion/scheduler.py:~518` — `_log_queue_item_resolved` (21 lines)
|
||||||
- `app/services/domains/arxiv/rate_limit.py:~199` — `_log_request_scheduled` (17 lines)
|
- `app/services/arxiv/rate_limit.py:~199` — `_log_request_scheduled` (17 lines)
|
||||||
- `app/services/domains/arxiv/rate_limit.py:~216` — `_log_request_completed` (19 lines)
|
- `app/services/arxiv/rate_limit.py:~216` — `_log_request_completed` (19 lines)
|
||||||
- `app/services/domains/arxiv/rate_limit.py:~235` — `_log_cooldown_activated` (13 lines)
|
- `app/services/arxiv/rate_limit.py:~235` — `_log_cooldown_activated` (13 lines)
|
||||||
- `app/services/domains/arxiv/client.py:~283` — `_log_cache_event` (16 lines)
|
- `app/services/arxiv/client.py:~283` — `_log_cache_event` (16 lines)
|
||||||
- `app/services/domains/arxiv/client.py:~299` — `_log_request_skipped_for_cooldown` (13 lines)
|
- `app/services/arxiv/client.py:~299` — `_log_request_skipped_for_cooldown` (13 lines)
|
||||||
- `app/services/domains/publications/pdf_resolution_pipeline.py:~148` — `_log_arxiv_skip` (11 lines)
|
- `app/services/publications/pdf_resolution_pipeline.py:~148` — `_log_arxiv_skip` (11 lines)
|
||||||
- `app/services/domains/unpaywall/application.py:~194` — `_log_resolution_summary` (21 lines)
|
- `app/services/unpaywall/application.py:~194` — `_log_resolution_summary` (21 lines)
|
||||||
- `app/api/routers/publications.py:~343` — `_log_retry_pdf_result` (24 lines)
|
- `app/api/routers/publications.py:~343` — `_log_retry_pdf_result` (24 lines)
|
||||||
- `app/api/routers/settings.py:~103` — `_log_settings_update` (15 lines)
|
- `app/api/routers/settings.py:~103` — `_log_settings_update` (15 lines)
|
||||||
- `app/main.py:~53` — `_log_startup_build_marker` (13 lines)
|
- `app/main.py:~53` — `_log_startup_build_marker` (13 lines)
|
||||||
|
|
@ -203,7 +203,7 @@ You are working on the scholarr repository. Slice 3 migrated `ingestion/applicat
|
||||||
- Remove `metric_name`/`metric_value` from all calls
|
- Remove `metric_name`/`metric_value` from all calls
|
||||||
- Keep `logger.exception()` calls but remove the duplicated `"event"` key from their extra dicts
|
- Keep `logger.exception()` calls but remove the duplicated `"event"` key from their extra dicts
|
||||||
|
|
||||||
3. In `app/services/domains/openalex/client.py`, the lines that log `response.text` (lines ~87, ~137):
|
3. In `app/services/openalex/client.py`, the lines that log `response.text` (lines ~87, ~137):
|
||||||
- Truncate: `response.text[:500]`
|
- Truncate: `response.text[:500]`
|
||||||
|
|
||||||
4. Do NOT change any business logic. Only logging call sites.
|
4. Do NOT change any business logic. Only logging call sites.
|
||||||
|
|
@ -221,8 +221,8 @@ Commit message: "refactor: migrate all remaining logging to structured_log, remo
|
||||||
**Why:** Self-hosted users need clear, actionable logs. Current logging has verbose debug noise (per-HTTP-request, per-publication), overly long event names, and no console display optimization.
|
**Why:** Self-hosted users need clear, actionable logs. Current logging has verbose debug noise (per-HTTP-request, per-publication), overly long event names, and no console display optimization.
|
||||||
|
|
||||||
**Files to modify:**
|
**Files to modify:**
|
||||||
- `app/services/domains/scholar/source.py`
|
- `app/services/scholar/source.py`
|
||||||
- `app/services/domains/ingestion/application.py`
|
- `app/services/ingestion/application.py`
|
||||||
- `app/logging_config.py`
|
- `app/logging_config.py`
|
||||||
- Any files with event names longer than ~40 chars
|
- Any files with event names longer than ~40 chars
|
||||||
|
|
||||||
|
|
@ -231,8 +231,8 @@ Commit message: "refactor: migrate all remaining logging to structured_log, remo
|
||||||
You are working on the scholarr repository. The logging has been migrated to `structured_log()` (slices 2-4). Now audit log RELEVANCE and READABILITY for self-hosted users running this in Docker.
|
You are working on the scholarr repository. The logging has been migrated to `structured_log()` (slices 2-4). Now audit log RELEVANCE and READABILITY for self-hosted users running this in Docker.
|
||||||
|
|
||||||
1. **Remove redundant debug logs:**
|
1. **Remove redundant debug logs:**
|
||||||
- `app/services/domains/scholar/source.py`: Remove the 3 debug-level HTTP fetch events (fetch_started, search_fetch_started, publication_fetch_started). Keep only `fetch_succeeded` at DEBUG. The HTTP middleware already logs request.started/completed.
|
- `app/services/scholar/source.py`: Remove the 3 debug-level HTTP fetch events (fetch_started, search_fetch_started, publication_fetch_started). Keep only `fetch_succeeded` at DEBUG. The HTTP middleware already logs request.started/completed.
|
||||||
- `app/services/domains/ingestion/application.py`: Remove per-publication `publication.discovered` and `publication.created` DEBUG logs. The run_completed summary already reports totals.
|
- `app/services/ingestion/application.py`: Remove per-publication `publication.discovered` and `publication.created` DEBUG logs. The run_completed summary already reports totals.
|
||||||
|
|
||||||
2. **Shorten overly long event names** (search all structured_log calls in app/):
|
2. **Shorten overly long event names** (search all structured_log calls in app/):
|
||||||
- `ingestion.request_delay_coerced_to_policy_floor` → `ingestion.delay_coerced`
|
- `ingestion.request_delay_coerced_to_policy_floor` → `ingestion.delay_coerced`
|
||||||
|
|
@ -396,7 +396,7 @@ Commit message: "ci: add CodeQL security scanning and Dependabot"
|
||||||
|
|
||||||
**Files to modify:**
|
**Files to modify:**
|
||||||
- `pyproject.toml`
|
- `pyproject.toml`
|
||||||
- `app/services/domains/crossref/application.py`
|
- `app/services/crossref/application.py`
|
||||||
|
|
||||||
**Files to create:**
|
**Files to create:**
|
||||||
- `.github/workflows/release.yml`
|
- `.github/workflows/release.yml`
|
||||||
|
|
@ -405,7 +405,7 @@ Commit message: "ci: add CodeQL security scanning and Dependabot"
|
||||||
```
|
```
|
||||||
You are working on the scholarr repository. Set up automated versioning with python-semantic-release.
|
You are working on the scholarr repository. Set up automated versioning with python-semantic-release.
|
||||||
|
|
||||||
1. Fix hardcoded version in `app/services/domains/crossref/application.py` (line ~294):
|
1. Fix hardcoded version in `app/services/crossref/application.py` (line ~294):
|
||||||
```python
|
```python
|
||||||
# Before:
|
# Before:
|
||||||
etiquette = Etiquette(settings.app_name, "0.1.0", "https://scholarr.local", email)
|
etiquette = Etiquette(settings.app_name, "0.1.0", "https://scholarr.local", email)
|
||||||
|
|
@ -546,18 +546,18 @@ Commit message: "docs: rebuild documentation from scratch with clean IA"
|
||||||
**Why:** After logging cleanup (slices 3+5), the file is ~2,700-2,800 lines. Still the largest source file by 3x. This begins decomposition with the safest extract: safety gate logic.
|
**Why:** After logging cleanup (slices 3+5), the file is ~2,700-2,800 lines. Still the largest source file by 3x. This begins decomposition with the safest extract: safety gate logic.
|
||||||
|
|
||||||
**Files to modify:**
|
**Files to modify:**
|
||||||
- `app/services/domains/ingestion/application.py`
|
- `app/services/ingestion/application.py`
|
||||||
|
|
||||||
**Files to create:**
|
**Files to create:**
|
||||||
- `app/services/domains/ingestion/safety.py`
|
- `app/services/ingestion/safety.py`
|
||||||
|
|
||||||
**Prompt:**
|
**Prompt:**
|
||||||
```
|
```
|
||||||
You are working on the scholarr repository. `app/services/domains/ingestion/application.py` is ~2,700-2,800 lines (after logging cleanup). It contains the entire ingestion orchestration in a single class.
|
You are working on the scholarr repository. `app/services/ingestion/application.py` is ~2,700-2,800 lines (after logging cleanup). It contains the entire ingestion orchestration in a single class.
|
||||||
|
|
||||||
Extract the FIRST logical chunk: safety gate logic.
|
Extract the FIRST logical chunk: safety gate logic.
|
||||||
|
|
||||||
1. Read `app/services/domains/ingestion/application.py` fully.
|
1. Read `app/services/ingestion/application.py` fully.
|
||||||
|
|
||||||
2. Identify safety/cooldown methods:
|
2. Identify safety/cooldown methods:
|
||||||
- `_enforce_safety_gate`
|
- `_enforce_safety_gate`
|
||||||
|
|
@ -565,7 +565,7 @@ Extract the FIRST logical chunk: safety gate logic.
|
||||||
- Cooldown activation/clearing methods
|
- Cooldown activation/clearing methods
|
||||||
- Alert threshold evaluation methods
|
- Alert threshold evaluation methods
|
||||||
|
|
||||||
3. Create `app/services/domains/ingestion/safety.py`:
|
3. Create `app/services/ingestion/safety.py`:
|
||||||
- Move identified methods to standalone functions or a small class
|
- Move identified methods to standalone functions or a small class
|
||||||
- Accept explicit parameters (no implicit self.xxx state)
|
- Accept explicit parameters (no implicit self.xxx state)
|
||||||
- Keep same function signatures where possible
|
- Keep same function signatures where possible
|
||||||
|
|
@ -577,7 +577,7 @@ Extract the FIRST logical chunk: safety gate logic.
|
||||||
|
|
||||||
Constraints from `agents.md`:
|
Constraints from `agents.md`:
|
||||||
- Max 50 lines per function
|
- Max 50 lines per function
|
||||||
- No flat files in `app/services/` root — all within `app/services/domains/ingestion/`
|
- No flat files in `app/services/` root — all within `app/services/ingestion/`
|
||||||
- Fail fast, early returns, guard clauses
|
- Fail fast, early returns, guard clauses
|
||||||
|
|
||||||
Commit message: "refactor: extract ingestion safety gate logic to dedicated module"
|
Commit message: "refactor: extract ingestion safety gate logic to dedicated module"
|
||||||
|
|
|
||||||
|
|
@ -35,7 +35,7 @@ These limits prevent IP bans and are not to be optimized away.
|
||||||
|
|
||||||
## 5. Domain Service Boundaries
|
## 5. Domain Service Boundaries
|
||||||
|
|
||||||
- **Strict Modularity:** Flat files in the `app/services/` root are strictly prohibited. All business logic and routing must reside exclusively within `app/services/domains/`.
|
- **Strict Modularity:** Flat files in the `app/services/` root are strictly prohibited. All business logic and routing must reside exclusively within `app/services/<domain>/`.
|
||||||
|
|
||||||
## 6. UI rules
|
## 6. UI rules
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -10,8 +10,8 @@ from app.api.deps import get_api_current_user
|
||||||
from app.api.errors import ApiException
|
from app.api.errors import ApiException
|
||||||
from app.db.models import User
|
from app.db.models import User
|
||||||
from app.db.session import get_db_session
|
from app.db.session import get_db_session
|
||||||
from app.services.domains.scholars import application as scholar_service
|
from app.services.scholars import application as scholar_service
|
||||||
from app.services.domains.scholars import uploads as scholar_uploads
|
from app.services.scholars import uploads as scholar_uploads
|
||||||
from app.settings import settings
|
from app.settings import settings
|
||||||
|
|
||||||
router = APIRouter(tags=["media"])
|
router = APIRouter(tags=["media"])
|
||||||
|
|
|
||||||
|
|
@ -21,7 +21,7 @@ from app.auth.service import AuthService
|
||||||
from app.db.models import User
|
from app.db.models import User
|
||||||
from app.db.session import get_db_session
|
from app.db.session import get_db_session
|
||||||
from app.logging_utils import structured_log
|
from app.logging_utils import structured_log
|
||||||
from app.services.domains.users import application as user_service
|
from app.services.users import application as user_service
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -22,13 +22,13 @@ from app.api.schemas import (
|
||||||
from app.db.models import DataRepairJob, User
|
from app.db.models import DataRepairJob, User
|
||||||
from app.db.session import get_db_session
|
from app.db.session import get_db_session
|
||||||
from app.logging_utils import structured_log
|
from app.logging_utils import structured_log
|
||||||
from app.services.domains.dbops import (
|
from app.services.dbops import (
|
||||||
collect_integrity_report,
|
collect_integrity_report,
|
||||||
list_repair_jobs,
|
list_repair_jobs,
|
||||||
run_publication_link_repair,
|
run_publication_link_repair,
|
||||||
run_publication_near_duplicate_repair,
|
run_publication_near_duplicate_repair,
|
||||||
)
|
)
|
||||||
from app.services.domains.publications import application as publication_service
|
from app.services.publications import application as publication_service
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -26,7 +26,7 @@ from app.db.models import User
|
||||||
from app.db.session import get_db_session
|
from app.db.session import get_db_session
|
||||||
from app.logging_utils import structured_log
|
from app.logging_utils import structured_log
|
||||||
from app.security.csrf import ensure_csrf_token
|
from app.security.csrf import ensure_csrf_token
|
||||||
from app.services.domains.users import application as user_service
|
from app.services.users import application as user_service
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -23,9 +23,9 @@ from app.api.schemas import (
|
||||||
from app.db.models import User
|
from app.db.models import User
|
||||||
from app.db.session import get_db_session
|
from app.db.session import get_db_session
|
||||||
from app.logging_utils import structured_log
|
from app.logging_utils import structured_log
|
||||||
from app.services.domains.publication_identifiers import application as identifier_service
|
from app.services.publication_identifiers import application as identifier_service
|
||||||
from app.services.domains.publications import application as publication_service
|
from app.services.publications import application as publication_service
|
||||||
from app.services.domains.scholars import application as scholar_service
|
from app.services.scholars import application as scholar_service
|
||||||
from app.settings import settings
|
from app.settings import settings
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
|
||||||
|
|
@ -25,11 +25,11 @@ from app.api.schemas import (
|
||||||
from app.db.models import RunStatus, RunTriggerType, User
|
from app.db.models import RunStatus, RunTriggerType, User
|
||||||
from app.db.session import get_db_session
|
from app.db.session import get_db_session
|
||||||
from app.logging_utils import structured_log
|
from app.logging_utils import structured_log
|
||||||
from app.services.domains.ingestion import application as ingestion_service
|
from app.services.ingestion import application as ingestion_service
|
||||||
from app.services.domains.ingestion import safety as run_safety_service
|
from app.services.ingestion import safety as run_safety_service
|
||||||
from app.services.domains.runs import application as run_service
|
from app.services.runs import application as run_service
|
||||||
from app.services.domains.runs.events import event_generator
|
from app.services.runs.events import event_generator
|
||||||
from app.services.domains.settings import application as user_settings_service
|
from app.services.settings import application as user_settings_service
|
||||||
from app.settings import settings
|
from app.settings import settings
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
|
||||||
|
|
@ -25,12 +25,12 @@ from app.api.schemas import (
|
||||||
from app.db.models import User
|
from app.db.models import User
|
||||||
from app.db.session import get_db_session
|
from app.db.session import get_db_session
|
||||||
from app.logging_utils import structured_log
|
from app.logging_utils import structured_log
|
||||||
from app.services.domains.ingestion import queue as ingestion_queue_service
|
from app.services.ingestion import queue as ingestion_queue_service
|
||||||
from app.services.domains.portability import application as import_export_service
|
from app.services.portability import application as import_export_service
|
||||||
from app.services.domains.scholar import rate_limit as scholar_rate_limit
|
from app.services.scholar import rate_limit as scholar_rate_limit
|
||||||
from app.services.domains.scholar.source import ScholarSource
|
from app.services.scholar.source import ScholarSource
|
||||||
from app.services.domains.scholars import application as scholar_service
|
from app.services.scholars import application as scholar_service
|
||||||
from app.services.domains.scholars import search_hints as scholar_search_hints
|
from app.services.scholars import search_hints as scholar_search_hints
|
||||||
from app.settings import settings
|
from app.settings import settings
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
|
||||||
|
|
@ -12,8 +12,8 @@ from app.api.schemas import SettingsEnvelope, SettingsUpdateRequest
|
||||||
from app.db.models import User
|
from app.db.models import User
|
||||||
from app.db.session import get_db_session
|
from app.db.session import get_db_session
|
||||||
from app.logging_utils import structured_log
|
from app.logging_utils import structured_log
|
||||||
from app.services.domains.ingestion import safety as run_safety_service
|
from app.services.ingestion import safety as run_safety_service
|
||||||
from app.services.domains.settings import application as user_settings_service
|
from app.services.settings import application as user_settings_service
|
||||||
from app.settings import settings as settings_module
|
from app.settings import settings as settings_module
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
|
||||||
|
|
@ -2,8 +2,8 @@ from __future__ import annotations
|
||||||
|
|
||||||
from fastapi import Depends
|
from fastapi import Depends
|
||||||
|
|
||||||
from app.services.domains.ingestion import application as ingestion_service
|
from app.services.ingestion import application as ingestion_service
|
||||||
from app.services.domains.scholar import source as scholar_source_service
|
from app.services.scholar import source as scholar_source_service
|
||||||
|
|
||||||
|
|
||||||
def get_scholar_source() -> scholar_source_service.ScholarSource:
|
def get_scholar_source() -> scholar_source_service.ScholarSource:
|
||||||
|
|
|
||||||
|
|
@ -9,7 +9,7 @@ from app.auth.session import clear_session_user, get_session_user, set_session_u
|
||||||
from app.db.models import User
|
from app.db.models import User
|
||||||
from app.logging_utils import structured_log
|
from app.logging_utils import structured_log
|
||||||
from app.security.csrf import CSRF_SESSION_KEY
|
from app.security.csrf import CSRF_SESSION_KEY
|
||||||
from app.services.domains.users import application as user_service
|
from app.services.users import application as user_service
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -21,7 +21,7 @@ from app.http.middleware import (
|
||||||
from app.logging_config import configure_logging, parse_redact_fields
|
from app.logging_config import configure_logging, parse_redact_fields
|
||||||
from app.logging_utils import structured_log
|
from app.logging_utils import structured_log
|
||||||
from app.security.csrf import CSRFMiddleware
|
from app.security.csrf import CSRFMiddleware
|
||||||
from app.services.domains.ingestion.scheduler import SchedulerService
|
from app.services.ingestion.scheduler import SchedulerService
|
||||||
from app.settings import settings
|
from app.settings import settings
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
|
||||||
|
|
@ -2,13 +2,13 @@ from __future__ import annotations
|
||||||
|
|
||||||
from typing import TYPE_CHECKING
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
from app.services.domains.arxiv.gateway import (
|
from app.services.arxiv.gateway import (
|
||||||
build_arxiv_query,
|
build_arxiv_query,
|
||||||
get_arxiv_gateway,
|
get_arxiv_gateway,
|
||||||
)
|
)
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem
|
from app.services.publications.types import PublicationListItem, UnreadPublicationItem
|
||||||
|
|
||||||
|
|
||||||
def _build_arxiv_query(title: str, author_surname: str | None) -> str | None:
|
def _build_arxiv_query(title: str, author_surname: str | None) -> str | None:
|
||||||
|
|
|
||||||
|
|
@ -12,8 +12,8 @@ from sqlalchemy import delete, func, select
|
||||||
|
|
||||||
from app.db.models import ArxivQueryCacheEntry
|
from app.db.models import ArxivQueryCacheEntry
|
||||||
from app.db.session import get_session_factory
|
from app.db.session import get_session_factory
|
||||||
from app.services.domains.arxiv.constants import ARXIV_CACHE_FINGERPRINT_VERSION
|
from app.services.arxiv.constants import ARXIV_CACHE_FINGERPRINT_VERSION
|
||||||
from app.services.domains.arxiv.types import ArxivEntry, ArxivFeed, ArxivOpenSearchMeta
|
from app.services.arxiv.types import ArxivEntry, ArxivFeed, ArxivOpenSearchMeta
|
||||||
|
|
||||||
_INFLIGHT_LOCK = asyncio.Lock()
|
_INFLIGHT_LOCK = asyncio.Lock()
|
||||||
_INFLIGHT_FEEDS: dict[str, asyncio.Future[ArxivFeed]] = {}
|
_INFLIGHT_FEEDS: dict[str, asyncio.Future[ArxivFeed]] = {}
|
||||||
|
|
|
||||||
|
|
@ -6,21 +6,21 @@ from collections.abc import Awaitable, Callable
|
||||||
import httpx
|
import httpx
|
||||||
|
|
||||||
from app.logging_utils import structured_log
|
from app.logging_utils import structured_log
|
||||||
from app.services.domains.arxiv.cache import (
|
from app.services.arxiv.cache import (
|
||||||
build_query_fingerprint,
|
build_query_fingerprint,
|
||||||
get_cached_feed,
|
get_cached_feed,
|
||||||
run_with_inflight_dedupe,
|
run_with_inflight_dedupe,
|
||||||
set_cached_feed,
|
set_cached_feed,
|
||||||
)
|
)
|
||||||
from app.services.domains.arxiv.constants import (
|
from app.services.arxiv.constants import (
|
||||||
ARXIV_SOURCE_PATH_LOOKUP_IDS,
|
ARXIV_SOURCE_PATH_LOOKUP_IDS,
|
||||||
ARXIV_SOURCE_PATH_SEARCH,
|
ARXIV_SOURCE_PATH_SEARCH,
|
||||||
ARXIV_SOURCE_PATH_UNKNOWN,
|
ARXIV_SOURCE_PATH_UNKNOWN,
|
||||||
)
|
)
|
||||||
from app.services.domains.arxiv.errors import ArxivClientValidationError, ArxivRateLimitError
|
from app.services.arxiv.errors import ArxivClientValidationError, ArxivRateLimitError
|
||||||
from app.services.domains.arxiv.parser import parse_arxiv_feed
|
from app.services.arxiv.parser import parse_arxiv_feed
|
||||||
from app.services.domains.arxiv.rate_limit import get_arxiv_cooldown_status, run_with_global_arxiv_limit
|
from app.services.arxiv.rate_limit import get_arxiv_cooldown_status, run_with_global_arxiv_limit
|
||||||
from app.services.domains.arxiv.types import ArxivFeed
|
from app.services.arxiv.types import ArxivFeed
|
||||||
from app.settings import settings
|
from app.settings import settings
|
||||||
|
|
||||||
_ARXIV_API_URL = "https://export.arxiv.org/api/query"
|
_ARXIV_API_URL = "https://export.arxiv.org/api/query"
|
||||||
|
|
|
||||||
|
|
@ -6,13 +6,13 @@ import unicodedata
|
||||||
from typing import TYPE_CHECKING, Protocol
|
from typing import TYPE_CHECKING, Protocol
|
||||||
|
|
||||||
from app.logging_utils import structured_log
|
from app.logging_utils import structured_log
|
||||||
from app.services.domains.arxiv.client import ArxivClient
|
from app.services.arxiv.client import ArxivClient
|
||||||
from app.services.domains.arxiv.errors import ArxivRateLimitError
|
from app.services.arxiv.errors import ArxivRateLimitError
|
||||||
from app.services.domains.arxiv.types import ArxivFeed
|
from app.services.arxiv.types import ArxivFeed
|
||||||
from app.settings import settings
|
from app.settings import settings
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem
|
from app.services.publications.types import PublicationListItem, UnreadPublicationItem
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -3,17 +3,17 @@ from __future__ import annotations
|
||||||
import re
|
import re
|
||||||
from typing import TYPE_CHECKING
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
from app.services.domains.arxiv.constants import (
|
from app.services.arxiv.constants import (
|
||||||
ARXIV_STRONG_IDENTIFIER_CONFIDENCE,
|
ARXIV_STRONG_IDENTIFIER_CONFIDENCE,
|
||||||
ARXIV_TITLE_MIN_ALPHA_TOKENS,
|
ARXIV_TITLE_MIN_ALPHA_TOKENS,
|
||||||
ARXIV_TITLE_MIN_TOKENS,
|
ARXIV_TITLE_MIN_TOKENS,
|
||||||
ARXIV_TITLE_TOKEN_MIN_LENGTH,
|
ARXIV_TITLE_TOKEN_MIN_LENGTH,
|
||||||
)
|
)
|
||||||
from app.services.domains.doi.normalize import normalize_doi
|
from app.services.doi.normalize import normalize_doi
|
||||||
from app.services.domains.publication_identifiers.normalize import normalize_arxiv_id
|
from app.services.publication_identifiers.normalize import normalize_arxiv_id
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem
|
from app.services.publications.types import PublicationListItem, UnreadPublicationItem
|
||||||
|
|
||||||
_TITLE_TOKEN_RE = re.compile(r"[a-z0-9]+")
|
_TITLE_TOKEN_RE = re.compile(r"[a-z0-9]+")
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -2,9 +2,9 @@ from __future__ import annotations
|
||||||
|
|
||||||
import xml.etree.ElementTree as ET
|
import xml.etree.ElementTree as ET
|
||||||
|
|
||||||
from app.services.domains.arxiv.errors import ArxivParseError
|
from app.services.arxiv.errors import ArxivParseError
|
||||||
from app.services.domains.arxiv.types import ArxivEntry, ArxivFeed, ArxivOpenSearchMeta
|
from app.services.arxiv.types import ArxivEntry, ArxivFeed, ArxivOpenSearchMeta
|
||||||
from app.services.domains.publication_identifiers.normalize import normalize_arxiv_id
|
from app.services.publication_identifiers.normalize import normalize_arxiv_id
|
||||||
|
|
||||||
_NAMESPACES = {
|
_NAMESPACES = {
|
||||||
"atom": "http://www.w3.org/2005/Atom",
|
"atom": "http://www.w3.org/2005/Atom",
|
||||||
|
|
|
||||||
|
|
@ -13,13 +13,13 @@ from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
from app.db.models import ArxivRuntimeState
|
from app.db.models import ArxivRuntimeState
|
||||||
from app.db.session import get_session_factory
|
from app.db.session import get_session_factory
|
||||||
from app.logging_utils import structured_log
|
from app.logging_utils import structured_log
|
||||||
from app.services.domains.arxiv.constants import (
|
from app.services.arxiv.constants import (
|
||||||
ARXIV_RATE_LIMIT_LOCK_KEY,
|
ARXIV_RATE_LIMIT_LOCK_KEY,
|
||||||
ARXIV_RATE_LIMIT_LOCK_NAMESPACE,
|
ARXIV_RATE_LIMIT_LOCK_NAMESPACE,
|
||||||
ARXIV_RUNTIME_STATE_KEY,
|
ARXIV_RUNTIME_STATE_KEY,
|
||||||
ARXIV_SOURCE_PATH_UNKNOWN,
|
ARXIV_SOURCE_PATH_UNKNOWN,
|
||||||
)
|
)
|
||||||
from app.services.domains.arxiv.errors import ArxivRateLimitError
|
from app.services.arxiv.errors import ArxivRateLimitError
|
||||||
from app.settings import settings
|
from app.settings import settings
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,5 @@
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from app.services.domains.crossref.application import discover_doi_for_publication
|
from app.services.crossref.application import discover_doi_for_publication
|
||||||
|
|
||||||
__all__ = ["discover_doi_for_publication"]
|
__all__ = ["discover_doi_for_publication"]
|
||||||
|
|
|
||||||
|
|
@ -1,9 +1,9 @@
|
||||||
from app.services.domains.dbops.application import run_publication_link_repair
|
from app.services.dbops.application import run_publication_link_repair
|
||||||
from app.services.domains.dbops.integrity import collect_integrity_report
|
from app.services.dbops.integrity import collect_integrity_report
|
||||||
from app.services.domains.dbops.near_duplicate_repair import (
|
from app.services.dbops.near_duplicate_repair import (
|
||||||
run_publication_near_duplicate_repair,
|
run_publication_near_duplicate_repair,
|
||||||
)
|
)
|
||||||
from app.services.domains.dbops.query import list_repair_jobs
|
from app.services.dbops.query import list_repair_jobs
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
"collect_integrity_report",
|
"collect_integrity_report",
|
||||||
|
|
|
||||||
|
|
@ -6,7 +6,7 @@ from typing import Any
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
from app.db.models import DataRepairJob
|
from app.db.models import DataRepairJob
|
||||||
from app.services.domains.publications import dedup as dedup_service
|
from app.services.publications import dedup as dedup_service
|
||||||
|
|
||||||
REPAIR_STATUS_PLANNED = "planned"
|
REPAIR_STATUS_PLANNED = "planned"
|
||||||
REPAIR_STATUS_RUNNING = "running"
|
REPAIR_STATUS_RUNNING = "running"
|
||||||
|
|
|
||||||
|
|
@ -20,11 +20,11 @@ from app.db.models import (
|
||||||
ScholarPublication,
|
ScholarPublication,
|
||||||
)
|
)
|
||||||
from app.logging_utils import structured_log
|
from app.logging_utils import structured_log
|
||||||
from app.services.domains.arxiv.errors import ArxivRateLimitError
|
from app.services.arxiv.errors import ArxivRateLimitError
|
||||||
from app.services.domains.doi.normalize import first_doi_from_texts
|
from app.services.doi.normalize import first_doi_from_texts
|
||||||
from app.services.domains.ingestion import queue as queue_service
|
from app.services.ingestion import queue as queue_service
|
||||||
from app.services.domains.ingestion import safety as run_safety_service
|
from app.services.ingestion import safety as run_safety_service
|
||||||
from app.services.domains.ingestion.constants import (
|
from app.services.ingestion.constants import (
|
||||||
FAILED_STATES,
|
FAILED_STATES,
|
||||||
FAILURE_BUCKET_BLOCKED,
|
FAILURE_BUCKET_BLOCKED,
|
||||||
FAILURE_BUCKET_INGESTION,
|
FAILURE_BUCKET_INGESTION,
|
||||||
|
|
@ -35,7 +35,7 @@ from app.services.domains.ingestion.constants import (
|
||||||
RESUMABLE_PARTIAL_REASONS,
|
RESUMABLE_PARTIAL_REASONS,
|
||||||
RUN_LOCK_NAMESPACE,
|
RUN_LOCK_NAMESPACE,
|
||||||
)
|
)
|
||||||
from app.services.domains.ingestion.fingerprints import (
|
from app.services.ingestion.fingerprints import (
|
||||||
_build_body_excerpt,
|
_build_body_excerpt,
|
||||||
_dedupe_publication_candidates,
|
_dedupe_publication_candidates,
|
||||||
_next_cstart_value,
|
_next_cstart_value,
|
||||||
|
|
@ -45,7 +45,7 @@ from app.services.domains.ingestion.fingerprints import (
|
||||||
canonical_title_for_dedup,
|
canonical_title_for_dedup,
|
||||||
normalize_title,
|
normalize_title,
|
||||||
)
|
)
|
||||||
from app.services.domains.ingestion.types import (
|
from app.services.ingestion.types import (
|
||||||
PagedLoopState,
|
PagedLoopState,
|
||||||
PagedParseResult,
|
PagedParseResult,
|
||||||
RunAlertSummary,
|
RunAlertSummary,
|
||||||
|
|
@ -56,17 +56,17 @@ from app.services.domains.ingestion.types import (
|
||||||
RunProgress,
|
RunProgress,
|
||||||
ScholarProcessingOutcome,
|
ScholarProcessingOutcome,
|
||||||
)
|
)
|
||||||
from app.services.domains.publication_identifiers import application as identifier_service
|
from app.services.publication_identifiers import application as identifier_service
|
||||||
from app.services.domains.runs.events import run_events
|
from app.services.runs.events import run_events
|
||||||
from app.services.domains.scholar.parser import (
|
from app.services.scholar.parser import (
|
||||||
ParsedProfilePage,
|
ParsedProfilePage,
|
||||||
ParseState,
|
ParseState,
|
||||||
PublicationCandidate,
|
PublicationCandidate,
|
||||||
ScholarParserError,
|
ScholarParserError,
|
||||||
parse_profile_page,
|
parse_profile_page,
|
||||||
)
|
)
|
||||||
from app.services.domains.scholar.source import FetchResult, ScholarSource
|
from app.services.scholar.source import FetchResult, ScholarSource
|
||||||
from app.services.domains.settings import application as user_settings_service
|
from app.services.settings import application as user_settings_service
|
||||||
from app.settings import settings
|
from app.settings import settings
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
@ -2369,12 +2369,12 @@ class ScholarIngestionService:
|
||||||
Stops immediately on budget exhaustion (429 with $0 remaining).
|
Stops immediately on budget exhaustion (429 with $0 remaining).
|
||||||
Sleeps 60s and continues on transient rate limits.
|
Sleeps 60s and continues on transient rate limits.
|
||||||
"""
|
"""
|
||||||
from app.services.domains.openalex.client import (
|
from app.services.openalex.client import (
|
||||||
OpenAlexBudgetExhaustedError,
|
OpenAlexBudgetExhaustedError,
|
||||||
OpenAlexClient,
|
OpenAlexClient,
|
||||||
OpenAlexRateLimitError,
|
OpenAlexRateLimitError,
|
||||||
)
|
)
|
||||||
from app.services.domains.openalex.matching import find_best_match
|
from app.services.openalex.matching import find_best_match
|
||||||
|
|
||||||
run_result = await db_session.execute(select(CrawlRun.user_id).where(CrawlRun.id == run_id))
|
run_result = await db_session.execute(select(CrawlRun.user_id).where(CrawlRun.id == run_id))
|
||||||
user_id = run_result.scalar_one()
|
user_id = run_result.scalar_one()
|
||||||
|
|
@ -2480,7 +2480,7 @@ class ScholarIngestionService:
|
||||||
|
|
||||||
await db_session.flush()
|
await db_session.flush()
|
||||||
|
|
||||||
from app.services.domains.publications.dedup import sweep_identifier_duplicates
|
from app.services.publications.dedup import sweep_identifier_duplicates
|
||||||
|
|
||||||
merge_count = await sweep_identifier_duplicates(db_session)
|
merge_count = await sweep_identifier_duplicates(db_session)
|
||||||
if merge_count:
|
if merge_count:
|
||||||
|
|
@ -2579,8 +2579,8 @@ class ScholarIngestionService:
|
||||||
if not publications:
|
if not publications:
|
||||||
return publications
|
return publications
|
||||||
|
|
||||||
from app.services.domains.openalex.client import OpenAlexClient
|
from app.services.openalex.client import OpenAlexClient
|
||||||
from app.services.domains.openalex.matching import find_best_match
|
from app.services.openalex.matching import find_best_match
|
||||||
|
|
||||||
client = OpenAlexClient(api_key=settings.openalex_api_key, mailto=settings.crossref_api_mailto)
|
client = OpenAlexClient(api_key=settings.openalex_api_key, mailto=settings.crossref_api_mailto)
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -2,7 +2,7 @@ from __future__ import annotations
|
||||||
|
|
||||||
import re
|
import re
|
||||||
|
|
||||||
from app.services.domains.scholar.parser import ParseState
|
from app.services.scholar.parser import ParseState
|
||||||
|
|
||||||
TITLE_ALNUM_RE = re.compile(r"[^a-z0-9]+")
|
TITLE_ALNUM_RE = re.compile(r"[^a-z0-9]+")
|
||||||
WORD_RE = re.compile(r"[a-z0-9]+")
|
WORD_RE = re.compile(r"[a-z0-9]+")
|
||||||
|
|
|
||||||
|
|
@ -7,14 +7,14 @@ import unicodedata
|
||||||
from typing import Any
|
from typing import Any
|
||||||
from urllib.parse import urljoin
|
from urllib.parse import urljoin
|
||||||
|
|
||||||
from app.services.domains.ingestion.constants import (
|
from app.services.ingestion.constants import (
|
||||||
HTML_TAG_RE,
|
HTML_TAG_RE,
|
||||||
INITIAL_PAGE_FINGERPRINT_MAX_PUBLICATIONS,
|
INITIAL_PAGE_FINGERPRINT_MAX_PUBLICATIONS,
|
||||||
SPACE_RE,
|
SPACE_RE,
|
||||||
TITLE_ALNUM_RE,
|
TITLE_ALNUM_RE,
|
||||||
WORD_RE,
|
WORD_RE,
|
||||||
)
|
)
|
||||||
from app.services.domains.scholar.parser import ParsedProfilePage, ParseState, PublicationCandidate
|
from app.services.scholar.parser import ParsedProfilePage, ParseState, PublicationCandidate
|
||||||
|
|
||||||
# Scholar-specific noise patterns stripped before canonical comparison.
|
# Scholar-specific noise patterns stripped before canonical comparison.
|
||||||
# Applied in order; each targets a different Scholar metadata injection style.
|
# Applied in order; each targets a different Scholar metadata injection style.
|
||||||
|
|
|
||||||
|
|
@ -18,14 +18,14 @@ from app.db.models import (
|
||||||
)
|
)
|
||||||
from app.db.session import get_session_factory
|
from app.db.session import get_session_factory
|
||||||
from app.logging_utils import structured_log
|
from app.logging_utils import structured_log
|
||||||
from app.services.domains.ingestion import queue as queue_service
|
from app.services.ingestion import queue as queue_service
|
||||||
from app.services.domains.ingestion.application import (
|
from app.services.ingestion.application import (
|
||||||
RunAlreadyInProgressError,
|
RunAlreadyInProgressError,
|
||||||
RunBlockedBySafetyPolicyError,
|
RunBlockedBySafetyPolicyError,
|
||||||
ScholarIngestionService,
|
ScholarIngestionService,
|
||||||
)
|
)
|
||||||
from app.services.domains.scholar.source import LiveScholarSource
|
from app.services.scholar.source import LiveScholarSource
|
||||||
from app.services.domains.settings import application as user_settings_service
|
from app.services.settings import application as user_settings_service
|
||||||
from app.settings import settings
|
from app.settings import settings
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
@ -589,7 +589,7 @@ class SchedulerService:
|
||||||
await self._finalize_queue_job_after_run(job, run_summary)
|
await self._finalize_queue_job_after_run(job, run_summary)
|
||||||
|
|
||||||
async def _drain_pdf_queue(self) -> None:
|
async def _drain_pdf_queue(self) -> None:
|
||||||
from app.services.domains.publications.pdf_queue import drain_ready_jobs
|
from app.services.publications.pdf_queue import drain_ready_jobs
|
||||||
|
|
||||||
session_factory = get_session_factory()
|
session_factory = get_session_factory()
|
||||||
async with session_factory() as session:
|
async with session_factory() as session:
|
||||||
|
|
|
||||||
|
|
@ -4,8 +4,8 @@ from dataclasses import dataclass, field
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from app.db.models import RunStatus
|
from app.db.models import RunStatus
|
||||||
from app.services.domains.scholar.parser import ParsedProfilePage, PublicationCandidate
|
from app.services.scholar.parser import ParsedProfilePage, PublicationCandidate
|
||||||
from app.services.domains.scholar.source import FetchResult
|
from app.services.scholar.source import FetchResult
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
|
|
|
||||||
|
|
@ -8,7 +8,7 @@ from tenacity import (
|
||||||
wait_exponential,
|
wait_exponential,
|
||||||
)
|
)
|
||||||
|
|
||||||
from app.services.domains.openalex.types import OpenAlexWork
|
from app.services.openalex.types import OpenAlexWork
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -3,7 +3,7 @@ import re
|
||||||
|
|
||||||
from rapidfuzz import fuzz
|
from rapidfuzz import fuzz
|
||||||
|
|
||||||
from app.services.domains.openalex.types import OpenAlexWork
|
from app.services.openalex.types import OpenAlexWork
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,21 +1,21 @@
|
||||||
from app.services.domains.portability.application import (
|
from app.services.portability.application import (
|
||||||
EXPORT_SCHEMA_VERSION as EXPORT_SCHEMA_VERSION,
|
EXPORT_SCHEMA_VERSION as EXPORT_SCHEMA_VERSION,
|
||||||
)
|
)
|
||||||
from app.services.domains.portability.application import (
|
from app.services.portability.application import (
|
||||||
MAX_IMPORT_PUBLICATIONS as MAX_IMPORT_PUBLICATIONS,
|
MAX_IMPORT_PUBLICATIONS as MAX_IMPORT_PUBLICATIONS,
|
||||||
)
|
)
|
||||||
from app.services.domains.portability.application import (
|
from app.services.portability.application import (
|
||||||
MAX_IMPORT_SCHOLARS as MAX_IMPORT_SCHOLARS,
|
MAX_IMPORT_SCHOLARS as MAX_IMPORT_SCHOLARS,
|
||||||
)
|
)
|
||||||
from app.services.domains.portability.application import (
|
from app.services.portability.application import (
|
||||||
ImportedPublicationInput as ImportedPublicationInput,
|
ImportedPublicationInput as ImportedPublicationInput,
|
||||||
)
|
)
|
||||||
from app.services.domains.portability.application import (
|
from app.services.portability.application import (
|
||||||
ImportExportError as ImportExportError,
|
ImportExportError as ImportExportError,
|
||||||
)
|
)
|
||||||
from app.services.domains.portability.application import (
|
from app.services.portability.application import (
|
||||||
export_user_data as export_user_data,
|
export_user_data as export_user_data,
|
||||||
)
|
)
|
||||||
from app.services.domains.portability.application import (
|
from app.services.portability.application import (
|
||||||
import_user_data as import_user_data,
|
import_user_data as import_user_data,
|
||||||
)
|
)
|
||||||
|
|
|
||||||
|
|
@ -5,20 +5,20 @@ from typing import Any
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
from app.db.models import Publication
|
from app.db.models import Publication
|
||||||
from app.services.domains.portability.constants import (
|
from app.services.portability.constants import (
|
||||||
EXPORT_SCHEMA_VERSION,
|
EXPORT_SCHEMA_VERSION,
|
||||||
MAX_IMPORT_PUBLICATIONS,
|
MAX_IMPORT_PUBLICATIONS,
|
||||||
MAX_IMPORT_SCHOLARS,
|
MAX_IMPORT_SCHOLARS,
|
||||||
)
|
)
|
||||||
from app.services.domains.portability.exporting import export_user_data
|
from app.services.portability.exporting import export_user_data
|
||||||
from app.services.domains.portability.normalize import _validate_import_sizes
|
from app.services.portability.normalize import _validate_import_sizes
|
||||||
from app.services.domains.portability.publication_import import (
|
from app.services.portability.publication_import import (
|
||||||
_build_imported_publication_input,
|
_build_imported_publication_input,
|
||||||
_initialize_import_counters,
|
_initialize_import_counters,
|
||||||
_upsert_imported_publication,
|
_upsert_imported_publication,
|
||||||
)
|
)
|
||||||
from app.services.domains.portability.scholar_import import _upsert_imported_scholars
|
from app.services.portability.scholar_import import _upsert_imported_scholars
|
||||||
from app.services.domains.portability.types import ImportedPublicationInput, ImportExportError
|
from app.services.portability.types import ImportedPublicationInput, ImportExportError
|
||||||
|
|
||||||
|
|
||||||
async def import_user_data(
|
async def import_user_data(
|
||||||
|
|
|
||||||
|
|
@ -7,7 +7,7 @@ from sqlalchemy import select
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
from app.db.models import Publication, ScholarProfile, ScholarPublication
|
from app.db.models import Publication, ScholarProfile, ScholarPublication
|
||||||
from app.services.domains.portability.constants import EXPORT_SCHEMA_VERSION
|
from app.services.portability.constants import EXPORT_SCHEMA_VERSION
|
||||||
|
|
||||||
|
|
||||||
def _exported_at_iso() -> str:
|
def _exported_at_iso() -> str:
|
||||||
|
|
|
||||||
|
|
@ -3,14 +3,14 @@ from __future__ import annotations
|
||||||
import hashlib
|
import hashlib
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from app.services.domains.ingestion.application import normalize_title
|
from app.services.ingestion.application import normalize_title
|
||||||
from app.services.domains.portability.constants import (
|
from app.services.portability.constants import (
|
||||||
MAX_IMPORT_PUBLICATIONS,
|
MAX_IMPORT_PUBLICATIONS,
|
||||||
MAX_IMPORT_SCHOLARS,
|
MAX_IMPORT_SCHOLARS,
|
||||||
SHA256_RE,
|
SHA256_RE,
|
||||||
WORD_RE,
|
WORD_RE,
|
||||||
)
|
)
|
||||||
from app.services.domains.portability.types import ImportExportError
|
from app.services.portability.types import ImportExportError
|
||||||
|
|
||||||
|
|
||||||
def _normalize_optional_text(value: Any) -> str | None:
|
def _normalize_optional_text(value: Any) -> str | None:
|
||||||
|
|
|
||||||
|
|
@ -6,15 +6,15 @@ from sqlalchemy import select
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
from app.db.models import Publication, ScholarProfile, ScholarPublication
|
from app.db.models import Publication, ScholarProfile, ScholarPublication
|
||||||
from app.services.domains.doi.normalize import normalize_doi
|
from app.services.doi.normalize import normalize_doi
|
||||||
from app.services.domains.ingestion.application import build_publication_url, normalize_title
|
from app.services.ingestion.application import build_publication_url, normalize_title
|
||||||
from app.services.domains.portability.normalize import (
|
from app.services.portability.normalize import (
|
||||||
_normalize_citation_count,
|
_normalize_citation_count,
|
||||||
_normalize_optional_text,
|
_normalize_optional_text,
|
||||||
_normalize_optional_year,
|
_normalize_optional_year,
|
||||||
_resolve_fingerprint,
|
_resolve_fingerprint,
|
||||||
)
|
)
|
||||||
from app.services.domains.portability.types import ImportedPublicationInput
|
from app.services.portability.types import ImportedPublicationInput
|
||||||
|
|
||||||
|
|
||||||
async def _find_publication_by_cluster(
|
async def _find_publication_by_cluster(
|
||||||
|
|
|
||||||
|
|
@ -6,8 +6,8 @@ from sqlalchemy import select
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
from app.db.models import ScholarProfile
|
from app.db.models import ScholarProfile
|
||||||
from app.services.domains.portability.normalize import _normalize_optional_text
|
from app.services.portability.normalize import _normalize_optional_text
|
||||||
from app.services.domains.scholars import application as scholar_service
|
from app.services.scholars import application as scholar_service
|
||||||
|
|
||||||
|
|
||||||
async def _load_user_scholar_map(
|
async def _load_user_scholar_map(
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,4 @@
|
||||||
from app.services.domains.publication_identifiers.application import (
|
from app.services.publication_identifiers.application import (
|
||||||
DisplayIdentifier,
|
DisplayIdentifier,
|
||||||
derive_display_identifier_from_values,
|
derive_display_identifier_from_values,
|
||||||
display_identifier_for_publication_id,
|
display_identifier_for_publication_id,
|
||||||
|
|
|
||||||
|
|
@ -7,22 +7,22 @@ from sqlalchemy import select
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
from app.db.models import Publication, PublicationIdentifier
|
from app.db.models import Publication, PublicationIdentifier
|
||||||
from app.services.domains.arxiv.guards import arxiv_skip_reason_for_item
|
from app.services.arxiv.guards import arxiv_skip_reason_for_item
|
||||||
from app.services.domains.doi.normalize import normalize_doi
|
from app.services.doi.normalize import normalize_doi
|
||||||
from app.services.domains.publication_identifiers.normalize import (
|
from app.services.publication_identifiers.normalize import (
|
||||||
normalize_arxiv_id,
|
normalize_arxiv_id,
|
||||||
normalize_pmcid,
|
normalize_pmcid,
|
||||||
normalize_pmid,
|
normalize_pmid,
|
||||||
)
|
)
|
||||||
from app.services.domains.publication_identifiers.types import (
|
from app.services.publication_identifiers.types import (
|
||||||
DisplayIdentifier,
|
DisplayIdentifier,
|
||||||
IdentifierCandidate,
|
IdentifierCandidate,
|
||||||
IdentifierKind,
|
IdentifierKind,
|
||||||
)
|
)
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from app.services.domains.publications.pdf_queue import PdfQueueListItem
|
from app.services.publications.pdf_queue import PdfQueueListItem
|
||||||
from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem
|
from app.services.publications.types import PublicationListItem, UnreadPublicationItem
|
||||||
|
|
||||||
CONFIDENCE_HIGH = 0.98
|
CONFIDENCE_HIGH = 0.98
|
||||||
CONFIDENCE_MEDIUM = 0.9
|
CONFIDENCE_MEDIUM = 0.9
|
||||||
|
|
@ -163,7 +163,7 @@ def _identifier_lookup_item(
|
||||||
publication: Publication,
|
publication: Publication,
|
||||||
scholar_label: str,
|
scholar_label: str,
|
||||||
) -> UnreadPublicationItem:
|
) -> UnreadPublicationItem:
|
||||||
from app.services.domains.publications.types import UnreadPublicationItem
|
from app.services.publications.types import UnreadPublicationItem
|
||||||
|
|
||||||
return UnreadPublicationItem(
|
return UnreadPublicationItem(
|
||||||
publication_id=int(publication.id),
|
publication_id=int(publication.id),
|
||||||
|
|
@ -184,7 +184,7 @@ async def _discover_crossref_doi(
|
||||||
publication_id: int,
|
publication_id: int,
|
||||||
item: UnreadPublicationItem,
|
item: UnreadPublicationItem,
|
||||||
) -> bool:
|
) -> bool:
|
||||||
from app.services.domains.crossref import application as crossref_service
|
from app.services.crossref import application as crossref_service
|
||||||
|
|
||||||
discovered_doi = await crossref_service.discover_doi_for_publication(item=item)
|
discovered_doi = await crossref_service.discover_doi_for_publication(item=item)
|
||||||
normalized_doi = normalize_doi(discovered_doi)
|
normalized_doi = normalize_doi(discovered_doi)
|
||||||
|
|
@ -212,7 +212,7 @@ async def _discover_arxiv_identifier(
|
||||||
publication_id: int,
|
publication_id: int,
|
||||||
item: UnreadPublicationItem,
|
item: UnreadPublicationItem,
|
||||||
) -> None:
|
) -> None:
|
||||||
from app.services.domains.arxiv import application as arxiv_service
|
from app.services.arxiv import application as arxiv_service
|
||||||
|
|
||||||
discovered_arxiv = await arxiv_service.discover_arxiv_id_for_publication(item=item)
|
discovered_arxiv = await arxiv_service.discover_arxiv_id_for_publication(item=item)
|
||||||
normalized_arxiv = normalize_arxiv_id(discovered_arxiv)
|
normalized_arxiv = normalize_arxiv_id(discovered_arxiv)
|
||||||
|
|
|
||||||
|
|
@ -3,8 +3,8 @@ from __future__ import annotations
|
||||||
import re
|
import re
|
||||||
from urllib.parse import urlparse
|
from urllib.parse import urlparse
|
||||||
|
|
||||||
from app.services.domains.doi.normalize import normalize_doi
|
from app.services.doi.normalize import normalize_doi
|
||||||
from app.services.domains.publication_identifiers.types import IdentifierKind
|
from app.services.publication_identifiers.types import IdentifierKind
|
||||||
|
|
||||||
ARXIV_ABS_RE = re.compile(r"\barxiv:\s*([a-z-]+/\d{7}|\d{4}\.\d{4,5})(v\d+)?\b", re.I)
|
ARXIV_ABS_RE = re.compile(r"\barxiv:\s*([a-z-]+/\d{7}|\d{4}\.\d{4,5})(v\d+)?\b", re.I)
|
||||||
ARXIV_RAW_RE = re.compile(r"^([a-z-]+/\d{7}|\d{4}\.\d{4,5})(v\d+)?$", re.I)
|
ARXIV_RAW_RE = re.compile(r"^([a-z-]+/\d{7}|\d{4}\.\d{4,5})(v\d+)?$", re.I)
|
||||||
|
|
|
||||||
|
|
@ -1,84 +1,84 @@
|
||||||
from app.services.domains.publications.application import (
|
from app.services.publications.application import (
|
||||||
MODE_ALL as MODE_ALL,
|
MODE_ALL as MODE_ALL,
|
||||||
)
|
)
|
||||||
from app.services.domains.publications.application import (
|
from app.services.publications.application import (
|
||||||
MODE_LATEST as MODE_LATEST,
|
MODE_LATEST as MODE_LATEST,
|
||||||
)
|
)
|
||||||
from app.services.domains.publications.application import (
|
from app.services.publications.application import (
|
||||||
MODE_NEW as MODE_NEW,
|
MODE_NEW as MODE_NEW,
|
||||||
)
|
)
|
||||||
from app.services.domains.publications.application import (
|
from app.services.publications.application import (
|
||||||
MODE_UNREAD as MODE_UNREAD,
|
MODE_UNREAD as MODE_UNREAD,
|
||||||
)
|
)
|
||||||
from app.services.domains.publications.application import (
|
from app.services.publications.application import (
|
||||||
PublicationListItem as PublicationListItem,
|
PublicationListItem as PublicationListItem,
|
||||||
)
|
)
|
||||||
from app.services.domains.publications.application import (
|
from app.services.publications.application import (
|
||||||
UnreadPublicationItem as UnreadPublicationItem,
|
UnreadPublicationItem as UnreadPublicationItem,
|
||||||
)
|
)
|
||||||
from app.services.domains.publications.application import (
|
from app.services.publications.application import (
|
||||||
count_favorite_for_user as count_favorite_for_user,
|
count_favorite_for_user as count_favorite_for_user,
|
||||||
)
|
)
|
||||||
from app.services.domains.publications.application import (
|
from app.services.publications.application import (
|
||||||
count_for_user as count_for_user,
|
count_for_user as count_for_user,
|
||||||
)
|
)
|
||||||
from app.services.domains.publications.application import (
|
from app.services.publications.application import (
|
||||||
count_latest_for_user as count_latest_for_user,
|
count_latest_for_user as count_latest_for_user,
|
||||||
)
|
)
|
||||||
from app.services.domains.publications.application import (
|
from app.services.publications.application import (
|
||||||
count_pdf_queue_items as count_pdf_queue_items,
|
count_pdf_queue_items as count_pdf_queue_items,
|
||||||
)
|
)
|
||||||
from app.services.domains.publications.application import (
|
from app.services.publications.application import (
|
||||||
count_unread_for_user as count_unread_for_user,
|
count_unread_for_user as count_unread_for_user,
|
||||||
)
|
)
|
||||||
from app.services.domains.publications.application import (
|
from app.services.publications.application import (
|
||||||
enqueue_all_missing_pdf_jobs as enqueue_all_missing_pdf_jobs,
|
enqueue_all_missing_pdf_jobs as enqueue_all_missing_pdf_jobs,
|
||||||
)
|
)
|
||||||
from app.services.domains.publications.application import (
|
from app.services.publications.application import (
|
||||||
enqueue_retry_pdf_job_for_publication_id as enqueue_retry_pdf_job_for_publication_id,
|
enqueue_retry_pdf_job_for_publication_id as enqueue_retry_pdf_job_for_publication_id,
|
||||||
)
|
)
|
||||||
from app.services.domains.publications.application import (
|
from app.services.publications.application import (
|
||||||
get_latest_run_id_for_user as get_latest_run_id_for_user,
|
get_latest_run_id_for_user as get_latest_run_id_for_user,
|
||||||
)
|
)
|
||||||
from app.services.domains.publications.application import (
|
from app.services.publications.application import (
|
||||||
get_publication_item_for_user as get_publication_item_for_user,
|
get_publication_item_for_user as get_publication_item_for_user,
|
||||||
)
|
)
|
||||||
from app.services.domains.publications.application import (
|
from app.services.publications.application import (
|
||||||
hydrate_pdf_enrichment_state as hydrate_pdf_enrichment_state,
|
hydrate_pdf_enrichment_state as hydrate_pdf_enrichment_state,
|
||||||
)
|
)
|
||||||
from app.services.domains.publications.application import (
|
from app.services.publications.application import (
|
||||||
list_for_user as list_for_user,
|
list_for_user as list_for_user,
|
||||||
)
|
)
|
||||||
from app.services.domains.publications.application import (
|
from app.services.publications.application import (
|
||||||
list_pdf_queue_items as list_pdf_queue_items,
|
list_pdf_queue_items as list_pdf_queue_items,
|
||||||
)
|
)
|
||||||
from app.services.domains.publications.application import (
|
from app.services.publications.application import (
|
||||||
list_pdf_queue_page as list_pdf_queue_page,
|
list_pdf_queue_page as list_pdf_queue_page,
|
||||||
)
|
)
|
||||||
from app.services.domains.publications.application import (
|
from app.services.publications.application import (
|
||||||
list_unread_for_user as list_unread_for_user,
|
list_unread_for_user as list_unread_for_user,
|
||||||
)
|
)
|
||||||
from app.services.domains.publications.application import (
|
from app.services.publications.application import (
|
||||||
mark_all_unread_as_read_for_user as mark_all_unread_as_read_for_user,
|
mark_all_unread_as_read_for_user as mark_all_unread_as_read_for_user,
|
||||||
)
|
)
|
||||||
from app.services.domains.publications.application import (
|
from app.services.publications.application import (
|
||||||
mark_selected_as_read_for_user as mark_selected_as_read_for_user,
|
mark_selected_as_read_for_user as mark_selected_as_read_for_user,
|
||||||
)
|
)
|
||||||
from app.services.domains.publications.application import (
|
from app.services.publications.application import (
|
||||||
publications_query as publications_query,
|
publications_query as publications_query,
|
||||||
)
|
)
|
||||||
from app.services.domains.publications.application import (
|
from app.services.publications.application import (
|
||||||
resolve_publication_view_mode as resolve_publication_view_mode,
|
resolve_publication_view_mode as resolve_publication_view_mode,
|
||||||
)
|
)
|
||||||
from app.services.domains.publications.application import (
|
from app.services.publications.application import (
|
||||||
retry_pdf_for_user as retry_pdf_for_user,
|
retry_pdf_for_user as retry_pdf_for_user,
|
||||||
)
|
)
|
||||||
from app.services.domains.publications.application import (
|
from app.services.publications.application import (
|
||||||
schedule_missing_pdf_enrichment_for_user as schedule_missing_pdf_enrichment_for_user,
|
schedule_missing_pdf_enrichment_for_user as schedule_missing_pdf_enrichment_for_user,
|
||||||
)
|
)
|
||||||
from app.services.domains.publications.application import (
|
from app.services.publications.application import (
|
||||||
schedule_retry_pdf_enrichment_for_row as schedule_retry_pdf_enrichment_for_row,
|
schedule_retry_pdf_enrichment_for_row as schedule_retry_pdf_enrichment_for_row,
|
||||||
)
|
)
|
||||||
from app.services.domains.publications.application import (
|
from app.services.publications.application import (
|
||||||
set_publication_favorite_for_user as set_publication_favorite_for_user,
|
set_publication_favorite_for_user as set_publication_favorite_for_user,
|
||||||
)
|
)
|
||||||
|
|
|
||||||
|
|
@ -1,46 +1,46 @@
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from app.services.domains.publications.counts import (
|
from app.services.publications.counts import (
|
||||||
count_favorite_for_user,
|
count_favorite_for_user,
|
||||||
count_for_user,
|
count_for_user,
|
||||||
count_latest_for_user,
|
count_latest_for_user,
|
||||||
count_unread_for_user,
|
count_unread_for_user,
|
||||||
)
|
)
|
||||||
from app.services.domains.publications.enrichment import (
|
from app.services.publications.enrichment import (
|
||||||
hydrate_pdf_enrichment_state,
|
hydrate_pdf_enrichment_state,
|
||||||
schedule_missing_pdf_enrichment_for_user,
|
schedule_missing_pdf_enrichment_for_user,
|
||||||
schedule_retry_pdf_enrichment_for_row,
|
schedule_retry_pdf_enrichment_for_row,
|
||||||
)
|
)
|
||||||
from app.services.domains.publications.listing import (
|
from app.services.publications.listing import (
|
||||||
list_for_user,
|
list_for_user,
|
||||||
list_unread_for_user,
|
list_unread_for_user,
|
||||||
retry_pdf_for_user,
|
retry_pdf_for_user,
|
||||||
)
|
)
|
||||||
from app.services.domains.publications.modes import (
|
from app.services.publications.modes import (
|
||||||
MODE_ALL,
|
MODE_ALL,
|
||||||
MODE_LATEST,
|
MODE_LATEST,
|
||||||
MODE_NEW,
|
MODE_NEW,
|
||||||
MODE_UNREAD,
|
MODE_UNREAD,
|
||||||
resolve_publication_view_mode,
|
resolve_publication_view_mode,
|
||||||
)
|
)
|
||||||
from app.services.domains.publications.pdf_queue import (
|
from app.services.publications.pdf_queue import (
|
||||||
count_pdf_queue_items,
|
count_pdf_queue_items,
|
||||||
enqueue_all_missing_pdf_jobs,
|
enqueue_all_missing_pdf_jobs,
|
||||||
enqueue_retry_pdf_job_for_publication_id,
|
enqueue_retry_pdf_job_for_publication_id,
|
||||||
list_pdf_queue_items,
|
list_pdf_queue_items,
|
||||||
list_pdf_queue_page,
|
list_pdf_queue_page,
|
||||||
)
|
)
|
||||||
from app.services.domains.publications.queries import (
|
from app.services.publications.queries import (
|
||||||
get_latest_run_id_for_user,
|
get_latest_run_id_for_user,
|
||||||
get_publication_item_for_user,
|
get_publication_item_for_user,
|
||||||
publications_query,
|
publications_query,
|
||||||
)
|
)
|
||||||
from app.services.domains.publications.read_state import (
|
from app.services.publications.read_state import (
|
||||||
mark_all_unread_as_read_for_user,
|
mark_all_unread_as_read_for_user,
|
||||||
mark_selected_as_read_for_user,
|
mark_selected_as_read_for_user,
|
||||||
set_publication_favorite_for_user,
|
set_publication_favorite_for_user,
|
||||||
)
|
)
|
||||||
from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem
|
from app.services.publications.types import PublicationListItem, UnreadPublicationItem
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
"MODE_ALL",
|
"MODE_ALL",
|
||||||
|
|
|
||||||
|
|
@ -6,13 +6,13 @@ from sqlalchemy import distinct, func, select
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
from app.db.models import Publication, ScholarProfile, ScholarPublication
|
from app.db.models import Publication, ScholarProfile, ScholarPublication
|
||||||
from app.services.domains.publications.modes import (
|
from app.services.publications.modes import (
|
||||||
MODE_ALL,
|
MODE_ALL,
|
||||||
MODE_LATEST,
|
MODE_LATEST,
|
||||||
MODE_UNREAD,
|
MODE_UNREAD,
|
||||||
resolve_publication_view_mode,
|
resolve_publication_view_mode,
|
||||||
)
|
)
|
||||||
from app.services.domains.publications.queries import get_latest_run_id_for_user
|
from app.services.publications.queries import get_latest_run_id_for_user
|
||||||
|
|
||||||
|
|
||||||
async def count_for_user(
|
async def count_for_user(
|
||||||
|
|
|
||||||
|
|
@ -11,7 +11,7 @@ from sqlalchemy.orm import aliased
|
||||||
|
|
||||||
from app.db.models import Publication, PublicationIdentifier, ScholarPublication
|
from app.db.models import Publication, PublicationIdentifier, ScholarPublication
|
||||||
from app.logging_utils import structured_log
|
from app.logging_utils import structured_log
|
||||||
from app.services.domains.ingestion.fingerprints import (
|
from app.services.ingestion.fingerprints import (
|
||||||
canonical_title_text_for_dedup,
|
canonical_title_text_for_dedup,
|
||||||
canonical_title_tokens_for_dedup,
|
canonical_title_tokens_for_dedup,
|
||||||
normalize_title,
|
normalize_title,
|
||||||
|
|
|
||||||
|
|
@ -5,12 +5,12 @@ import logging
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
from app.logging_utils import structured_log
|
from app.logging_utils import structured_log
|
||||||
from app.services.domains.publications.pdf_queue import (
|
from app.services.publications.pdf_queue import (
|
||||||
enqueue_missing_pdf_jobs,
|
enqueue_missing_pdf_jobs,
|
||||||
enqueue_retry_pdf_job,
|
enqueue_retry_pdf_job,
|
||||||
overlay_pdf_job_state,
|
overlay_pdf_job_state,
|
||||||
)
|
)
|
||||||
from app.services.domains.publications.types import PublicationListItem
|
from app.services.publications.types import PublicationListItem
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -4,20 +4,20 @@ from datetime import datetime
|
||||||
|
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
from app.services.domains.publication_identifiers import application as identifier_service
|
from app.services.publication_identifiers import application as identifier_service
|
||||||
from app.services.domains.publications.modes import (
|
from app.services.publications.modes import (
|
||||||
MODE_ALL,
|
MODE_ALL,
|
||||||
MODE_UNREAD,
|
MODE_UNREAD,
|
||||||
resolve_publication_view_mode,
|
resolve_publication_view_mode,
|
||||||
)
|
)
|
||||||
from app.services.domains.publications.queries import (
|
from app.services.publications.queries import (
|
||||||
get_latest_run_id_for_user,
|
get_latest_run_id_for_user,
|
||||||
get_publication_item_for_user,
|
get_publication_item_for_user,
|
||||||
publication_list_item_from_row,
|
publication_list_item_from_row,
|
||||||
publications_query,
|
publications_query,
|
||||||
unread_item_from_row,
|
unread_item_from_row,
|
||||||
)
|
)
|
||||||
from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem
|
from app.services.publications.types import PublicationListItem, UnreadPublicationItem
|
||||||
|
|
||||||
|
|
||||||
async def list_for_user(
|
async def list_for_user(
|
||||||
|
|
|
||||||
|
|
@ -18,13 +18,13 @@ from app.db.models import (
|
||||||
)
|
)
|
||||||
from app.db.session import get_session_factory
|
from app.db.session import get_session_factory
|
||||||
from app.logging_utils import structured_log
|
from app.logging_utils import structured_log
|
||||||
from app.services.domains.publication_identifiers import application as identifier_service
|
from app.services.publication_identifiers import application as identifier_service
|
||||||
from app.services.domains.publication_identifiers.types import DisplayIdentifier
|
from app.services.publication_identifiers.types import DisplayIdentifier
|
||||||
from app.services.domains.publications.pdf_resolution_pipeline import (
|
from app.services.publications.pdf_resolution_pipeline import (
|
||||||
resolve_publication_pdf_outcome_for_row,
|
resolve_publication_pdf_outcome_for_row,
|
||||||
)
|
)
|
||||||
from app.services.domains.publications.types import PublicationListItem
|
from app.services.publications.types import PublicationListItem
|
||||||
from app.services.domains.unpaywall.application import (
|
from app.services.unpaywall.application import (
|
||||||
FAILURE_RESOLUTION_EXCEPTION,
|
FAILURE_RESOLUTION_EXCEPTION,
|
||||||
OaResolutionOutcome,
|
OaResolutionOutcome,
|
||||||
)
|
)
|
||||||
|
|
@ -450,7 +450,7 @@ async def _resolve_publication_row(
|
||||||
openalex_api_key: str | None = None,
|
openalex_api_key: str | None = None,
|
||||||
allow_arxiv_lookup: bool = True,
|
allow_arxiv_lookup: bool = True,
|
||||||
) -> bool:
|
) -> bool:
|
||||||
from app.services.domains.openalex.client import OpenAlexBudgetExhaustedError
|
from app.services.openalex.client import OpenAlexBudgetExhaustedError
|
||||||
|
|
||||||
await _mark_attempt_started(publication_id=row.publication_id, user_id=user_id)
|
await _mark_attempt_started(publication_id=row.publication_id, user_id=user_id)
|
||||||
try:
|
try:
|
||||||
|
|
@ -493,8 +493,8 @@ async def _run_resolution_task(
|
||||||
request_email: str | None,
|
request_email: str | None,
|
||||||
rows: list[PublicationListItem],
|
rows: list[PublicationListItem],
|
||||||
) -> None:
|
) -> None:
|
||||||
from app.services.domains.openalex.client import OpenAlexBudgetExhaustedError
|
from app.services.openalex.client import OpenAlexBudgetExhaustedError
|
||||||
from app.services.domains.settings import application as user_settings_service
|
from app.services.settings import application as user_settings_service
|
||||||
|
|
||||||
# Resolve the best available API key: per-user setting → env var fallback.
|
# Resolve the best available API key: per-user setting → env var fallback.
|
||||||
openalex_api_key: str | None = None
|
openalex_api_key: str | None = None
|
||||||
|
|
|
||||||
|
|
@ -5,11 +5,11 @@ from dataclasses import dataclass
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from app.logging_utils import structured_log
|
from app.logging_utils import structured_log
|
||||||
from app.services.domains.arxiv.errors import ArxivRateLimitError
|
from app.services.arxiv.errors import ArxivRateLimitError
|
||||||
from app.services.domains.arxiv.guards import arxiv_skip_reason_for_item
|
from app.services.arxiv.guards import arxiv_skip_reason_for_item
|
||||||
from app.services.domains.openalex.client import OpenAlexBudgetExhaustedError
|
from app.services.openalex.client import OpenAlexBudgetExhaustedError
|
||||||
from app.services.domains.publications.types import PublicationListItem
|
from app.services.publications.types import PublicationListItem
|
||||||
from app.services.domains.unpaywall.application import OaResolutionOutcome, resolve_publication_oa_outcomes
|
from app.services.unpaywall.application import OaResolutionOutcome, resolve_publication_oa_outcomes
|
||||||
from app.settings import settings
|
from app.settings import settings
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
@ -59,8 +59,8 @@ async def _openalex_outcome(
|
||||||
request_email: str | None,
|
request_email: str | None,
|
||||||
openalex_api_key: str | None = None,
|
openalex_api_key: str | None = None,
|
||||||
) -> OaResolutionOutcome | None:
|
) -> OaResolutionOutcome | None:
|
||||||
from app.services.domains.openalex.client import OpenAlexClient
|
from app.services.openalex.client import OpenAlexClient
|
||||||
from app.services.domains.openalex.matching import find_best_match
|
from app.services.openalex.matching import find_best_match
|
||||||
|
|
||||||
if not row.title:
|
if not row.title:
|
||||||
return None
|
return None
|
||||||
|
|
@ -105,7 +105,7 @@ async def _arxiv_outcome(
|
||||||
request_email: str | None,
|
request_email: str | None,
|
||||||
allow_lookup: bool = True,
|
allow_lookup: bool = True,
|
||||||
) -> OaResolutionOutcome | None:
|
) -> OaResolutionOutcome | None:
|
||||||
from app.services.domains.arxiv.application import discover_arxiv_id_for_publication
|
from app.services.arxiv.application import discover_arxiv_id_for_publication
|
||||||
|
|
||||||
if not allow_lookup:
|
if not allow_lookup:
|
||||||
structured_log(
|
structured_log(
|
||||||
|
|
|
||||||
|
|
@ -13,8 +13,8 @@ from app.db.models import (
|
||||||
ScholarProfile,
|
ScholarProfile,
|
||||||
ScholarPublication,
|
ScholarPublication,
|
||||||
)
|
)
|
||||||
from app.services.domains.publications.modes import MODE_LATEST, MODE_UNREAD
|
from app.services.publications.modes import MODE_LATEST, MODE_UNREAD
|
||||||
from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem
|
from app.services.publications.types import PublicationListItem, UnreadPublicationItem
|
||||||
|
|
||||||
|
|
||||||
def _normalized_citation_count(value: object) -> int:
|
def _normalized_citation_count(value: object) -> int:
|
||||||
|
|
|
||||||
|
|
@ -3,7 +3,7 @@ from __future__ import annotations
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
|
|
||||||
from app.services.domains.publication_identifiers.types import DisplayIdentifier
|
from app.services.publication_identifiers.types import DisplayIdentifier
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
|
|
|
||||||
|
|
@ -1,51 +1,51 @@
|
||||||
from app.services.domains.runs.application import (
|
from app.services.runs.application import (
|
||||||
QUEUE_STATUS_DROPPED as QUEUE_STATUS_DROPPED,
|
QUEUE_STATUS_DROPPED as QUEUE_STATUS_DROPPED,
|
||||||
)
|
)
|
||||||
from app.services.domains.runs.application import (
|
from app.services.runs.application import (
|
||||||
QUEUE_STATUS_QUEUED as QUEUE_STATUS_QUEUED,
|
QUEUE_STATUS_QUEUED as QUEUE_STATUS_QUEUED,
|
||||||
)
|
)
|
||||||
from app.services.domains.runs.application import (
|
from app.services.runs.application import (
|
||||||
QUEUE_STATUS_RETRYING as QUEUE_STATUS_RETRYING,
|
QUEUE_STATUS_RETRYING as QUEUE_STATUS_RETRYING,
|
||||||
)
|
)
|
||||||
from app.services.domains.runs.application import (
|
from app.services.runs.application import (
|
||||||
QueueClearResult as QueueClearResult,
|
QueueClearResult as QueueClearResult,
|
||||||
)
|
)
|
||||||
from app.services.domains.runs.application import (
|
from app.services.runs.application import (
|
||||||
QueueListItem as QueueListItem,
|
QueueListItem as QueueListItem,
|
||||||
)
|
)
|
||||||
from app.services.domains.runs.application import (
|
from app.services.runs.application import (
|
||||||
QueueTransitionError as QueueTransitionError,
|
QueueTransitionError as QueueTransitionError,
|
||||||
)
|
)
|
||||||
from app.services.domains.runs.application import (
|
from app.services.runs.application import (
|
||||||
clear_queue_item_for_user as clear_queue_item_for_user,
|
clear_queue_item_for_user as clear_queue_item_for_user,
|
||||||
)
|
)
|
||||||
from app.services.domains.runs.application import (
|
from app.services.runs.application import (
|
||||||
drop_queue_item_for_user as drop_queue_item_for_user,
|
drop_queue_item_for_user as drop_queue_item_for_user,
|
||||||
)
|
)
|
||||||
from app.services.domains.runs.application import (
|
from app.services.runs.application import (
|
||||||
extract_run_summary as extract_run_summary,
|
extract_run_summary as extract_run_summary,
|
||||||
)
|
)
|
||||||
from app.services.domains.runs.application import (
|
from app.services.runs.application import (
|
||||||
get_manual_run_by_idempotency_key as get_manual_run_by_idempotency_key,
|
get_manual_run_by_idempotency_key as get_manual_run_by_idempotency_key,
|
||||||
)
|
)
|
||||||
from app.services.domains.runs.application import (
|
from app.services.runs.application import (
|
||||||
get_queue_item_for_user as get_queue_item_for_user,
|
get_queue_item_for_user as get_queue_item_for_user,
|
||||||
)
|
)
|
||||||
from app.services.domains.runs.application import (
|
from app.services.runs.application import (
|
||||||
get_run_for_user as get_run_for_user,
|
get_run_for_user as get_run_for_user,
|
||||||
)
|
)
|
||||||
from app.services.domains.runs.application import (
|
from app.services.runs.application import (
|
||||||
list_queue_items_for_user as list_queue_items_for_user,
|
list_queue_items_for_user as list_queue_items_for_user,
|
||||||
)
|
)
|
||||||
from app.services.domains.runs.application import (
|
from app.services.runs.application import (
|
||||||
list_recent_runs_for_user as list_recent_runs_for_user,
|
list_recent_runs_for_user as list_recent_runs_for_user,
|
||||||
)
|
)
|
||||||
from app.services.domains.runs.application import (
|
from app.services.runs.application import (
|
||||||
list_runs_for_user as list_runs_for_user,
|
list_runs_for_user as list_runs_for_user,
|
||||||
)
|
)
|
||||||
from app.services.domains.runs.application import (
|
from app.services.runs.application import (
|
||||||
queue_status_counts_for_user as queue_status_counts_for_user,
|
queue_status_counts_for_user as queue_status_counts_for_user,
|
||||||
)
|
)
|
||||||
from app.services.domains.runs.application import (
|
from app.services.runs.application import (
|
||||||
retry_queue_item_for_user as retry_queue_item_for_user,
|
retry_queue_item_for_user as retry_queue_item_for_user,
|
||||||
)
|
)
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from app.services.domains.runs.queue_service import (
|
from app.services.runs.queue_service import (
|
||||||
clear_queue_item_for_user,
|
clear_queue_item_for_user,
|
||||||
drop_queue_item_for_user,
|
drop_queue_item_for_user,
|
||||||
get_queue_item_for_user,
|
get_queue_item_for_user,
|
||||||
|
|
@ -8,14 +8,14 @@ from app.services.domains.runs.queue_service import (
|
||||||
queue_status_counts_for_user,
|
queue_status_counts_for_user,
|
||||||
retry_queue_item_for_user,
|
retry_queue_item_for_user,
|
||||||
)
|
)
|
||||||
from app.services.domains.runs.runs_service import (
|
from app.services.runs.runs_service import (
|
||||||
get_manual_run_by_idempotency_key,
|
get_manual_run_by_idempotency_key,
|
||||||
get_run_for_user,
|
get_run_for_user,
|
||||||
list_recent_runs_for_user,
|
list_recent_runs_for_user,
|
||||||
list_runs_for_user,
|
list_runs_for_user,
|
||||||
)
|
)
|
||||||
from app.services.domains.runs.summary import extract_run_summary
|
from app.services.runs.summary import extract_run_summary
|
||||||
from app.services.domains.runs.types import (
|
from app.services.runs.types import (
|
||||||
QUEUE_STATUS_DROPPED,
|
QUEUE_STATUS_DROPPED,
|
||||||
QUEUE_STATUS_QUEUED,
|
QUEUE_STATUS_QUEUED,
|
||||||
QUEUE_STATUS_RETRYING,
|
QUEUE_STATUS_RETRYING,
|
||||||
|
|
|
||||||
|
|
@ -3,7 +3,7 @@ from __future__ import annotations
|
||||||
from sqlalchemy import and_, select
|
from sqlalchemy import and_, select
|
||||||
|
|
||||||
from app.db.models import IngestionQueueItem, ScholarProfile
|
from app.db.models import IngestionQueueItem, ScholarProfile
|
||||||
from app.services.domains.runs.types import QueueListItem
|
from app.services.runs.types import QueueListItem
|
||||||
|
|
||||||
|
|
||||||
def queue_item_columns() -> tuple:
|
def queue_item_columns() -> tuple:
|
||||||
|
|
|
||||||
|
|
@ -4,9 +4,9 @@ from sqlalchemy import case, func, select
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
from app.db.models import IngestionQueueItem
|
from app.db.models import IngestionQueueItem
|
||||||
from app.services.domains.ingestion import queue as queue_mutations
|
from app.services.ingestion import queue as queue_mutations
|
||||||
from app.services.domains.runs.queue_queries import queue_item_select, queue_list_item_from_row
|
from app.services.runs.queue_queries import queue_item_select, queue_list_item_from_row
|
||||||
from app.services.domains.runs.types import (
|
from app.services.runs.types import (
|
||||||
QUEUE_STATUS_DROPPED,
|
QUEUE_STATUS_DROPPED,
|
||||||
QUEUE_STATUS_QUEUED,
|
QUEUE_STATUS_QUEUED,
|
||||||
QUEUE_STATUS_RETRYING,
|
QUEUE_STATUS_RETRYING,
|
||||||
|
|
|
||||||
|
|
@ -5,9 +5,9 @@ from html.parser import HTMLParser
|
||||||
from typing import Any
|
from typing import Any
|
||||||
from urllib.parse import parse_qs, urlparse
|
from urllib.parse import parse_qs, urlparse
|
||||||
|
|
||||||
from app.services.domains.scholar.parser_constants import AUTHOR_SEARCH_MARKER_KEYS
|
from app.services.scholar.parser_constants import AUTHOR_SEARCH_MARKER_KEYS
|
||||||
from app.services.domains.scholar.parser_types import ScholarSearchCandidate
|
from app.services.scholar.parser_types import ScholarSearchCandidate
|
||||||
from app.services.domains.scholar.parser_utils import (
|
from app.services.scholar.parser_utils import (
|
||||||
attr_class,
|
attr_class,
|
||||||
attr_href,
|
attr_href,
|
||||||
attr_src,
|
attr_src,
|
||||||
|
|
|
||||||
|
|
@ -1,30 +1,30 @@
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from app.services.domains.scholar.author_rows import (
|
from app.services.scholar.author_rows import (
|
||||||
ScholarAuthorSearchParser,
|
ScholarAuthorSearchParser,
|
||||||
count_author_search_markers,
|
count_author_search_markers,
|
||||||
)
|
)
|
||||||
from app.services.domains.scholar.parser_constants import SCRIPT_STYLE_RE
|
from app.services.scholar.parser_constants import SCRIPT_STYLE_RE
|
||||||
from app.services.domains.scholar.parser_types import (
|
from app.services.scholar.parser_types import (
|
||||||
ParsedAuthorSearchPage,
|
ParsedAuthorSearchPage,
|
||||||
ParsedProfilePage,
|
ParsedProfilePage,
|
||||||
PublicationCandidate,
|
PublicationCandidate,
|
||||||
ScholarDomInvariantError,
|
ScholarDomInvariantError,
|
||||||
ScholarMalformedDataError,
|
ScholarMalformedDataError,
|
||||||
)
|
)
|
||||||
from app.services.domains.scholar.parser_types import (
|
from app.services.scholar.parser_types import (
|
||||||
ParseState as ParseState,
|
ParseState as ParseState,
|
||||||
)
|
)
|
||||||
from app.services.domains.scholar.parser_types import (
|
from app.services.scholar.parser_types import (
|
||||||
ScholarParserError as ScholarParserError,
|
ScholarParserError as ScholarParserError,
|
||||||
)
|
)
|
||||||
from app.services.domains.scholar.parser_types import (
|
from app.services.scholar.parser_types import (
|
||||||
ScholarSearchCandidate as ScholarSearchCandidate,
|
ScholarSearchCandidate as ScholarSearchCandidate,
|
||||||
)
|
)
|
||||||
from app.services.domains.scholar.parser_utils import (
|
from app.services.scholar.parser_utils import (
|
||||||
strip_tags,
|
strip_tags,
|
||||||
)
|
)
|
||||||
from app.services.domains.scholar.profile_rows import (
|
from app.services.scholar.profile_rows import (
|
||||||
count_markers,
|
count_markers,
|
||||||
extract_articles_range,
|
extract_articles_range,
|
||||||
extract_profile_image_url,
|
extract_profile_image_url,
|
||||||
|
|
@ -33,8 +33,8 @@ from app.services.domains.scholar.profile_rows import (
|
||||||
has_show_more_button,
|
has_show_more_button,
|
||||||
parse_publications,
|
parse_publications,
|
||||||
)
|
)
|
||||||
from app.services.domains.scholar.source import FetchResult
|
from app.services.scholar.source import FetchResult
|
||||||
from app.services.domains.scholar.state_detection import (
|
from app.services.scholar.state_detection import (
|
||||||
detect_author_search_state,
|
detect_author_search_state,
|
||||||
detect_state,
|
detect_state,
|
||||||
)
|
)
|
||||||
|
|
|
||||||
|
|
@ -2,7 +2,7 @@ from __future__ import annotations
|
||||||
|
|
||||||
from html import unescape
|
from html import unescape
|
||||||
|
|
||||||
from app.services.domains.scholar.parser_constants import TAG_RE
|
from app.services.scholar.parser_constants import TAG_RE
|
||||||
|
|
||||||
|
|
||||||
def normalize_space(value: str) -> str:
|
def normalize_space(value: str) -> str:
|
||||||
|
|
|
||||||
|
|
@ -5,12 +5,12 @@ from html.parser import HTMLParser
|
||||||
from typing import Any
|
from typing import Any
|
||||||
from urllib.parse import parse_qs, urlparse
|
from urllib.parse import parse_qs, urlparse
|
||||||
|
|
||||||
from app.services.domains.scholar.parser_constants import (
|
from app.services.scholar.parser_constants import (
|
||||||
MARKER_KEYS,
|
MARKER_KEYS,
|
||||||
SHOW_MORE_BUTTON_RE,
|
SHOW_MORE_BUTTON_RE,
|
||||||
)
|
)
|
||||||
from app.services.domains.scholar.parser_types import PublicationCandidate
|
from app.services.scholar.parser_types import PublicationCandidate
|
||||||
from app.services.domains.scholar.parser_utils import (
|
from app.services.scholar.parser_utils import (
|
||||||
attr_class,
|
attr_class,
|
||||||
attr_href,
|
attr_href,
|
||||||
build_absolute_scholar_url,
|
build_absolute_scholar_url,
|
||||||
|
|
|
||||||
|
|
@ -10,7 +10,7 @@ from urllib.parse import urlencode
|
||||||
from urllib.request import Request, urlopen
|
from urllib.request import Request, urlopen
|
||||||
|
|
||||||
from app.logging_utils import structured_log
|
from app.logging_utils import structured_log
|
||||||
from app.services.domains.scholar import rate_limit as scholar_rate_limit
|
from app.services.scholar import rate_limit as scholar_rate_limit
|
||||||
from app.settings import settings
|
from app.settings import settings
|
||||||
|
|
||||||
SCHOLAR_PROFILE_URL = "https://scholar.google.com/citations"
|
SCHOLAR_PROFILE_URL = "https://scholar.google.com/citations"
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from app.services.domains.scholar.parser_constants import (
|
from app.services.scholar.parser_constants import (
|
||||||
BLOCKED_KEYWORDS,
|
BLOCKED_KEYWORDS,
|
||||||
NETWORK_DNS_ERROR_KEYWORDS,
|
NETWORK_DNS_ERROR_KEYWORDS,
|
||||||
NETWORK_TIMEOUT_KEYWORDS,
|
NETWORK_TIMEOUT_KEYWORDS,
|
||||||
|
|
@ -8,8 +8,8 @@ from app.services.domains.scholar.parser_constants import (
|
||||||
NO_AUTHOR_RESULTS_KEYWORDS,
|
NO_AUTHOR_RESULTS_KEYWORDS,
|
||||||
NO_RESULTS_KEYWORDS,
|
NO_RESULTS_KEYWORDS,
|
||||||
)
|
)
|
||||||
from app.services.domains.scholar.parser_types import ParseState, PublicationCandidate, ScholarSearchCandidate
|
from app.services.scholar.parser_types import ParseState, PublicationCandidate, ScholarSearchCandidate
|
||||||
from app.services.domains.scholar.source import FetchResult
|
from app.services.scholar.source import FetchResult
|
||||||
|
|
||||||
|
|
||||||
def classify_network_error_reason(fetch_error: str | None) -> str:
|
def classify_network_error_reason(fetch_error: str | None) -> str:
|
||||||
|
|
|
||||||
|
|
@ -15,15 +15,15 @@ from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
from app.db.models import AuthorSearchCacheEntry, AuthorSearchRuntimeState, ScholarProfile
|
from app.db.models import AuthorSearchCacheEntry, AuthorSearchRuntimeState, ScholarProfile
|
||||||
from app.logging_utils import structured_log
|
from app.logging_utils import structured_log
|
||||||
from app.services.domains.scholar.parser import (
|
from app.services.scholar.parser import (
|
||||||
ParsedAuthorSearchPage,
|
ParsedAuthorSearchPage,
|
||||||
ParseState,
|
ParseState,
|
||||||
ScholarParserError,
|
ScholarParserError,
|
||||||
parse_author_search_page,
|
parse_author_search_page,
|
||||||
parse_profile_page,
|
parse_profile_page,
|
||||||
)
|
)
|
||||||
from app.services.domains.scholar.source import ScholarSource
|
from app.services.scholar.source import ScholarSource
|
||||||
from app.services.domains.scholars.constants import (
|
from app.services.scholars.constants import (
|
||||||
ALLOWED_IMAGE_UPLOAD_CONTENT_TYPES,
|
ALLOWED_IMAGE_UPLOAD_CONTENT_TYPES,
|
||||||
AUTHOR_SEARCH_LOCK_KEY,
|
AUTHOR_SEARCH_LOCK_KEY,
|
||||||
AUTHOR_SEARCH_LOCK_NAMESPACE,
|
AUTHOR_SEARCH_LOCK_NAMESPACE,
|
||||||
|
|
@ -41,18 +41,18 @@ from app.services.domains.scholars.constants import (
|
||||||
SEARCH_COOLDOWN_REASON,
|
SEARCH_COOLDOWN_REASON,
|
||||||
SEARCH_DISABLED_REASON,
|
SEARCH_DISABLED_REASON,
|
||||||
)
|
)
|
||||||
from app.services.domains.scholars.exceptions import ScholarServiceError
|
from app.services.scholars.exceptions import ScholarServiceError
|
||||||
from app.services.domains.scholars.search_hints import (
|
from app.services.scholars.search_hints import (
|
||||||
_merge_warnings,
|
_merge_warnings,
|
||||||
_policy_blocked_author_search_result,
|
_policy_blocked_author_search_result,
|
||||||
_trim_author_search_result,
|
_trim_author_search_result,
|
||||||
)
|
)
|
||||||
from app.services.domains.scholars.uploads import (
|
from app.services.scholars.uploads import (
|
||||||
_ensure_upload_root,
|
_ensure_upload_root,
|
||||||
_resolve_upload_path,
|
_resolve_upload_path,
|
||||||
_safe_remove_upload,
|
_safe_remove_upload,
|
||||||
)
|
)
|
||||||
from app.services.domains.scholars.validators import (
|
from app.services.scholars.validators import (
|
||||||
normalize_display_name,
|
normalize_display_name,
|
||||||
normalize_profile_image_url,
|
normalize_profile_image_url,
|
||||||
validate_scholar_id,
|
validate_scholar_id,
|
||||||
|
|
@ -205,7 +205,7 @@ def _deserialize_parsed_author_search_page(payload: object) -> ParsedAuthorSearc
|
||||||
|
|
||||||
marker_counts = _payload_marker_counts(payload)
|
marker_counts = _payload_marker_counts(payload)
|
||||||
warnings = _payload_warnings(payload)
|
warnings = _payload_warnings(payload)
|
||||||
from app.services.domains.scholar.parser import ScholarSearchCandidate
|
from app.services.scholar.parser import ScholarSearchCandidate
|
||||||
|
|
||||||
normalized_candidates = _deserialize_candidates(payload)
|
normalized_candidates = _deserialize_candidates(payload)
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,8 +1,8 @@
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from app.db.models import ScholarProfile
|
from app.db.models import ScholarProfile
|
||||||
from app.services.domains.scholar.parser import ParsedAuthorSearchPage, ParseState
|
from app.services.scholar.parser import ParsedAuthorSearchPage, ParseState
|
||||||
from app.services.domains.scholars.constants import (
|
from app.services.scholars.constants import (
|
||||||
MAX_AUTHOR_SEARCH_LIMIT,
|
MAX_AUTHOR_SEARCH_LIMIT,
|
||||||
STATE_REASON_HINTS,
|
STATE_REASON_HINTS,
|
||||||
)
|
)
|
||||||
|
|
|
||||||
|
|
@ -2,7 +2,7 @@ from __future__ import annotations
|
||||||
|
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from app.services.domains.scholars.exceptions import ScholarServiceError
|
from app.services.scholars.exceptions import ScholarServiceError
|
||||||
|
|
||||||
|
|
||||||
def _ensure_upload_root(upload_dir: str, *, create: bool) -> Path:
|
def _ensure_upload_root(upload_dir: str, *, create: bool) -> Path:
|
||||||
|
|
|
||||||
|
|
@ -2,8 +2,8 @@ from __future__ import annotations
|
||||||
|
|
||||||
from urllib.parse import urlparse
|
from urllib.parse import urlparse
|
||||||
|
|
||||||
from app.services.domains.scholars.constants import MAX_IMAGE_URL_LENGTH, SCHOLAR_ID_PATTERN
|
from app.services.scholars.constants import MAX_IMAGE_URL_LENGTH, SCHOLAR_ID_PATTERN
|
||||||
from app.services.domains.scholars.exceptions import ScholarServiceError
|
from app.services.scholars.exceptions import ScholarServiceError
|
||||||
|
|
||||||
|
|
||||||
def validate_scholar_id(value: str) -> str:
|
def validate_scholar_id(value: str) -> str:
|
||||||
|
|
|
||||||
|
|
@ -2,7 +2,7 @@ from __future__ import annotations
|
||||||
|
|
||||||
|
|
||||||
async def resolve_publication_pdf_urls(*args, **kwargs):
|
async def resolve_publication_pdf_urls(*args, **kwargs):
|
||||||
from app.services.domains.unpaywall.application import resolve_publication_pdf_urls as _impl
|
from app.services.unpaywall.application import resolve_publication_pdf_urls as _impl
|
||||||
|
|
||||||
return await _impl(*args, **kwargs)
|
return await _impl(*args, **kwargs)
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -7,17 +7,17 @@ from typing import TYPE_CHECKING
|
||||||
from urllib.parse import unquote
|
from urllib.parse import unquote
|
||||||
|
|
||||||
from app.logging_utils import structured_log
|
from app.logging_utils import structured_log
|
||||||
from app.services.domains.crossref.application import discover_doi_for_publication
|
from app.services.crossref.application import discover_doi_for_publication
|
||||||
from app.services.domains.doi.normalize import normalize_doi
|
from app.services.doi.normalize import normalize_doi
|
||||||
from app.services.domains.unpaywall.pdf_discovery import (
|
from app.services.unpaywall.pdf_discovery import (
|
||||||
looks_like_pdf_url,
|
looks_like_pdf_url,
|
||||||
resolve_pdf_from_landing_page,
|
resolve_pdf_from_landing_page,
|
||||||
)
|
)
|
||||||
from app.services.domains.unpaywall.rate_limit import wait_for_unpaywall_slot
|
from app.services.unpaywall.rate_limit import wait_for_unpaywall_slot
|
||||||
from app.settings import settings
|
from app.settings import settings
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem
|
from app.services.publications.types import PublicationListItem, UnreadPublicationItem
|
||||||
|
|
||||||
DOI_PATTERN = re.compile(r"10\.\d{4,9}/[-._;()/:A-Z0-9]+", re.I)
|
DOI_PATTERN = re.compile(r"10\.\d{4,9}/[-._;()/:A-Z0-9]+", re.I)
|
||||||
DOI_PREFIX_RE = re.compile(r"\bdoi\s*[:=]\s*(10\.\d{4,9}/[-._;()/:A-Z0-9]+)", re.I)
|
DOI_PREFIX_RE = re.compile(r"\bdoi\s*[:=]\s*(10\.\d{4,9}/[-._;()/:A-Z0-9]+)", re.I)
|
||||||
|
|
|
||||||
|
|
@ -4,7 +4,7 @@ import re
|
||||||
from html.parser import HTMLParser
|
from html.parser import HTMLParser
|
||||||
from urllib.parse import urljoin, urlparse
|
from urllib.parse import urljoin, urlparse
|
||||||
|
|
||||||
from app.services.domains.unpaywall.rate_limit import wait_for_unpaywall_slot
|
from app.services.unpaywall.rate_limit import wait_for_unpaywall_slot
|
||||||
from app.settings import settings
|
from app.settings import settings
|
||||||
|
|
||||||
PDF_MIME = "application/pdf"
|
PDF_MIME = "application/pdf"
|
||||||
|
|
|
||||||
|
|
@ -6,7 +6,7 @@ import asyncio
|
||||||
import json
|
import json
|
||||||
|
|
||||||
from app.db.session import get_session_factory
|
from app.db.session import get_session_factory
|
||||||
from app.services.domains.dbops import collect_integrity_report
|
from app.services.dbops import collect_integrity_report
|
||||||
|
|
||||||
|
|
||||||
def build_parser() -> argparse.ArgumentParser:
|
def build_parser() -> argparse.ArgumentParser:
|
||||||
|
|
|
||||||
|
|
@ -6,7 +6,7 @@ import asyncio
|
||||||
import json
|
import json
|
||||||
|
|
||||||
from app.db.session import get_session_factory
|
from app.db.session import get_session_factory
|
||||||
from app.services.domains.dbops import run_publication_link_repair
|
from app.services.dbops import run_publication_link_repair
|
||||||
|
|
||||||
|
|
||||||
def build_parser() -> argparse.ArgumentParser:
|
def build_parser() -> argparse.ArgumentParser:
|
||||||
|
|
|
||||||
|
|
@ -11,9 +11,9 @@ from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
from app.api.runtime_deps import get_scholar_source
|
from app.api.runtime_deps import get_scholar_source
|
||||||
from app.main import app
|
from app.main import app
|
||||||
from app.services.domains.publications.types import PublicationListItem
|
from app.services.publications.types import PublicationListItem
|
||||||
from app.services.domains.scholar.source import FetchResult
|
from app.services.scholar.source import FetchResult
|
||||||
from app.services.domains.settings import application as user_settings_service
|
from app.services.settings import application as user_settings_service
|
||||||
from app.settings import settings
|
from app.settings import settings
|
||||||
from tests.integration.helpers import insert_user, login_user
|
from tests.integration.helpers import insert_user, login_user
|
||||||
|
|
||||||
|
|
@ -473,7 +473,7 @@ async def test_api_admin_dbops_pdf_queue_requeue_endpoint(
|
||||||
fingerprint=f"{(target_user_id + 73):064x}",
|
fingerprint=f"{(target_user_id + 73):064x}",
|
||||||
)
|
)
|
||||||
monkeypatch.setattr(
|
monkeypatch.setattr(
|
||||||
"app.services.domains.publications.pdf_queue._schedule_rows",
|
"app.services.publications.pdf_queue._schedule_rows",
|
||||||
lambda **_kwargs: None,
|
lambda **_kwargs: None,
|
||||||
)
|
)
|
||||||
client = TestClient(app)
|
client = TestClient(app)
|
||||||
|
|
@ -517,7 +517,7 @@ async def test_api_admin_dbops_pdf_queue_requeue_all_endpoint(
|
||||||
fingerprint=f"{(target_user_id + 83):064x}",
|
fingerprint=f"{(target_user_id + 83):064x}",
|
||||||
)
|
)
|
||||||
monkeypatch.setattr(
|
monkeypatch.setattr(
|
||||||
"app.services.domains.publications.pdf_queue._schedule_rows",
|
"app.services.publications.pdf_queue._schedule_rows",
|
||||||
lambda **_kwargs: None,
|
lambda **_kwargs: None,
|
||||||
)
|
)
|
||||||
client = TestClient(app)
|
client = TestClient(app)
|
||||||
|
|
@ -1290,12 +1290,16 @@ async def test_api_settings_get_and_update(db_session: AsyncSession) -> None:
|
||||||
_assert_safety_state_contract(settings_payload["safety_state"])
|
_assert_safety_state_contract(settings_payload["safety_state"])
|
||||||
assert settings_payload["safety_state"]["cooldown_active"] is False
|
assert settings_payload["safety_state"]["cooldown_active"] is False
|
||||||
|
|
||||||
|
policy = settings_payload["policy"]
|
||||||
|
run_interval_minutes = max(45, int(policy["min_run_interval_minutes"]))
|
||||||
|
request_delay_seconds = max(6, int(policy["min_request_delay_seconds"]))
|
||||||
|
|
||||||
update_response = client.put(
|
update_response = client.put(
|
||||||
"/api/v1/settings",
|
"/api/v1/settings",
|
||||||
json={
|
json={
|
||||||
"auto_run_enabled": True,
|
"auto_run_enabled": True,
|
||||||
"run_interval_minutes": 45,
|
"run_interval_minutes": run_interval_minutes,
|
||||||
"request_delay_seconds": 6,
|
"request_delay_seconds": request_delay_seconds,
|
||||||
"nav_visible_pages": ["dashboard", "scholars", "publications", "settings", "runs"],
|
"nav_visible_pages": ["dashboard", "scholars", "publications", "settings", "runs"],
|
||||||
},
|
},
|
||||||
headers=headers,
|
headers=headers,
|
||||||
|
|
@ -1303,8 +1307,8 @@ async def test_api_settings_get_and_update(db_session: AsyncSession) -> None:
|
||||||
assert update_response.status_code == 200
|
assert update_response.status_code == 200
|
||||||
updated = update_response.json()["data"]
|
updated = update_response.json()["data"]
|
||||||
assert updated["auto_run_enabled"] is True
|
assert updated["auto_run_enabled"] is True
|
||||||
assert updated["run_interval_minutes"] == 45
|
assert updated["run_interval_minutes"] == run_interval_minutes
|
||||||
assert updated["request_delay_seconds"] == 6
|
assert updated["request_delay_seconds"] == request_delay_seconds
|
||||||
assert updated["nav_visible_pages"] == [
|
assert updated["nav_visible_pages"] == [
|
||||||
"dashboard",
|
"dashboard",
|
||||||
"scholars",
|
"scholars",
|
||||||
|
|
@ -2404,7 +2408,7 @@ async def test_api_publications_list_schedules_background_enrichment(
|
||||||
return 1
|
return 1
|
||||||
|
|
||||||
monkeypatch.setattr(
|
monkeypatch.setattr(
|
||||||
"app.services.domains.publications.application.schedule_missing_pdf_enrichment_for_user",
|
"app.services.publications.application.schedule_missing_pdf_enrichment_for_user",
|
||||||
_fake_schedule,
|
_fake_schedule,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
@ -2484,7 +2488,7 @@ async def test_api_publication_retry_pdf_queues_resolution_job(
|
||||||
return True
|
return True
|
||||||
|
|
||||||
monkeypatch.setattr(
|
monkeypatch.setattr(
|
||||||
"app.services.domains.publications.application.schedule_retry_pdf_enrichment_for_row",
|
"app.services.publications.application.schedule_retry_pdf_enrichment_for_row",
|
||||||
_fake_retry_scheduler,
|
_fake_retry_scheduler,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
@ -2514,7 +2518,7 @@ async def test_api_publication_retry_pdf_queues_resolution_job(
|
||||||
]
|
]
|
||||||
|
|
||||||
monkeypatch.setattr(
|
monkeypatch.setattr(
|
||||||
"app.services.domains.publications.application.hydrate_pdf_enrichment_state",
|
"app.services.publications.application.hydrate_pdf_enrichment_state",
|
||||||
_fake_hydrate,
|
_fake_hydrate,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -4,7 +4,7 @@ import pytest
|
||||||
from sqlalchemy import text
|
from sqlalchemy import text
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
from app.services.domains.dbops import run_publication_link_repair
|
from app.services.dbops import run_publication_link_repair
|
||||||
from tests.integration.helpers import insert_user
|
from tests.integration.helpers import insert_user
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -4,7 +4,7 @@ import pytest
|
||||||
from sqlalchemy import text
|
from sqlalchemy import text
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
from app.services.domains.dbops.integrity import collect_integrity_report
|
from app.services.dbops.integrity import collect_integrity_report
|
||||||
|
|
||||||
|
|
||||||
def _check_counts(report: dict) -> dict[str, int]:
|
def _check_counts(report: dict) -> dict[str, int]:
|
||||||
|
|
|
||||||
|
|
@ -8,8 +8,8 @@ from sqlalchemy import select
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
from app.db.models import CrawlRun, Publication, RunStatus, RunTriggerType, ScholarProfile, ScholarPublication
|
from app.db.models import CrawlRun, Publication, RunStatus, RunTriggerType, ScholarProfile, ScholarPublication
|
||||||
from app.services.domains.ingestion.application import ScholarIngestionService
|
from app.services.ingestion.application import ScholarIngestionService
|
||||||
from app.services.domains.openalex.types import OpenAlexWork
|
from app.services.openalex.types import OpenAlexWork
|
||||||
from tests.integration.helpers import insert_user
|
from tests.integration.helpers import insert_user
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -84,9 +84,9 @@ async def test_deferred_enrichment_sweeps_previous_runs(db_session: AsyncSession
|
||||||
|
|
||||||
# We patch the client at its source, and also mock arXiv to avoid real HTTP calls
|
# We patch the client at its source, and also mock arXiv to avoid real HTTP calls
|
||||||
with (
|
with (
|
||||||
patch("app.services.domains.openalex.client.OpenAlexClient") as MockClient,
|
patch("app.services.openalex.client.OpenAlexClient") as MockClient,
|
||||||
patch(
|
patch(
|
||||||
"app.services.domains.arxiv.application.discover_arxiv_id_for_publication", new=AsyncMock(return_value=None)
|
"app.services.arxiv.application.discover_arxiv_id_for_publication", new=AsyncMock(return_value=None)
|
||||||
),
|
),
|
||||||
):
|
):
|
||||||
mock_instance = MockClient.return_value
|
mock_instance = MockClient.return_value
|
||||||
|
|
|
||||||
|
|
@ -8,8 +8,8 @@ from sqlalchemy import text
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
from app.db.models import RunStatus, RunTriggerType
|
from app.db.models import RunStatus, RunTriggerType
|
||||||
from app.services.domains.ingestion.application import ScholarIngestionService
|
from app.services.ingestion.application import ScholarIngestionService
|
||||||
from app.services.domains.scholar.source import FetchResult
|
from app.services.scholar.source import FetchResult
|
||||||
from tests.integration.helpers import insert_user
|
from tests.integration.helpers import insert_user
|
||||||
|
|
||||||
REGRESSION_FIXTURE_DIR = Path("tests/fixtures/scholar/regression")
|
REGRESSION_FIXTURE_DIR = Path("tests/fixtures/scholar/regression")
|
||||||
|
|
|
||||||
|
|
@ -12,7 +12,7 @@ from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||||
from app.api.runtime_deps import get_ingestion_service
|
from app.api.runtime_deps import get_ingestion_service
|
||||||
from app.db.models import CrawlRun, Publication, RunStatus, RunTriggerType
|
from app.db.models import CrawlRun, Publication, RunStatus, RunTriggerType
|
||||||
from app.main import app
|
from app.main import app
|
||||||
from app.services.domains.ingestion.application import ScholarIngestionService
|
from app.services.ingestion.application import ScholarIngestionService
|
||||||
from tests.integration.helpers import insert_user, login_user
|
from tests.integration.helpers import insert_user, login_user
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -214,7 +214,7 @@ async def test_background_enrichment_preserves_canceled_status(
|
||||||
return []
|
return []
|
||||||
|
|
||||||
monkeypatch.setattr(
|
monkeypatch.setattr(
|
||||||
"app.services.domains.openalex.client.OpenAlexClient",
|
"app.services.openalex.client.OpenAlexClient",
|
||||||
_OpenAlexClientStub,
|
_OpenAlexClientStub,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
@ -339,7 +339,7 @@ async def test_partial_discovery_exception_keeps_new_pub_count_consistent(
|
||||||
monkeypatch.setattr(service, "_resolve_publication", _resolve_publication_stub)
|
monkeypatch.setattr(service, "_resolve_publication", _resolve_publication_stub)
|
||||||
|
|
||||||
from app.db.models import ScholarProfile
|
from app.db.models import ScholarProfile
|
||||||
from app.services.domains.scholar.parser_types import PublicationCandidate
|
from app.services.scholar.parser_types import PublicationCandidate
|
||||||
|
|
||||||
scholar = await db_session.get(ScholarProfile, scholar_profile_id)
|
scholar = await db_session.get(ScholarProfile, scholar_profile_id)
|
||||||
assert scholar is not None
|
assert scholar is not None
|
||||||
|
|
|
||||||
|
|
@ -9,13 +9,13 @@ from sqlalchemy import select
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
from app.db.models import ArxivQueryCacheEntry
|
from app.db.models import ArxivQueryCacheEntry
|
||||||
from app.services.domains.arxiv.cache import (
|
from app.services.arxiv.cache import (
|
||||||
build_query_fingerprint,
|
build_query_fingerprint,
|
||||||
get_cached_feed,
|
get_cached_feed,
|
||||||
run_with_inflight_dedupe,
|
run_with_inflight_dedupe,
|
||||||
set_cached_feed,
|
set_cached_feed,
|
||||||
)
|
)
|
||||||
from app.services.domains.arxiv.types import ArxivEntry, ArxivFeed, ArxivOpenSearchMeta
|
from app.services.arxiv.types import ArxivEntry, ArxivFeed, ArxivOpenSearchMeta
|
||||||
|
|
||||||
|
|
||||||
def _sample_feed(arxiv_id: str = "1234.5678") -> ArxivFeed:
|
def _sample_feed(arxiv_id: str = "1234.5678") -> ArxivFeed:
|
||||||
|
|
|
||||||
|
|
@ -7,10 +7,10 @@ import httpx
|
||||||
import pytest
|
import pytest
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
from app.services.domains.arxiv import client as arxiv_client_module
|
from app.services.arxiv import client as arxiv_client_module
|
||||||
from app.services.domains.arxiv.client import ArxivClient
|
from app.services.arxiv.client import ArxivClient
|
||||||
from app.services.domains.arxiv.errors import ArxivClientValidationError, ArxivRateLimitError
|
from app.services.arxiv.errors import ArxivClientValidationError, ArxivRateLimitError
|
||||||
from app.services.domains.arxiv.rate_limit import ArxivCooldownStatus
|
from app.services.arxiv.rate_limit import ArxivCooldownStatus
|
||||||
|
|
||||||
_CLIENT_FEED_XML = """<?xml version="1.0" encoding="UTF-8"?>
|
_CLIENT_FEED_XML = """<?xml version="1.0" encoding="UTF-8"?>
|
||||||
<feed xmlns="http://www.w3.org/2005/Atom"
|
<feed xmlns="http://www.w3.org/2005/Atom"
|
||||||
|
|
@ -160,7 +160,7 @@ async def test_client_logs_cache_hit_and_miss(
|
||||||
def _capture_log(_msg: str, *args, **kwargs) -> None:
|
def _capture_log(_msg: str, *args, **kwargs) -> None:
|
||||||
logged.append({"event": _msg, **(kwargs.get("extra") or {})})
|
logged.append({"event": _msg, **(kwargs.get("extra") or {})})
|
||||||
|
|
||||||
monkeypatch.setattr("app.services.domains.arxiv.client.logger.info", _capture_log)
|
monkeypatch.setattr("app.services.arxiv.client.logger.info", _capture_log)
|
||||||
client = ArxivClient(request_fn=_request_fn, cache_enabled=True)
|
client = ArxivClient(request_fn=_request_fn, cache_enabled=True)
|
||||||
await client.search(query="ti:test-cache")
|
await client.search(query="ti:test-cache")
|
||||||
await client.search(query="ti:test-cache")
|
await client.search(query="ti:test-cache")
|
||||||
|
|
@ -196,7 +196,7 @@ async def test_request_feed_skips_live_call_when_global_cooldown_is_active(
|
||||||
|
|
||||||
monkeypatch.setattr(arxiv_client_module, "get_arxiv_cooldown_status", _cooldown_status)
|
monkeypatch.setattr(arxiv_client_module, "get_arxiv_cooldown_status", _cooldown_status)
|
||||||
monkeypatch.setattr(arxiv_client_module, "run_with_global_arxiv_limit", _unexpected_limit_call)
|
monkeypatch.setattr(arxiv_client_module, "run_with_global_arxiv_limit", _unexpected_limit_call)
|
||||||
monkeypatch.setattr("app.services.domains.arxiv.client.logger.warning", _capture_warning)
|
monkeypatch.setattr("app.services.arxiv.client.logger.warning", _capture_warning)
|
||||||
|
|
||||||
with pytest.raises(ArxivRateLimitError):
|
with pytest.raises(ArxivRateLimitError):
|
||||||
await arxiv_client_module._request_arxiv_feed(
|
await arxiv_client_module._request_arxiv_feed(
|
||||||
|
|
|
||||||
|
|
@ -4,9 +4,9 @@ from types import SimpleNamespace
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from app.services.domains.arxiv import application as arxiv_application
|
from app.services.arxiv import application as arxiv_application
|
||||||
from app.services.domains.arxiv import gateway as arxiv_gateway
|
from app.services.arxiv import gateway as arxiv_gateway
|
||||||
from app.services.domains.arxiv.types import ArxivEntry, ArxivFeed, ArxivOpenSearchMeta
|
from app.services.arxiv.types import ArxivEntry, ArxivFeed, ArxivOpenSearchMeta
|
||||||
from app.settings import settings
|
from app.settings import settings
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -2,9 +2,9 @@ from __future__ import annotations
|
||||||
|
|
||||||
from datetime import UTC, datetime
|
from datetime import UTC, datetime
|
||||||
|
|
||||||
from app.services.domains.arxiv.guards import arxiv_skip_reason_for_item
|
from app.services.arxiv.guards import arxiv_skip_reason_for_item
|
||||||
from app.services.domains.publication_identifiers.types import DisplayIdentifier
|
from app.services.publication_identifiers.types import DisplayIdentifier
|
||||||
from app.services.domains.publications.types import PublicationListItem
|
from app.services.publications.types import PublicationListItem
|
||||||
|
|
||||||
|
|
||||||
def _item(
|
def _item(
|
||||||
|
|
|
||||||
|
|
@ -2,8 +2,8 @@ from __future__ import annotations
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from app.services.domains.arxiv.errors import ArxivParseError
|
from app.services.arxiv.errors import ArxivParseError
|
||||||
from app.services.domains.arxiv.parser import parse_arxiv_feed
|
from app.services.arxiv.parser import parse_arxiv_feed
|
||||||
|
|
||||||
_VALID_FEED_XML = """<?xml version="1.0" encoding="UTF-8"?>
|
_VALID_FEED_XML = """<?xml version="1.0" encoding="UTF-8"?>
|
||||||
<feed xmlns="http://www.w3.org/2005/Atom"
|
<feed xmlns="http://www.w3.org/2005/Atom"
|
||||||
|
|
|
||||||
|
|
@ -9,9 +9,9 @@ from sqlalchemy import select
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
from app.db.models import ArxivRuntimeState
|
from app.db.models import ArxivRuntimeState
|
||||||
from app.services.domains.arxiv.constants import ARXIV_RUNTIME_STATE_KEY
|
from app.services.arxiv.constants import ARXIV_RUNTIME_STATE_KEY
|
||||||
from app.services.domains.arxiv.errors import ArxivRateLimitError
|
from app.services.arxiv.errors import ArxivRateLimitError
|
||||||
from app.services.domains.arxiv.rate_limit import get_arxiv_cooldown_status, run_with_global_arxiv_limit
|
from app.services.arxiv.rate_limit import get_arxiv_cooldown_status, run_with_global_arxiv_limit
|
||||||
from app.settings import settings
|
from app.settings import settings
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -98,7 +98,7 @@ async def test_arxiv_rate_limit_logs_request_scheduled_and_completed(
|
||||||
def _capture_log(_msg: str, *args, **kwargs) -> None:
|
def _capture_log(_msg: str, *args, **kwargs) -> None:
|
||||||
logged.append({"event": _msg, **(kwargs.get("extra") or {})})
|
logged.append({"event": _msg, **(kwargs.get("extra") or {})})
|
||||||
|
|
||||||
monkeypatch.setattr("app.services.domains.arxiv.rate_limit.logger.info", _capture_log)
|
monkeypatch.setattr("app.services.arxiv.rate_limit.logger.info", _capture_log)
|
||||||
await run_with_global_arxiv_limit(fetch=_fetch, source_path="search")
|
await run_with_global_arxiv_limit(fetch=_fetch, source_path="search")
|
||||||
|
|
||||||
scheduled = [entry for entry in logged if entry.get("event") == "arxiv.request_scheduled"]
|
scheduled = [entry for entry in logged if entry.get("event") == "arxiv.request_scheduled"]
|
||||||
|
|
@ -128,7 +128,7 @@ async def test_arxiv_rate_limit_logs_cooldown_activation(
|
||||||
def _capture_warning(_msg: str, *args, **kwargs) -> None:
|
def _capture_warning(_msg: str, *args, **kwargs) -> None:
|
||||||
logged_warning.append({"event": _msg, **(kwargs.get("extra") or {})})
|
logged_warning.append({"event": _msg, **(kwargs.get("extra") or {})})
|
||||||
|
|
||||||
monkeypatch.setattr("app.services.domains.arxiv.rate_limit.logger.warning", _capture_warning)
|
monkeypatch.setattr("app.services.arxiv.rate_limit.logger.warning", _capture_warning)
|
||||||
with pytest.raises(ArxivRateLimitError):
|
with pytest.raises(ArxivRateLimitError):
|
||||||
await run_with_global_arxiv_limit(fetch=_fetch, source_path="lookup_ids")
|
await run_with_global_arxiv_limit(fetch=_fetch, source_path="lookup_ids")
|
||||||
finally:
|
finally:
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,4 @@
|
||||||
from app.services.domains.openalex.types import OpenAlexWork
|
from app.services.openalex.types import OpenAlexWork
|
||||||
|
|
||||||
|
|
||||||
def test_parse_openalex_work_from_api_dict() -> None:
|
def test_parse_openalex_work_from_api_dict() -> None:
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,5 @@
|
||||||
from app.services.domains.openalex.matching import find_best_match
|
from app.services.openalex.matching import find_best_match
|
||||||
from app.services.domains.openalex.types import OpenAlexWork
|
from app.services.openalex.types import OpenAlexWork
|
||||||
|
|
||||||
|
|
||||||
def test_find_best_match_exact_title():
|
def test_find_best_match_exact_title():
|
||||||
|
|
|
||||||
|
|
@ -8,8 +8,8 @@ from unittest.mock import AsyncMock, MagicMock, patch
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from app.db.models import ScholarPublication
|
from app.db.models import ScholarPublication
|
||||||
from app.services.domains.publications import dedup as dedup_service
|
from app.services.publications import dedup as dedup_service
|
||||||
from app.services.domains.publications.dedup import (
|
from app.services.publications.dedup import (
|
||||||
NearDuplicateCluster,
|
NearDuplicateCluster,
|
||||||
NearDuplicateMember,
|
NearDuplicateMember,
|
||||||
find_identifier_duplicate_pairs,
|
find_identifier_duplicate_pairs,
|
||||||
|
|
@ -87,15 +87,15 @@ async def test_merge_duplicate_publication_migrates_links_and_identifiers() -> N
|
||||||
|
|
||||||
with (
|
with (
|
||||||
patch(
|
patch(
|
||||||
"app.services.domains.publications.dedup._load_publication",
|
"app.services.publications.dedup._load_publication",
|
||||||
new=AsyncMock(side_effect=[winner, dup]),
|
new=AsyncMock(side_effect=[winner, dup]),
|
||||||
),
|
),
|
||||||
patch(
|
patch(
|
||||||
"app.services.domains.publications.dedup._migrate_scholar_links",
|
"app.services.publications.dedup._migrate_scholar_links",
|
||||||
new=AsyncMock(),
|
new=AsyncMock(),
|
||||||
) as mock_links,
|
) as mock_links,
|
||||||
patch(
|
patch(
|
||||||
"app.services.domains.publications.dedup._migrate_identifiers",
|
"app.services.publications.dedup._migrate_identifiers",
|
||||||
new=AsyncMock(),
|
new=AsyncMock(),
|
||||||
) as mock_identifiers,
|
) as mock_identifiers,
|
||||||
):
|
):
|
||||||
|
|
@ -112,7 +112,7 @@ async def test_merge_duplicate_publication_rejects_missing_publications() -> Non
|
||||||
|
|
||||||
with (
|
with (
|
||||||
patch(
|
patch(
|
||||||
"app.services.domains.publications.dedup._load_publication",
|
"app.services.publications.dedup._load_publication",
|
||||||
new=AsyncMock(side_effect=[None, None]),
|
new=AsyncMock(side_effect=[None, None]),
|
||||||
),
|
),
|
||||||
pytest.raises(ValueError),
|
pytest.raises(ValueError),
|
||||||
|
|
@ -160,7 +160,7 @@ async def test_migrate_scholar_links_drops_conflicts() -> None:
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_sweep_returns_zero_when_no_pairs() -> None:
|
async def test_sweep_returns_zero_when_no_pairs() -> None:
|
||||||
with patch(
|
with patch(
|
||||||
"app.services.domains.publications.dedup.find_identifier_duplicate_pairs",
|
"app.services.publications.dedup.find_identifier_duplicate_pairs",
|
||||||
new=AsyncMock(return_value=[]),
|
new=AsyncMock(return_value=[]),
|
||||||
):
|
):
|
||||||
session = AsyncMock()
|
session = AsyncMock()
|
||||||
|
|
@ -174,11 +174,11 @@ async def test_sweep_returns_zero_when_no_pairs() -> None:
|
||||||
async def test_sweep_returns_merge_count() -> None:
|
async def test_sweep_returns_merge_count() -> None:
|
||||||
with (
|
with (
|
||||||
patch(
|
patch(
|
||||||
"app.services.domains.publications.dedup.find_identifier_duplicate_pairs",
|
"app.services.publications.dedup.find_identifier_duplicate_pairs",
|
||||||
new=AsyncMock(return_value=[(1, 2), (3, 4)]),
|
new=AsyncMock(return_value=[(1, 2), (3, 4)]),
|
||||||
),
|
),
|
||||||
patch(
|
patch(
|
||||||
"app.services.domains.publications.dedup.merge_duplicate_publication",
|
"app.services.publications.dedup.merge_duplicate_publication",
|
||||||
new=AsyncMock(),
|
new=AsyncMock(),
|
||||||
) as mock_merge,
|
) as mock_merge,
|
||||||
):
|
):
|
||||||
|
|
@ -194,11 +194,11 @@ async def test_sweep_returns_merge_count() -> None:
|
||||||
async def test_sweep_merges_each_dup_only_once() -> None:
|
async def test_sweep_merges_each_dup_only_once() -> None:
|
||||||
with (
|
with (
|
||||||
patch(
|
patch(
|
||||||
"app.services.domains.publications.dedup.find_identifier_duplicate_pairs",
|
"app.services.publications.dedup.find_identifier_duplicate_pairs",
|
||||||
new=AsyncMock(return_value=[(1, 2), (1, 2)]),
|
new=AsyncMock(return_value=[(1, 2), (1, 2)]),
|
||||||
),
|
),
|
||||||
patch(
|
patch(
|
||||||
"app.services.domains.publications.dedup.merge_duplicate_publication",
|
"app.services.publications.dedup.merge_duplicate_publication",
|
||||||
new=AsyncMock(),
|
new=AsyncMock(),
|
||||||
) as mock_merge,
|
) as mock_merge,
|
||||||
):
|
):
|
||||||
|
|
@ -227,7 +227,7 @@ async def test_find_near_duplicate_clusters_groups_similar_titles() -> None:
|
||||||
assert second is not None
|
assert second is not None
|
||||||
|
|
||||||
with patch(
|
with patch(
|
||||||
"app.services.domains.publications.dedup._load_near_duplicate_candidates",
|
"app.services.publications.dedup._load_near_duplicate_candidates",
|
||||||
new=AsyncMock(return_value=[first, second]),
|
new=AsyncMock(return_value=[first, second]),
|
||||||
):
|
):
|
||||||
clusters = await find_near_duplicate_clusters(AsyncMock())
|
clusters = await find_near_duplicate_clusters(AsyncMock())
|
||||||
|
|
@ -255,7 +255,7 @@ async def test_find_near_duplicate_clusters_skips_unrelated_titles() -> None:
|
||||||
assert second is not None
|
assert second is not None
|
||||||
|
|
||||||
with patch(
|
with patch(
|
||||||
"app.services.domains.publications.dedup._load_near_duplicate_candidates",
|
"app.services.publications.dedup._load_near_duplicate_candidates",
|
||||||
new=AsyncMock(return_value=[first, second]),
|
new=AsyncMock(return_value=[first, second]),
|
||||||
):
|
):
|
||||||
clusters = await find_near_duplicate_clusters(AsyncMock())
|
clusters = await find_near_duplicate_clusters(AsyncMock())
|
||||||
|
|
@ -277,7 +277,7 @@ async def test_merge_near_duplicate_cluster_merges_non_winner_members() -> None:
|
||||||
)
|
)
|
||||||
|
|
||||||
with patch(
|
with patch(
|
||||||
"app.services.domains.publications.dedup.merge_duplicate_publication",
|
"app.services.publications.dedup.merge_duplicate_publication",
|
||||||
new=AsyncMock(),
|
new=AsyncMock(),
|
||||||
) as mock_merge:
|
) as mock_merge:
|
||||||
merged = await merge_near_duplicate_cluster(AsyncMock(), cluster=cluster)
|
merged = await merge_near_duplicate_cluster(AsyncMock(), cluster=cluster)
|
||||||
|
|
|
||||||
|
|
@ -4,7 +4,7 @@ from types import SimpleNamespace
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from app.services.domains.crossref import application as crossref_app
|
from app.services.crossref import application as crossref_app
|
||||||
|
|
||||||
|
|
||||||
def _item(*, title: str, year: int | None, scholar_label: str = "Shinya Yamanaka"):
|
def _item(*, title: str, year: int | None, scholar_label: str = "Shinya Yamanaka"):
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from app.services.domains.doi.normalize import first_doi_from_texts, normalize_doi
|
from app.services.doi.normalize import first_doi_from_texts, normalize_doi
|
||||||
|
|
||||||
|
|
||||||
def test_normalize_doi_extracts_and_lowercases() -> None:
|
def test_normalize_doi_extracts_and_lowercases() -> None:
|
||||||
|
|
|
||||||
|
|
@ -1,12 +1,12 @@
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from app.services.domains.ingestion.fingerprints import (
|
from app.services.ingestion.fingerprints import (
|
||||||
_dedupe_publication_candidates,
|
_dedupe_publication_candidates,
|
||||||
canonical_title_for_dedup,
|
canonical_title_for_dedup,
|
||||||
fuzzy_titles_match,
|
fuzzy_titles_match,
|
||||||
normalize_title,
|
normalize_title,
|
||||||
)
|
)
|
||||||
from app.services.domains.scholar.parser import PublicationCandidate
|
from app.services.scholar.parser import PublicationCandidate
|
||||||
|
|
||||||
|
|
||||||
def _candidate(
|
def _candidate(
|
||||||
|
|
|
||||||
|
|
@ -4,9 +4,9 @@ from types import SimpleNamespace
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from app.services.domains.arxiv.errors import ArxivRateLimitError
|
from app.services.arxiv.errors import ArxivRateLimitError
|
||||||
from app.services.domains.ingestion.application import ScholarIngestionService
|
from app.services.ingestion.application import ScholarIngestionService
|
||||||
from app.services.domains.publication_identifiers import application as identifier_service
|
from app.services.publication_identifiers import application as identifier_service
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
|
|
|
||||||
|
|
@ -2,8 +2,8 @@ from __future__ import annotations
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from app.services.domains.ingestion.application import ScholarIngestionService
|
from app.services.ingestion.application import ScholarIngestionService
|
||||||
from app.services.domains.ingestion.types import RunProgress, ScholarProcessingOutcome
|
from app.services.ingestion.types import RunProgress, ScholarProcessingOutcome
|
||||||
|
|
||||||
|
|
||||||
def _outcome(*, scholar_profile_id: int, outcome_label: str) -> ScholarProcessingOutcome:
|
def _outcome(*, scholar_profile_id: int, outcome_label: str) -> ScholarProcessingOutcome:
|
||||||
|
|
|
||||||
|
|
@ -5,10 +5,10 @@ from sqlalchemy import select
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
from app.db.models import Publication, PublicationIdentifier
|
from app.db.models import Publication, PublicationIdentifier
|
||||||
from app.services.domains.arxiv import application as arxiv_service
|
from app.services.arxiv import application as arxiv_service
|
||||||
from app.services.domains.publication_identifiers import application as identifier_service
|
from app.services.publication_identifiers import application as identifier_service
|
||||||
from app.services.domains.publication_identifiers.types import IdentifierKind
|
from app.services.publication_identifiers.types import IdentifierKind
|
||||||
from app.services.domains.publications.types import UnreadPublicationItem
|
from app.services.publications.types import UnreadPublicationItem
|
||||||
|
|
||||||
|
|
||||||
def test_derive_display_identifier_prefers_doi_over_arxiv() -> None:
|
def test_derive_display_identifier_prefers_doi_over_arxiv() -> None:
|
||||||
|
|
@ -47,7 +47,7 @@ def test_derive_display_identifier_uses_pmcid_when_present() -> None:
|
||||||
|
|
||||||
|
|
||||||
def test_normalize_arxiv_id_handles_urls() -> None:
|
def test_normalize_arxiv_id_handles_urls() -> None:
|
||||||
from app.services.domains.publication_identifiers.normalize import normalize_arxiv_id
|
from app.services.publication_identifiers.normalize import normalize_arxiv_id
|
||||||
|
|
||||||
assert normalize_arxiv_id("https://arxiv.org/abs/1504.08025") == "1504.08025"
|
assert normalize_arxiv_id("https://arxiv.org/abs/1504.08025") == "1504.08025"
|
||||||
assert normalize_arxiv_id("http://arxiv.org/pdf/1504.08025v2.pdf") == "1504.08025v2"
|
assert normalize_arxiv_id("http://arxiv.org/pdf/1504.08025v2.pdf") == "1504.08025v2"
|
||||||
|
|
@ -59,7 +59,7 @@ def test_normalize_arxiv_id_handles_urls() -> None:
|
||||||
|
|
||||||
|
|
||||||
def test_normalize_arxiv_id_handles_raw_text() -> None:
|
def test_normalize_arxiv_id_handles_raw_text() -> None:
|
||||||
from app.services.domains.publication_identifiers.normalize import normalize_arxiv_id
|
from app.services.publication_identifiers.normalize import normalize_arxiv_id
|
||||||
|
|
||||||
assert normalize_arxiv_id("1504.08025v1") == "1504.08025v1"
|
assert normalize_arxiv_id("1504.08025v1") == "1504.08025v1"
|
||||||
assert normalize_arxiv_id("arXiv:1504.08025") == "1504.08025"
|
assert normalize_arxiv_id("arXiv:1504.08025") == "1504.08025"
|
||||||
|
|
@ -69,7 +69,7 @@ def test_normalize_arxiv_id_handles_raw_text() -> None:
|
||||||
|
|
||||||
|
|
||||||
def test_normalize_pmcid_handles_urls_and_text() -> None:
|
def test_normalize_pmcid_handles_urls_and_text() -> None:
|
||||||
from app.services.domains.publication_identifiers.normalize import normalize_pmcid
|
from app.services.publication_identifiers.normalize import normalize_pmcid
|
||||||
|
|
||||||
assert normalize_pmcid("https://pmc.ncbi.nlm.nih.gov/articles/PMC2175868/") == "PMC2175868"
|
assert normalize_pmcid("https://pmc.ncbi.nlm.nih.gov/articles/PMC2175868/") == "PMC2175868"
|
||||||
assert normalize_pmcid("http://www.ncbi.nlm.nih.gov/pmc/articles/PMC1234567") == "PMC1234567"
|
assert normalize_pmcid("http://www.ncbi.nlm.nih.gov/pmc/articles/PMC1234567") == "PMC1234567"
|
||||||
|
|
@ -79,7 +79,7 @@ def test_normalize_pmcid_handles_urls_and_text() -> None:
|
||||||
|
|
||||||
|
|
||||||
def test_normalize_pmid_handles_urls() -> None:
|
def test_normalize_pmid_handles_urls() -> None:
|
||||||
from app.services.domains.publication_identifiers.normalize import normalize_pmid
|
from app.services.publication_identifiers.normalize import normalize_pmid
|
||||||
|
|
||||||
assert normalize_pmid("https://pubmed.ncbi.nlm.nih.gov/12345678/") == "12345678"
|
assert normalize_pmid("https://pubmed.ncbi.nlm.nih.gov/12345678/") == "12345678"
|
||||||
assert normalize_pmid("http://pubmed.ncbi.nlm.nih.gov/12345678") == "12345678"
|
assert normalize_pmid("http://pubmed.ncbi.nlm.nih.gov/12345678") == "12345678"
|
||||||
|
|
@ -117,11 +117,11 @@ async def test_discover_and_sync_skips_arxiv_when_crossref_finds_doi(
|
||||||
raise AssertionError("arXiv should be skipped after strong DOI evidence.")
|
raise AssertionError("arXiv should be skipped after strong DOI evidence.")
|
||||||
|
|
||||||
monkeypatch.setattr(
|
monkeypatch.setattr(
|
||||||
"app.services.domains.crossref.application.discover_doi_for_publication",
|
"app.services.crossref.application.discover_doi_for_publication",
|
||||||
_fake_crossref,
|
_fake_crossref,
|
||||||
)
|
)
|
||||||
monkeypatch.setattr(
|
monkeypatch.setattr(
|
||||||
"app.services.domains.arxiv.application.discover_arxiv_id_for_publication",
|
"app.services.arxiv.application.discover_arxiv_id_for_publication",
|
||||||
_fail_arxiv,
|
_fail_arxiv,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
@ -158,11 +158,11 @@ async def test_discover_and_sync_skips_arxiv_for_low_quality_title(
|
||||||
return "1234.5678"
|
return "1234.5678"
|
||||||
|
|
||||||
monkeypatch.setattr(
|
monkeypatch.setattr(
|
||||||
"app.services.domains.crossref.application.discover_doi_for_publication",
|
"app.services.crossref.application.discover_doi_for_publication",
|
||||||
_fake_crossref,
|
_fake_crossref,
|
||||||
)
|
)
|
||||||
monkeypatch.setattr(
|
monkeypatch.setattr(
|
||||||
"app.services.domains.arxiv.application.discover_arxiv_id_for_publication",
|
"app.services.arxiv.application.discover_arxiv_id_for_publication",
|
||||||
_fake_arxiv,
|
_fake_arxiv,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
@ -190,11 +190,11 @@ async def test_discover_and_sync_calls_arxiv_when_item_is_eligible(
|
||||||
return "2501.00001v2"
|
return "2501.00001v2"
|
||||||
|
|
||||||
monkeypatch.setattr(
|
monkeypatch.setattr(
|
||||||
"app.services.domains.crossref.application.discover_doi_for_publication",
|
"app.services.crossref.application.discover_doi_for_publication",
|
||||||
_fake_crossref,
|
_fake_crossref,
|
||||||
)
|
)
|
||||||
monkeypatch.setattr(
|
monkeypatch.setattr(
|
||||||
"app.services.domains.arxiv.application.discover_arxiv_id_for_publication",
|
"app.services.arxiv.application.discover_arxiv_id_for_publication",
|
||||||
_fake_arxiv,
|
_fake_arxiv,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -6,9 +6,9 @@ from types import SimpleNamespace
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from app.db.models import PublicationPdfJob
|
from app.db.models import PublicationPdfJob
|
||||||
from app.services.domains.publications import pdf_queue
|
from app.services.publications import pdf_queue
|
||||||
from app.services.domains.publications.pdf_resolution_pipeline import PipelineOutcome
|
from app.services.publications.pdf_resolution_pipeline import PipelineOutcome
|
||||||
from app.services.domains.unpaywall.application import OaResolutionOutcome
|
from app.services.unpaywall.application import OaResolutionOutcome
|
||||||
|
|
||||||
|
|
||||||
def _job(
|
def _job(
|
||||||
|
|
|
||||||
|
|
@ -5,10 +5,10 @@ from types import SimpleNamespace
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from app.services.domains.arxiv.errors import ArxivRateLimitError
|
from app.services.arxiv.errors import ArxivRateLimitError
|
||||||
from app.services.domains.publication_identifiers.types import DisplayIdentifier
|
from app.services.publication_identifiers.types import DisplayIdentifier
|
||||||
from app.services.domains.publications import pdf_resolution_pipeline as pipeline
|
from app.services.publications import pdf_resolution_pipeline as pipeline
|
||||||
from app.services.domains.unpaywall.application import OaResolutionOutcome
|
from app.services.unpaywall.application import OaResolutionOutcome
|
||||||
|
|
||||||
|
|
||||||
def _row(*, display_identifier: DisplayIdentifier | None = None) -> SimpleNamespace:
|
def _row(*, display_identifier: DisplayIdentifier | None = None) -> SimpleNamespace:
|
||||||
|
|
@ -165,7 +165,7 @@ async def test_arxiv_outcome_skips_when_strong_doi_identifier(
|
||||||
raise AssertionError("arXiv lookup should be skipped when DOI evidence is strong.")
|
raise AssertionError("arXiv lookup should be skipped when DOI evidence is strong.")
|
||||||
|
|
||||||
monkeypatch.setattr(
|
monkeypatch.setattr(
|
||||||
"app.services.domains.arxiv.application.discover_arxiv_id_for_publication",
|
"app.services.arxiv.application.discover_arxiv_id_for_publication",
|
||||||
_fail_discover,
|
_fail_discover,
|
||||||
)
|
)
|
||||||
outcome = await pipeline._arxiv_outcome(row, request_email="user@example.com")
|
outcome = await pipeline._arxiv_outcome(row, request_email="user@example.com")
|
||||||
|
|
@ -183,7 +183,7 @@ async def test_arxiv_outcome_skips_when_title_quality_is_low(
|
||||||
raise AssertionError("arXiv lookup should be skipped for low-quality titles.")
|
raise AssertionError("arXiv lookup should be skipped for low-quality titles.")
|
||||||
|
|
||||||
monkeypatch.setattr(
|
monkeypatch.setattr(
|
||||||
"app.services.domains.arxiv.application.discover_arxiv_id_for_publication",
|
"app.services.arxiv.application.discover_arxiv_id_for_publication",
|
||||||
_fail_discover,
|
_fail_discover,
|
||||||
)
|
)
|
||||||
outcome = await pipeline._arxiv_outcome(row, request_email="user@example.com")
|
outcome = await pipeline._arxiv_outcome(row, request_email="user@example.com")
|
||||||
|
|
@ -198,7 +198,7 @@ async def test_arxiv_outcome_calls_arxiv_when_eligible(
|
||||||
return "1234.5678"
|
return "1234.5678"
|
||||||
|
|
||||||
monkeypatch.setattr(
|
monkeypatch.setattr(
|
||||||
"app.services.domains.arxiv.application.discover_arxiv_id_for_publication",
|
"app.services.arxiv.application.discover_arxiv_id_for_publication",
|
||||||
_fake_discover,
|
_fake_discover,
|
||||||
)
|
)
|
||||||
row = _row()
|
row = _row()
|
||||||
|
|
|
||||||
|
|
@ -2,8 +2,8 @@ from __future__ import annotations
|
||||||
|
|
||||||
from datetime import UTC, datetime
|
from datetime import UTC, datetime
|
||||||
|
|
||||||
from app.services.domains.ingestion import scheduler as scheduler_module
|
from app.services.ingestion import scheduler as scheduler_module
|
||||||
from app.services.domains.ingestion.application import ScholarIngestionService
|
from app.services.ingestion.application import ScholarIngestionService
|
||||||
from app.settings import settings
|
from app.settings import settings
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -3,7 +3,7 @@ from __future__ import annotations
|
||||||
from datetime import UTC, datetime, timedelta
|
from datetime import UTC, datetime, timedelta
|
||||||
|
|
||||||
from app.db.models import UserSetting
|
from app.db.models import UserSetting
|
||||||
from app.services.domains.ingestion import safety as run_safety
|
from app.services.ingestion import safety as run_safety
|
||||||
|
|
||||||
|
|
||||||
def test_apply_run_safety_outcome_triggers_blocked_cooldown() -> None:
|
def test_apply_run_safety_outcome_triggers_blocked_cooldown() -> None:
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from app.services.domains.runs.application import extract_run_summary
|
from app.services.runs.application import extract_run_summary
|
||||||
|
|
||||||
|
|
||||||
def test_extract_run_summary_includes_extended_metrics() -> None:
|
def test_extract_run_summary_includes_extended_metrics() -> None:
|
||||||
|
|
|
||||||
|
|
@ -4,13 +4,13 @@ from pathlib import Path
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from app.services.domains.scholar.parser import (
|
from app.services.scholar.parser import (
|
||||||
ParseState,
|
ParseState,
|
||||||
ScholarDomInvariantError,
|
ScholarDomInvariantError,
|
||||||
parse_author_search_page,
|
parse_author_search_page,
|
||||||
parse_profile_page,
|
parse_profile_page,
|
||||||
)
|
)
|
||||||
from app.services.domains.scholar.source import FetchResult
|
from app.services.scholar.source import FetchResult
|
||||||
|
|
||||||
|
|
||||||
def _fixture(name: str) -> str:
|
def _fixture(name: str) -> str:
|
||||||
|
|
|
||||||
|
|
@ -3,9 +3,9 @@ from __future__ import annotations
|
||||||
import pytest
|
import pytest
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
from app.services.domains.scholar.parser import ParseState
|
from app.services.scholar.parser import ParseState
|
||||||
from app.services.domains.scholar.source import FetchResult
|
from app.services.scholar.source import FetchResult
|
||||||
from app.services.domains.scholars import application as scholar_service
|
from app.services.scholars import application as scholar_service
|
||||||
|
|
||||||
pytestmark = [pytest.mark.integration, pytest.mark.db]
|
pytestmark = [pytest.mark.integration, pytest.mark.db]
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,7 @@
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from app.services.domains.scholar import rate_limit as scholar_rate_limit
|
from app.services.scholar import rate_limit as scholar_rate_limit
|
||||||
from app.services.domains.scholar.source import FetchResult, LiveScholarSource, _build_profile_url
|
from app.services.scholar.source import FetchResult, LiveScholarSource, _build_profile_url
|
||||||
|
|
||||||
|
|
||||||
def test_build_profile_url_includes_pagesize_for_initial_page() -> None:
|
def test_build_profile_url_includes_pagesize_for_initial_page() -> None:
|
||||||
|
|
|
||||||
|
|
@ -2,7 +2,7 @@ from __future__ import annotations
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from app.services.domains.unpaywall import pdf_discovery
|
from app.services.unpaywall import pdf_discovery
|
||||||
|
|
||||||
|
|
||||||
def test_looks_like_pdf_url_detects_path_and_query_variants() -> None:
|
def test_looks_like_pdf_url_detects_path_and_query_variants() -> None:
|
||||||
|
|
|
||||||
|
|
@ -5,9 +5,9 @@ from datetime import UTC, datetime
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from app.services.domains.publication_identifiers.types import DisplayIdentifier
|
from app.services.publication_identifiers.types import DisplayIdentifier
|
||||||
from app.services.domains.publications.types import PublicationListItem
|
from app.services.publications.types import PublicationListItem
|
||||||
from app.services.domains.unpaywall import application as unpaywall_app
|
from app.services.unpaywall import application as unpaywall_app
|
||||||
|
|
||||||
|
|
||||||
class _DummyAsyncClient:
|
class _DummyAsyncClient:
|
||||||
|
|
|
||||||
|
|
@ -2,7 +2,7 @@ from __future__ import annotations
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from app.services.domains.settings.application import (
|
from app.services.settings.application import (
|
||||||
DEFAULT_NAV_VISIBLE_PAGES,
|
DEFAULT_NAV_VISIBLE_PAGES,
|
||||||
HARD_MIN_REQUEST_DELAY_SECONDS,
|
HARD_MIN_REQUEST_DELAY_SECONDS,
|
||||||
HARD_MIN_RUN_INTERVAL_MINUTES,
|
HARD_MIN_RUN_INTERVAL_MINUTES,
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue