justinzeus/scholarr
1
0
Fork 0
Code Issues 1 Pull requests 18 Projects Releases Packages Wiki Activity Actions

ci: add ruff linting and mypy type checking

Browse source 
Add ruff and mypy to dev dependencies with configuration in pyproject.toml.
Add a lint CI job that runs ruff check, ruff format --check, and mypy.
Auto-fix import sorting and formatting across the codebase. Exclude
alembic/versions from linting (auto-generated migrations). Ignore B008
(FastAPI Depends pattern) and RUF001 (unicode in user-facing strings).

21 ruff lint errors and 50 mypy errors remain for manual review.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Justin Visser 2026-02-26 22:11:41 +01:00
parent 399276ea69
commit bf04c77aa9
137 changed files with 3066 additions and 1900 deletions
Download patch file Download diff file Expand all files Collapse all files

9
app/services/domains/scholar/author_rows.py
View file

@ -79,9 +79,7 @@ class ScholarAuthorSearchParser(HTMLParser):
return
affiliation = normalize_space("".join(self._candidate["aff_parts"])) or None
email_domain = _extract_verified_email_domain(
normalize_space("".join(self._candidate["eml_parts"])) or None
)
email_domain = _extract_verified_email_domain(normalize_space("".join(self._candidate["eml_parts"])) or None)
cited_by_text = normalize_space("".join(self._candidate["cby_parts"]))
cited_by_match = re.search(r"\d+", cited_by_text)
cited_by_count = int(cited_by_match.group(0)) if cited_by_match else None
@ -97,10 +95,7 @@ class ScholarAuthorSearchParser(HTMLParser):
profile_url = build_absolute_scholar_url(self._candidate["name_href"])
if not profile_url:
profile_url = (
"https://scholar.google.com/citations"
f"?hl=en&user={scholar_id}"
)
profile_url = f"https://scholar.google.com/citations?hl=en&user={scholar_id}"
self.candidates.append(
ScholarSearchCandidate(

11
app/services/domains/scholar/parser.py
View file

@ -3,40 +3,29 @@ from __future__ import annotations
from app.services.domains.scholar.author_rows import (
ScholarAuthorSearchParser,
count_author_search_markers,
parse_scholar_id_from_href,
)
from app.services.domains.scholar.parser_constants import SCRIPT_STYLE_RE
from app.services.domains.scholar.parser_types import (
ParseState,
ParsedAuthorSearchPage,
ParsedProfilePage,
PublicationCandidate,
ScholarDomInvariantError,
ScholarMalformedDataError,
ScholarParserError,
ScholarSearchCandidate,
)
from app.services.domains.scholar.parser_utils import (
strip_tags,
)
from app.services.domains.scholar.profile_rows import (
ScholarRowParser,
count_markers,
extract_articles_range,
extract_profile_image_url,
extract_profile_name,
extract_rows,
has_operation_error_banner,
has_show_more_button,
parse_citation_count,
parse_cluster_id_from_href,
parse_publications,
parse_year,
)
from app.services.domains.scholar.source import FetchResult
from app.services.domains.scholar.state_detection import (
classify_block_or_captcha_reason,
classify_network_error_reason,
detect_author_search_state,
detect_state,
)

3
app/services/domains/scholar/profile_rows.py
View file

@ -12,7 +12,6 @@ from app.services.domains.scholar.parser_types import PublicationCandidate
from app.services.domains.scholar.parser_utils import (
attr_class,
attr_href,
attr_src,
build_absolute_scholar_url,
normalize_space,
strip_tags,
@ -260,7 +259,7 @@ def has_show_more_button(html: str) -> bool:
def has_operation_error_banner(html: str) -> bool:
lowered = html.lower()
if "id=\"gsc_a_err\"" not in lowered and "id='gsc_a_err'" not in lowered:
if 'id="gsc_a_err"' not in lowered and "id='gsc_a_err'" not in lowered:
return False
return "can't perform the operation now" in lowered or "cannot perform the operation now" in lowered

31
app/services/domains/scholar/source.py
View file

@ -17,18 +17,12 @@ SCHOLAR_PROFILE_URL = "https://scholar.google.com/citations"
DEFAULT_PAGE_SIZE = 100
DEFAULT_USER_AGENTS = [
(
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
"(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
),
("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"),
(
"Mozilla/5.0 (Macintosh; Intel Mac OS X 14_7) AppleWebKit/605.1.15 "
"(KHTML, like Gecko) Version/18.1 Safari/605.1.15"
),
(
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:131.0) "
"Gecko/20100101 Firefox/131.0"
),
("Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:131.0) Gecko/20100101 Firefox/131.0"),
]
logger = logging.getLogger(__name__)
@ -44,8 +38,7 @@ class FetchResult:
class ScholarSource(Protocol):
async def fetch_profile_html(self, scholar_id: str) -> FetchResult:
...
async def fetch_profile_html(self, scholar_id: str) -> FetchResult: ...
async def fetch_profile_page_html(
self,
@ -53,16 +46,14 @@ class ScholarSource(Protocol):
*,
cstart: int,
pagesize: int,
) -> FetchResult:
...
) -> FetchResult: ...
async def fetch_author_search_html(
self,
query: str,
*,
start: int,
) -> FetchResult:
...
) -> FetchResult: ...
class LiveScholarSource:
@ -145,7 +136,9 @@ class LiveScholarSource:
@staticmethod
def _network_error_result(requested_url: str, exc: URLError) -> FetchResult:
structured_log(
logger, "warning", "scholar_source.fetch_network_error",
logger,
"warning",
"scholar_source.fetch_network_error",
requested_url=requested_url,
)
return FetchResult(
@ -159,7 +152,9 @@ class LiveScholarSource:
@staticmethod
def _http_error_result(requested_url: str, exc: HTTPError) -> FetchResult:
structured_log(
logger, "warning", "scholar_source.fetch_http_error",
logger,
"warning",
"scholar_source.fetch_http_error",
requested_url=requested_url,
status_code=exc.code,
)
@ -176,7 +171,9 @@ class LiveScholarSource:
body = response.read().decode("utf-8", errors="replace")
status_code = getattr(response, "status", 200)
structured_log(
logger, "debug", "scholar_source.fetch_succeeded",
logger,
"debug",
"scholar_source.fetch_succeeded",
requested_url=requested_url,
status_code=status_code,
)