justinzeus/scholarr
1
0
Fork 0
Code Issues 1 Pull requests 18 Projects Releases Packages Wiki Activity Actions

First product

Browse source
This commit is contained in:
Justin Visser 2026-02-17 14:51:25 +01:00
parent 778da9e2fc
commit 4433d7d2c4
157 changed files with 23975 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

24
planning/scholar_probe_tmp/fixtures/run_20260216T182029Z/robots.txt Normal file
View file

@ -0,0 +1,24 @@
User-agent: *
Disallow: /search
Disallow: /index.html
Disallow: /scholar
Disallow: /citations?
Allow: /citations?user=
Disallow: /citations?*cstart=
Disallow: /citations?user=*%40
Disallow: /citations?user=*@
Allow: /citations?view_op=list_classic_articles
Allow: /citations?view_op=mandates_leaderboard
Allow: /citations?view_op=metrics_intro
Allow: /citations?view_op=new_profile
Allow: /citations?view_op=sitemap
Allow: /citations?view_op=top_venues
User-agent: Twitterbot
Disallow:
User-agent: facebookexternalhit
Disallow:
User-agent: PetalBot
Disallow: /