AaronLuo00.github.io/robots.txt at main · AaronLuo00/AaronLuo00.github.io · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
User-agent: *
Allow: /
Sitemap: {{ site.url }}{{ site.baseurl }}/sitemap.xml

# Allow crawling of all content
Allow: /*.html
Allow: /*.js
Allow: /*.css
Allow: /*.jpg
Allow: /*.jpeg
Allow: /*.png
Allow: /*.gif

# Specific paths to ensure indexing
Allow: /about/
Allow: /publications/
Allow: /research/
Allow: /teaching/
Allow: /blog/

# Prevent duplicate content
Disallow: /assets/css/
Disallow: /assets/js/
Disallow: /_site/
Disallow: /_drafts/
Disallow: /_includes/
Disallow: /_layouts/
Disallow: /_sass/
Disallow: /google_scholar_crawler/