diff --git a/static/robots.txt b/static/robots.txt index 7dd3caf3e..04040f2c9 100644 --- a/static/robots.txt +++ b/static/robots.txt @@ -1 +1,7 @@ Sitemap: https://docs.rs/sitemap.xml +# Semver-based URL are always redirects, and sometimes +# confuse Google's duplicate detection, so we block crawling them. +# https://docs.rs/about/redirections +User-Agent: * +Disallow: */^ +Disallow: */~