# for all bots
User-Agent: *
Disallow: /
# Allow: /web/gopubmed/sitemap
# Allow: /web/gopubmed/statistics/
# Allow: /web/go3r/sitemap
Allow: /web/go3r/statistics/
# Allow: /sitemap
# Allow: /statistics/

# Limit requests
Request-rate: 1/20 #1 request per 20 seconds
Crawl-delay: 20    #1 request per 20 seconds

# Sitemap locations
# Sitemap: http://www.gopubmed.org/web/gopubmed/sitemap.xml