# Consult the guide on how to open your Dataverse installation to crawling by 
# search engine bots: 
# /guides/installation/config.html#letting-search-engines-crawl-your-installation 


# Accessible to all the crawler bots
User-agent: *
Sitemap: https://dataverse.lib.virginia.edu/sitemap/sitemap.xml
# Allow homepage
Allow: /$
# Allow dataset pages
Allow: /dataset.xhtml
# Allow Dataverse pages
Allow: /dataverse/
# Allow sitemaps
Allow: /sitemap/
Allow: /sitemap.xml$
# The following lines are for the Facebook/Twitter/Linkedin preview bots:
Allow: /api/datasets/:persistentId/thumbnail
Allow: /logos/*/*
Allow: /javax.faces.resource/images/
# Disallow pages with jsessionid
Disallow: /*;jsessionid
# Disallow crawling via search API
Disallow: /dataverse/*?q
Disallow: /dataverse/*/search
Disallow: /dataset.xhtml?*&version=&q=
# Disallow all other pages
Disallow: /
Crawl-delay: 20