User-Agent: Googlebot
Disallow: /var

User-Agent: Googlebot
Disallow: /srv

User-Agent: Googlebot
Disallow: /spec/shutdown

User-Agent: Googlebot
Disallow: /memory.usage_in_bytes

User-Agent: Googlebot
Disallow: /tmp

User-Agent: Googlebot
Disallow: /nfsfileshare

User-Agent: Googlebot
Disallow: /mnt

User-agent: *
Allow: /

Sitemap: https://www.crunchydata.com/sitemap.xml