#block Gomez agent
User-agent: GomezAgent
Disallow: /

#block YandexBot
User-agent: YandexBot
Disallow: /

#block SemrushBot
User-agent: SemrushBot
Disallow: /

#allow googlebot
User-agent: Googlebot
Disallow:

#allow Google Images bot
User-agent: Googlebot-Image
Disallow: 

# Crawlers Setup
User-agent: *
Crawl-delay: 10

# Amazon Aranhabot
User-agent: Aranhabot
Crawl-delay: 10

# Directories
Disallow: /404/
Disallow: /app/
Disallow: /bin/
Disallow: /bollman-cs/
Disallow: /cgi-bin/
Disallow: /channeladvisorapi/
Disallow: /edi/
Disallow: /dev/
Disallow: /lib/
Disallow: /magento/
Disallow: /pub/media/
Disallow: /report/
Disallow: /skin/
Disallow: /stats/
Disallow: /setup/
Disallow: /update/
Disallow: /var/
Disallow: /vendor/
Disallow: /customer/
Disallow: /catalog/product/view/_ignore_category/
Disallow: /checkout/


#define sitemap path
Sitemap: https://www.hats.com/media/sitemap.xml

#TEST
Sitemap: https://www.hats.com/media/sitemap.xml