# Defender

User-agent: *
Crawl-delay: 1

## M2
Disallow: /index.php/
Disallow: /checkout/
Disallow: /app/
Disallow: /lib/
Disallow: /*.php$
Disallow: /pkginfo/
Disallow: /report/
Disallow: /var/
Disallow: /catalog/
Disallow: /customer/
Disallow: /sendfriend/
Disallow: /review/
Disallow: /*SID=

## Do not crawl common Magento technical folders
Disallow: /cgi-bin/
Disallow: /app/
Disallow: /downloader/
Disallow: /errors/
Disallow: /includes/
Disallow: /pkginfo/
Disallow: /shell/
Disallow: /var/

## Do not crawl common Magento files
Disallow: /api.php
Disallow: /cron.php
Disallow: /cron.sh
Disallow: /error_log
Disallow: /get.php
Disallow: /install.php
Disallow: /index.php/install/
Disallow: /LICENSE.html
Disallow: /LICENSE.txt
Disallow: /LICENSE_AFL.txt
Disallow: /README.txt
Disallow: /RELEASE_NOTES.txt

## Do not crawl checkout and user account pages
Disallow: /customer/
Disallow: /customer/account/
Disallow: /customer/account/login/

## Do not crawl seach pages and not-SEO optimized catalog links
Disallow: /catalogsearch/
Disallow: /catalog/product_compare/
Disallow: /catalog/category/view/
Disallow: /catalog/product/view/
Disallow: /poll/
Disallow: /review/
Disallow: /sendfriend/
Disallow: /tag/
Disallow: /wishlist/

## Do not crawl OEM section for now
Disallow: /oem-parts#/*

## Do not crawl easy returns confirmation page
Disallow: /easy-return-success


## Crawl Budget Modifications for Layered Navigation

# Disallow URLs with '?availability[0]=' at the beginning of the query string
Disallow: /*?availability[0]=*

# Disallow URLs with '?fulfilment[0]=' at the beginning of the query string
Disallow: /*?fulfilment[0]=*

# Disallow URLs with '&availability[0]=' as a query parameter (not at the beginning)
Disallow: /*&availability[0]=*

# Disallow URLs with '&fulfillment[0]=' as a query parameter (not at the beginning)
Disallow: /*&fulfillment[0]=*

# Disallow URLs with '&principal_category[0]=' as a query parameter (not at the beginning)
Disallow: /*&principal_category[0]=*

# Disallow URLs with '?*' followed by 'availability[0]=' (wildcard before query parameter)
Disallow: /*?*availability[0]=*

# Disallow URLs with '?*' followed by 'fulfilment[0]=' (wildcard before query parameter)
Disallow: /*?*fulfilment[0]=*

# Disallow URLs with '?*' followed by 'principal_category[0]=' (wildcard before query parameter)
Disallow: /*?*principal_category[0]=*

# Disallow URLs with '?manufacturer[0]=' followed by '&*' (wildcard after query parameter)
Disallow: /*?manufacturer[0]=*&*

# Disallow URLs with '&manufacturer[0]=' followed by '&*' (wildcard after query parameter)
Disallow: /*&manufacturer[0]=*&*

# Disallow URLs with '&*' followed by 'manufacturer[0]=' (wildcard before query parameter)
Disallow: /*&*manufacturer[0]=*

# Disallow URLs with '?principal_category[0]=' followed by '&*' (wildcard after query parameter)
Disallow: /*?principal_category[0]=*&*

# Disallow URLs with '&principal_category[0]=' followed by '&*' (wildcard after query parameter)
Disallow: /*&principal_category[0]=*&*

# Disallow URLs with '&*' followed by 'principal_category[0]=' (wildcard before query parameter)
Disallow: /*&*principal_category[0]=*

# Allow URLs with '?manufacturer[0]=' at the end and no other parameters
Allow: /*?manufacturer[0]=$

# Allow URLs with '?principal_category[0]=' at the end and no other parameters
Allow: /*?principal_category[0]=$


user-agent: Pinterestbot
disallow: /

## Sitemap
Sitemap: https://defender.com/sitemaps/sitemap_defender.xml