User-agent: Yandex
# Disallow: /
Disallow: /_OlD
Disallow: /awstats
Disallow: /admin
Disallow: /css
Disallow: /catalog
Disallow: /cgi-bin
Disallow: /download
Disallow: /image/cache/data
Disallow: /js
Disallow: /override
Disallow: /system
Disallow: /temp_excelport
Disallow: /vqmod
Disallow: /vendors
Disallow: /*filter=
Disallow: /*filters=
Disallow: /*login
Disallow: /*route=account/
Disallow: /*route=affiliate/
Disallow: /*route=checkout/
Disallow: /*route=product/search
Disallow: /*search
Disallow: /*sort=
Disallow: /*sorting=
Disallow: /*tag=
Disallow: /*page=
Disallow: */category&path=
Disallow: */product&path=
Disallow: /*route=product/product&product_id=
Disallow: /*&route=product/product
# Disallow: /*product/category
Disallow: /*product/compare
Disallow: /*limit=
Disallow: /*manufacturer_id=
Disallow: /*tracking=
Clean-param: tracking
# Crawl-delay: 30 # задает таймаут в сек.(35-40 - норм)

User-agent: AhrefsBot 
Disallow: /

User-agent: Baiduspider
Disallow: /

User-agent: Baiduspider+
Disallow: /

User-agent: bingbot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: crawler4j
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Cliqzbot
Disallow: /

User-agent: dataprovider
Disallow: /

User-agent: DomainCrawler
Disallow: /

User-agent: info@domaincrawler.com
Disallow: /

User-agent: dotbot
Disallow: /

User-agent: ExtLinksBot 
Disallow: /

User-agent: Linguee
Disallow: /

User-agent: MegaIndex
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: SafeDNSBot
Disallow: /

User-agent: SiteExplorer
Disallow: /

User-agent: SemrushBot
Disallow: /

User-Agent: serpstatbot
Disallow: /

User-agent: TurnitinBot
Disallow: /

User-agent: svetabot
Disallow: /

User-agent: VelenPublicWebCrawler
Disallow: /

User-agent: wotbox
Disallow: /

User-agent: *
#Disallow: /
Disallow: /_OlD
Disallow: /awstats
Disallow: /admin
Disallow: /css
Disallow: /catalog
Disallow: /cgi-bin
Disallow: /download
Disallow: /image/cache/data
Disallow: /js
Disallow: /override
Disallow: /system
Disallow: /temp_excelport
Disallow: /vqmod
Disallow: /vendors
Disallow: /*filter=
Disallow: /*filters=
Disallow: /*login
Disallow: /*route=account/
Disallow: /*route=affiliate/
Disallow: /*route=checkout/
Disallow: /*route=product/search
Disallow: /*search
Disallow: /*sort=
Disallow: /*sorting=
Disallow: /*tag=
Disallow: /*page=
Disallow: */category&path=
Disallow: */product&path=
Disallow: /*route=product/product&product_id=
#Disallow: /*product/category
Disallow: /*product/compare
Disallow: /*limit=
Disallow: /*manufacturer_id=
Disallow: /*tracking=
Crawl-delay: 30 # задает таймаут в сек.
Host: https://kazpipe.ru
Sitemap: https://kazpipe.ru/sitemap.xml