sitemap: https://www.bluediamondattachments.com/sitemaps-1-sitemap.xml sitemap: https://dealers.bluediamondattachments.com/sitemaps-1-sitemap.xml # live - don't allow web crawlers to index cpresources/ or vendor/ User-agent: * Disallow: /cpresources/ Disallow: /vendor/ Disallow: /.env Disallow: /cache/ # even good bots don't need to crawl the site with random search queries -Stuart Disallow: /search/ # Disallow ChatGPT bot, as there's no benefit to allowing it to index your site User-agent: GPTBot Disallow: / # Disallow Google Bard and Vertex AI bots, as there's no benefit to allowing it to index your site User-agent: Google-Extended Disallow: / User-agent: msnbot-media Disallow: / User-agent: psbot Disallow: / User-agent: Yahoo-MMCrawler Disallow: / #Baiduspider User-agent: Baiduspider Disallow: / User-agent: Baiduspider+ Disallow: / User-agent: Baiduspider+(+http://www.baidu.com/search/spider.htm) Disallow: / User-agent: Baiduspider/2.0;+http://www.baidu.com/search/spider.html Disallow: / User-agent: Baiduspider/2.0 Disallow: / User-agent: +Baiduspider Disallow: / User-agent: +Baiduspider/2.0 Disallow: / User-agent: +Baiduspider/2.0;++http://www.baidu.com/search/spider.html Disallow: / User-agent: Mozilla/5.0(compatible; Baiduspider/2.0;+http://www.baidu.com/search/spider.html) Disallow: / #Yandex User-agent: spider Disallow: / User-agent: Yandex Disallow: / User-agent: Yandex.com Disallow: / User-agent: YandexBot/3.0 Disallow: / User-agent: YandexImages Disallow: / User-agent: MJ12bot Disallow: / User-agent: Blekkobot Disallow: / User-agent: Exabot Disallow: / # Block BlexBot User-agent: BLEXBot Disallow: / # # Block MegaIndex.ru User-agent: MegaIndex.ru/2.0 Disallow: / User-agent: MegaIndex.ru Disallow: / User-agent: megaIndex.ru Disallow: / # Block Ahrefs User-agent: AhrefsBot Disallow: / # Block MJ12bot as it is just noise User-agent: MJ12bot Disallow: / #-stuart User-agent: SemrushBot Crawl-delay: 2 User-agent: SiteAuditBot Crawl-delay: 2 # Block Go-http-client as it's causing resource issues User-agent: Go-http-client Disallow: / # Block Bytespider User-agent: Bytespider Disallow: / # Slow down SEBot-WA. Hard block next time it behaves badly User-agent: SEBot-WA Crawl-delay: 3 # Slow down SiteCheckerPro, crawled cached pages at very high rate User-agent: SiteCheckerBotCrawler Crawl-delay: 1 #Slow down Scrapy, crawled whole site high rate multiple days in a row User-agent: Scrapy Crawl-delay: 3