# Direct link to Sitemap.xml Sitemap: https://domain.com/sitemap_index.xml # General Crawl Directives User-agent: * Allow: /wp/wp-admin/admin-ajax.php Disallow: /wp/wp-admin/ Disallow: /wp/wp-includes/ # Allow All Known Bots User-agent: Googlebot Allow: / User-agent: Googlebot-Image Allow: /app/uploads/ User-agent: Mediapartners-Google Allow: / User-agent: Adsbot-Google Allow: / User-agent: Googlebot-Mobile Allow: / User-agent: Bingbot Allow: / User-agent: MSNBot Allow: / User-agent: MSNBot-Media Allow: /app/uploads/ User-agent: Applebot Allow: / User-agent: YandexBot Allow: / User-agent: YandexImages Allow: /app/uploads/ User-agent: Slurp Allow: / User-agent: Baiduspider Allow: / User-agent: Baiduspider-image Allow: /app/uploads/ User-agent: DuckDuckBot Allow: / User-agent: Qwantify Allow: / # Disallow Spam Bots User-agent: DotBot Disallow: / User-agent: GiftGhostBot Disallow: / User-agent: Seznam Disallow: / User-agent: PaperLiBot Disallow: / User-agent: Genieo Disallow: / User-agent: Dataprovider/6.101 Disallow: / User-agent: DataproviderSiteExplorer Disallow: / User-agent: Dazoobot/1.0 Disallow: / User-agent: Diffbot Disallow: / User-agent: DomainStatsBot/1.0 Disallow: / User-agent: DotBot/1.1 Disallow: / User-agent: dubaiindex Disallow: / User-agent: eCommerceBot Disallow: / User-agent: ExpertSearchSpider Disallow: / User-agent: Feedbin Disallow: / User-agent: Fetch/2.0a Disallow: / User-agent: FFbot/1.0 Disallow: / User-agent: focusbot/1.1 Disallow: / User-agent: HuaweiSymantecSpider Disallow: / User-agent: HuaweiSymantecSpider/1.0 Disallow: / User-agent: JobdiggerSpider Disallow: / User-agent: LemurWebCrawler Disallow: / User-agent: LipperheyLinkExplorer Disallow: / User-agent: LSSRocketCrawler/1.0 Disallow: / User-agent: LYT.SRv1.5 Disallow: / User-agent: MiaDev/0.0.1 Disallow: / User-agent: Najdi.si/3.1 Disallow: / User-agent: BountiiBot Disallow: / User-agent: Experibot_v1 Disallow: / # Define Crawl Delay Crawl-delay: 5