User-agent: *
Disallow: /cdn-cgi/l/email-protection
Allow: /


# Disallow Marketing bots
User-agent: SemrushBot-SA
Disallow: / #Semrush
Allow: /ads.txt

User-agent: SemrushBot
Disallow: / #Semrush
Allow: /ads.txt

User-agent: rogerbot
Disallow: / #MOZ
Allow: /ads.txt

User-agent: dotbot
Disallow: / #MOZ
Allow: /ads.txt

User-agent: BLEXBot
Disallow: / #Webmeup.com
Allow: /ads.txt

User-agent: spbot
Disallow: / #Openlinkprofiler
Allow: /ads.txt

user-agent: SEOdiver
disallow: / #SEOdiver
Allow: /ads.txt

User-agent: dataprovider
Disallow: / #DataProvider.com
Allow: /ads.txt

User-agent: magpie-crawler
Disallow: / #BrandWatch.com
Allow: /ads.txt

User-agent: GetIntent Crawler
Disallow: /
Allow: /ads.txt

User-agent: grapeshot
Disallow: /

User-agent: Doubleverify
Disallow: /

User-agent: White ops
Disallow: /

User-agent: moatbot
Disallow: /

User-agent: ias_crawler
Disallow: /

User-agent: forensiq
Disallow: /

User-agent: duckduckbot
Disallow: /

User-agent: Leikibot
Disallow: /

User-agent: Baidu-YunGuanCe-ScanBot(ce.baidu.com)
Disallow: /

User-agent: Baidu-YunGuanCe-SLABot(ce.baidu.com)
Disallow: /

User-agent: Baidu-YunGuanCe-PerfBot(ce.baidu.com)
Disallow: /

User-agent: Baidu-YunGuanCe-VSBot(ce.baidu.com)
Disallow: /

#Baiduspider
User-agent: Baiduspider
Disallow: /

#Yandex
User-agent: Yandex
Disallow: /

#Disallow exotic search engine crawlers

User-agent: SeznamBot
Disallow: /
Allow: /ads.txt

User-agent: Sogou web spider
Disallow: /
Allow: /ads.txt

User-agent: 360spider
Disallow: /
Allow: /ads.txt

User-agent: Baiduspider
Disallow: /
Allow: /ads.txt

User-agent: NaverBot
Disallow: /
Allow: /ads.txt

User-agent: Yeti
Disallow: /
Allow: /ads.txt

User-agent: coccocbot-web
Disallow: /
Allow: /ads.txt

User-agent: Qwantify
Disallow: /
Allow: /ads.txt

User-agent: Exabot
Disallow: /
Allow: /ads.txt

#Disallow other crawlers

User-agent: Linguee
Disallow: / #Language tool
Allow: /ads.txt

User-agent: SurdotlyBot
Disallow: / #Sur.ly
Allow: /ads.txt

User-agent: BUbiNG
Disallow: / #Bubing academic crawler
Allow: /ads.txt

# Good bots whitelisting:

User-agent: Twitterbot
Disallow:

User-agent: Mediapartners-Google
Disallow:

User-agent: Facebot
Disallow:

#Other bots
#Neticle Crawler v1.0 ( http://bot.neticle.hu/ ) https://bot.neticle.hu/ - brand monitoring
#Mega https://megaindex.com/crawler - link indexer tool (supports directives in user-agent:*)
#Obot - IBM X-Force service
#SafeDNSBot (https://www.safedns.com/searchbot)

Sitemap: https://xsongs.net/sitemap.xml