# Block BLEXBot to prevent excessive, non-human traffic that may trigger ad limits
User-agent: BLEXBot
Disallow: /

# Block SE Ranking Bot
User-agent: SERankingBacklinksBot
Disallow: /

# Block Anthropic's ClaudeBot
User-agent: ClaudeBot
Disallow: /

# Google's main crawler
User-agent: Googlebot
Disallow: /nogooglebot/
# You can add other specific exclusions here if needed.

# Google Image Crawler
User-agent: Googlebot-Image
# Allow all images by default

# Google News Crawler
User-agent: Googlebot-News
# Allow all news content by default

# Explicitly allow AdSense crawler to ensure ads can be verified
User-agent: Mediapartners-Google
Allow: /

# General rule for ALL other bots and crawlers (including Bing, Yandex, etc.)
# By default, allow all content for other search engines
User-agent: *
Allow: /

# Optional: Add the path to your main sitemap file
# Replace example.com with your actual domain
# Sitemap: https://www.example.com/sitemap.xml