# YieldScope — public crypto-yield comparison site.
#
# We want to be FOUND — in classic search AND in AI answers. Being cited by
# ChatGPT, Perplexity, Claude and Google AI Overviews is a primary growth
# channel for a comparison site, so we ALLOW retrieval / answer crawlers.
# We do not want our content bulk-scraped purely to train models, so a few
# training-only crawlers stay blocked and we signal ai-train=no.
#
# NOTE: for this file to take full effect, Cloudflare's managed "Block AI bots"
# / AI Crawl Control must be turned off for this zone — otherwise Cloudflare
# prepends its own block and overrides these rules.

User-agent: *
Content-Signal: search=yes, ai-train=no
Allow: /

# Raw data files — large, not useful for SEO.
Disallow: /data/

# Training-only / bulk scrapers (no user-facing citation benefit) — keep blocked.
# Answer engines use separate crawlers (e.g. OAI-SearchBot, ChatGPT-User,
# PerplexityBot, ClaudeBot, Google-Extended) which remain allowed via the rule
# above, so we stay visible in AI answers while opting out of bulk training.
User-agent: GPTBot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /

Sitemap: https://yieldscope.io/sitemap-index.xml