# robots.txt — collegehelpguide.com
# Goal: let real search engines + AI crawlers + our own SEO tools (Ahrefs,
# Semrush) crawl, while blocking zero-value scrapers and param/crawl-trap paths
# that waste crawl budget and edge requests. (Hosted on Cloudflare Workers.)

# Default: everything is crawlable except admin, API, and search param traps.
User-agent: *
Allow: /
Allow: /api/og
Disallow: /admin/
Disallow: /api/
Disallow: /search
Disallow: /*?

# --- Zero-value / abusive scrapers (no SEO or tooling benefit, high crawl volume) ---
User-agent: Bytespider
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: dotbot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: SeekportBot
Disallow: /

User-agent: MegaIndex
Disallow: /

User-agent: SerpstatBot
Disallow: /

# Note: AhrefsBot and SemrushBot are intentionally NOT blocked — they power our
# own site audits. AI crawlers (GPTBot, ClaudeBot, Google-Extended, CCBot, etc.)
# are intentionally ALLOWED — blocking them would cut AI-answer visibility.
# IMPORTANT: Cloudflare's managed-robots.txt + ai_bots_protection were turned OFF
# for this zone (2026-06-24) so the dashboard doesn't re-inject AI "Disallow: /"
# rules on top of this file. Re-enabling "Block AI bots" in Cloudflare will undo
# that. See OPERATIONS-LOG.

# Sitemaps
Sitemap: https://www.collegehelpguide.com/sitemap.xml
Sitemap: https://www.collegehelpguide.com/schools/sitemap.xml