# ============================================================
# robots.txt for mykuan.app
# Last updated: 2026-05-12
# ============================================================

# Default: allow all crawlers (search engines need this)
User-agent: *
Allow: /

# ============================================================
# AI Crawlers — explicitly ALLOWED for 2026 LLM citation visibility
# (Switch to Disallow if you want to prevent training-data scraping;
# keep allowed to be cited in ChatGPT, Claude, Perplexity, Gemini, etc.)
# ============================================================

User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: Bytespider
Allow: /

# ============================================================
# Aggressive SEO crawlers — BLOCKED
# These consume server resources without providing value to you.
# Well-behaved bots respect this; malicious ones ignore it
# (but Cloudflare Bot Fight Mode handles those).
# ============================================================

User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: SeekportBot
Disallow: /

User-agent: serpstatbot
Disallow: /

User-agent: SearchAtlas
Disallow: /

User-agent: Barkrowler
Disallow: /

# ============================================================
# Honeypot — real users never see this link
# (Add a hidden link in your HTML pointing here; any User-Agent
# that requests this is a crawler ignoring CSS/aria-hidden.)
# ============================================================

User-agent: *
Disallow: /honeypot-do-not-follow

# ============================================================
# Sitemap
# ============================================================

Sitemap: https://mykuan.app/sitemap.xml