User-agent: *
Allow: /

# Important public pages
Allow: /
Allow: /subscribe
Allow: /about
Allow: /privacy
Allow: /terms

# Block admin and protected routes
Disallow: /api/
Disallow: /archon
Disallow: /devdocs
Disallow: /devlog
Disallow: /ghosts
Disallow: /settings
Disallow: /login
Disallow: /sign-up
Disallow: /unsubscribe
Disallow: /verify

# AI-specific directives
# Allow AI search and agent use
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: PerplexityBot
User-agent: FirecrawlAgent
User-agent: AndiBot
User-agent: ExaBot
User-agent: PhindBot
User-agent: YouBot
Allow: /

# Disallow AI training data collection
User-agent: GPTBot
User-agent: CCBot
User-agent: Google-Extended
Disallow: /

# Resource locations
Sitemap: https://killfeed.ai/sitemap.xml
LLMs.txt: https://killfeed.ai/llms.txt