# Robots.txt for AI and standard crawlers
# Data owner: The Stow Brothers

# Allow all crawlers by default
User-agent: *
Allow: /
Crawl-delay: 1

# ===================
# AI Training Crawlers
# ===================

# OpenAI
User-agent: GPTBot
Allow: /

# Anthropic
User-agent: ClaudeBot
Allow: /

# Anthropic (alternate)
User-agent: anthropic-ai
Allow: /

# Common Crawl (used by many AI training datasets)
User-agent: CCBot
Allow: /

# ===================
# AI Search/RAG Crawlers
# ===================

# Perplexity
User-agent: PerplexityBot
Allow: /

# Google AI (Gemini)
User-agent: Google-Extended
Allow: /

# ByteDance
User-agent: Bytespider
Allow: /

# Apple Intelligence
User-agent: Applebot-Extended
Allow: /

# Cohere
User-agent: cohere-ai
Allow: /

# Meta AI
User-agent: FacebookBot
Allow: /

# ===================
# AI-Friendly Resources
# ===================

# LLM-optimized content guide (llms.txt standard)
# See: https://ai.stowbrothers.com/llms.txt

# ===================
# Sitemaps
# ===================
Sitemap: https://ai.stowbrothers.com/sitemap.xml
Sitemap: https://ai.stowbrothers.com/sitemap-ai.xml