# =============================================================
# Runcrate robots.txt — AI-visibility-optimized
# =============================================================

# Default: allow all well-behaved crawlers
User-agent: *
Allow: /
Disallow: /dashboard/
Disallow: /admin/
Disallow: /admin-runcrate/
Disallow: /auth/
Disallow: /api/

# ---- OpenAI ----
User-agent: GPTBot
Allow: /
Disallow: /dashboard/
Disallow: /admin/
Disallow: /admin-runcrate/
Disallow: /auth/
Disallow: /api/

User-agent: OAI-SearchBot
Allow: /
Disallow: /dashboard/
Disallow: /admin/
Disallow: /admin-runcrate/
Disallow: /auth/
Disallow: /api/

User-agent: ChatGPT-User
Allow: /
Disallow: /dashboard/
Disallow: /admin/
Disallow: /auth/

# ---- Anthropic ----
User-agent: ClaudeBot
Allow: /
Disallow: /dashboard/
Disallow: /admin/
Disallow: /admin-runcrate/
Disallow: /auth/
Disallow: /api/
Crawl-delay: 1

User-agent: Claude-SearchBot
Allow: /
Disallow: /dashboard/
Disallow: /admin/
Disallow: /auth/

User-agent: Claude-User
Allow: /
Disallow: /dashboard/
Disallow: /admin/
Disallow: /auth/

# ---- Perplexity ----
User-agent: PerplexityBot
Allow: /
Disallow: /dashboard/
Disallow: /admin/
Disallow: /admin-runcrate/
Disallow: /auth/
Disallow: /api/

# ---- Google AI training opt-in ----
User-agent: Google-Extended
Allow: /

# ---- Apple AI training opt-in ----
User-agent: Applebot-Extended
Allow: /

# ---- Amazon ----
User-agent: Amazonbot
Allow: /
Disallow: /dashboard/
Disallow: /admin/
Disallow: /auth/

# ---- DuckDuckGo AI ----
User-agent: DuckAssistBot
Allow: /
Disallow: /dashboard/
Disallow: /admin/
Disallow: /auth/

# ---- Meta / Facebook ----
User-agent: Meta-ExternalAgent
Allow: /
Disallow: /dashboard/
Disallow: /admin/
Disallow: /auth/

# ---- Common Crawl (feeds many open-source models) ----
User-agent: CCBot
Allow: /
Disallow: /dashboard/
Disallow: /admin/
Disallow: /auth/

# ---- Mistral ----
User-agent: MistralAI-User
Allow: /
Disallow: /dashboard/
Disallow: /admin/
Disallow: /auth/

# ---- Cohere ----
User-agent: cohere-ai
Allow: /
Disallow: /dashboard/
Disallow: /admin/
Disallow: /auth/

# ---- ByteDance (allow training — competitive advantage) ----
User-agent: Bytespider
Allow: /
Disallow: /dashboard/
Disallow: /admin/
Disallow: /auth/

# ---- Content Signals (draft-romm-aipref-contentsignals) ----
Content-Signal: ai-train=yes, search=yes, ai-input=yes

# ---- Sitemaps ----
Sitemap: https://runcrate.ai/sitemap.xml
Sitemap: https://runcrate.ai/docs/sitemap.xml