# ── Flux robots.txt ──────────────────────────────────────────────────
# Multi-bot aware: explicit rules for LLM retrieval/search crawlers,
# training crawlers, and traditional search engines.
#
# Retrieval bots (OAI-SearchBot, ChatGPT-User, Claude-SearchBot, etc.)
# power AI search results — blocking them removes Flux from AI answers.
# Training bots (GPTBot, ClaudeBot) build model knowledge — allow public
# surfaces, block private ones.

# ── Traditional search engines ───────────────────────────────────────
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Applebot
Allow: /

User-agent: Yandex
Allow: /

# ── AI retrieval / search crawlers (visibility-critical) ─────────────
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: Claude-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: Google-Extended
Allow: /

# ── AI training crawlers (allow public, block private) ───────────────
User-agent: GPTBot
Allow: /
Allow: /mcp
Allow: /llms.txt
Allow: /api-docs
Allow: /sign-up
Allow: /sign-in
Disallow: /api/
Disallow: /dashboard
Disallow: /board/
Disallow: /multiboard
Disallow: /settings
Disallow: /activity
Disallow: /w/
Disallow: /u/
Disallow: /import
Disallow: /invite
Disallow: /connect/
Disallow: /oauth/

User-agent: ClaudeBot
Allow: /
Allow: /mcp
Allow: /llms.txt
Allow: /api-docs
Allow: /sign-up
Allow: /sign-in
Disallow: /api/
Disallow: /dashboard
Disallow: /board/
Disallow: /multiboard
Disallow: /settings
Disallow: /activity
Disallow: /w/
Disallow: /u/
Disallow: /import
Disallow: /invite
Disallow: /connect/
Disallow: /oauth/

# ── Aggressive / low-value crawlers ──────────────────────────────────
User-agent: Bytespider
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Amazonbot
Disallow: /

# ── Default (everything else) ────────────────────────────────────────
User-agent: *
Allow: /
Allow: /sign-in
Allow: /sign-up
Allow: /api-docs
Allow: /mcp
Allow: /llms.txt
Disallow: /api/
Disallow: /dashboard
Disallow: /board/
Disallow: /multiboard
Disallow: /settings
Disallow: /activity
Disallow: /w/
Disallow: /u/
Disallow: /import
Disallow: /invite
Disallow: /connect/
Disallow: /oauth/

Sitemap: https://flux.umin.ai/sitemap-index.xml