# leolabs.me · robots.txt
# Strategy: allow AI search & user-triggered fetches, block training crawlers.
# Reference: Tw93's GEO field notes (https://x.com/HiTw93/status/2050189572999618982)

# ---- Default: allow indexing for traditional search engines ----
User-agent: *
Allow: /

# ---- AI search & retrieval (real-time citation) — ALLOW ----
User-agent: OAI-SearchBot
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: PerplexityBot
Allow: /

# ---- User-triggered (user pastes URL into chat) — ALLOW ----
User-agent: ChatGPT-User
Allow: /

User-agent: Claude-User
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: Google-Agent
Allow: /

# ---- Training crawlers — BLOCK ----
User-agent: GPTBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

User-agent: CCBot
Disallow: /

# ---- Opt-out tokens for training datasets — BLOCK ----
User-agent: Google-Extended
Disallow: /

User-agent: Applebot-Extended
Disallow: /

# ---- Undeclared / non-compliant — BLOCK ----
User-agent: Bytespider
Disallow: /

Sitemap: https://leolabs.me/sitemap.xml
LLMs: https://leolabs.me/llms.txt