# Generative AI Maturity Framework — genaimaturity.net

# Default policy for all crawlers
User-agent: *
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /dashboard/
Disallow: /assessment/anonymous/
Disallow: /auth/

# OpenAI crawlers
# GPTBot: training-data crawler
# ChatGPT-User: browses the web when a user asks ChatGPT to visit a page
# OAI-SearchBot: builds the ChatGPT-with-search index
User-agent: GPTBot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /dashboard/
Disallow: /assessment/anonymous/
Disallow: /auth/

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

# Anthropic crawlers
User-agent: ClaudeBot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /dashboard/
Disallow: /assessment/anonymous/
Disallow: /auth/

User-agent: Claude-Web
Allow: /

# Perplexity crawler
User-agent: PerplexityBot
Allow: /

# Google-Extended: separate opt-in for Gemini / Vertex AI training and AI Overviews
User-agent: Google-Extended
Allow: /

# Common Crawl (feeds many LLM training pipelines and search indexes)
User-agent: CCBot
Allow: /

# Anthropic training-focused agent (distinct from ClaudeBot)
User-agent: anthropic-ai
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /dashboard/
Disallow: /assessment/anonymous/
Disallow: /auth/

# Claude retrieval bot (used for search-augmented answers)
User-agent: Claude-SearchBot
Allow: /

# Perplexity user-agent (browses on user request)
User-agent: Perplexity-User
Allow: /

# DuckDuckGo AI assistant
User-agent: DuckAssistBot
Allow: /

# Apple Intelligence / Siri opt-in
User-agent: Applebot-Extended
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /dashboard/
Disallow: /assessment/anonymous/
Disallow: /auth/

# ByteDance / Doubao
User-agent: Bytespider
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /dashboard/
Disallow: /assessment/anonymous/
Disallow: /auth/

# Amazon (Alexa+, Bedrock retrieval)
User-agent: Amazonbot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /dashboard/
Disallow: /assessment/anonymous/
Disallow: /auth/

# Meta AI (Llama training / Meta AI assistant)
User-agent: Meta-ExternalAgent
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /dashboard/
Disallow: /assessment/anonymous/
Disallow: /auth/

# Cohere training crawler
User-agent: cohere-ai
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /dashboard/
Disallow: /assessment/anonymous/
Disallow: /auth/

# Diffbot structured-data extractor (feeds many downstream AI products)
User-agent: Diffbot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /dashboard/
Disallow: /assessment/anonymous/
Disallow: /auth/

# Allen Institute for AI (open research corpus)
User-agent: Ai2Bot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /dashboard/
Disallow: /assessment/anonymous/
Disallow: /auth/

# Timpi (independent AI search index)
User-agent: Timpibot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /dashboard/
Disallow: /assessment/anonymous/
Disallow: /auth/

Sitemap: https://genaimaturity.net/sitemap.xml
Sitemap: https://genaimaturity.net/sitemap-index.xml

# llmstxt.org curated index for AI crawlers (short) and full corpus (long).
# See https://llmstxt.org/ for the emerging spec that generative engines use to
# locate a site's canonical, LLM-friendly content overview.
# llms.txt: https://genaimaturity.net/llms.txt
# llms-full.txt: https://genaimaturity.net/llms-full.txt