# houstonaivisibility.com — all crawlers welcome, AI crawlers explicitly.
#
# Tokens below are taken from each vendor's own published crawler docs.
# Verified 2026-08-16. Note that "User-agent: *  Allow: /" already permits
# everything; the explicit groups exist so the policy is legible to humans
# and to the agents that read this file as a signal.

User-agent: *
Allow: /

# OpenAI — developers.openai.com/api/docs/bots
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-AdsBot
Allow: /

# Anthropic — support.claude.com (ClaudeBot / Claude-User / Claude-SearchBot)
User-agent: ClaudeBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

# Perplexity — docs.perplexity.ai/guides/bots
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# Google
# Googlebot is the crawler behind Google Search, AI Overviews, and AI Mode.
# Google-Extended is a separate token that governs Gemini app training and
# grounding only — per Google's docs it "does not impact a site's inclusion
# in Google Search nor is it used as a ranking signal." Allowing Google-Extended
# is NOT what gets a site into AI Overviews; Googlebot access is.
User-agent: Googlebot
Allow: /

User-agent: Google-Extended
Allow: /

# xAI — xAI publishes no official crawler documentation. These tokens are
# third-party-attested only, and xAI's fetchers frequently present generic
# browser or Go-http-client user agents that match no robots.txt group.
User-agent: GrokBot
Allow: /

User-agent: xAI-Grok
Allow: /

User-agent: Grok-DeepSearch
Allow: /

# Microsoft / Bing — ChatGPT's search grounding runs against Bing's index.
User-agent: bingbot
Allow: /

# Apple (Siri / Apple Intelligence)
User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

# Meta AI
User-agent: meta-externalagent
Allow: /

User-agent: FacebookBot
Allow: /

# Amazon (Alexa / Rufus)
User-agent: Amazonbot
Allow: /

# Cohere
User-agent: cohere-ai
Allow: /

# DuckDuckGo DuckAssist
User-agent: DuckAssistBot
Allow: /

# Mistral
User-agent: MistralAI-User
Allow: /

# You.com
User-agent: YouBot
Allow: /

# Common Crawl (training corpora many models draw from)
User-agent: CCBot
Allow: /

# AI context files
# llms.txt:      https://houstonaivisibility.com/llms.txt
# llms-full.txt: https://houstonaivisibility.com/llms-full.txt
# pricing.md:    https://houstonaivisibility.com/pricing.md

Sitemap: https://houstonaivisibility.com/sitemap.xml