# ---------------------------------------------------------------------------
# robots.txt — Neuro Wellness TMS Centers of America
# Allow all major search engines AND AI answer engines to crawl the full site.
# CSS, JS, and images are never blocked for any bot (required for rendering).
# ---------------------------------------------------------------------------

# --- Traditional search engines ---
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: YandexBot
Allow: /

User-agent: Applebot
Allow: /

# --- AI answer engines / training crawlers ---
# GPTBot — OpenAI's web crawler (powers ChatGPT search & training).
User-agent: GPTBot
Allow: /

# OAI-SearchBot — OpenAI's search crawler (powers ChatGPT search results, not training).
User-agent: OAI-SearchBot
Allow: /

# ChatGPT-User — OpenAI's live fetch user agent (real-time ChatGPT page reads).
User-agent: ChatGPT-User
Allow: /

# ClaudeBot — Anthropic's web crawler (powers Claude answers & training).
User-agent: ClaudeBot
Allow: /

# Claude-User — Anthropic's live fetch user agent (real-time Claude page reads).
User-agent: Claude-User
Allow: /

# anthropic-ai — Anthropic's AI crawler.
User-agent: anthropic-ai
Allow: /

# PerplexityBot — Perplexity's crawler (powers answer engine results).
User-agent: PerplexityBot
Allow: /

# Perplexity-User — Perplexity's live fetch user agent (real-time page fetches).
User-agent: Perplexity-User
Allow: /

# CCBot — Common Crawl (open dataset used by many AI projects).
User-agent: CCBot
Allow: /

# Google-Extended — Google's crawler for Gemini/Vertex AI training & Bard/SGE.
User-agent: Google-Extended
Allow: /

# Applebot-Extended — Apple's crawler for Apple Intelligence / generative features.
User-agent: Applebot-Extended
Allow: /

# Meta-ExternalAgent — Meta's AI crawler (powers Meta AI features).
User-agent: Meta-ExternalAgent
Allow: /

# Meta-ExternalFetcher — Meta's live fetch agent (real-time page reads).
User-agent: Meta-ExternalFetcher
Allow: /

# Bytespider — ByteDance's crawler (powers AI training & search).
User-agent: Bytespider
Allow: /

# cohere-ai — Cohere's AI crawler.
User-agent: cohere-ai
Allow: /

# AI2Bot — Allen Institute for AI crawler.
User-agent: AI2Bot
Allow: /

# Diffbot — Diffbot's crawler (knowledge graph & AI applications).
User-agent: Diffbot
Allow: /

# Omgilibot / Omgili — AI research crawler.
User-agent: Omgilibot
Allow: /

# ImagesiftBot — Image-based AI crawler.
User-agent: ImagesiftBot
Allow: /

# --- Social / sharing crawlers ---
User-agent: Twitterbot
Allow: /

User-agent: facebookexternalhit
Allow: /

# --- Catch-all: allow everything for any unlisted bot ---
User-agent: *
Allow: /

# --- Private / admin pages (block all bots) ---
User-agent: *
Disallow: /admin-links

# Sitemap
Sitemap: https://neurowellnesstms.com/sitemap.xml