# NeuronPathway Robots.txt — AI Search Friendly, Copyblocker Enabled # Last updated: 2026-04-16 # ══════════════════════════════════════════════════════════════════════════════ # BLOCK AI TRAINING SCRAPERS (No Model Training On Our Content) # ══════════════════════════════════════════════════════════════════════════════ # Bots that scrape for training data, reverse engineering, or unauthorized cloning User-agent: CCBot Disallow: / User-agent: Diffbot Disallow: / User-agent: Bytespider Disallow: / User-agent: img2dataset Disallow: / User-agent: omgili Disallow: / User-agent: GPT-Scraper Disallow: / User-agent: anthropic-scraper Disallow: / User-agent: GPT4All Disallow: / # 2026 additions — AI training data scrapers confirmed active User-agent: AI2Bot Disallow: / User-agent: DataForSeoBot Disallow: / User-agent: cohere-ai Disallow: / User-agent: Kangaroo Disallow: / User-agent: Webzio-Extended Disallow: / User-agent: TimpiBot Disallow: / User-agent: Scrapy Disallow: / # SemrushBot + AhrefsBot: allowed to crawl public pages only. # These are SEO research bots — we need them to index us so we can appear in # competitor reports and use their tools for our own keyword/backlink research. # Not training data scrapers. User-agent: SemrushBot Allow: / Allow: /sitemap.xml Disallow: /admin Disallow: /dashboard Disallow: /api Crawl-delay: 2 User-agent: AhrefsBot Allow: / Allow: /sitemap.xml Disallow: /admin Disallow: /dashboard Disallow: /api Crawl-delay: 2 User-agent: MJ12bot Disallow: / # ══════════════════════════════════════════════════════════════════════════════ # ALLOW AI SEARCH ENGINES (Indexing & Citation) # ══════════════════════════════════════════════════════════════════════════════ # These bots can crawl, index, and cite our content in AI-generated answers. User-agent: PerplexityBot Allow: / Allow: /sitemap.xml Allow: /llms.txt Allow: /llms-full.txt Allow: /ai.txt Allow: /.well-known/mcp.json Crawl-delay: 1 User-agent: ClaudeBot Allow: / Allow: /sitemap.xml Allow: /llms.txt Allow: /llms-full.txt Allow: /ai.txt Allow: /.well-known/mcp.json Crawl-delay: 1 User-agent: Claude-Web Allow: / Allow: /sitemap.xml Allow: /llms.txt Allow: /llms-full.txt Allow: /ai.txt Allow: /.well-known/mcp.json Crawl-delay: 1 User-agent: Claude-SearchBot Allow: / Allow: /sitemap.xml Allow: /llms.txt Allow: /llms-full.txt Allow: /ai.txt Allow: /.well-known/mcp.json Crawl-delay: 1 User-agent: Claude-User Allow: / Allow: /sitemap.xml Allow: /llms.txt Allow: /llms-full.txt Allow: /ai.txt Allow: /.well-known/mcp.json Crawl-delay: 1 User-agent: GPTBot Allow: / Allow: /sitemap.xml Allow: /llms.txt Allow: /llms-full.txt Allow: /ai.txt Allow: /.well-known/mcp.json Crawl-delay: 1 User-agent: ChatGPT-User Allow: / Allow: /sitemap.xml Allow: /llms.txt Allow: /llms-full.txt Allow: /ai.txt Allow: /.well-known/mcp.json Crawl-delay: 1 User-agent: OAI-SearchBot Allow: / Allow: /sitemap.xml Allow: /llms.txt Allow: /llms-full.txt Allow: /ai.txt Allow: /.well-known/mcp.json Crawl-delay: 1 User-agent: Google-Extended Allow: / Allow: /sitemap.xml Allow: /llms.txt Allow: /llms-full.txt Disallow: /admin Disallow: /dashboard Disallow: /api Crawl-delay: 1 User-agent: anthropic-ai Allow: / Allow: /sitemap.xml Allow: /llms.txt Allow: /llms-full.txt Allow: /ai.txt Allow: /.well-known/mcp.json Crawl-delay: 1 User-agent: Gemini-Bot Allow: / Allow: /sitemap.xml Allow: /llms.txt Allow: /llms-full.txt Allow: /ai.txt Allow: /.well-known/mcp.json Crawl-delay: 1 User-agent: GrokBot Allow: / Allow: /sitemap.xml Allow: /llms.txt Allow: /llms-full.txt Allow: /ai.txt Allow: /.well-known/mcp.json Crawl-delay: 1 User-agent: Perplexity-User Allow: / Allow: /sitemap.xml Allow: /llms.txt Allow: /llms-full.txt Allow: /ai.txt Allow: /.well-known/mcp.json Crawl-delay: 1 User-agent: GoogleOther Allow: / Crawl-delay: 1 User-agent: Meta-ExternalAgent Allow: / Crawl-delay: 1 User-agent: Applebot Allow: / Crawl-delay: 1 User-agent: Amazonbot Allow: / Crawl-delay: 1 # ══════════════════════════════════════════════════════════════════════════════ # STANDARD SEARCH ENGINES (Full Access) # ══════════════════════════════════════════════════════════════════════════════ User-agent: Googlebot Allow: / Allow: /sitemap.xml Allow: /llms.txt Allow: /llms-full.txt Allow: /ai.txt Allow: /.well-known/mcp.json User-agent: Googlebot-Image Allow: / User-agent: Googlebot-Video Allow: / User-agent: Bingbot Allow: / Allow: /sitemap.xml Allow: /llms.txt Allow: /llms-full.txt Allow: /ai.txt Allow: /.well-known/mcp.json User-agent: Slurp Allow: / User-agent: DuckDuckBot Allow: / User-agent: Baiduspider Allow: / User-agent: YandexBot Allow: / User-agent: Sogou Allow: / User-agent: facebookexternalhit Allow: / User-agent: FacebookExternalHit Allow: / User-agent: Twitterbot Allow: / User-agent: LinkedInBot Allow: / # ══════════════════════════════════════════════════════════════════════════════ # ALL OTHER BOTS — Public pages allowed, private areas blocked # ══════════════════════════════════════════════════════════════════════════════ User-agent: * Allow: / Allow: /sitemap.xml Allow: /geo-sitemap.xml Allow: /llms.txt Allow: /llms-full.txt Allow: /ai.txt Allow: /.well-known/mcp.json # Disallow private user areas Disallow: /api/ Disallow: /dashboard Disallow: /dashboard/ Disallow: /settings Disallow: /auth Disallow: /clients/ Disallow: /my-plan/ Disallow: /sessions Disallow: /messages Disallow: /templates Disallow: /classes Disallow: /payments Disallow: /nutrition Disallow: /partner-dashboard Disallow: /subscription Disallow: /gyms Disallow: /admin Disallow: /agent-office Disallow: /workout-session Disallow: /workouts Disallow: /achievements Disallow: /revenue Disallow: /partner-onboarding Disallow: /offline.html Disallow: /AI_Architecture_Presentation.html Disallow: /Architecture_Secure.html Disallow: /ai-architecture-presentation Disallow: /architecture-secure Disallow: /login Disallow: /signup Disallow: /reset-password Disallow: /onboarding Disallow: /wearables Disallow: /my-progress Disallow: /check-ins Disallow: /notifications Disallow: /form-analysis Disallow: /recovery Disallow: /affiliate-dashboard Disallow: /nutrition-hub Disallow: /recovery-hub Disallow: /wearable-hub # Sitemap locations Sitemap: https://www.neuronpathway.app/sitemap.xml Sitemap: https://www.neuronpathway.app/geo-sitemap.xml # AI / LLM discovery (GEO): machine-readable site index for crawlers # https://www.neuronpathway.app/llms.txt # https://www.neuronpathway.app/llms-full.txt # https://www.neuronpathway.app/ai.txt