# Help My Medical Practice — robots.txt
# Doctor Marketing Agency | Medical Marketing | Website Developer for Doctors
# Slogan: "Marketing That Heals Your Practice"
# Last Updated: 2026-10-20
#
# ACCESS NOTICE (contractual — not merely a suggestion):
# Search-engine crawling and indexing, link previews, and AI search/answer
# crawlers that cite and link back are PERMITTED on public pages per the rules
# below. AI-model training, dataset assembly, bulk extraction, mirroring and
# republication are PROHIBITED. See our Terms of Use
# section 8A ("Prohibition on Scraping, Crawling & Automated Access") at
# https://helpmymdpractice.com/terms-of-service#no-scraping — access implies
# acceptance of those Terms. Enforcement contact: info@helpmymdpractice.com.
# Sitemap & LLM resource directory declared at the bottom of this file.


# ─────────────────────────────────────────────────────────────
# 1. PRIMARY SEARCH ENGINES — full crawl, no delay
# ─────────────────────────────────────────────────────────────
User-agent: Googlebot
Disallow: /internal-admin-panel
Disallow: /admin
Disallow: /auth
Disallow: /doctor-portal
Disallow: /reset-password
Disallow: /thank-you
Disallow: /unsubscribe
Disallow: /practice-portal/
Disallow: /my-call/
Disallow: /schedule-consultation/
Disallow: /appointment-confirmed
Disallow: /honeypot-trap
Disallow: /wp-admin
Disallow: /wp-login.php
Disallow: /*?*utm_
Disallow: /*?*fbclid
Disallow: /*?*gclid
Allow: /

User-agent: Googlebot-Image
Disallow: /admin
Disallow: /auth
Disallow: /doctor-portal
Disallow: /reset-password
Disallow: /internal-admin-panel
Allow: /
Allow: /*.jpg$
Allow: /*.jpeg$
Allow: /*.png$
Allow: /*.webp$
Allow: /*.svg$

# A crawler obeys only the most specific matching User-agent group, so the
# sensitive-path Disallows from the "*" group are repeated in every block.
User-agent: Googlebot-News
Allow: /blog/
Allow: /media/
Disallow: /admin
Disallow: /auth
Disallow: /doctor-portal
Disallow: /reset-password
Disallow: /internal-admin-panel

User-agent: Googlebot-Video
Disallow: /admin
Disallow: /auth
Disallow: /doctor-portal
Disallow: /reset-password
Disallow: /internal-admin-panel
Allow: /

User-agent: AdsBot-Google
Disallow: /admin
Disallow: /auth
Disallow: /doctor-portal
Disallow: /reset-password
Disallow: /internal-admin-panel
Allow: /


User-agent: Bingbot
Disallow: /admin
Disallow: /auth
Disallow: /doctor-portal
Disallow: /reset-password
Disallow: /internal-admin-panel
Disallow: /thank-you
Disallow: /unsubscribe
Disallow: /practice-portal/
Disallow: /my-call/
Disallow: /schedule-consultation/
Disallow: /appointment-confirmed
Allow: /
Crawl-delay: 1

User-agent: Slurp
Disallow: /admin
Disallow: /auth
Disallow: /doctor-portal
Disallow: /reset-password
Disallow: /internal-admin-panel
Allow: /

User-agent: DuckDuckBot
Disallow: /admin
Disallow: /auth
Disallow: /doctor-portal
Disallow: /reset-password
Allow: /

User-agent: Yandex
Disallow: /admin
Disallow: /auth
Disallow: /doctor-portal
Disallow: /reset-password
Allow: /

User-agent: Baiduspider
Disallow: /admin
Disallow: /auth
Disallow: /doctor-portal
Disallow: /reset-password
Allow: /

User-agent: BraveBot
Disallow: /admin
Allow: /

User-agent: Applebot
Disallow: /admin
Allow: /

# ─────────────────────────────────────────────────────────────
# 2. SOCIAL MEDIA PREVIEW CRAWLERS — must access OG images
# ─────────────────────────────────────────────────────────────
User-agent: Twitterbot
Allow: /

User-agent: facebookexternalhit
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: Pinterestbot
Allow: /

User-agent: WhatsApp
Allow: /

User-agent: TelegramBot
Allow: /

User-agent: Slackbot
Allow: /

User-agent: Discordbot
Allow: /

# ─────────────────────────────────────────────────────────────
# 3. AI SEARCH / ANSWER CRAWLERS — allowed on public pages
# These fetch pages to show cited, linked answers (ChatGPT search, Claude,
# Perplexity, Copilot, etc.). They are NOT model-training crawlers.
# ─────────────────────────────────────────────────────────────
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: DuckAssistBot
User-agent: MistralAI-User
User-agent: Meta-ExternalFetcher
User-agent: YouBot
User-agent: Amazonbot
Disallow: /admin
Disallow: /auth
Disallow: /doctor-portal
Disallow: /reset-password
Disallow: /internal-admin-panel
Disallow: /thank-you
Disallow: /unsubscribe
Disallow: /practice-portal/
Disallow: /my-call/
Disallow: /schedule-consultation/
Disallow: /appointment-confirmed
Disallow: /honeypot-trap
Allow: /

# ─────────────────────────────────────────────────────────────
# 4. AI MODEL-TRAINING & BULK DATASET CRAWLERS — blocked
# No licence is granted to train, fine-tune or evaluate models, or to build
# datasets from this site (Terms of Use §8A). Blocking these does NOT affect
# Google/Bing search or the AI answer crawlers in section 3.
# ─────────────────────────────────────────────────────────────
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: anthropic-ai
User-agent: Claude-Web
User-agent: Google-Extended
User-agent: GoogleOther
User-agent: Applebot-Extended
User-agent: Meta-ExternalAgent
User-agent: FacebookBot
User-agent: CCBot
User-agent: Bytespider
User-agent: cohere-ai
User-agent: cohere-training-data-crawler
User-agent: Diffbot
User-agent: ImagesiftBot
User-agent: Omgilibot
User-agent: Omgili
User-agent: Webzio-Extended
User-agent: PiplBot
User-agent: AI2Bot
User-agent: Timpibot
User-agent: iaskspider
User-agent: Kangaroo Bot
User-agent: DeepSeekBot
User-agent: PanguBot
User-agent: img2dataset
Disallow: /

# ─────────────────────────────────────────────────────────────
# 5. SEO SNOOPS — blocked (backlink/keyword harvesters, not search engines)
# ─────────────────────────────────────────────────────────────
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /


User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: rogerbot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

# ─────────────────────────────────────────────────────────────
# 6. ABUSIVE / VULNERABILITY SCANNERS — blocked
# ─────────────────────────────────────────────────────────────
User-agent: SiteAuditBot
Disallow: /

User-agent: ZoominfoBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: serpstatbot
Disallow: /

# Generic scrapers, automation frameworks, and script clients — blocked.
# Also enumerated in .htaccess (see deployment/htaccess-prerender-io-snippet.txt)
# for server-side enforcement independent of robots.txt honor.
User-agent: HTTrack
Disallow: /

User-agent: WebCopier
Disallow: /

User-agent: WebZIP
Disallow: /

User-agent: Xenu
Disallow: /

User-agent: SiteSnagger
Disallow: /

User-agent: ProWebWalker
Disallow: /

User-agent: CheeseBot
Disallow: /

User-agent: NetMechanic
Disallow: /

User-agent: EmailCollector
Disallow: /

User-agent: EmailSiphon
Disallow: /

User-agent: EmailWolf
Disallow: /

User-agent: ExtractorPro
Disallow: /

User-agent: python-requests
Disallow: /

User-agent: python-urllib
Disallow: /

User-agent: Go-http-client
Disallow: /

User-agent: node-fetch
Disallow: /

User-agent: axios
Disallow: /

User-agent: Scrapy
Disallow: /

User-agent: HeadlessChrome
Disallow: /

User-agent: PhantomJS
Disallow: /

User-agent: Puppeteer
Disallow: /

User-agent: Playwright
Disallow: /

User-agent: Selenium
Disallow: /

User-agent: curl
Disallow: /

User-agent: Wget
Disallow: /

User-agent: libwww-perl
Disallow: /

User-agent: Java
Disallow: /


# ─────────────────────────────────────────────────────────────
# ─────────────────────────────────────────────────────────────
# 7. DEFAULT — other well-behaved crawlers may index public pages.
#    Training/bulk extraction remains prohibited by Terms of Use §8A. Sensitive/internal routes stay blocked below.
# ─────────────────────────────────────────────────────────────
User-agent: *
Disallow: /admin
Disallow: /auth
Disallow: /doctor-portal
Disallow: /reset-password
Disallow: /thank-you
Disallow: /unsubscribe
Disallow: /practice-portal/
Disallow: /my-call/
Disallow: /schedule-consultation/
Disallow: /appointment-confirmed
Disallow: /internal-admin-panel
Disallow: /wp-admin
Disallow: /wp-login.php
Disallow: /honeypot-trap
Disallow: /lp/
Disallow: /*?*utm_
Disallow: /*?*fbclid
Disallow: /*?*gclid
Allow: /

# Critical SEO / discovery assets remain accessible to ANY UA so that
# verification probes (Google Search Console, Bing Webmaster, IndexNow,
# llms.txt directories) keep working even under strict allowlist mode.
Allow: /sitemap.xml
Allow: /sitemap-index.xml
Allow: /sitemap-*.xml
Allow: /robots.txt
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /ai.txt
Allow: /humans.txt
Allow: /favicon.ico
Allow: /manifest.json
Allow: /og-image.jpg
Allow: /icons/
Allow: /.well-known/ai-plugin.json
Allow: /.well-known/openapi.yaml

# ─────────────────────────────────────────────────────────────
# 8. SITEMAPS — canonical, finalized set
# ─────────────────────────────────────────────────────────────
# Canonical sitemap directory for public search and AI resource discovery.
# Primary aggregator — index-aware crawlers (Google, Bing, Yandex) follow
# every child sitemap listed inside sitemap-index.xml automatically.
Sitemap: https://helpmymdpractice.com/sitemap-index.xml
# Legacy master sitemap — kept for non-index-aware crawlers and the build validator.
Sitemap: https://helpmymdpractice.com/sitemap.xml
# Blog sitemap — refreshed through explicit SEO maintenance.
Sitemap: https://helpmymdpractice.com/sitemap-blog.xml
# Blog category hubs — crawlable ?category= filter URLs, canonical to themselves.
Sitemap: https://helpmymdpractice.com/sitemap-blog-categories.xml
# Answer pages — /answers/:slug detail URLs (dynamic route, one entry per published answer).
Sitemap: https://helpmymdpractice.com/sitemap-answers.xml
# Remaining topical child sitemaps, listed explicitly for non-index-aware crawlers.
Sitemap: https://helpmymdpractice.com/sitemap-main.xml
Sitemap: https://helpmymdpractice.com/sitemap-services.xml
Sitemap: https://helpmymdpractice.com/sitemap-specialties.xml
Sitemap: https://helpmymdpractice.com/sitemap-geo.xml
Sitemap: https://helpmymdpractice.com/sitemap-cities.xml
Sitemap: https://helpmymdpractice.com/sitemap-media.xml
Sitemap: https://helpmymdpractice.com/sitemap-legal.xml
Sitemap: https://helpmymdpractice.com/sitemap-prerendered.xml

# ─────────────────────────────────────────────────────────────
# 9. AI / LLM DISCOVERY DIRECTORIES
# ─────────────────────────────────────────────────────────────
# LLMs.txt:        https://helpmymdpractice.com/llms.txt
# LLMs-full.txt:   https://helpmymdpractice.com/llms-full.txt
# AI plugin:       https://helpmymdpractice.com/.well-known/ai-plugin.json
# OpenAPI:         https://helpmymdpractice.com/.well-known/openapi.yaml
# AI policy:       https://helpmymdpractice.com/ai.txt
# Knowledge base:  https://helpmymdpractice.com/knowledge-base

# ─────────────────────────────────────────────────────────────
# Keywords: doctor marketing agency, medical marketing agency, doctor marketing,
# website design for doctors, medical SEO,
# urology marketing, orthopedic marketing, sports medicine marketing, regenerative
# medicine marketing, age management marketing, peptide marketing, stem cell
# marketing, dermatology marketing, medispa marketing, DPC marketing, concierge
# marketing, OB-GYN marketing, telemedicine marketing, lead generation for doctors,
# practice development, HIPAA-compliant website design, marketing that heals your practice