# ADD DOMAIN NAME HERE - KEEP SITEMAPS AT TOP
Sitemap: https://pensacolamensrehab.com/sitemap_index.xml

# ======================================================
# EXPLICITLY ALLOWED BOTS
# ======================================================

User-agent: Googlebot
Disallow:

User-agent: Bingbot
Disallow:

User-agent: DuckDuckBot
Disallow:

User-agent: BraveBot
Disallow:

User-agent: OAI-SearchBot
Disallow:

User-agent: ChatGPT-User
Disallow:

User-agent: GPTBot
Disallow:

User-agent: GoogleOther
Disallow:

User-agent: AhrefsBot
Disallow:

User-agent: Claude-SearchBot
Disallow:

User-agent: Claude-User
Disallow:

User-agent: ClaudeBot
Disallow:

User-agent: PerplexityBot
Disallow:

User-agent: xAI-SearchBot
Disallow:

User-agent: Applebot
Disallow:

User-agent: Baiduspider
Disallow:

User-agent: YandexBot
Disallow:

User-agent: CCBot
Disallow:

User-agent: FacebookBot
Disallow:

User-agent: LinkedInBot
Disallow:

User-agent: Twitterbot
Disallow:

User-agent: Slackbot
Disallow:

User-agent: AhrefsBot
Disallow:

User-agent: SemrushBot
Disallow:

User-agent: GoogleOther
Disallow:

User-agent: Googlebot-Image
Disallow:

User-agent: GoogleOther-Image
Disallow:

User-agent: MicrosoftPreview
Disallow:

User-agent: Perplexity-User
Disallow:

# ======================================================
# EXPLICITLY DISALLOWED BOTS
# ======================================================

User-agent: Ai2Bot-Dolma
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: ChatGLM-Spider
Disallow: /

User-agent: CloudVertexBot
Disallow: /

User-agent: cohere-training-data-crawler
Disallow: /

User-agent: Cotoyogi
Disallow: /

User-agent: laion-huggingface-processor
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

User-agent: PanguBot
Disallow: /

# ======================================================
# COMMERCIAL SCRAPERS & DATA AGGREGATORS (DISALLOW)
# ======================================================

User-agent: Diffbot
Disallow: /

User-agent: webzio-extended
Disallow: /

User-agent: omgili
Disallow: /

User-agent: Timpibot
Disallow: /

User-agent: VelenPublicWebCrawler
Disallow: /

User-agent: Datenbank Crawler
Disallow: /

User-agent: ICC-Crawler
Disallow: /

User-agent: Kangaroo Bot
Disallow: /

User-agent: netEstate Imprint Crawler
Disallow: /

User-agent: SBIntuitionsBot
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: Google-CloudVertexBot
Disallow: /

User-agent: ImagesiftBot
Disallow: /

User-agent: ImageSift
Disallow: /

# ======================================================
# LOW-VALUE SEO / INDEXING CRAWLERS (DISALLOW)
# ======================================================

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: SeekportBot
Disallow: /

User-agent: PetalBot
Disallow: /


# ======================================================
# SECURITY / INTERNET SCANNERS (DISALLOW)
# ======================================================

User-agent: CensysInspect
Disallow: /

# ======================================================
# --- UNIVERSAL CRAWLER RULES ---
# ======================================================

User-agent: *

# AI policy access
Allow: /llms.txt
Allow: /.well-known/ai.txt
Allow: /ai.txt

# WordPress core security paths
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-includes/
Disallow: /xmlrpc.php
Disallow: /trackback/
Disallow: /upgrade/
Disallow: /mu-plugins/
Disallow: /ai1wm-backups/
Disallow: /EMS-SITE-BACKUPS/
Disallow: /php_errorlog
Disallow: /readme.*$
Disallow: /license.txt
Disallow: /wp-login.php
Disallow: /wp-links-opml.php
Disallow: /?rest_route=
Disallow: /wp-json/

# Low-value / duplicate content
Disallow: /search/
Disallow: /*?s=
Disallow: /*?replytocom
Disallow: /author/
Disallow: /comments/
Disallow: /feed/
Disallow: /tag/
Disallow: /category/

# Account and commerce areas (if present)
Disallow: /cart/
Disallow: /checkout/
Disallow: /account/

# Allow all major asset folders (critical for SEO, indexing, and AI crawlers)
Allow: /wp-content/uploads/
Allow: /wp-content/cache/
Allow: /wp-content/themes/
Allow: /wp-content/plugins/
Allow: /fonts/
Allow: /newfonts/
Allow: /assets/
Allow: /images/
Allow: /media/

# Allow essential file types (indexable content, media, design assets)
Allow: /*.css$
Allow: /*.js$
Allow: /*.jpg$
Allow: /*.jpeg$
Allow: /*.png$
Allow: /*.gif$
Allow: /*.webp$
Allow: /*.svg$
Allow: /*.avif$
Allow: /*.ico$
Allow: /*.bmp$
Allow: /*.tif$
Allow: /*.tiff$
Allow: /*.txt$
Allow: /*.mp4$
Allow: /*.webm$
Allow: /*.mov$
Allow: /*.m4v$
Allow: /*.ogv$
Allow: /*.woff$
Allow: /*.woff2$
Allow: /*.ttf$
Allow: /*.otf$
Allow: /*.eot$
Allow: /*.mp3$
Allow: /*.wav$
Allow: /*.ogg$