# Robots.txt for AMZprep.com
# Last Updated: October 2026
# Purpose: AI-optimized and SEO-ready crawl management
# Strategy: Allow all legitimate crawlers (including AI) to crawl content,
#           block admin, system, search and tracking-parameter URLs for every bot

# ============================================
# ALL LEGITIMATE CRAWLERS: SEARCH ENGINES, AI BOTS, SEO TOOLS
# ============================================
# AI search and assistant bots are named explicitly to signal they are welcome.
# They share this group, so the same Disallow rules apply to them.
# Any bot not named here (Googlebot, Bingbot, GPTBot, Claude-SearchBot,
# Applebot, etc.) falls under User-agent: * and follows the same rules.
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: PerplexityBot
User-agent: Google-Extended
User-agent: *
Allow: /llms.txt

# Block WordPress admin area
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php

# Block WordPress system files
Disallow: /wp-login.php
Disallow: /wp-register.php
Disallow: /xmlrpc.php

# Block sensitive WP JSON endpoints (general API stays open)
Disallow: /wp-json/wp/v2/users
Disallow: /wp-json/wp/v2/settings

# Block internal search results (duplicate content)
Disallow: /?s=
Disallow: /*?s=
Disallow: /*&s=
Disallow: /search/

# Block tracking parameters (duplicate content)
# "?*" matches the parameter whether it appears first or later in the query string
Disallow: /*?*utm_source=
Disallow: /*?*utm_medium=
Disallow: /*?*utm_campaign=
Disallow: /*?*utm_term=
Disallow: /*?*utm_content=
Disallow: /*?*ref=
Disallow: /*?*fbclid=
Disallow: /*?*gclid=
Disallow: /*?*msclkid=

# Block HubSpot tracking parameters
Disallow: /*?*__hstc=
Disallow: /*?*__hssc=
Disallow: /*?*__hsfp=
Disallow: /*?*_hsenc=
Disallow: /*?*_hsmi=
Disallow: /*?*hsCtaTracking=

# Block AMP/nonamp parameter variants
Disallow: /*?*nonamp=
# Block vCard download URLs
Disallow: /*?amz_vcard=
Disallow: /*?*&amz_vcard=
# ============================================
# BLOCK HARMFUL BOTS
# ============================================
# Note: badly behaved bots often ignore robots.txt.
# Enforce real blocking in SiteGround Site Tools > Security.

# Content scrapers
User-agent: HTTrack
User-agent: Wget
User-agent: Offline Explorer
User-agent: Teleport
User-agent: TeleportPro
User-agent: WebStripper
User-agent: WebCopier
User-agent: WebZIP
User-agent: Website.eXtractor
User-agent: WebReaper
# Email harvesters
User-agent: EmailCollector
User-agent: EmailSiphon
User-agent: EmailWolf
User-agent: ExtractorPro
# Malicious bots
User-agent: CherryPicker
User-agent: WebBandit
User-agent: BackDoorBot
User-agent: Cogentbot
User-agent: EroCrawler
User-agent: Zeus
User-agent: MSIECrawler
# Aggressive link checkers
User-agent: Xenu
User-agent: Xenu's Link Sleuth
User-agent: LinkWalker
Disallow: /

# ============================================
# SITEMAP
# ============================================
# Rank Math's index lists the post, page and other sitemaps
Sitemap: https://amzprep.com/sitemap_index.xml