# Mr. Woodini - robots.txt
# Last updated: Aug 2026
# Optimized for small ecommerce (WooCommerce + WPML)

# ── General crawlers ──────────────────────────────
User-agent: *

# WordPress core
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /wp-json/
Allow: /wp-admin/admin-ajax.php

# WooCommerce functional pages
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /lost-password/
Disallow: /wc-api/
Disallow: /wishlist/
Disallow: /wp-content/uploads/woocommerce_uploads/

# Duplicate content — WooCommerce parameters
Disallow: /*?add-to-cart=*
Disallow: /*add-to-cart=*
Disallow: /*?orderby=*
Disallow: /*?currency=*
Disallow: /*?min_price=*
Disallow: /*?max_price=*
Disallow: /*?variation_id=*
Disallow: /*?attribute_pa_*
Disallow: /*?filter_color=*
Disallow: /*?filter_size=*

# Duplicate content — wishlist
Disallow: /*add_to_wishlist=*
Disallow: /*remove_from_wishlist=*

# Duplicate content — search & pagination
Disallow: /search/
Disallow: /?s=
Disallow: /shop/page/

# Duplicate content — author archives
Disallow: /author/

# WPML language parameter (handled via directories, not params)
Disallow: /*?lang=*

Allow: /wp-content/uploads/

# ── Search engines (drive real traffic — no throttle) ─────
User-agent: Googlebot
Allow: /

User-agent: Googlebot-Image
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Slurp
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Baiduspider
Allow: /

User-agent: YandexBot
Allow: /

User-agent: Sogou
Allow: /

User-agent: Exabot
Allow: /

# ── AI crawlers ────────────────────────────────────────────
# CHANGED: added Crawl-delay to every AI crawler. These don't
# need real-time freshness like a search engine does — throttling
# them costs nothing and reduces concurrent load on the server
# (relevant given our PHP-FPM worker pool is limited).
User-agent: GPTBot
Crawl-delay: 10
Allow: /

User-agent: ChatGPT-User
Crawl-delay: 10
Allow: /

User-agent: OAI-SearchBot
Crawl-delay: 10
Allow: /

User-agent: ClaudeBot
Crawl-delay: 10
Allow: /

User-agent: Claude-Web
Crawl-delay: 10
Allow: /

User-agent: anthropic-ai
Crawl-delay: 10
Allow: /

User-agent: Google-Extended
Crawl-delay: 10
Allow: /

User-agent: GoogleOther
Crawl-delay: 10
Allow: /

User-agent: PerplexityBot
Crawl-delay: 10
Allow: /

User-agent: cohere-ai
Crawl-delay: 10
Allow: /

User-agent: Diffbot
Crawl-delay: 10
Allow: /

User-agent: YouBot
Crawl-delay: 10
Allow: /

User-agent: FacebookBot
Crawl-delay: 10
Allow: /

User-agent: Meta-ExternalAgent
Crawl-delay: 10
Allow: /

User-agent: Applebot-Extended
Crawl-delay: 10
Allow: /

User-agent: ImagesiftBot
Crawl-delay: 10
Allow: /

# CHANGED: Bytespider (ByteDance/TikTok) blocked entirely —
# known in the industry as an unusually heavy/aggressive crawler
# with little benefit for a store like this. Remove this block
# and add "Crawl-delay: 10 / Allow: /" instead if you specifically
# want visibility in TikTok's AI products.
User-agent: Bytespider
Disallow: /

# ── SEO tools ──────────────────────────────────────────────
# CHANGED: added Crawl-delay. If you don't personally use Ahrefs
# or Semrush to monitor this site, consider replacing "Crawl-delay:
# 10 / Allow: /" with "Disallow: /" for these three — they provide
# no benefit to you and only consume server resources.
User-agent: Ahrefsbot
Crawl-delay: 10
Allow: /

User-agent: SemrushBot
Crawl-delay: 10
Allow: /

User-agent: MJ12bot
Crawl-delay: 10
Allow: /

# ── Aggressive crawlers — throttle ────────────────
User-agent: DotBot
Crawl-delay: 10

User-agent: BLEXBot
Crawl-delay: 10

User-agent: Rogerbot
Crawl-delay: 10

User-agent: SiteBot
Crawl-delay: 10

User-agent: MegaIndex
Crawl-delay: 10

# ── Sitemap ───────────────────────────────────────
Sitemap: https://www.mrwoodini.com/sitemap_index.xml