# robots.txt — calze.co.il (WordPress + WooCommerce)
# Updated: 2026-08-03
#
# IMPORTANT: a crawler obeys ONE group only — the most specific
# one that matches its name. If a bot has its own "User-agent:"
# block, it ignores everything in the "User-agent: *" block.
# That is why the allowed bots below have NO group of their own:
# they inherit the full ruleset instead of bypassing it.


# =========================================================
# DEFAULT — applies to Googlebot, Bingbot, Applebot,
# Google-Extended, OAI-SearchBot, ChatGPT-User,
# Claude-SearchBot, Claude-User, PerplexityBot,
# meta-externalagent, Applebot-Extended and everyone else
# not named further down. All of these remain ALLOWED to
# crawl the site; they just no longer bypass the rules.
# =========================================================
User-agent: *

# --- WordPress internals ---
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /xmlrpc.php
Disallow: /wp-json/
Disallow: /*?rest_route=

# --- WooCommerce transactional pages (uncacheable, zero SEO value) ---
# NOTE: if these pages use Hebrew slugs, add the Hebrew paths too.
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /*add-to-cart=
Disallow: /*?wc-ajax=
Disallow: /*?removed_item=
Disallow: /*?added-to-cart=

# --- Faceted navigation / filters / sorting ---
# This is normally the single biggest source of crawl load on a
# WooCommerce store: every filter combination is a unique URL.
# Confirm the exact parameter names in GSC > Settings > Crawl stats.
Disallow: /*?filter_
Disallow: /*?query_type_
Disallow: /*?min_price=
Disallow: /*?max_price=
Disallow: /*?rating_filter=
Disallow: /*?orderby=
Disallow: /*?product_orderby=
Disallow: /*?product_count=
Disallow: /*?per_page=
Disallow: /*?jsf=
Disallow: /*?e-filter
Disallow: /*?e-page

# --- Internal search & other crawl traps ---
Disallow: /?s=
Disallow: /*?s=
Disallow: /search/
Disallow: /*?p=
Disallow: /*?attachment_id=
Disallow: /*?replytocom=
Disallow: /*/trackback/
Disallow: /*/feed/
Disallow: /comments/feed/

# --- Tracking parameters (duplicate URLs) ---
Disallow: /*?utm_
Disallow: /*?fbclid=
Disallow: /*?gclid=

# --- Keep these crawlable ---
Allow: /wp-content/uploads/
Allow: /*.css$
Allow: /*.js$

# Ignored by Googlebot, honoured by Bing / Yandex / others.
Crawl-delay: 10


# =========================================================
# AI TRAINING CRAWLERS — not permitted
# (unchanged from your original policy)
# =========================================================
User-agent: GPTBot
Disallow: /

User-agent: ClaudeBot
Disallow: /


# =========================================================
# SEO / DATA-MINING CRAWLERS — heavy load, no traffic value.
# Remove AhrefsBot and SemrushBot if you audit this site
# with those tools.
# =========================================================
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: Barkrowler
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: ImagesiftBot
Disallow: /


Sitemap: https://calze.co.il/sitemap_index.xml