# ===== Sitemap =====
Sitemap: https://www.beyondeast.com/sitemap.xml

# ===== Allow Major Search Bots =====
User-agent: Googlebot
Disallow:

User-agent: Googlebot-Image
Disallow:

User-agent: Bingbot
Crawl-delay: 5
Disallow: /admin
Disallow: /cart
Disallow: /carts
Disallow: /orders
Disallow: /account
Disallow: /checkouts

User-agent: Applebot
Allow: /
Crawl-delay: 5

User-agent: Pinterest
Crawl-delay: 10

User-agent: AdsBot-Google
Disallow: /cart
Disallow: /checkouts
Disallow: /orders
Disallow: /account
Disallow: /*?*utm_
Disallow: /preview_theme_id
Disallow: /preview_script_id

# ===== Allow AI Bots =====
User-agent: GPTBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: AnthropicBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: GrokBot
Allow: /

User-agent: CCBot
Allow: /

# ===== All Other Bots =====
User-agent: *
Disallow: /admin
Disallow: /cart
Disallow: /carts
Disallow: /orders
Disallow: /account
Disallow: /checkouts
Disallow: /checkout
Disallow: /.well-known/shopify/monorail
Disallow: /wpm@*
Disallow: /custom/web-pixel-shopify-custom-pixel*

# --- Duplicate/Thin URLs (filters, sorting, pagination) ---
Disallow: /collections/sort_by
Disallow: /collections/+
Disallow: /collections/%2B
Disallow: /collections/filter
Disallow: /collections/tag=
Disallow: /blogs/+
Disallow: /blogs/%2B

# --- Parameter-based Duplicate URLs ---
Disallow: /*?*utm_
Disallow: /*?variant=
Disallow: /*?pr_prod_strat=
Disallow: /*?pr_rec_id=
Disallow: /*?pr_rec_pid=
Disallow: /*?pr_ref_pid=
Disallow: /*?pr_seq=
Disallow: /*?srsltid=
Disallow: /*?start=
Disallow: /*?sz=
Disallow: /*?cgid=
Disallow: /pmin
Disallow: /pmax
Disallow: /prefn1
Disallow: /prefv1
Disallow: /srule
Disallow: /selectedUrl
Disallow: /search?q=

# --- Locale duplication ---
Disallow: /en-us/

# --- Search Pages ---
Disallow: /search
Disallow: /search-result-page

# --- Atom Feeds ---
Disallow: /*.atom$

# ===== Throttle or Block Specific Bots =====
User-agent: facebookexternalhit/1.1
Crawl-delay: 0
User-agent: facebookexternalhit/1.0
Crawl-delay: 15
User-agent: facebookexternalhit/*
Crawl-delay: 15

User-agent: AhrefsBot
Crawl-delay: 10
Disallow: /admin
Disallow: /cart
Disallow: /checkouts
Disallow: /account

User-agent: MJ12bot
Disallow: /

# ===== Block Known Bad Bots =====
User-agent: BLEXBot
Disallow: /
User-agent: Baiduspider
Disallow: /
User-agent: EasouSpider
Disallow: /
User-agent: Exabot
Disallow: /
User-agent: trovitbot
Disallow: /
User-agent: sitecheck.internetseer.com
Disallow: /
User-agent: Zealbot
Disallow: /
User-agent: MSIECrawler
Disallow: /
User-agent: SiteSnagger
Disallow: /
User-agent: WebStripper
Disallow: /
User-agent: WebCopier
Disallow: /
User-agent: Fetch
Disallow: /
User-agent: Offline Explorer
Disallow: /
User-agent: Teleport
Disallow: /
User-agent: TeleportPro
Disallow: /
User-agent: WebZIP
Disallow: /
User-agent: linko
Disallow: /
User-agent: HTTrack
Disallow: /
User-agent: Xenu
Disallow: /
User-agent: larbin
Disallow: /
User-agent: libwww
Disallow: /
User-agent: ZyBORG
Disallow: /
User-agent: Download Ninja
Disallow: /
User-agent: WebReaper
Disallow: /
User-agent: ZumBot
Disallow: /

# ===== Nutch Crawler =====
User-agent: Nutch
Disallow: /