# robots.txt for Subconex - Enterprise E-commerce Platform
# Last updated: 2024
# Environment: Production

# Global rules for all crawlers
User-agent: *
Allow: /
Allow: /*/products/
Allow: /*/categories/
Allow: /*/blog/
Allow: /*/about/
Allow: /*/features/
Allow: /*/howitworks/
Allow: /*/faq/
Allow: /*/pricing/
Allow: /en/
Allow: /fr/
Allow: /it/
Allow: /es/
Allow: /de/

# Disallow sensitive and dynamic content
Disallow: /admin/
Disallow: /dashboard/
Disallow: /api/
Disallow: /private/
Disallow: /tmp/
Disallow: /backup/
Disallow: /config/
Disallow: /logs/
Disallow: /*/cart
Disallow: /*/checkout
Disallow: /*/account/
Disallow: /*/order/
Disallow: /*/profile/
Disallow: /*/wishlist
Disallow: /*/search?*
Disallow: /*.json$
Disallow: /*.js$
Disallow: /*.css$
Disallow: /*?*
Disallow: /*/print/
Disallow: /*/preview/
Disallow: /*/draft/
Disallow: /*/pending/

# Googlebot specific rules
User-agent: Googlebot
Allow: /
Allow: /*/products/
Allow: /*/blog/
Allow: /*/sitemap*.xml
Disallow: /*/cart
Disallow: /*/checkout
Disallow: /admin/
Disallow: /api/
Crawl-delay: 1

# Googlebot Image
User-agent: Googlebot-Image
Allow: /*.jpg
Allow: /*.jpeg
Allow: /*.png
Allow: /*.gif
Allow: /*.webp
Allow: /*.svg
Disallow: /admin/

# Googlebot Video
User-agent: Googlebot-Video
Allow: /*.mp4
Allow: /*.webm
Allow: /*.ogg

# Bingbot
User-agent: Bingbot
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /*/cart
Disallow: /*/checkout
Crawl-delay: 1

# Yandex
User-agent: Yandex
Allow: /
Allow: /*/products/
Allow: /*/blog/
Allow: /sitemap*.xml
Disallow: /admin/
Disallow: /api/
Disallow: /*/cart
Disallow: /*/checkout
Crawl-delay: 2

# Baidu
User-agent: Baiduspider
Allow: /
Disallow: /admin/
Disallow: /api/
Crawl-delay: 2

# DuckDuckGo
User-agent: DuckDuckBot
Allow: /
Disallow: /admin/
Disallow: /api/

# Archive.org
User-agent: archive.org_bot
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /*/cart
Disallow: /*/checkout

# Common crawlers that should respect rules
User-agent: Slurp
Allow: /
Disallow: /admin/
Disallow: /api/

User-agent: YandexBot
Allow: /
Disallow: /admin/
Disallow: /api/

# SEO-friendly crawlers
User-agent: SemrushBot
Allow: /
Disallow: /admin/
Disallow: /api/
Crawl-delay: 5

User-agent: AhrefsBot
Allow: /
Disallow: /admin/
Disallow: /api/
Crawl-delay: 5

User-agent: MJ12bot
Allow: /
Disallow: /admin/
Disallow: /api/
Crawl-delay: 2

# Block aggressive crawlers
User-agent: DotBot
Disallow: /

User-agent: MauiBot
Disallow: /

User-agent: GPTBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

# Sitemap locations
Sitemap: https://subconex.com/sitemap.xml

# Host directive (for Yandex)
Host: https://subconex.com

# Crawl-delay for all bots (optional - adjust based on server capacity)
# Crawl-delay: 2