User-Agent: Yandex
Disallow: /gxp-admin/
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/themes/
Disallow: /stats/
Disallow: /en/author/
Disallow: /novosti/
Disallow: /novosti-gxp-mobile/
Disallow: /events/tag/
Disallow: /author/*page/
Disallow: /events/*page/
Disallow: /*?s=
Disallow: /comments/feed/
Allow: /wp-admin/admin-ajax.php
Allow: /wp-content/uploads/
Clean-param: did&s&_gl&etext&from&tribe-bar-date&trk&tribe_event_display&tribe_paged&sid&tribe_events&url&utm&yadclid&yadordid&i&a&isappinstalled&hl&rsd&tdbLoadDataFromId&clid&erid&eventDate&eventDisplay&noamp
Clean-param: __FB_PRIVATE_TRACKING__&_ym_status-check&_ym_lang&_ym_debug&adv&bsa_pro_stats&bsa_pro_email&bsa_pro_id&category&lang&post_type&noredir&nw&tribe-bar-search&ref&td_action&post_id&tdbTemplateType
Clean-param: amp&utm_order_number&posts_per_page

User-Agent: Googlebot
Disallow: /gxp-admin/
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/themes/
Disallow: /stats/
Disallow: /en/author/
Disallow: /novosti/
Disallow: /novosti-gxp-mobile/
Disallow: /events/tag/
Disallow: /author/*page/
Disallow: /events/*page/
Disallow: /*?s=
Disallow: /comments/feed/
Allow: /wp-admin/admin-ajax.p

User-Agent: *
Disallow: /gxp-admin/
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/themes/
Disallow: /stats/
Disallow: /en/author/
Disallow: /novosti/
Disallow: /novosti-gxp-mobile/
Disallow: /events/tag/
Disallow: /author/*page/
Disallow: /events/*page/
Disallow: /*?s=
Disallow: /comments/feed/
Allow: /wp-admin/admin-ajax.php
Allow: /wp-content/uploads/

User-agent: Googlebot-Image
Crawl-delay: 1
Disallow: /gxp-admin/


# ====================================================================
# === AI-БОТЫ С ОТКРЫТЫМ ДОСТУПОМ ===
# ====================================================================

# === OpenAI ChatGPT ===
# GPTBot - для обучения моделей и SearchGPT
User-agent: GPTBot
Disallow: /gxp-admin/
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/themes/
Disallow: /stats/
Disallow: /comments/feed/
Disallow: /*?s=
Disallow: /events/*page/

# ChatGPT-User - для ответов пользователям в ChatGPT
User-agent: ChatGPT-User
Disallow: /gxp-admin/
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/themes/
Disallow: /stats/
Disallow: /comments/feed/
Disallow: /*?s=
Disallow: /events/*page/

# OAI-SearchBot - SearchGPT
User-agent: OAI-SearchBot
Disallow: /gxp-admin/
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/themes/
Disallow: /stats/

# === Perplexity AI ===
User-agent: PerplexityBot
Disallow: /gxp-admin/
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/themes/
Disallow: /stats/
Disallow: /comments/feed/
Disallow: /*?s=

# === You.com AI-поисковик ===
User-agent: YouBot
Disallow: /gxp-admin/
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/themes/
Disallow: /stats/

# === Anthropic Claude ===
User-agent: anthropic-ai
Disallow: /gxp-admin/
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/themes/
Disallow: /stats/

User-agent: Claude-Web
Disallow: /gxp-admin/
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/themes/
Disallow: /stats/

# ClaudeBot - основной краулер Anthropic
User-agent: ClaudeBot
Disallow: /gxp-admin/
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/themes/
Disallow: /stats/

# === DeepSeek AI ===
User-agent: DeepSeek
Disallow: /gxp-admin/
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/themes/
Disallow: /stats/

User-agent: DeepSeekBot
Disallow: /gxp-admin/
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/themes/
Disallow: /stats/

# === Google Gemini/Bard ===
User-agent: Google-Extended
Disallow: /gxp-admin/
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/themes/
Disallow: /stats/
Disallow: /comments/feed/
Disallow: /*?s=
Disallow: /events/*page/

# === Apple Intelligence ===
User-agent: Applebot-Extended
Disallow: /gxp-admin/
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/themes/
Disallow: /stats/

# === Meta AI ===
User-agent: FacebookBot
Disallow: /wp-admin/
Disallow: /gxp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/themes/
Disallow: /*?*

User-agent: Meta-ExternalAgent
Crawl-delay: 15
Disallow: /wp-admin/
Disallow: /gxp-admin/
Disallow: /wp-includes/

# === Amazon Alexa / AWS AI ===
User-agent: Amazonbot
Disallow: /gxp-admin/
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/themes/
Disallow: /stats/

# === Mistral AI ===
User-agent: MistralBot
Disallow: /gxp-admin/
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /stats/

User-agent: mistral-crawler
Disallow: /gxp-admin/
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /stats/

# === xAI Grok ===
User-agent: Grokbot
Disallow: /gxp-admin/
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /stats/

# ========================================
# === ПОЛНАЯ БЛОКИРОВКА ВРЕДНЫХ AI-БОТОВ ===
# ========================================

# Common Crawl (только для датасетов, не приносит пользы)
User-agent: CCBot
Disallow: /

# Cohere AI (только обучение)
User-agent: cohere-ai
Disallow: /

# Bytedance/TikTok (агрессивный краулер)
User-agent: Bytespider
Disallow: /

# Коммерческие парсеры данных
User-agent: Omgilibot
Disallow: /

User-agent: Diffbot
Disallow: /

# Img2Dataset (парсинг изображений для обучения моделей)
User-agent: img2dataset
Disallow: /

# === SEO-ИНСТРУМЕНТЫ (с умеренной задержкой) ===

User-agent: AhrefsBot
Crawl-delay: 1
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /*?*

User-agent: SemrushBot
Crawl-delay: 1
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /*?*

User-agent: MJ12bot
Crawl-delay: 1
Disallow: /*?*

User-agent: DotBot
Crawl-delay: 1

User-agent: Turnitin
Disallow: /

User-agent: Screaming Frog SEO Spider
Disallow: /

# === АГРЕССИВНЫЕ ПАРСЕРЫ (полная блокировка) ===

User-agent: SiteSnagger
Disallow: /

User-agent: WebZIP
Disallow: /

User-agent: WebCopier
Disallow: /

User-agent: HTTrack
Disallow: /

User-agent: Offline Explorer
Disallow: /

User-agent: Teleport
Disallow: /

User-agent: WebReaper
Disallow: /

User-agent: WebStripper
Disallow: /

User-agent: webprosbot
Disallow: /

User-agent: ExtractorPro
Disallow: /


# === ДРУГИЕ ПОИСКОВИКИ ===

User-agent: DuckDuckBot
Disallow: /gxp-admin/
Disallow: /wp-admin/
Disallow: /wp-includes/

User-agent: Baiduspider
Disallow: /gxp-admin/
Disallow: /wp-admin/
Disallow: /wp-includes/

User-agent: Mail.RU_Bot
Crawl-delay: 1
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /gxp-admin/