47 lines
804 B
Plaintext
47 lines
804 B
Plaintext
# =============================================
|
||
# robots.txt для oakazanin.ru
|
||
# IT-блог на Hugo | /posts/ | RU
|
||
# =============================================
|
||
|
||
# --- Яндекс ---
|
||
User-agent: Yandex
|
||
Allow: /
|
||
Allow: /posts/
|
||
Disallow: /tags/
|
||
Disallow: /categories/
|
||
Disallow: /page/
|
||
Crawl-delay: 1.0
|
||
|
||
# --- Google ---
|
||
User-agent: Googlebot
|
||
Allow: /
|
||
Allow: /posts/
|
||
Disallow: /tags/
|
||
Disallow: /categories/
|
||
Disallow: /page/
|
||
|
||
# --- AI-боты ---
|
||
User-agent: GPTBot
|
||
Allow: /
|
||
|
||
User-agent: PerplexityBot
|
||
Allow: /
|
||
|
||
User-agent: anthropic-ai
|
||
Allow: /
|
||
|
||
User-agent: Claude-Web
|
||
Allow: /
|
||
|
||
# --- Все остальные боты ---
|
||
User-agent: *
|
||
Allow: /
|
||
Allow: /posts/
|
||
Disallow: /tags/
|
||
Disallow: /categories/
|
||
Disallow: /page/
|
||
Crawl-delay: 2.0
|
||
|
||
# --- Sitemap ---
|
||
Sitemap: https://oakazanin.ru/sitemap.xml
|