55 lines
774 B
Plaintext
55 lines
774 B
Plaintext
# --- Yandex ---
|
|
User-agent: Yandex
|
|
Allow: /
|
|
Allow: /posts/
|
|
Disallow: /tags/
|
|
Disallow: /categories/
|
|
Disallow: /page/
|
|
Crawl-delay: 1.0
|
|
|
|
# --- Google ---
|
|
User-agent: Googlebot
|
|
Allow: /
|
|
Allow: /posts/
|
|
Disallow: /tags/
|
|
Disallow: /categories/
|
|
Disallow: /page/
|
|
|
|
# --- AI-bots ---
|
|
User-agent: GPTBot
|
|
Allow: /
|
|
|
|
User-agent: PerplexityBot
|
|
Allow: /
|
|
|
|
User-agent: anthropic-ai
|
|
Allow: /
|
|
|
|
User-agent: Claude-Web
|
|
Allow: /
|
|
|
|
User-agent: ClaudeBot
|
|
Allow: /
|
|
|
|
User-agent: Google-Extended
|
|
Allow: /
|
|
|
|
# --- Google AI Overview ---
|
|
User-agent: Google-Extended
|
|
Allow: /
|
|
|
|
User-agent: ClaudeBot
|
|
Allow: /
|
|
|
|
# --- All other bots ---
|
|
User-agent: *
|
|
Allow: /
|
|
Allow: /posts/
|
|
Disallow: /tags/
|
|
Disallow: /categories/
|
|
Disallow: /page/
|
|
Crawl-delay: 2.0
|
|
|
|
# --- Sitemap ---
|
|
Sitemap: https://oakazanin.ru/sitemap.xml
|