feat: обновил robots.txt

2026-02-27 13:58:47 +00:00
parent b63ff31c66
commit dc532ca592
63 changed files with 560 additions and 172 deletions
@@ -0,0 +1,46 @@
+# =============================================
+# robots.txt для oakazanin.ru
+# IT-блог на Hugo | /posts/ | RU
+# =============================================
+
+# --- Яндекс ---
+User-agent: Yandex
+Allow: /
+Allow: /posts/
+Disallow: /tags/
+Disallow: /categories/
+Disallow: /page/
+Crawl-delay: 1.0
+
+# --- Google ---
+User-agent: Googlebot
+Allow: /
+Allow: /posts/
+Disallow: /tags/
+Disallow: /categories/
+Disallow: /page/
+
+# --- AI-боты ---
+User-agent: GPTBot
+Allow: /
+
+User-agent: PerplexityBot
+Allow: /
+
+User-agent: anthropic-ai
+Allow: /
+
+User-agent: Claude-Web
+Allow: /
+
+# --- Все остальные боты ---
+User-agent: *
+Allow: /
+Allow: /posts/
+Disallow: /tags/
+Disallow: /categories/
+Disallow: /page/
+Crawl-delay: 2.0
+
+# --- Sitemap ---
+Sitemap: https://oakazanin.ru/sitemap.xml