fix: дополнил robots.txt

2026-02-27 14:19:52 +00:00
parent dc532ca592
commit 1cbbb645fb
1 changed files with 16 additions and 8 deletions
@@ -1,9 +1,4 @@
-# =============================================
-# robots.txt для oakazanin.ru
-# IT-блог на Hugo | /posts/ | RU
-# =============================================
-
-# --- Яндекс ---
+# --- Yandex ---
 User-agent: Yandex
 Allow: /
 Allow: /posts/
@@ -20,7 +15,7 @@ Disallow: /tags/
 Disallow: /categories/
 Disallow: /page/

-# --- AI-боты ---
+# --- AI-bots ---
 User-agent: GPTBot
 Allow: /

@@ -33,7 +28,20 @@ Allow: /
 User-agent: Claude-Web
 Allow: /

-# --- Все остальные боты ---
+User-agent: ClaudeBot
+Allow: /
+
+User-agent: Google-Extended
+Allow: /
+
+# --- Google AI Overview ---
+User-agent: Google-Extended
+Allow: /
+
+User-agent: ClaudeBot
+Allow: /
+
+# --- All other bots ---
 User-agent: *
 Allow: /
 Allow: /posts/