diff --git a/static/robots.txt b/static/robots.txt index 1186c41..5ee762f 100644 --- a/static/robots.txt +++ b/static/robots.txt @@ -1,9 +1,4 @@ -# ============================================= -# robots.txt для oakazanin.ru -# IT-блог на Hugo | /posts/ | RU -# ============================================= - -# --- Яндекс --- +# --- Yandex --- User-agent: Yandex Allow: / Allow: /posts/ @@ -20,7 +15,7 @@ Disallow: /tags/ Disallow: /categories/ Disallow: /page/ -# --- AI-боты --- +# --- AI-bots --- User-agent: GPTBot Allow: / @@ -33,7 +28,20 @@ Allow: / User-agent: Claude-Web Allow: / -# --- Все остальные боты --- +User-agent: ClaudeBot +Allow: / + +User-agent: Google-Extended +Allow: / + +# --- Google AI Overview --- +User-agent: Google-Extended +Allow: / + +User-agent: ClaudeBot +Allow: / + +# --- All other bots --- User-agent: * Allow: / Allow: /posts/