diff --git a/layouts/robots.txt b/layouts/robots.txt index f090dee..a7aafbb 100644 --- a/layouts/robots.txt +++ b/layouts/robots.txt @@ -1,7 +1,73 @@ User-agent: * Allow: / +User-agent: AdsBot-Google +Disallow: / + +User-agent: Amazonbot +Disallow: / + +User-agent: anthropic-ai +Disallow: / + +User-agent: AwarioRssBot +Disallow: / + +User-agent: AwarioSmartBot +Disallow: / + +User-agent: Bytespider +Disallow: / + +User-agent: CCBot +Disallow: / + +User-agent: ChatGPT-User +Disallow: / + +User-agent: ClaudeBot +Disallow: / + +User-agent: Claude-Web +Disallow: / + +User-agent: cohere-ai +Disallow: / + +User-agent: DataForSeoBot +Disallow: / + +User-agent: FacebookBot +Disallow: / + +User-agent: Google-Extended +Disallow: / + User-agent: GPTBot Disallow: / +User-agent: ImagesiftBot +Disallow: / + +User-agent: magpie-crawler +Disallow: / + +User-agent: omgili +Disallow: / + +User-agent: omgilibot +Disallow: / + +User-agent: peer39_crawler +Disallow: / + +User-agent: peer39_crawler/1.0 +Disallow: / + +User-agent: PerplexityBot +Disallow: / + +User-agent: YouBot +Disallow: / + Sitemap: {{ "/sitemap.xml" | absURL }}