diff --git a/robots.txt b/robots.txt index 3b3624b..8e0a755 100644 --- a/robots.txt +++ b/robots.txt @@ -4,6 +4,27 @@ # Reference: https://neil-clarke.com/block-the-bots-that-feed-ai-models-by-scraping-your-website/ # See: https://github.com/MattWilcox/native-base/blob/45f6e7a837104f5ad83a5c7e280fb9a4eb126219/robots.txt +# Block all crawlers by default +User-agent: * +Disallow: / + +# Search engines (explicit, for clarity) +User-agent: Googlebot +User-agent: Googlebot-Image +User-agent: Googlebot-News +User-agent: Googlebot-Video +User-agent: AdsBot-Google +User-agent: Bingbot +User-agent: Slurp +User-agent: DuckDuckBot +User-agent: Baiduspider +User-agent: YandexBot +User-agent: Sogou +User-agent: Exabot +User-agent: ia_archiver +Disallow: / + +# AI scrapers User-agent: CCBot User-agent: ChatGPT-User User-agent: GPTBot