{
    "question": "How many websites block AI crawlers or use llms.txt?",
    "answer": "4.5% of desktop websites named OpenAI's GPTBot in their robots.txt file in the HTTP Archive's 2025 crawl, up from 2.9% in 2024, and 3.6% named Anthropic's ClaudeBot. Naming a bot sets a rule for it, which can be a block or an allow. Only 2.13% of desktop sites had a valid llms.txt file.",
    "period": "2025 crawl",
    "key_numbers": [
        {
            "value": "4.5%",
            "label": "Desktop websites naming GPTBot in robots.txt, up from 2.9% in 2024",
            "note": "HTTP Archive Web Almanac, 2025 crawl",
            "stat": "ai-search/sites-naming-gptbot-in-robots-txt"
        },
        {
            "value": "3.6%",
            "label": "Desktop websites naming ClaudeBot in robots.txt, up from 1.9%",
            "note": "HTTP Archive Web Almanac, 2025 crawl",
            "stat": "ai-search/sites-naming-claudebot-in-robots-txt"
        },
        {
            "value": "2.13%",
            "label": "Desktop websites with a valid llms.txt file (2.10% of mobile)",
            "note": "HTTP Archive Web Almanac, 2025 crawl",
            "stat": "ai-search/sites-with-llms-txt"
        },
        {
            "value": "39.6%",
            "label": "llms.txt files made by the All in One SEO plugin",
            "note": "HTTP Archive Web Almanac, 2025 crawl",
            "stat": "ai-search/llms-txt-files-from-aioseo"
        },
        {
            "value": "97%",
            "label": "llms.txt files that got no requests at all in May 2026",
            "note": "Ahrefs, 38,000 domains with a valid file (vendor)"
        }
    ],
    "tables": [
        {
            "id": "bots-2025",
            "title": "Websites naming AI-related crawlers in robots.txt (2025 crawl)",
            "columns": [
                "Crawler (operator)",
                "Desktop sites",
                "Mobile sites"
            ],
            "rows": [
                [
                    "GPTBot (OpenAI)",
                    "4.5%",
                    "4.2%"
                ],
                [
                    "PetalBot (Huawei)",
                    "4.0%",
                    "4.4%"
                ],
                [
                    "ClaudeBot (Anthropic)",
                    "3.6%",
                    "3.4%"
                ],
                [
                    "CCBot (Common Crawl)",
                    "3.5%",
                    "3.2%"
                ],
                [
                    "Google-Extended (Google)",
                    "3.4%",
                    "3.0%"
                ],
                [
                    "Amazonbot (Amazon)",
                    "3.3%",
                    "3.0%"
                ],
                [
                    "FacebookBot (Meta)",
                    "2.9%",
                    "2.5%"
                ],
                [
                    "PerplexityBot (Perplexity)",
                    "2.8%",
                    "2.7%"
                ],
                [
                    "ChatGPT-User (OpenAI)",
                    "2.8%",
                    "2.5%"
                ],
                [
                    "Meta-ExternalAgent (Meta)",
                    "2.8%",
                    "2.5%"
                ]
            ],
            "row_ids": [
                "bot-gptbot",
                "bot-petalbot",
                "bot-claudebot",
                "bot-ccbot",
                "bot-google-extended",
                "bot-amazonbot",
                "bot-facebookbot",
                "bot-perplexitybot",
                "bot-chatgpt-user",
                "bot-meta-externalagent"
            ],
            "note": "HTTP Archive Web Almanac 2025, SEO chapter, figure 5.5. Counts sites whose robots.txt names the user agent, whether the rule blocks or allows it. The Almanac lists PetalBot among the AI-related crawlers.",
            "source": "http-archive"
        },
        {
            "id": "bots-2024-2025",
            "title": "AI crawlers named in robots.txt, 2024 and 2025",
            "columns": [
                "Crawler",
                "Desktop 2024",
                "Desktop 2025",
                "Mobile 2024",
                "Mobile 2025"
            ],
            "rows": [
                [
                    "GPTBot",
                    "2.9%",
                    "4.5%",
                    "2.7%",
                    "4.2%"
                ],
                [
                    "ClaudeBot",
                    "1.9%",
                    "3.6%",
                    "1.6%",
                    "3.4%"
                ],
                [
                    "CCBot",
                    "2.7%",
                    "3.5%",
                    "2.4%",
                    "3.2%"
                ],
                [
                    "ChatGPT-User",
                    "2.0%",
                    "2.8%",
                    "1.7%",
                    "2.5%"
                ]
            ],
            "row_ids": [
                "trend-gptbot",
                "trend-claudebot",
                "trend-ccbot",
                "trend-chatgpt-user"
            ],
            "note": "HTTP Archive Web Almanac 2025 and 2024 crawls. The 2024 data also listed anthropic-ai (2.0% desktop, 1.7% mobile), a broader Anthropic name; 2025 counts the more specific bots.",
            "source": "http-archive"
        },
        {
            "id": "llms-txt",
            "title": "llms.txt: who publishes it, and who reads it",
            "columns": [
                "Measure",
                "Value",
                "Source"
            ],
            "rows": [
                [
                    "Desktop websites with a valid llms.txt",
                    "2.13%",
                    "HTTP Archive, 2025 crawl"
                ],
                [
                    "Mobile websites with a valid llms.txt",
                    "2.10%",
                    "HTTP Archive, 2025 crawl"
                ],
                [
                    "Valid llms.txt files found across mobile sites",
                    "324,184",
                    "HTTP Archive, 2025 crawl"
                ],
                [
                    "llms.txt files made by All in One SEO",
                    "39.6%",
                    "HTTP Archive, 2025 crawl"
                ],
                [
                    "llms.txt files made by Yoast SEO",
                    "3.6%",
                    "HTTP Archive, 2025 crawl"
                ],
                [
                    "Domains using Ahrefs Web Analytics that publish llms.txt",
                    "28%",
                    "Ahrefs, 137K domains (upper bound)"
                ],
                [
                    "Of those files, the share that got no requests in May 2026",
                    "97%",
                    "Ahrefs, about 38,000 domains"
                ]
            ],
            "row_ids": [
                "llms-desktop",
                "llms-mobile",
                "llms-files",
                "llms-aioseo",
                "llms-yoast",
                "llms-ahrefs-share",
                "llms-ahrefs-unread"
            ],
            "note": "The plugin shares come from the default comments the plugins leave in the files. Ahrefs' sample is its own analytics customers, who are more SEO-aware than the web as a whole, so Ahrefs calls its 28% an upper bound."
        },
        {
            "id": "robots-status",
            "title": "What websites return when asked for robots.txt (2025 crawl)",
            "columns": [
                "Response",
                "Desktop sites",
                "Mobile sites"
            ],
            "rows": [
                [
                    "200 (file served)",
                    "84.9%",
                    "84.9%"
                ],
                [
                    "404 (no file)",
                    "13.3%",
                    "13.2%"
                ],
                [
                    "Timeout",
                    "1.0%",
                    "1.1%"
                ],
                [
                    "403 (forbidden)",
                    "0.5%",
                    "0.5%"
                ],
                [
                    "500 (server error)",
                    "0.1%",
                    "0.1%"
                ]
            ],
            "row_ids": [
                "status-200",
                "status-404",
                "status-timeout",
                "status-403",
                "status-500"
            ],
            "note": "HTTP Archive Web Almanac 2025, SEO chapter, figure 5.1. A site with no robots.txt sets no rules for any crawler.",
            "source": "http-archive"
        }
    ],
    "sources": [
        {
            "name": "Web Almanac (SEO chapter)",
            "publisher": "HTTP Archive",
            "type": "research",
            "url": "https://httparchive.org/"
        },
        {
            "name": "AI Overviews click-through studies",
            "publisher": "Ahrefs",
            "type": "vendor",
            "url": "https://ahrefs.com/"
        }
    ],
    "content_changed_on": "2026-10-31",
    "page_url": "https://digitaltreed.com/statistics/ai-search/ai-crawlers-robots-txt",
    "csv_url": "https://digitaltreed.com/statistics/ai-search/ai-crawlers-robots-txt.csv"
}