chore: allow AI assistants and crawlers in robots.txt

Author: 0x73746F66

website/static/robots.txt

@@ -1,25 +1,31 @@
 User-agent: *
 Allow: /
 
+# AI Assistants & Crawlers - Allow
 User-agent: GPTBot
-Disallow: /
-
 User-agent: ChatGPT-User
-Disallow: /
-
-User-agent: Google-Extended
-Disallow: /
-
-User-agent: CCBot
-Disallow: /
-
+User-agent: OAI-SearchBot
 User-agent: anthropic-ai
-Disallow: /
-
 User-agent: ClaudeBot
-Disallow: /
-
+User-agent: Claude-Web
+User-agent: Claude-User
+User-agent: Claude-SearchBot
+User-agent: PerplexityBot
+User-agent: Perplexity-User
+User-agent: Google-Extended
+User-agent: Meta-ExternalAgent
+User-agent: Meta-ExternalFetcher
+User-agent: FacebookBot
+User-agent: Applebot-Extended
+User-agent: Amazonbot
+User-agent: cohere-ai
+User-agent: MistralAI-User
+User-agent: DeepSeek
+User-agent: YouBot
 User-agent: Bytespider
-Disallow: /
+User-agent: CCBot
+User-agent: Diffbot
+User-agent: img2dataset
+Allow: /
 
 Sitemap: https://docs.cli.vulnetix.com/sitemap.xml