Merge pull request #2439 from madeline-underwood/ai_seo_ins_updates

Update robots.txt to allow additional AI crawlers and remove unnecessary disallow rule

Author: pareenaverma

themes/arm-design-system-hugo-theme/layouts/robots.txt

@@ -1,6 +1,28 @@
 User-agent: *
+Disallow:
+
+# Explicitly welcome AI crawlers
+User-agent: GPTBot
+Allow: /
+
+User-agent: Google-Extended
+Allow: /
+
+User-agent: ClaudeBot
+Allow: /
+
+User-agent: PerplexityBot
+Allow: /
+
+User-agent: anthropic-ai
+Allow: /
+
+User-agent: Bytespider
+Allow: /
+
+User-agent: CCBot
 Allow: /
-Disallow: /*?
 
+# Sitemaps help AI discovery
 Sitemap: https://learn.arm.com/sitemap.xml
-Image Sitemap: https://learn.arm.com/learn-image-sitemap.xml
+Sitemap: https://learn.arm.com/learn-image-sitemap.xml