Admiraldesvl.github.io/robot.txt at master · Admiraldesvl/Admiraldesvl.github.io · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
# Block OpenAI (ChatGPT/GPT-4)
User-agent: GPTBot
Disallow: /

# Block OpenAI User requests
User-agent: ChatGPT-User
Disallow: /

# Block Google Gemini & Vertex AI (Does not affect Google Search)
User-agent: Google-Extended
Disallow: /

# Block Anthropic (Claude)
User-agent: Claude-bot
Disallow: /

# Block Common Crawl (Used by many AI models for training)
User-agent: CCBot
Disallow: /

# Block Perplexity AI
User-agent: PerplexityBot
Disallow: /

# Block Meta (Llama/Meta AI)
User-agent: FacebookBot
Disallow: /

# Block ByteDance (TikTok/Crawl for AI)
User-agent: Bytespider
Disallow: /

# Block Amazon (AWS AI training)
User-agent: Amazonbot
Disallow: /

# Block Apple (Apple Intelligence/Applebot-Extended)
User-agent: Applebot-Extended
Disallow: /