appclust-support/robots.txt at main · Erratums/appclust-support · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
# robots.txt for AppClust Help Center
# https://help.appclust.com/robots.txt
# Last updated: 2025-01-27

User-agent: *
Allow: /
Allow: /files/
Allow: /images/
Allow: /article/
Allow: /sitemap.xml
Allow: /robots.txt

# Disallow hash-based URLs (not SEO-friendly)
Disallow: /*#article/
Disallow: /*#videos

# Sitemap location
Sitemap: https://help.appclust.com/sitemap.xml

# Crawl-delay (optional, helps prevent server overload)
Crawl-delay: 1

# Specific bot rules
User-agent: Googlebot
Allow: /
Allow: /files/
Allow: /images/
Allow: /article/

User-agent: Bingbot
Allow: /
Allow: /files/
Allow: /images/
Allow: /article/

User-agent: Slurp
Allow: /
Allow: /files/
Allow: /images/
Allow: /article/

# Block bad bots (optional)
User-agent: AhrefsBot
Crawl-delay: 10

User-agent: SemrushBot
Crawl-delay: 10