lcb-website/robots.txt at main · hyperpolymath/lcb-website · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
# SPDX-License-Identifier: PMPL-1.0-or-later
# robots.txt for nuj-lcb.org.uk
# Blocks AI training bots per AI usage policy
# See also: .well-known/ai.txt, .well-known/aibdp.json

# === AI Training Bots — DISALLOWED ===
# These bots scrape content for AI/ML training without consent.
# Per our AIBDP policy, training use requires explicit written consent.

User-agent: GPTBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Claude-Web
Disallow: /

User-agent: cohere-ai
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: FacebookBot
Disallow: /

# === General Crawlers — ALLOWED with restrictions ===
User-agent: *
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/uploads/private/
Disallow: /wp-content/plugins/
Disallow: /wp-json/wp/v2/users
Allow: /wp-admin/admin-ajax.php

Sitemap: https://nuj-lcb.org.uk/sitemap.xml