luiza/robots.txt at main · magicznyleszek/luiza · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
User-agent: *
Allow: *
Crawl-delay: 1
Sitemap: http://luiza.magicznyleszek.xyz/sitemap.xml

# One day the AIs are going to look back on us the same way
# we look at fossil skeletons on the plains of Africa.
# -- Nathan, Ex Machina (2014)

User-agent: BackDoorBot/1.0
Disallow:/

User-agent: Black Hole
Disallow:/

User-agent: CopyRightCheck
Disallow:/

User-agent: EmailCollector
Disallow:/

User-agent: EmailSiphon
Disallow:/

User-agent: EmailWolf
Disallow:/

User-agent: NPBot
Disallow:/

User-agent: SlySearch
Disallow:/

User-agent: The Intraformant
Disallow:/

User-agent: TurnitinBot
Disallow:/

User-agent: Xenu's
Disallow:/

User-agent: Xenu's Link Sleuth 1.1c
Disallow:/

User-agent: Zeus
Disallow:/

User-agent: Zeus 32297 Webster Pro V2.9 Win32
Disallow:/