SensorOrgNet.github.io/robots.txt at main · SensorOrgNet/SensorOrgNet.github.io · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
#UVic robots.txt
User-agent:*
Crawl-delay:30
#Googlebot specs https://developers.google.com/webmasters/control-crawl-index/docs/robots_txt
#More specific rules (the allows here) trump more general rules
Allow:*/wp-includes/*/*.css$
Allow:*/wp-includes/js/*.js$
Allow:*/wp-includes/js/*/*.js$

#WORDPRESS-1268 Discourage bots from indexing search pages in robots.txt
Disallow:*/search/
Disallow:*/globalsearch/

Disallow:*/cgi-bin/
Disallow:*/et_temp/
Disallow:*/exports/
Disallow:*/localextras/
Disallow:*/TOSPages/
Disallow:*/related-tags*
Disallow:*/trackback/
Disallow:*/server-status/
Disallow:*/wp-admin/
Disallow:*/wp-config*
Disallow:*/wp-content/cache/
Disallow:*/wp-content/themes/*.php
Disallow:*/wp-includes/
Disallow:*/xmlrpc.php

Disallow:/*.7z
Disallow:/*.7zip
Disallow:/*.asp*
Disallow:/*.bak
Disallow:/*.bz
Disallow:/*.bzip
Disallow:/*.cfm
Disallow:/*.cgi
Disallow:/*.gz
Disallow:/*.gzip
Disallow:/*.inc
Disallow:/*.ini
Disallow:/*.json
Disallow:/*.|jsp*
Disallow:/*.old
Disallow:/*.log
Disallow:/*.phar
Disallow:/*.php4
Disallow:/*.php5
Disallow:/*.php~
Disallow:/*.pl
Disallow:/*.py
Disallow:/*.rar
Disallow:/*.readme
Disallow:/readme.html
Disallow:/*.sh
Disallow:/*.sql
Disallow:/*.tar
Disallow:/*.tgz
Disallow:/*.txt
Disallow:/*.xml
Disallow:/*.zip
Disallow:/*.|jsp*

Allow:/*.css$
Allow:/*.js$


#Dicourage bots I do not care to support ====

User-agent:msnbot
Crawl-delay:1

# digg mirror
User-agent:duggmirror
Disallow:/

# MJ12bot http://mj12bot.com/
User-agent:MJ12bot
Disallow:/

User-agent:Sogouwebspider
Disallow:/*

User-agent:Sogouinstspider
Disallow:/*

User-agent:Sogouspider2
Disallow:/*

User-agent:Experibot*
Disallow:/*

#End of UVic robots.txt