diff options
Diffstat (limited to 'test/w3c/c89/robots.txt')
| -rw-r--r-- | test/w3c/c89/robots.txt | 103 |
1 files changed, 103 insertions, 0 deletions
diff --git a/test/w3c/c89/robots.txt b/test/w3c/c89/robots.txt new file mode 100644 index 0000000..fdd7340 --- /dev/null +++ b/test/w3c/c89/robots.txt @@ -0,0 +1,103 @@ +# +# robots.txt for http://www.w3.org/ +# +# $Id: robots.txt,v 1.74 2016/02/11 20:30:26 gerald Exp $ +# + +# For use by search.w3.org +User-agent: W3C-gsa +Disallow: /Out-Of-Date + +User-agent: W3T_SE +Disallow: /Out-Of-Date + +User-agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT; MS Search 4.0 Robot) +Disallow: / + +# W3C Link checker +User-agent: W3C-checklink +Disallow: + +# the following settings apply to all bots +User-agent: * +# Blogs - WordPress +# https://codex.wordpress.org/Search_Engine_Optimization_for_WordPress#Robots.txt_Optimization +Disallow: /*/wp-admin/ +Disallow: /*/wp-includes/ +Disallow: /*/wp-content/plugins/ +Disallow: /*/wp-content/cache/ +Disallow: /*/wp-content/themes/ +Disallow: /blog/*/trackback/ +Disallow: /blog/*/feed/ +Disallow: /blog/*/comments/ +Disallow: /blog/*/category/*/* +Disallow: /blog/*/*/trackback/ +Disallow: /blog/*/*/feed/ +Disallow: /blog/*/*/comments/ +Disallow: /blog/*/*? +Disallow: /community/trackback/ +Disallow: /community/feed/ +Disallow: /community/comments/ +Disallow: /community/category/*/* +Disallow: /community/*/trackback/ +Disallow: /community/*/feed/ +Disallow: /community/*/comments/ +Disallow: /community/*/category/*/* +Disallow: /community/*? +Disallow: /Consortium/Offices/trackback/ +Disallow: /Consortium/Offices/feed/ +Disallow: /Consortium/Offices/comments/ +Disallow: /Consortium/Offices/category/*/* +Disallow: /Consortium/Offices/*/trackback/ +Disallow: /Consortium/Offices/*/feed/ +Disallow: /Consortium/Offices/*/comments/ +Disallow: /Consortium/Offices/*? +# Wikis - Mediawiki +# https://www.mediawiki.org/wiki/Manual:Robots.txt +Disallow: /wiki/index.php? +Disallow: /wiki/index.php/Help +Disallow: /wiki/index.php/MediaWiki +Disallow: /wiki/index.php/Special: +Disallow: /wiki/index.php/Template +Disallow: /wiki/skins/ +Disallow: /*/wiki/index.php? +Disallow: /*/wiki/index.php/Help +Disallow: /*/wiki/index.php/MediaWiki +Disallow: /*/wiki/index.php/Special: +Disallow: /*/wiki/index.php/Template +# various other access-controlled or expensive areas +Disallow: /2004/ontaria/basic +Disallow: /Team/ +Disallow: /Project +Disallow: /Web +Disallow: /Systems +Disallow: /History +Disallow: /Out-Of-Date +Disallow: /2002/02/mid +Disallow: /mid/ +Disallow: /2005/06/blog/ +Disallow: /2004/08/W3CTalks +Disallow: /2007/11/Talks/search +Disallow: /People/all/ +Disallow: /RDF/Validator/ARPServlet +Disallow: /RDF/Validator/rdfval +Disallow: /2003/03/Translations/byLanguage +Disallow: /2003/03/Translations/byTechnology +Disallow: /2005/11/Translations/Query +Disallow: /2000/06/webdata/xslt +Disallow: /2000/09/webdata/xslt +Disallow: /2005/08/online_xslt/xslt +Disallow: /Bugs/ +Disallow: /Search/Mail/Public/ +Disallow: /2006/02/chartergen +Disallow: /2004/01/pp-impl +Disallow: /Consortium/supporters +Disallow: /2007/08/pyRdfa/ +Disallow: /2012/pyRdfa/extract +Disallow: /WAI/PF/comments/ +Disallow: /participate/conferences.xml +Disallow: /scripts/ +Disallow: /2005/01/yacker/ +Disallow: /2005/01/yacker? +Disallow: /2003/09/nschecker? + |