diff options
Diffstat (limited to 'test/w3c/promela/robots.txt')
-rw-r--r-- | test/w3c/promela/robots.txt | 96 |
1 files changed, 0 insertions, 96 deletions
diff --git a/test/w3c/promela/robots.txt b/test/w3c/promela/robots.txt deleted file mode 100644 index 9be2782..0000000 --- a/test/w3c/promela/robots.txt +++ /dev/null @@ -1,96 +0,0 @@ -# -# robots.txt for http://www.w3.org/ -# -# $Id: robots.txt,v 1.67 2014-06-25 13:06:01 ddavis Exp $ -# - -# For use by search.w3.org -User-agent: W3C-gsa -Disallow: /Out-Of-Date - -User-agent: W3T_SE -Disallow: /Out-Of-Date - -User-agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT; MS Search 4.0 Robot) -Disallow: / - -# W3C Link checker -User-agent: W3C-checklink -Disallow: - -# the following settings apply to all bots -User-agent: * -# Blogs - WordPress -# https://codex.wordpress.org/Search_Engine_Optimization_for_WordPress#Robots.txt_Optimization -Disallow: /*/wp-admin/ -Disallow: /*/wp-includes/ -Disallow: /*/wp-content/plugins/ -Disallow: /*/wp-content/cache/ -Disallow: /*/wp-content/themes/ -Disallow: /blog/*/trackback/ -Disallow: /blog/*/feed/ -Disallow: /blog/*/comments/ -Disallow: /blog/*/category/*/* -Disallow: /blog/*/*/trackback/ -Disallow: /blog/*/*/feed/ -Disallow: /blog/*/*/comments/ -Disallow: /blog/*/*? -Disallow: /community/trackback/ -Disallow: /community/feed/ -Disallow: /community/comments/ -Disallow: /community/category/*/* -Disallow: /community/*/trackback/ -Disallow: /community/*/feed/ -Disallow: /community/*/comments/ -Disallow: /community/*/category/*/* -Disallow: /community/*? -Disallow: /Consortium/Offices/trackback/ -Disallow: /Consortium/Offices/feed/ -Disallow: /Consortium/Offices/comments/ -Disallow: /Consortium/Offices/category/*/* -Disallow: /Consortium/Offices/*/trackback/ -Disallow: /Consortium/Offices/*/feed/ -Disallow: /Consortium/Offices/*/comments/ -Disallow: /Consortium/Offices/*? -# Wikis - Mediawiki -# https://www.mediawiki.org/wiki/Manual:Robots.txt -Disallow: /wiki/index.php? -Disallow: /wiki/index.php/Help -Disallow: /wiki/index.php/MediaWiki -Disallow: /wiki/index.php/Special: -Disallow: /wiki/index.php/Template -Disallow: /wiki/skins/ -Disallow: /*/wiki/index.php? -Disallow: /*/wiki/index.php/Help -Disallow: /*/wiki/index.php/MediaWiki -Disallow: /*/wiki/index.php/Special: -Disallow: /*/wiki/index.php/Template -# various other access-controlled or expensive areas -Disallow: /2004/ontaria/basic -Disallow: /Team/ -Disallow: /Project -Disallow: /Web -Disallow: /Systems -Disallow: /History -Disallow: /Out-Of-Date -Disallow: /2002/02/mid -Disallow: /mid/ -Disallow: /2005/06/blog/ -Disallow: /2004/08/W3CTalks -Disallow: /2007/11/Talks/search -Disallow: /People/all/ -Disallow: /RDF/Validator/ARPServlet -Disallow: /2003/03/Translations/byLanguage -Disallow: /2003/03/Translations/byTechnology -Disallow: /2005/11/Translations/Query -Disallow: /2000/06/webdata/xslt -Disallow: /2000/09/webdata/xslt -Disallow: /2005/08/online_xslt/xslt -Disallow: /Bugs/ -Disallow: /Search/Mail/Public/ -Disallow: /2006/02/chartergen -Disallow: /2004/01/pp-impl -Disallow: /Consortium/supporters -Disallow: /2007/08/pyRdfa/ -Disallow: /WAI/PF/comments/ -Disallow: /participate/conferences.xml |