commit c456fe4ce93d8ad3cc7182944ecaf54c5dac39cb parent 648f103f5d7279e4298bb98f38314c8bb8e073a9 Author: Christian Grothoff <christian@grothoff.org> Date: Wed, 19 Nov 2014 17:55:44 +0100 robots Diffstat:
| A | robots.txt | | | 20 | ++++++++++++++++++++ |
1 file changed, 20 insertions(+), 0 deletions(-)
diff --git a/robots.txt b/robots.txt @@ -0,0 +1,20 @@ +# +# robots.txt +# +# This file is to prevent the crawling and indexing of certain parts +# of your site by web crawlers and spiders run by sites like Yahoo! +# and Google. By telling these "robots" where not to go on your site, +# you save bandwidth and server resources. +# +# This file will be ignored unless it is at the root of your host: +# Used: http://example.com/robots.txt +# Ignored: http://example.com/site/robots.txt +# +# For more information about the robots.txt standard, see: +# http://www.robotstxt.org/robotstxt.html +# +# For syntax checking, see: +# http://www.frobee.com/robots-txt-check + +User-agent: * +Crawl-delay: 10