Customize robots.txt

2011-03-06 12:20:02 -08:00 · 2011-03-06 12:20:02 -08:00 · c2601c708a
parent 41e4303efb
commit c2601c708a
1 changed files with 3 additions and 5 deletions
--- a/public/robots.txt
+++ b/public/robots.txt
@ -1,5 +1,3 @@
-# See http://www.robotstxt.org/wc/norobots.html for documentation on how to use the robots.txt file
+User-Agent: *
-#
+Disallow: /
-# To ban all spiders from the entire site uncomment the next two lines:
+Allow: /$
 # User-Agent: *
 # Disallow: /