Updated the robots.txt to disallow admin and moved sitemap to top

just3ws · just3ws · commit fdb96caf653e · 2014-10-06T13:48:03.000-05:00
diff --git a/public/robots.txt b/public/robots.txt
@@ -1,9 +1,7 @@
-# See http://www.robotstxt.org/wc/norobots.html for documentation on how to use the robots.txt file
-#
-# To ban all spiders from the entire site uncomment the next two lines:
-# User-Agent: *
-# Disallow: /
+Sitemap: https://coderwall-assets-0.s3.amazonaws.com/sitemaps/sitemap.xml.gz
+
+User-agent: *
+Disallow: /admin/
 
 User-agent: EasouSpider
 Disallow: /
-Sitemap: https://coderwall-assets-0.s3.amazonaws.com/sitemaps/sitemap.xml.gz
