https://ahrefs.com/robot/ got stuck in our site.

2013-12-11 11:52:48 -05:00 · 2013-12-11 11:52:48 -05:00 · 9440f88d67
parent 2e0e282765
commit 9440f88d67
1 changed files with 3 additions and 1 deletions
--- a/frontend/templates/robots.txt
+++ b/frontend/templates/robots.txt
@ -1,7 +1,9 @@
 # Hi Robot, welcome to UnglueIt :-)
 {% ifequal request.META.HTTP_HOST "unglue.it" %}
 User-agent: *
-Disallow: /nothing--please-crawl-us
+Disallow: /accounts/
+Disallow: /feedback/
+Disallow: /socialauth/
 {% else %}
 User-agent: *
 Disallow: /