Tell crawlers to ignore .json and .png files

7 years ago · 2f44c1aa07
parent 5b9f97ce52
commit 2f44c1aa07
1 changed files with 5 additions and 0 deletions
--- a/public/robots.txt
+++ b/public/robots.txt
@ -3,3 +3,8 @@
 # To ban all spiders from the entire site uncomment the next two lines:
 # User-Agent: *
 # Disallow: /
+
+User-Agent: *
+Disallow: /a/*.json
+Disallow: /a/*.png
+Disallow: /a/*.gif