From 2f44c1aa07a4affe485d264069539e904bd06596 Mon Sep 17 00:00:00 2001 From: Marcin Kulik Date: Sun, 11 Jun 2017 15:31:43 +0200 Subject: [PATCH] Tell crawlers to ignore .json and .png files --- public/robots.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/public/robots.txt b/public/robots.txt index 085187f..ce3c656 100644 --- a/public/robots.txt +++ b/public/robots.txt @@ -3,3 +3,8 @@ # To ban all spiders from the entire site uncomment the next two lines: # User-Agent: * # Disallow: / + +User-Agent: * +Disallow: /a/*.json +Disallow: /a/*.png +Disallow: /a/*.gif