|
|
@ -45,7 +45,7 @@ func generateCapsTxt(desc, admin, geo string) []byte {
|
|
|
|
func generateRobotsTxt() []byte {
|
|
|
|
func generateRobotsTxt() []byte {
|
|
|
|
text := generatePolicyHeader("robots.txt")
|
|
|
|
text := generatePolicyHeader("robots.txt")
|
|
|
|
text += "\r\n"
|
|
|
|
text += "\r\n"
|
|
|
|
text += "Usage-agent: *" + "\r\n"
|
|
|
|
text += "User-agent: *" + "\r\n"
|
|
|
|
text += "Disallow: *" + "\r\n"
|
|
|
|
text += "Disallow: *" + "\r\n"
|
|
|
|
text += "\r\n"
|
|
|
|
text += "\r\n"
|
|
|
|
text += "Crawl-delay: 99999" + "\r\n"
|
|
|
|
text += "Crawl-delay: 99999" + "\r\n"
|
|
|
|