From 30c95dd0378e7fdf427428a2a49e7e4d6e92dcb8 Mon Sep 17 00:00:00 2001
From: bouncepaw <bouncepaw2@ya.ru>
Date: Tue, 29 Dec 2020 13:10:11 +0500
Subject: [PATCH] Add robots.txt

---
 main.go | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/main.go b/main.go
index d368bb0..0caa1d5 100644
--- a/main.go
+++ b/main.go
@@ -128,6 +128,17 @@ func handlerIcon(w http.ResponseWriter, rq *http.Request) {
 	}
 }
 
+func handlerRobotsTxt(w http.ResponseWriter, rq *http.Request) {
+	w.Header().Set("Content-Type", "text/plain")
+	w.WriteHeader(http.StatusOK)
+	w.Write([]byte(
+		`User-agent: *
+Allow: /page/
+Allow: /recent-changes
+Disallow: /
+Crawl-delay: 5`))
+}
+
 func main() {
 	log.Println("Running MycorrhizaWiki β")
 	parseCliArgs()
@@ -157,5 +168,6 @@ func main() {
 	http.HandleFunc("/", func(w http.ResponseWriter, rq *http.Request) {
 		http.Redirect(w, rq, "/page/"+util.HomePage, http.StatusSeeOther)
 	})
+	http.HandleFunc("/robots.txt", handlerRobotsTxt)
 	log.Fatal(http.ListenAndServe("0.0.0.0:"+util.ServerPort, nil))
 }