diff --git a/static/robots.txt b/static/robots.txt new file mode 100644 index 0000000..e5199a8 --- /dev/null +++ b/static/robots.txt @@ -0,0 +1,6 @@ +User-agent: * +Allow: /page/ +Allow: /hypha/ +Allow: /recent-changes +Disallow: / +Crawl-delay: 5 diff --git a/static/static.go b/static/static.go index 9c76563..e7850d6 100644 --- a/static/static.go +++ b/static/static.go @@ -6,7 +6,7 @@ import ( "os" ) -//go:embed *.css *.js icon +//go:embed *.css *.js *.txt icon var embedFS embed.FS // FS serves all static files. diff --git a/web/web.go b/web/web.go index 6d8125a..478562a 100644 --- a/web/web.go +++ b/web/web.go @@ -23,7 +23,7 @@ var stylesheets = []string{"default.css", "custom.css"} // httpErr is used by many handlers to signal errors in a compact way. func httpErr(w http.ResponseWriter, status int, name, title, errMsg string) { log.Println(errMsg, "for", name) - w.Header().Set("Content-Type", "text/html;charset=utf-8") + w.Header().Set("Content-Type", mime.TypeByExtension(".html")) w.WriteHeader(status) fmt.Fprint( w, @@ -54,21 +54,20 @@ func handlerStyle(w http.ResponseWriter, rq *http.Request) { } func handlerUserList(w http.ResponseWriter, rq *http.Request) { - w.Header().Set("Content-Type", "text/html;charset=utf-8") + w.Header().Set("Content-Type", mime.TypeByExtension(".html")) w.WriteHeader(http.StatusOK) w.Write([]byte(views.BaseHTML("User list", views.UserListHTML(), user.FromRequest(rq)))) } func handlerRobotsTxt(w http.ResponseWriter, rq *http.Request) { - w.Header().Set("Content-Type", "text/plain") - w.WriteHeader(http.StatusOK) - w.Write([]byte( - `User-agent: * -Allow: /page/ -Allow: /hypha/ -Allow: /recent-changes -Disallow: / -Crawl-delay: 5`)) + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + + file, err := static.FS.Open("robots.txt") + if err != nil { + return + } + io.Copy(w, file) + file.Close() } func Init() {