feat: Add default robots.txt (#2834)

Signed-off-by: m.nabokikh <maksim.nabokikh@flant.com>

server/server.go

@@ -252,7 +252,7 @@ func newServer(ctx context.Context, c Config, rotationStrategy rotationStrategy)
 		extra:     c.Web.Extra,
 	}
 
-	static, theme, tmpls, err := loadWebConfig(web)
+	static, theme, robots, tmpls, err := loadWebConfig(web)
 	if err != nil {
 		return nil, fmt.Errorf("server: failed to load web static: %v", err)
 	}
@@ -390,6 +390,8 @@ func newServer(ctx context.Context, c Config, rotationStrategy rotationStrategy)
 
 	handlePrefix("/static", static)
 	handlePrefix("/theme", theme)
+	handleFunc("/robots.txt", robots)
+
 	s.mux = r
 
 	s.startKeyRotation(ctx, rotationStrategy, now)

server/templates.go

@@ -88,7 +88,7 @@ func getFuncMap(c webConfig) (template.FuncMap, error) {
 //	|- themes
 //	|  |- (theme name)
 //	|- templates
-func loadWebConfig(c webConfig) (http.Handler, http.Handler, *templates, error) {
+func loadWebConfig(c webConfig) (http.Handler, http.Handler, http.HandlerFunc, *templates, error) {
 	// fallback to the default theme if the legacy theme name is provided
 	if c.theme == "coreos" || c.theme == "tectonic" {
 		c.theme = ""
@@ -105,18 +105,24 @@ func loadWebConfig(c webConfig) (http.Handler, http.Handler, *templates, error)
 
 	staticFiles, err := fs.Sub(c.webFS, "static")
 	if err != nil {
-		return nil, nil, nil, fmt.Errorf("read static dir: %v", err)
+		return nil, nil, nil, nil, fmt.Errorf("read static dir: %v", err)
 	}
 	themeFiles, err := fs.Sub(c.webFS, path.Join("themes", c.theme))
 	if err != nil {
-		return nil, nil, nil, fmt.Errorf("read themes dir: %v", err)
+		return nil, nil, nil, nil, fmt.Errorf("read themes dir: %v", err)
+	}
+	robotsContent, err := fs.ReadFile(c.webFS, "robots.txt")
+	if err != nil {
+		return nil, nil, nil, nil, fmt.Errorf("read robots.txt dir: %v", err)
 	}
 
 	static := http.FileServer(http.FS(staticFiles))
 	theme := http.FileServer(http.FS(themeFiles))
+	robots := func(w http.ResponseWriter, r *http.Request) { fmt.Fprint(w, string(robotsContent)) }
 
 	templates, err := loadTemplates(c, "templates")
-	return static, theme, templates, err
+
+	return static, theme, robots, templates, err
 }
 
 // loadTemplates parses the expected templates from the provided directory.

web/robots.txt

+User-agent: *
+Disallow: /

web/web.go

@@ -5,7 +5,7 @@ import (
 	"io/fs"
 )
 
-//go:embed static/* templates/* themes/*
+//go:embed static/* templates/* themes/* robots.txt
 var files embed.FS
 
 // FS returns a filesystem with the default web assets.