From fc0e2e93837df623fbf03ee18c33a91dcd9fe23c Mon Sep 17 00:00:00 2001
From: Maksim Nabokikh <maksim.nabokikh@flant.com>
Date: Thu, 23 Feb 2023 18:53:09 +0400
Subject: [PATCH] feat: Add default robots.txt (#2834)
Signed-off-by: m.nabokikh <maksim.nabokikh@flant.com>
---
server/server.go | 4 +++-
server/templates.go | 14 ++++++++++----
web/robots.txt | 2 ++
web/web.go | 2 +-
4 files changed, 16 insertions(+), 6 deletions(-)
create mode 100644 web/robots.txt
diff --git a/server/server.go b/server/server.go
index df16e655..0aac0a6c 100755
--- a/server/server.go
+++ b/server/server.go
@@ -252,7 +252,7 @@ func newServer(ctx context.Context, c Config, rotationStrategy rotationStrategy)
extra: c.Web.Extra,
}
- static, theme, tmpls, err := loadWebConfig(web)
+ static, theme, robots, tmpls, err := loadWebConfig(web)
if err != nil {
return nil, fmt.Errorf("server: failed to load web static: %v", err)
}
@@ -390,6 +390,8 @@ func newServer(ctx context.Context, c Config, rotationStrategy rotationStrategy)
handlePrefix("/static", static)
handlePrefix("/theme", theme)
+ handleFunc("/robots.txt", robots)
+
s.mux = r
s.startKeyRotation(ctx, rotationStrategy, now)
diff --git a/server/templates.go b/server/templates.go
index e6ab3a79..b77663e1 100644
--- a/server/templates.go
+++ b/server/templates.go
@@ -88,7 +88,7 @@ func getFuncMap(c webConfig) (template.FuncMap, error) {
// |- themes
// | |- (theme name)
// |- templates
-func loadWebConfig(c webConfig) (http.Handler, http.Handler, *templates, error) {
+func loadWebConfig(c webConfig) (http.Handler, http.Handler, http.HandlerFunc, *templates, error) {
// fallback to the default theme if the legacy theme name is provided
if c.theme == "coreos" || c.theme == "tectonic" {
c.theme = ""
@@ -105,18 +105,24 @@ func loadWebConfig(c webConfig) (http.Handler, http.Handler, *templates, error)
staticFiles, err := fs.Sub(c.webFS, "static")
if err != nil {
- return nil, nil, nil, fmt.Errorf("read static dir: %v", err)
+ return nil, nil, nil, nil, fmt.Errorf("read static dir: %v", err)
}
themeFiles, err := fs.Sub(c.webFS, path.Join("themes", c.theme))
if err != nil {
- return nil, nil, nil, fmt.Errorf("read themes dir: %v", err)
+ return nil, nil, nil, nil, fmt.Errorf("read themes dir: %v", err)
+ }
+ robotsContent, err := fs.ReadFile(c.webFS, "robots.txt")
+ if err != nil {
+ return nil, nil, nil, nil, fmt.Errorf("read robots.txt dir: %v", err)
}
static := http.FileServer(http.FS(staticFiles))
theme := http.FileServer(http.FS(themeFiles))
+ robots := func(w http.ResponseWriter, r *http.Request) { fmt.Fprint(w, string(robotsContent)) }
templates, err := loadTemplates(c, "templates")
- return static, theme, templates, err
+
+ return static, theme, robots, templates, err
}
// loadTemplates parses the expected templates from the provided directory.
diff --git a/web/robots.txt b/web/robots.txt
new file mode 100644
index 00000000..1f53798b
--- /dev/null
+++ b/web/robots.txt
@@ -0,0 +1,2 @@
+User-agent: *
+Disallow: /
diff --git a/web/web.go b/web/web.go
index c5ff7514..0c7e9873 100644
--- a/web/web.go
+++ b/web/web.go
@@ -5,7 +5,7 @@ import (
"io/fs"
)
-//go:embed static/* templates/* themes/*
+//go:embed static/* templates/* themes/* robots.txt
var files embed.FS
// FS returns a filesystem with the default web assets.
--
GitLab