feat: Add serve_ignore option (#15)

This commit introduces a new option `serve_ignore` to the
Defender middleware. When enabled, it serves a robots.txt
file with a "Disallow: /" directive to discourage search
engine crawlers.

The robots.txt file now allows specific bots like
Googlebot, Bingbot, and DuckDuckBot while blocking others.
This helps in managing how search engines interact with
the protected resource.

Signed-off-by: Jason Cameron <git@jasoncameron.dev>

examples/garbage/Caddyfile

@@ -9,6 +9,7 @@
 
 	defender garbage {
 		ranges private
+    	serve_ignore
 	}
 	respond "This is what a human sees"
 }

middleware.go

@@ -12,12 +12,28 @@ import (
 // serveIgnore is a helper function to serve a robots.txt file if the ServeIgnore option is enabled.
 // It returns true if the request was handled, false otherwise.
 func (m Defender) serveGitignore(w http.ResponseWriter, r *http.Request) bool {
-	if !m.ServeIgnore && r.URL.Path != "/robots.txt" && r.Method != http.MethodGet {
+	m.log.Debug("ServeIgnore", zap.Bool("serveIgnore", m.ServeIgnore), zap.String("path", r.URL.Path), zap.String("method", r.Method))
+	// Serve robots.txt only if ServeIgnore is enabled, the path is "/robots.txt", and the method is GET.
+	if !m.ServeIgnore || r.URL.Path != "/robots.txt" || r.Method != http.MethodGet {
 		return false
 	}
 	w.Header().Set("Content-Type", "text/plain")
 	w.WriteHeader(http.StatusOK)
-	w.Write([]byte("User-agent: *\nDisallow: /\n"))
+	// Build the robots.txt content to allow specific bots and block others.
+	robotsTxt := `
+User-agent: Googlebot
+Disallow:
+
+User-agent: Bingbot
+Disallow:
+
+User-agent: DuckDuckBot
+Disallow:
+
+User-agent: *
+Disallow: /
+`
+	w.Write([]byte(robotsTxt))
 	return true
 }
 

plugin.go

@@ -77,6 +77,9 @@ type Defender struct {
 	// Required. Must be one of: "block", "garbage", "custom"
 	RawResponder string `json:"raw_responder,omitempty"`
 
+	// ServeIgnore specifies whether to serve a robots.txt file with a "Disallow: /" directive
+	// Default: false
+	ServeIgnore bool `json:"serve_ignore,omitempty"`
 	// responder is the internal implementation of the response strategy
 	responder responders.Responder
 	ipChecker *ip.IPChecker