临时增加 dump 完整请求包

Author: Becivells

go.mod

@@ -4,6 +4,7 @@ go 1.17
 
 require (
 	github.com/go-redis/redis/v8 v8.11.3
+	github.com/google/uuid v1.1.2
 	github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible
 	github.com/olivere/elastic v6.2.37+incompatible
 	github.com/olivere/elastic/v7 v7.0.29

go.sum

@@ -71,6 +71,7 @@ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
 github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=

internal/plugin/dump/determine.go

@@ -3,7 +3,7 @@ package dump
 import (
 	"bytes"
 	"goblin/pkg/utils"
-	"io/ioutil"
+	"io"
 	"net/http"
 	"strconv"
 	"strings"
@@ -58,17 +58,38 @@ func (dump *Dump) Determine(maxContentLength int, response *http.Response) (dete
 				return true, dump.Notice
 			}
 
-			body, err := ioutil.ReadAll(response.Body)
+			body, err := io.ReadAll(response.Body)
 			if err != nil {
 				log.Info("%s", err.Error())
 				return false, dump.Notice
 			}
 			if bytes.Contains(body, []byte(dump.Response.Body)) {
 				return true, dump.Notice
 			}
-			response.Body = ioutil.NopCloser(bytes.NewReader(body))
+			response.Body = io.NopCloser(bytes.NewReader(body))
 		}
 	}
 
 	return false, dump.Notice
 }
+
+func (dump *Dump) NeedCache(r *http.Request) bool {
+	start := time.Now()
+	defer log.Info("[time] url: %s, dump cache hand time: %v", r.RequestURI, time.Since(start))
+	if dump == nil {
+		return false
+	}
+
+	// 如何没有任何请求方式支持直接返回
+	if len(dump.Request.Method) == 0 {
+		return false
+	}
+	// 判断 method 是否符合 dump 规则
+	if utils.EleInArray(r.Method, dump.Request.Method) {
+		//为 nil 不匹配  Response
+		return true
+
+	}
+
+	return false
+}

internal/plugin/types.go

@@ -26,7 +26,7 @@ type BasePlugin struct {
 	UseBody     bool    `yaml:"-"` //响应body解包一次
 }
 
-//Rule 规则结构体
+// Rule 规则结构体
 type Rule struct {
 	URL      string
 	Match    string             `yaml:"Match"`

internal/reverse/response.go

@@ -26,14 +26,14 @@ func (reverse *Reverse) ModifyResponse(shost string) func(response *http.Respons
 		response.Header.Del("X-XSS-Protection")
 		//https://stackoverflow.com/questions/27358966/how-to-set-x-frame-options-on-iframe
 		response.Header.Del("X-Frame-Options")
-		
+
 		response.Header.Del("Content-Security-Policy-Report-Only")
 
 		// 删除缓存策略
 		response.Header.Del("Expires")
 		response.Header.Del("Last-Modified")
 		response.Header.Del("Date")
-		
+
 		if response.Header.Get("Access-Control-Allow-Origin") != "" {
 			//https://stackoverflow.com/questions/1653308/access-control-allow-origin-multiple-origin-domains
 			if response.Request.Header.Get("Origin") != "" {
@@ -101,7 +101,14 @@ func (reverse *Reverse) ModifyResponse(shost string) func(response *http.Respons
 								dete, msg := dp.Determine(reverse.MaxContentLength, response)
 								start := time.Now()
 								if dete {
-									dplog := dumpJson(response.Request)
+									uid := strings.Join(response.Request.Header["X-Request-ID"], "")
+									dplog, isCache := cache.DumpCache.Get(uid)
+									cache.DumpCache.Delete(uid)
+									if !isCache {
+										log.Warn("[Plugin:%s.%s]not cache : %s\n", rules.Name, rule.URL, dplog)
+										dplog = dumpJson(response.Request)
+									}
+
 									logging.AccLogger.WithFields(logrus.Fields{
 										"method":     response.Request.Method,
 										"url":        response.Request.URL.RequestURI(),

internal/reverse/reverse.go

@@ -52,11 +52,22 @@ func (reverse *Reverse) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 				"type":       "clientReq",
 			}).Warn(reqraw)
 		}
+
 		log.Info("[c->p] host: %s,RemoteAddr: %s,URI: %s", host, GetClientIP(r), r.RequestURI)
 		//response
 		//插件系统 rule 处理响应数据
 		if rules, ok := plugin.Plugins[host]; ok {
+			//dump
+
 			for _, rule := range rules.Rule {
+				for _, dp := range rule.Dump {
+					if dp.NeedCache(r) {
+						uuidstr := utils.GenerateUUID()
+						r.Header["X-Request-ID"] = []string{uuidstr}
+						cache.DumpCache.Set(uuidstr, dumpJson(r), 60*time.Second)
+					}
+				}
+
 				urlmatch := false
 				// url 匹配规则
 				switch strings.ToLower(rule.Match) {

pkg/cache/cache.go

@@ -3,6 +3,7 @@ package cache
 import (
 	"context"
 	"fmt"
+	cacheNew "github.com/patrickmn/go-cache"
 	"time"
 
 	log "unknwon.dev/clog/v2"
@@ -15,6 +16,7 @@ import (
 var ctx = context.Background()
 
 var GlobalCache *Cache
+var DumpCache *cacheNew.Cache
 
 func (db *Config) Init() {
 	c := &Cache{
@@ -32,6 +34,7 @@ func (db *Config) Init() {
 		log.Fatal("unsupported database type: %s", db.Type)
 	}
 	GlobalCache = c
+	DumpCache = cacheNew.New(15*time.Second, 60*time.Second)
 }
 
 func (cache *Cache) Set(key string, v interface{}) {
@@ -72,3 +75,21 @@ func (cache *Cache) Get(key string) (interface{}, error) {
 	}
 	return nil, fmt.Errorf("no cache type")
 }
+
+func (cache *Cache) GetOnce(key string) (interface{}, error) {
+	switch cache.Type {
+	case "self":
+		if val, hasKey := cache.Self.Get(key); hasKey {
+			cache.Self.Delete(key)
+			return val, nil
+		}
+		return nil, fmt.Errorf("no cache")
+	case "redis":
+		result, err := cache.Redis.Get(ctx, key).Result()
+		cache.Redis.Del(ctx, key).Result()
+		return result, err
+	case "none":
+		return nil, fmt.Errorf("no cache")
+	}
+	return nil, fmt.Errorf("no cache type")
+}

pkg/ipinfo/qqwry/download.go

@@ -1,5 +1,5 @@
 // Package qqwry
-//nolint
+// nolint
 package qqwry
 
 import (

pkg/utils/password.go

@@ -1,6 +1,9 @@
 package utils
 
-import "golang.org/x/crypto/bcrypt"
+import (
+	"github.com/google/uuid"
+	"golang.org/x/crypto/bcrypt"
+)
 
 // HashAndSalt 加密密码
 func HashAndSalt(pwd string) (string, error) {
@@ -16,3 +19,9 @@ func ValidatePWD(hashedPwd, plainPwd string) bool {
 	err := bcrypt.CompareHashAndPassword(byteHash, []byte(plainPwd))
 	return err == nil
 }
+
+func GenerateUUID() string {
+	// 使用标准库中的 uuid 包生成 UUID
+	id := uuid.New()
+	return id.String()
+}

vendor/github.com/google/uuid/.travis.yml

@@ -20,6 +20,9 @@ github.com/go-redis/redis/v8/internal/pool
 github.com/go-redis/redis/v8/internal/proto
 github.com/go-redis/redis/v8/internal/rand
 github.com/go-redis/redis/v8/internal/util
+# github.com/google/uuid v1.1.2
+## explicit
+github.com/google/uuid
 # github.com/jonboulle/clockwork v0.2.2
 ## explicit; go 1.13
 # github.com/josharian/intern v1.0.0