diff --git a/config.example.yaml b/config.example.yaml index 79a242c..16d30f3 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -6,6 +6,8 @@ no_start_message: false diff_mode: false white_keywords: [ ] black_keywords: [ ] +skip_tls_verify: false +proxy: "" pusher: - type: dingding diff --git a/ctrl/config.go b/ctrl/config.go index 3df72dd..31b9195 100644 --- a/ctrl/config.go +++ b/ctrl/config.go @@ -22,6 +22,8 @@ type WatchVulnAppConfig struct { WhiteKeywords []string `yaml:"white_keywords" json:"white_keywords"` BlackKeywords []string `yaml:"black_keywords" json:"black_keywords"` Pusher []map[string]string `yaml:"pusher" json:"pusher"` + Proxy string `yaml:"proxy" json:"proxy"` + SkipTLSVerify bool `yaml:"skip_tls_verify" json:"skip_tls_verify"` NoFilter bool `yaml:"-" json:"-"` Version string `yaml:"-" json:"-"` @@ -54,6 +56,27 @@ func (c *WatchVulnAppConfig) Init() { } if len(c.Sources) == 0 { c.Sources = []string{"avd", "chaitin", "nox", "oscs", "threatbook", "seebug", "struts2", "kev", "venustech"} + + } + + if c.Proxy != "" { + fmt.Println(c.Proxy) + must(os.Setenv("HTTP_PROXY", c.Proxy)) + must(os.Setenv("HTTPS_PROXY", c.Proxy)) + } + if os.Getenv("HTTPS_PROXY") != "" { + must(os.Setenv("HTTP_PROXY", os.Getenv("HTTPS_PROXY"))) + } + + if c.SkipTLSVerify { + // 这个环境变量仅内部使用,go 本身并不支持 + must(os.Setenv("GO_SKIP_TLS_CHECK", "1")) + } +} + +func must(err error) { + if err != nil { + panic(err) } } diff --git a/go.mod b/go.mod index 3070291..13e0cdc 100644 --- a/go.mod +++ b/go.mod @@ -21,8 +21,8 @@ require ( github.com/stretchr/testify v1.8.4 github.com/urfave/cli/v2 v2.26.0 github.com/vimsucks/wxwork-bot-go v0.0.0-20221213061339-fcbcd88ede1c - golang.org/x/net v0.31.0 - golang.org/x/sync v0.9.0 + golang.org/x/net v0.34.0 + golang.org/x/sync v0.10.0 gopkg.in/yaml.v3 v3.0.1 modernc.org/sqlite v1.33.1 ) @@ -69,12 +69,12 @@ require ( github.com/xrash/smetrics v0.0.0-20231213231151-1d8dd44e695e // indirect github.com/zclconf/go-cty v1.13.1 // indirect go.uber.org/mock v0.4.0 // indirect - golang.org/x/crypto v0.29.0 // indirect + golang.org/x/crypto v0.32.0 // indirect golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 // indirect golang.org/x/mod v0.17.0 // indirect golang.org/x/oauth2 v0.8.0 // indirect - golang.org/x/sys v0.27.0 // indirect - golang.org/x/text v0.20.0 // indirect + golang.org/x/sys v0.29.0 // indirect + golang.org/x/text v0.21.0 // indirect golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.33.0 // indirect diff --git a/go.sum b/go.sum index 058a41f..3e70b64 100644 --- a/go.sum +++ b/go.sum @@ -174,8 +174,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= -golang.org/x/crypto v0.29.0 h1:L5SG1JTTXupVV3n6sUqMTeWbjAyfPwoda2DLX8J8FrQ= -golang.org/x/crypto v0.29.0/go.mod h1:+F4F4N5hv6v38hfeYwTdx20oUvLLc+QfrE9Ax9HtgRg= +golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc= +golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc= golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 h1:ESSUROHIBHg7USnszlcdmjBEwdMj9VUvU+OPk4yl2mc= golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8/go.mod h1:/lliqkxwWAhPjf5oSOIJup2XcqJaw8RGS6k3TGEc7GI= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= @@ -193,15 +193,15 @@ golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= -golang.org/x/net v0.31.0 h1:68CPQngjLL0r2AlUKiSxtQFKvzRVbnzLwMUn5SzcLHo= -golang.org/x/net v0.31.0/go.mod h1:P4fl1q7dY2hnZFxEk4pPSkDHF+QqjitcnDjUQyMM+pM= +golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0= +golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k= golang.org/x/oauth2 v0.8.0 h1:6dkIjl3j3LtZ/O3sTgZTMsLKSftL/B8Zgq4huOIIUu8= golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.9.0 h1:fEo0HyrW1GIgZdpbhCRO0PkJajUS5H9IFUztCgEo2jQ= -golang.org/x/sync v0.9.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -216,8 +216,8 @@ golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s= -golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= +golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= @@ -234,8 +234,8 @@ golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.20.0 h1:gK/Kv2otX8gz+wn7Rmb3vT96ZwuoxnQlY+HlJVj7Qug= -golang.org/x/text v0.20.0/go.mod h1:D4IsuqiFMhST5bX19pQ9ikHC2GsaKyk/oF+pn3ducp4= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/grab/avd.go b/grab/avd.go index edef489..436a087 100644 --- a/grab/avd.go +++ b/grab/avd.go @@ -51,7 +51,7 @@ func NewAVDCrawler() Grabber { if err != nil { return errors.Wrap(err, "waf bypass failed") } - crawler.log.Infof("got new url %s", newUrl) + crawler.log.Debugf("got new url %s", newUrl) req.RawURL = newUrl return nil }) diff --git a/main.go b/main.go index 76529df..862c652 100644 --- a/main.go +++ b/main.go @@ -305,10 +305,10 @@ func initConfigFromCli(c *cli.Context) (*ctrl.WatchVulnAppConfig, error) { debug := c.Bool("debug") iv := c.String("interval") db := c.String("db") - proxy := c.String("proxy") diff := c.Bool("diff") whitelistFile := c.String("whitelist-file") blacklistFile := c.String("blacklist-file") + proxy := c.String("proxy") insecure := c.Bool("insecure") if os.Getenv("INTERVAL") != "" { @@ -332,18 +332,6 @@ func initConfigFromCli(c *cli.Context) (*ctrl.WatchVulnAppConfig, error) { if os.Getenv("DB_CONN") != "" { db = os.Getenv("DB_CONN") } - if proxy != "" { - must(os.Setenv("HTTP_PROXY", proxy)) - must(os.Setenv("HTTPS_PROXY", proxy)) - } - if os.Getenv("HTTPS_PROXY") != "" { - must(os.Setenv("HTTP_PROXY", os.Getenv("HTTPS_PROXY"))) - } - - if insecure { - // 这个环境变量仅内部使用,go 本身并不支持 - must(os.Setenv("GO_SKIP_TLS_CHECK", "1")) - } log.Infof("config: INTERVAL=%s, NO_FILTER=%v, NO_START_MESSAGE=%v, NO_GITHUB_SEARCH=%v, ENABLE_CVE_FILTER=%v", iv, noFilter, noStartMessage, noGithubSearch, cveFilter) @@ -394,6 +382,8 @@ func initConfigFromCli(c *cli.Context) (*ctrl.WatchVulnAppConfig, error) { WhiteKeywords: whiteKeywords, BlackKeywords: blackKeywords, Pusher: pusher, + Proxy: proxy, + SkipTLSVerify: insecure, } return config, nil } @@ -540,12 +530,6 @@ func signalCtx() (context.Context, func()) { return ctx, cancel } -func must(err error) { - if err != nil { - panic(err) - } -} - func splitLines(path string) ([]string, error) { var products []string if path != "" {