diff --git a/go.mod b/go.mod
index 3f65384..9149c95 100644
--- a/go.mod
+++ b/go.mod
@@ -2,4 +2,4 @@ module github.com/solarhell/ZhihuZhuanlanCrawler
go 1.12
-require github.com/imroc/req v0.2.4
+require github.com/imroc/req v0.3.0
diff --git a/go.sum b/go.sum
index 497e9af..0f3680c 100644
--- a/go.sum
+++ b/go.sum
@@ -1,2 +1,4 @@
github.com/imroc/req v0.2.4 h1:8XbvaQpERLAJV6as/cB186DtH5f0m5zAOtHEaTQ4ac0=
github.com/imroc/req v0.2.4/go.mod h1:J9FsaNHDTIVyW/b5r6/Df5qKEEEq2WzZKIgKSajd1AE=
+github.com/imroc/req v0.3.0 h1:3EioagmlSG+z+KySToa+Ylo3pTFZs+jh3Brl7ngU12U=
+github.com/imroc/req v0.3.0/go.mod h1:F+NZ+2EFSo6EFXdeIbpfE9hcC233id70kf0byW97Caw=
diff --git a/vendor/github.com/imroc/req/README.md b/vendor/github.com/imroc/req/README.md
index c41d5ae..7d54deb 100644
--- a/vendor/github.com/imroc/req/README.md
+++ b/vendor/github.com/imroc/req/README.md
@@ -67,7 +67,8 @@ Examples
[Cookie](#Cookie)
[Set Timeout](#Set-Timeout)
[Set Proxy](#Set-Proxy)
-[Customize Client](#Customize-Client)
+[Customize Client](#Customize-Client)
+[Set context.Context](#Context)
## Basic
``` go
@@ -104,6 +105,25 @@ header.Set("Accept", "application/json")
req.Get("https://www.baidu.com", header)
```
+You can also set header from struct, use `HeaderFromStruct` func to parse your struct
+``` go
+type HeaderStruct struct {
+ UserAgent string `json:"User-Agent"`
+ Authorization string `json:"Authorization"`
+}
+
+func main(){
+ h := HeaderStruct{
+ "V1.0.0",
+ "roc",
+ }
+
+ authHeader := req.HeaderFromStruct(h)
+ req.Get("https://www.baidu.com", authHeader, req.Header{"User-Agent": "V1.1"})
+}
+```
+> Note: Please add tag 'json' to your argument in struct to let you customize the key name of your header
+
## Set Param
Use `req.Param` (it is actually a `map[string]interface{}`)
``` go
@@ -163,7 +183,7 @@ Output in simple way (default format)
``` go
r, _ := req.Get(url, param)
log.Printf("%v\n", r) // GET http://foo.bar/api?name=roc&cmd=add {"code":"0","msg":"success"}
-log.Prinln(r) // smae as above
+log.Prinln(r) // same as above
```
### `%-v` or `%-s`
@@ -281,6 +301,12 @@ Set a simple proxy (use fixed proxy url for every request)
req.SetProxyUrl("http://my.proxy.com:23456")
```
+## Set context.Context
+You can pass context.Context in simple way:
+```go
+r, _ := req.Get(url, context.Background())
+```
+
## Customize Client
Use `SetClient` to change the default underlying `*http.Client`
``` go
diff --git a/vendor/github.com/imroc/req/go.mod b/vendor/github.com/imroc/req/go.mod
new file mode 100644
index 0000000..433bcc0
--- /dev/null
+++ b/vendor/github.com/imroc/req/go.mod
@@ -0,0 +1,3 @@
+module github.com/imroc/req
+
+go 1.12
diff --git a/vendor/github.com/imroc/req/header.go b/vendor/github.com/imroc/req/header.go
new file mode 100644
index 0000000..ac1be32
--- /dev/null
+++ b/vendor/github.com/imroc/req/header.go
@@ -0,0 +1,41 @@
+/*
+ GoLang code created by Jirawat Harnsiriwatanakit https://github.com/kazekim
+*/
+
+package req
+
+import "encoding/json"
+
+// Header represents http request header
+type Header map[string]string
+
+func (h Header) Clone() Header {
+ if h == nil {
+ return nil
+ }
+ hh := Header{}
+ for k, v := range h {
+ hh[k] = v
+ }
+ return hh
+}
+
+// ParseStruct parse struct into header
+func ParseStruct(h Header, v interface{}) Header {
+ data, err := json.Marshal(v)
+ if err != nil {
+ return h
+ }
+
+ err = json.Unmarshal(data, &h)
+ return h
+}
+
+// HeaderFromStruct init header from struct
+func HeaderFromStruct(v interface{}) Header {
+
+ var header Header
+ header = ParseStruct(header, v)
+ return header
+}
+
diff --git a/vendor/github.com/imroc/req/req.go b/vendor/github.com/imroc/req/req.go
index d1b3e71..01937f0 100644
--- a/vendor/github.com/imroc/req/req.go
+++ b/vendor/github.com/imroc/req/req.go
@@ -34,20 +34,6 @@ const (
LstdFlags = LreqHead | LreqBody | LrespHead | LrespBody
)
-// Header represents http request header
-type Header map[string]string
-
-func (h Header) Clone() Header {
- if h == nil {
- return nil
- }
- hh := Header{}
- for k, v := range h {
- hh[k] = v
- }
- return hh
-}
-
// Param represents http request param
type Param map[string]interface{}
@@ -121,15 +107,17 @@ func BodyXML(v interface{}) *bodyXml {
// Req is a convenient client for initiating requests
type Req struct {
- client *http.Client
- jsonEncOpts *jsonEncOpts
- xmlEncOpts *xmlEncOpts
- flag int
+ client *http.Client
+ jsonEncOpts *jsonEncOpts
+ xmlEncOpts *xmlEncOpts
+ flag int
+ progressInterval time.Duration
}
// New create a new *Req
func New() *Req {
- return &Req{flag: LstdFlags}
+ // default progress reporting interval is 200 milliseconds
+ return &Req{flag: LstdFlags, progressInterval: 200 * time.Millisecond}
}
type param struct {
@@ -277,9 +265,10 @@ func (r *Req) Do(method, rawurl string, vs ...interface{}) (resp *Resp, err erro
up = UploadProgress(progress)
}
multipartHelper := &multipartHelper{
- form: formParam.Values,
- uploads: uploads,
- uploadProgress: up,
+ form: formParam.Values,
+ uploads: uploads,
+ uploadProgress: up,
+ progressInterval: resp.r.progressInterval,
}
multipartHelper.Upload(req)
resp.multipartHelper = multipartHelper
@@ -484,10 +473,11 @@ func (b *bodyWrapper) Read(p []byte) (n int, err error) {
}
type multipartHelper struct {
- form url.Values
- uploads []FileUpload
- dump []byte
- uploadProgress UploadProgress
+ form url.Values
+ uploads []FileUpload
+ dump []byte
+ uploadProgress UploadProgress
+ progressInterval time.Duration
}
func (m *multipartHelper) Upload(req *http.Request) {
@@ -514,6 +504,11 @@ func (m *multipartHelper) Upload(req *http.Request) {
var current int64
buf := make([]byte, 1024)
var lastTime time.Time
+
+ defer func() {
+ m.uploadProgress(current, total)
+ }()
+
upload = func(w io.Writer, r io.Reader) error {
for {
n, err := r.Read(buf)
@@ -523,7 +518,7 @@ func (m *multipartHelper) Upload(req *http.Request) {
return _err
}
current += int64(n)
- if now := time.Now(); now.Sub(lastTime) > 200*time.Millisecond {
+ if now := time.Now(); now.Sub(lastTime) > m.progressInterval {
lastTime = now
m.uploadProgress(current, total)
}
diff --git a/vendor/github.com/imroc/req/resp.go b/vendor/github.com/imroc/req/resp.go
index eb56b1b..b464c2b 100644
--- a/vendor/github.com/imroc/req/resp.go
+++ b/vendor/github.com/imroc/req/resp.go
@@ -124,6 +124,11 @@ func (r *Resp) download(file *os.File) error {
total := r.resp.ContentLength
var current int64
var lastTime time.Time
+
+ defer func() {
+ r.downloadProgress(current, total)
+ }()
+
for {
l, err := b.Read(p)
if l > 0 {
@@ -132,7 +137,7 @@ func (r *Resp) download(file *os.File) error {
return _err
}
current += int64(l)
- if now := time.Now(); now.Sub(lastTime) > 200*time.Millisecond {
+ if now := time.Now(); now.Sub(lastTime) > r.r.progressInterval {
lastTime = now
r.downloadProgress(current, total)
}
diff --git a/vendor/github.com/imroc/req/setting.go b/vendor/github.com/imroc/req/setting.go
index 74235f3..ee771e0 100644
--- a/vendor/github.com/imroc/req/setting.go
+++ b/vendor/github.com/imroc/req/setting.go
@@ -234,3 +234,15 @@ func (r *Req) SetXMLIndent(prefix, indent string) {
func SetXMLIndent(prefix, indent string) {
std.SetXMLIndent(prefix, indent)
}
+
+// SetProgressInterval sets the progress reporting interval of both
+// UploadProgress and DownloadProgress handler
+func (r *Req) SetProgressInterval(interval time.Duration) {
+ r.progressInterval = interval
+}
+
+// SetProgressInterval sets the progress reporting interval of both
+// UploadProgress and DownloadProgress handler for the default client
+func SetProgressInterval(interval time.Duration) {
+ std.SetProgressInterval(interval)
+}
diff --git a/vendor/modules.txt b/vendor/modules.txt
index 8c94f58..7379466 100644
--- a/vendor/modules.txt
+++ b/vendor/modules.txt
@@ -1,2 +1,2 @@
-# github.com/imroc/req v0.2.4
+# github.com/imroc/req v0.3.0
github.com/imroc/req