diff --git a/README.md b/README.md index 4b44851..f670902 100644 --- a/README.md +++ b/README.md @@ -10,34 +10,15 @@ package main import ( "fmt" - "github.com/solarhell/ZhihuZhuanlanCrawler" "log" - "net/http" "os" - "time" + Zhihu "github.com/solarhell/ZhihuZhuanlanCrawler" ) func main() { - log.SetFlags(log.LstdFlags | log.Lshortfile) - - const debug = true - - c := ZhihuZhuanlanCrawler.NewClient(&http.Client{ - Timeout: 30 * time.Second, - Transport: &ZhihuZhuanlanCrawler.DebugRequestTransport{ - RequestHeader: debug, - RequestBody: debug, - ResponseHeader: debug, - ResponseBody: debug, - Transport: &http.Transport{ - IdleConnTimeout: 30 * time.Second, - }, - }, - }) - const columnName = "OTalk" - pinnedArticlePidAndAuthor, err := c.GetPinnedArticlePidAndAuthor(columnName) + pinnedArticlePidAndAuthor, err := Zhihu.GetPinnedArticlePidAndAuthor(columnName) if err != nil { log.Println(err) os.Exit(1) @@ -45,7 +26,7 @@ func main() { fmt.Printf("%+v\n", *pinnedArticlePidAndAuthor) - pinnedArticle, err := c.GetSingleArticle(pinnedArticlePidAndAuthor.ID) + pinnedArticle, err := Zhihu.GetSingleArticle(pinnedArticlePidAndAuthor.ID) if err != nil { log.Println(err) os.Exit(1) @@ -53,7 +34,7 @@ func main() { fmt.Printf("%+v\n", *pinnedArticle) - pids, err := c.GetArticlesListPids(columnName) + pids, err := Zhihu.GetArticlesListPids(columnName) if err != nil { log.Println(err) os.Exit(1) @@ -63,7 +44,7 @@ func main() { if pid == pinnedArticle.ID { continue } - article, err := c.GetSingleArticle(pid) + article, err := Zhihu.GetSingleArticle(pid) if err != nil { log.Println(err) os.Exit(1) @@ -72,8 +53,3 @@ func main() { } } ``` - -## credits - -httpClient的代码来自 https://github.com/mozillazg/go-cos/blob/master/debug/http.go 感谢🙏 - diff --git a/client.go b/client.go index 408f89a..feecfaa 100644 --- a/client.go +++ b/client.go @@ -1,50 +1,18 @@ package ZhihuZhuanlanCrawler import ( - "io/ioutil" - "log" - "net/http" + "github.com/imroc/req" ) -type Client struct { - client *http.Client -} - -func NewClient(httpClient *http.Client) *Client { - if httpClient == nil { - httpClient = &http.Client{} - } - - c := &Client{ - client: httpClient, - } - - return c -} - -func (c *Client) SendNewZhihuRequest(u string) ([]byte, error) { - req, err := http.NewRequest("GET", u, nil) - if err != nil { - return nil, err - } - - req.Header.Add("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.75 Safari/537.36") - req.Header.Add("Host", "zhuanlan.zhihu.com") - req.Header.Add("Referer", "https://zhuanlan.zhihu.com/") - - res, err := c.client.Do(req) - if err != nil { - log.Println(err) - return nil, err - } - - defer res.Body.Close() +func sendNewZhihuRequest(u string) ([]byte, error) { + r, err := req.Get(u, req.Header{ + "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.75 Safari/537.36", + "Referer": "https://zhuanlan.zhihu.com/", + }, nil) - bodyByte, err := ioutil.ReadAll(res.Body) if err != nil { - log.Println(err) return nil, err } - return bodyByte, nil + return r.ToBytes() } diff --git a/crawler.go b/crawler.go index 01ffdd2..09134bc 100644 --- a/crawler.go +++ b/crawler.go @@ -6,12 +6,12 @@ import ( "log" ) -func (c *Client) GetPinnedArticlePidAndAuthor(columnName string) (*PinnedArticleAndAuthor, error) { +func GetPinnedArticlePidAndAuthor(columnName string) (*PinnedArticleAndAuthor, error) { if columnName == "" { return nil, ColumnNameCanNotBeEmpty } u := fmt.Sprintf("https://zhuanlan.zhihu.com/api/columns/%s/pinned-article", columnName) - res, err := c.SendNewZhihuRequest(u) + res, err := sendNewZhihuRequest(u) if err != nil { log.Println(err) return nil, err @@ -27,12 +27,12 @@ func (c *Client) GetPinnedArticlePidAndAuthor(columnName string) (*PinnedArticle return &pinnedArticleAndAuthor, nil } -func (c *Client) GetSingleArticle(pid int) (*Article, error) { +func GetSingleArticle(pid int) (*Article, error) { if pid == 0 { return nil, PidCanNotBeEmpty } u := fmt.Sprintf("https://api.zhihu.com/articles/%d", pid) - res, err := c.SendNewZhihuRequest(u) + res, err := sendNewZhihuRequest(u) if err != nil { log.Println(err) return nil, err @@ -48,7 +48,7 @@ func (c *Client) GetSingleArticle(pid int) (*Article, error) { return &article, nil } -func (c *Client) GetArticlesListPids(columnName string) ([]int, error) { +func GetArticlesListPids(columnName string) ([]int, error) { if columnName == "" { return nil, ColumnNameCanNotBeEmpty } @@ -57,7 +57,7 @@ func (c *Client) GetArticlesListPids(columnName string) ([]int, error) { var offset = 0 u := fmt.Sprintf("https://zhuanlan.zhihu.com/api/columns/%s/articles?limit=%d&offset=%d", columnName, limit, offset) - res, err := c.SendNewZhihuRequest(u) + res, err := sendNewZhihuRequest(u) if err != nil { log.Println(err) return nil, err @@ -78,7 +78,7 @@ func (c *Client) GetArticlesListPids(columnName string) ([]int, error) { for offset = offset + limit; offset < articleList.Paging.Totals; offset = offset + limit { u := fmt.Sprintf("https://zhuanlan.zhihu.com/api/columns/%s/articles?limit=%d&offset=%d", columnName, limit, offset) - res, err := c.SendNewZhihuRequest(u) + res, err := sendNewZhihuRequest(u) if err != nil { log.Println(err) return nil, err diff --git a/crawler_test.go b/crawler_test.go index 65145a6..e90d4a2 100644 --- a/crawler_test.go +++ b/crawler_test.go @@ -3,36 +3,25 @@ package ZhihuZhuanlanCrawler import "testing" const columnName = "OTalk" // https://zhuanlan.zhihu.com/Otalk -const pid = 41604227 // https://zhuanlan.zhihu.com/p/41604227 - +const pid = 60968502 // https://zhuanlan.zhihu.com/p/60968502 func TestClient_GetPinnedArticlePidAndAuthor(t *testing.T) { - c := NewClient(nil) - _, err := c.GetPinnedArticlePidAndAuthor(columnName) + _, err := GetPinnedArticlePidAndAuthor(columnName) if err != nil { t.Error(err.Error()) } - - t.Log("GetPinnedArticlePidAndAuthor ok") - } func TestClient_GetArticlesListPids(t *testing.T) { - c := NewClient(nil) - _, err := c.GetArticlesListPids(columnName) + _, err := GetArticlesListPids(columnName) if err != nil { t.Error(err.Error()) } - - t.Log("GetArticlesListPids ok") } func TestClient_GetSingleArticle(t *testing.T) { - c := NewClient(nil) - _, err := c.GetSingleArticle(pid) + _, err := GetSingleArticle(pid) if err != nil { t.Error(err.Error()) } - - t.Log("GetSingleArticle ok") } diff --git a/go.mod b/go.mod index 4cb7ec7..3f65384 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,5 @@ module github.com/solarhell/ZhihuZhuanlanCrawler go 1.12 + +require github.com/imroc/req v0.2.4 diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..497e9af --- /dev/null +++ b/go.sum @@ -0,0 +1,2 @@ +github.com/imroc/req v0.2.4 h1:8XbvaQpERLAJV6as/cB186DtH5f0m5zAOtHEaTQ4ac0= +github.com/imroc/req v0.2.4/go.mod h1:J9FsaNHDTIVyW/b5r6/Df5qKEEEq2WzZKIgKSajd1AE= diff --git a/httpclient.go b/httpclient.go deleted file mode 100644 index ae22344..0000000 --- a/httpclient.go +++ /dev/null @@ -1,70 +0,0 @@ -package ZhihuZhuanlanCrawler - -import ( - "fmt" - "io" - "net/http" - "net/http/httputil" - "os" -) - -// DebugRequestTransport 会打印请求和响应信息, 方便调试. -type DebugRequestTransport struct { - RequestHeader bool - RequestBody bool // RequestHeader 为 true 时,这个选项才会生效 - ResponseHeader bool - ResponseBody bool // ResponseHeader 为 true 时,这个选项才会生效 - - // debug 信息输出到 Writer 中, 默认是 os.Stderr - Writer io.Writer - - Transport http.RoundTripper -} - -// RoundTrip implements the RoundTripper interface. -func (t *DebugRequestTransport) RoundTrip(req *http.Request) (*http.Response, error) { - req = cloneRequest(req) // per RoundTrip contract - w := t.Writer - if w == nil { - w = os.Stderr - } - - if t.RequestHeader { - a, _ := httputil.DumpRequestOut(req, t.RequestBody) - fmt.Fprintf(w, "%s\n\n", string(a)) - } - - resp, err := t.transport().RoundTrip(req) - if err != nil { - return resp, err - } - - if t.ResponseHeader { - - b, _ := httputil.DumpResponse(resp, t.ResponseBody) - fmt.Fprintf(w, "%s\n", string(b)) - } - - return resp, err -} - -func (t *DebugRequestTransport) transport() http.RoundTripper { - if t.Transport != nil { - return t.Transport - } - return http.DefaultTransport -} - -// cloneRequest returns a clone of the provided *http.Request. The clone is a -// shallow copy of the struct and its Header map. -func cloneRequest(r *http.Request) *http.Request { - // shallow copy of the struct - r2 := new(http.Request) - *r2 = *r - // deep copy of the Header - r2.Header = make(http.Header, len(r.Header)) - for k, s := range r.Header { - r2.Header[k] = append([]string(nil), s...) - } - return r2 -} diff --git a/httpclient_test.go b/httpclient_test.go deleted file mode 100644 index 3f8f5e2..0000000 --- a/httpclient_test.go +++ /dev/null @@ -1,78 +0,0 @@ -package ZhihuZhuanlanCrawler - -import ( - "bytes" - "net/http" - "net/http/httptest" - "strings" - "testing" -) - -var ( - // mux is the HTTP request multiplexer used with the test server. - mux *http.ServeMux - - // server is a test HTTP server used to provide mock API responses. - server *httptest.Server -) - -// setup sets up a test HTTP server along with a cos.Client that is -// configured to talk to that test server. Tests should register handlers on -// mux which provide mock responses for the API method being tested. -func setup() { - // test server - mux = http.NewServeMux() - server = httptest.NewServer(mux) -} - -// teardown closes the test HTTP server. -func teardown() { - server.Close() -} - -func TestDebugRequestTransport(t *testing.T) { - setup() - defer teardown() - - mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { - w.Header().Add("X-Test-Response", "2333") - w.WriteHeader(http.StatusBadGateway) - w.Write([]byte("test response body")) - }) - - w := bytes.NewBufferString("") - client := http.Client{} - - client.Transport = &DebugRequestTransport{ - RequestHeader: true, - RequestBody: true, - ResponseHeader: true, - ResponseBody: true, - Writer: w, - } - - body := bytes.NewReader([]byte("test_request body")) - req, _ := http.NewRequest("GET", server.URL, body) - req.Header.Add("X-Test-Debug", "123") - client.Do(req) - - b := make([]byte, 800) - w.Read(b) - info := string(b) - if !strings.Contains(info, "GET / HTTP/1.1\r\n") || - !strings.Contains(info, "X-Test-Debug: 123\r\n") { - t.Errorf("DebugRequestTransport debug info %#v don't contains request header", info) - } - if !strings.Contains(info, "\r\n\r\ntest_request body") { - t.Errorf("DebugRequestTransport debug info %#v don't contains request body", info) - } - - if !strings.Contains(info, "HTTP/1.1 502 Bad Gateway\r\n") || - !strings.Contains(info, "X-Test-Response: 2333\r\n") { - t.Errorf("DebugRequestTransport debug info %#v don't contains response header", info) - } - - if !strings.Contains(info, "\r\n\r\ntest response body") { - t.Errorf("DebugRequestTransport debug info %#v don't contains response body", info) - } -} diff --git a/vendor/github.com/imroc/req/LICENSE b/vendor/github.com/imroc/req/LICENSE new file mode 100644 index 0000000..8dada3e --- /dev/null +++ b/vendor/github.com/imroc/req/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/vendor/github.com/imroc/req/README.md b/vendor/github.com/imroc/req/README.md new file mode 100644 index 0000000..c41d5ae --- /dev/null +++ b/vendor/github.com/imroc/req/README.md @@ -0,0 +1,302 @@ +# req +[![GoDoc](https://godoc.org/github.com/imroc/req?status.svg)](https://godoc.org/github.com/imroc/req) + +A golang http request library for humans + + + +Features +======== + +- Light weight +- Simple +- Easy play with JSON and XML +- Easy for debug and logging +- Easy file uploads and downloads +- Easy manage cookie +- Easy set up proxy +- Easy set timeout +- Easy customize http client + + +Document +======== +[中文](doc/README_cn.md) + + +Install +======= +``` sh +go get github.com/imroc/req +``` + +Overview +======= +`req` implements a friendly API over Go's existing `net/http` library. + +`Req` and `Resp` are two most important struct, you can think of `Req` as a client that initiate HTTP requests, `Resp` as a information container for the request and response. They all provide simple and convenient APIs that allows you to do a lot of things. +``` go +func (r *Req) Post(url string, v ...interface{}) (*Resp, error) +``` + +In most cases, only url is required, others are optional, like headers, params, files or body etc. + +There is a default `Req` object, all of its' public methods are wrapped by the `req` package, so you can also think of `req` package as a `Req` object +``` go +// use Req object to initiate requests. +r := req.New() +r.Get(url) + +// use req package to initiate request. +req.Get(url) +``` +You can use `req.New()` to create lots of `*Req` as client with independent configuration + +Examples +======= +[Basic](#Basic) +[Set Header](#Set-Header) +[Set Param](#Set-Param) +[Set Body](#Set-Body) +[Debug](#Debug) +[Output Format](#Format) +[ToJSON & ToXML](#ToJSON-ToXML) +[Get *http.Response](#Response) +[Upload](#Upload) +[Download](#Download) +[Cookie](#Cookie) +[Set Timeout](#Set-Timeout) +[Set Proxy](#Set-Proxy) +[Customize Client](#Customize-Client) + +## Basic +``` go +header := req.Header{ + "Accept": "application/json", + "Authorization": "Basic YWRtaW46YWRtaW4=", +} +param := req.Param{ + "name": "imroc", + "cmd": "add", +} +// only url is required, others are optional. +r, err = req.Post("http://foo.bar/api", header, param) +if err != nil { + log.Fatal(err) +} +r.ToJSON(&foo) // response => struct/map +log.Printf("%+v", r) // print info (try it, you may surprise) +``` + +## Set Header +Use `req.Header` (it is actually a `map[string]string`) +``` go +authHeader := req.Header{ + "Accept": "application/json", + "Authorization": "Basic YWRtaW46YWRtaW4=", +} +req.Get("https://www.baidu.com", authHeader, req.Header{"User-Agent": "V1.1"}) +``` +use `http.Header` +``` go +header := make(http.Header) +header.Set("Accept", "application/json") +req.Get("https://www.baidu.com", header) +``` + +## Set Param +Use `req.Param` (it is actually a `map[string]interface{}`) +``` go +param := req.Param{ + "id": "imroc", + "pwd": "roc", +} +req.Get("http://foo.bar/api", param) // http://foo.bar/api?id=imroc&pwd=roc +req.Post(url, param) // body => id=imroc&pwd=roc +``` +use `req.QueryParam` force to append params to the url (it is also actually a `map[string]interface{}`) +``` go +req.Post("http://foo.bar/api", req.Param{"name": "roc", "age": "22"}, req.QueryParam{"access_token": "fedledGF9Hg9ehTU"}) +/* +POST /api?access_token=fedledGF9Hg9ehTU HTTP/1.1 +Host: foo.bar +User-Agent: Go-http-client/1.1 +Content-Length: 15 +Content-Type: application/x-www-form-urlencoded;charset=UTF-8 +Accept-Encoding: gzip + +age=22&name=roc +*/ +``` + +## Set Body +Put `string`, `[]byte` and `io.Reader` as body directly. +``` go +req.Post(url, "id=roc&cmd=query") +``` +Put object as xml or json body (add `Content-Type` header automatically) +``` go +req.Post(url, req.BodyJSON(&foo)) +req.Post(url, req.BodyXML(&bar)) +``` + +## Debug +Set global variable `req.Debug` to true, it will print detail infomation for every request. +``` go +req.Debug = true +req.Post("http://localhost/test" "hi") +``` +![post](doc/post.png) + +## Output Format +You can use different kind of output format to log the request and response infomation in your log file in defferent scenarios. For example, use `%+v` output format in the development phase, it allows you to observe the details. Use `%v` or `%-v` output format in production phase, just log the information necessarily. + +### `%+v` or `%+s` +Output in detail +``` go +r, _ := req.Post(url, header, param) +log.Printf("%+v", r) // output the same format as Debug is enabled +``` + +### `%v` or `%s` +Output in simple way (default format) +``` go +r, _ := req.Get(url, param) +log.Printf("%v\n", r) // GET http://foo.bar/api?name=roc&cmd=add {"code":"0","msg":"success"} +log.Prinln(r) // smae as above +``` + +### `%-v` or `%-s` +Output in simple way and keep all in one line (request body or response body may have multiple lines, this format will replace `"\r"` or `"\n"` with `" "`, it's useful when doing some search in your log file) + +### Flag +You can call `SetFlags` to control the output content, decide which pieces can be output. +``` go +const ( + LreqHead = 1 << iota // output request head (request line and request header) + LreqBody // output request body + LrespHead // output response head (response line and response header) + LrespBody // output response body + Lcost // output time costed by the request + LstdFlags = LreqHead | LreqBody | LrespHead | LrespBody +) +``` +``` go +req.SetFlags(req.LreqHead | req.LreqBody | req.LrespHead) +``` + +### Monitoring time consuming +``` go +req.SetFlags(req.LstdFlags | req.Lcost) // output format add time costed by request +r,_ := req.Get(url) +log.Println(r) // http://foo.bar/api 3.260802ms {"code":0 "msg":"success"} +if r.Cost() > 3 * time.Second { // check cost + log.Println("WARN: slow request:", r) +} +``` + +## ToJSON & ToXML +``` go +r, _ := req.Get(url) +r.ToJSON(&foo) +r, _ = req.Post(url, req.BodyXML(&bar)) +r.ToXML(&baz) +``` + +## Get *http.Response +```go +// func (r *Req) Response() *http.Response +r, _ := req.Get(url) +resp := r.Response() +fmt.Println(resp.StatusCode) +``` + +## Upload +Use `req.File` to match files +``` go +req.Post(url, req.File("imroc.png"), req.File("/Users/roc/Pictures/*.png")) +``` +Use `req.FileUpload` to fully control +``` go +file, _ := os.Open("imroc.png") +req.Post(url, req.FileUpload{ + File: file, + FieldName: "file", // FieldName is form field name + FileName: "avatar.png", //Filename is the name of the file that you wish to upload. We use this to guess the mimetype as well as pass it onto the server +}) +``` +Use `req.UploadProgress` to listen upload progress +```go +progress := func(current, total int64) { + fmt.Println(float32(current)/float32(total)*100, "%") +} +req.Post(url, req.File("/Users/roc/Pictures/*.png"), req.UploadProgress(progress)) +fmt.Println("upload complete") +``` + +## Download +``` go +r, _ := req.Get(url) +r.ToFile("imroc.png") +``` +Use `req.DownloadProgress` to listen download progress +```go +progress := func(current, total int64) { + fmt.Println(float32(current)/float32(total)*100, "%") +} +r, _ := req.Get(url, req.DownloadProgress(progress)) +r.ToFile("hello.mp4") +fmt.Println("download complete") +``` + +## Cookie +By default, the underlying `*http.Client` will manage your cookie(send cookie header to server automatically if server has set a cookie for you), you can disable it by calling this function : +``` go +req.EnableCookie(false) +``` +and you can set cookie in request just using `*http.Cookie` +``` go +cookie := new(http.Cookie) +// ...... +req.Get(url, cookie) +``` + +## Set Timeout +``` go +req.SetTimeout(50 * time.Second) +``` + +## Set Proxy +By default, req use proxy from system environment if `http_proxy` or `https_proxy` is specified, you can set a custom proxy or disable it by set `nil` +``` go +req.SetProxy(func(r *http.Request) (*url.URL, error) { + if strings.Contains(r.URL.Hostname(), "google") { + return url.Parse("http://my.vpn.com:23456") + } + return nil, nil +}) +``` +Set a simple proxy (use fixed proxy url for every request) +``` go +req.SetProxyUrl("http://my.proxy.com:23456") +``` + +## Customize Client +Use `SetClient` to change the default underlying `*http.Client` +``` go +req.SetClient(client) +``` +Specify independent http client for some requests +``` go +client := &http.Client{Timeout: 30 * time.Second} +req.Get(url, client) +``` +Change some properties of default client you want +``` go +req.Client().Jar, _ = cookiejar.New(nil) +trans, _ := req.Client().Transport.(*http.Transport) +trans.MaxIdleConns = 20 +trans.TLSHandshakeTimeout = 20 * time.Second +trans.DisableKeepAlives = true +trans.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} +``` diff --git a/vendor/github.com/imroc/req/dump.go b/vendor/github.com/imroc/req/dump.go new file mode 100644 index 0000000..ce6d3a5 --- /dev/null +++ b/vendor/github.com/imroc/req/dump.go @@ -0,0 +1,216 @@ +package req + +import ( + "bufio" + "bytes" + "fmt" + "io" + "io/ioutil" + "net" + "net/http" + "net/http/httputil" + "net/url" + "strings" + "time" +) + +// Debug enable debug mode if set to true +var Debug bool + +// dumpConn is a net.Conn which writes to Writer and reads from Reader +type dumpConn struct { + io.Writer + io.Reader +} + +func (c *dumpConn) Close() error { return nil } +func (c *dumpConn) LocalAddr() net.Addr { return nil } +func (c *dumpConn) RemoteAddr() net.Addr { return nil } +func (c *dumpConn) SetDeadline(t time.Time) error { return nil } +func (c *dumpConn) SetReadDeadline(t time.Time) error { return nil } +func (c *dumpConn) SetWriteDeadline(t time.Time) error { return nil } + +// delegateReader is a reader that delegates to another reader, +// once it arrives on a channel. +type delegateReader struct { + c chan io.Reader + r io.Reader // nil until received from c +} + +func (r *delegateReader) Read(p []byte) (int, error) { + if r.r == nil { + r.r = <-r.c + } + return r.r.Read(p) +} + +type dummyBody struct { + N int + off int +} + +func (d *dummyBody) Read(p []byte) (n int, err error) { + if d.N <= 0 { + err = io.EOF + return + } + left := d.N - d.off + if left <= 0 { + err = io.EOF + return + } + + if l := len(p); l > 0 { + if l >= left { + n = left + err = io.EOF + } else { + n = l + } + d.off += n + for i := 0; i < n; i++ { + p[i] = '*' + } + } + + return +} + +func (d *dummyBody) Close() error { + return nil +} + +type dumpBuffer struct { + bytes.Buffer +} + +func (b *dumpBuffer) Write(p []byte) { + if b.Len() > 0 { + b.Buffer.WriteString("\r\n\r\n") + } + b.Buffer.Write(p) +} + +func (b *dumpBuffer) WriteString(s string) { + b.Write([]byte(s)) +} + +func (r *Resp) dumpRequest(dump *dumpBuffer) { + head := r.r.flag&LreqHead != 0 + body := r.r.flag&LreqBody != 0 + + if head { + r.dumpReqHead(dump) + } + if body { + if r.multipartHelper != nil { + dump.Write(r.multipartHelper.Dump()) + } else if len(r.reqBody) > 0 { + dump.Write(r.reqBody) + } + } +} + +func (r *Resp) dumpReqHead(dump *dumpBuffer) { + reqSend := new(http.Request) + *reqSend = *r.req + if reqSend.URL.Scheme == "https" { + reqSend.URL = new(url.URL) + *reqSend.URL = *r.req.URL + reqSend.URL.Scheme = "http" + } + + if reqSend.ContentLength > 0 { + reqSend.Body = &dummyBody{N: int(reqSend.ContentLength)} + } else { + reqSend.Body = &dummyBody{N: 1} + } + + // Use the actual Transport code to record what we would send + // on the wire, but not using TCP. Use a Transport with a + // custom dialer that returns a fake net.Conn that waits + // for the full input (and recording it), and then responds + // with a dummy response. + var buf bytes.Buffer // records the output + pr, pw := io.Pipe() + defer pw.Close() + dr := &delegateReader{c: make(chan io.Reader)} + + t := &http.Transport{ + Dial: func(net, addr string) (net.Conn, error) { + return &dumpConn{io.MultiWriter(&buf, pw), dr}, nil + }, + } + defer t.CloseIdleConnections() + + client := new(http.Client) + *client = *r.client + client.Transport = t + + // Wait for the request before replying with a dummy response: + go func() { + req, err := http.ReadRequest(bufio.NewReader(pr)) + if err == nil { + // Ensure all the body is read; otherwise + // we'll get a partial dump. + io.Copy(ioutil.Discard, req.Body) + req.Body.Close() + } + + dr.c <- strings.NewReader("HTTP/1.1 204 No Content\r\nConnection: close\r\n\r\n") + pr.Close() + }() + + _, err := client.Do(reqSend) + if err != nil { + dump.WriteString(err.Error()) + } else { + reqDump := buf.Bytes() + if i := bytes.Index(reqDump, []byte("\r\n\r\n")); i >= 0 { + reqDump = reqDump[:i] + } + dump.Write(reqDump) + } +} + +func (r *Resp) dumpResponse(dump *dumpBuffer) { + head := r.r.flag&LrespHead != 0 + body := r.r.flag&LrespBody != 0 + if head { + respDump, err := httputil.DumpResponse(r.resp, false) + if err != nil { + dump.WriteString(err.Error()) + } else { + if i := bytes.Index(respDump, []byte("\r\n\r\n")); i >= 0 { + respDump = respDump[:i] + } + dump.Write(respDump) + } + } + if body && len(r.Bytes()) > 0 { + dump.Write(r.Bytes()) + } +} + +// Cost return the time cost of the request +func (r *Resp) Cost() time.Duration { + return r.cost +} + +// Dump dump the request +func (r *Resp) Dump() string { + dump := new(dumpBuffer) + if r.r.flag&Lcost != 0 { + dump.WriteString(fmt.Sprint(r.cost)) + } + r.dumpRequest(dump) + l := dump.Len() + if l > 0 { + dump.WriteString("=================================") + l = dump.Len() + } + + r.dumpResponse(dump) + + return dump.String() +} diff --git a/vendor/github.com/imroc/req/req.go b/vendor/github.com/imroc/req/req.go new file mode 100644 index 0000000..d1b3e71 --- /dev/null +++ b/vendor/github.com/imroc/req/req.go @@ -0,0 +1,688 @@ +package req + +import ( + "bytes" + "compress/gzip" + "context" + "encoding/json" + "encoding/xml" + "errors" + "fmt" + "io" + "io/ioutil" + "mime/multipart" + "net/http" + "net/textproto" + "net/url" + "os" + "path/filepath" + "strconv" + "strings" + "time" +) + +// default *Req +var std = New() + +// flags to decide which part can be outputed +const ( + LreqHead = 1 << iota // output request head (request line and request header) + LreqBody // output request body + LrespHead // output response head (response line and response header) + LrespBody // output response body + Lcost // output time costed by the request + LstdFlags = LreqHead | LreqBody | LrespHead | LrespBody +) + +// Header represents http request header +type Header map[string]string + +func (h Header) Clone() Header { + if h == nil { + return nil + } + hh := Header{} + for k, v := range h { + hh[k] = v + } + return hh +} + +// Param represents http request param +type Param map[string]interface{} + +// QueryParam is used to force append http request param to the uri +type QueryParam map[string]interface{} + +// Host is used for set request's Host +type Host string + +// FileUpload represents a file to upload +type FileUpload struct { + // filename in multipart form. + FileName string + // form field name + FieldName string + // file to uplaod, required + File io.ReadCloser +} + +type DownloadProgress func(current, total int64) + +type UploadProgress func(current, total int64) + +// File upload files matching the name pattern such as +// /usr/*/bin/go* (assuming the Separator is '/') +func File(patterns ...string) interface{} { + matches := []string{} + for _, pattern := range patterns { + m, err := filepath.Glob(pattern) + if err != nil { + return err + } + matches = append(matches, m...) + } + if len(matches) == 0 { + return errors.New("req: no file have been matched") + } + uploads := []FileUpload{} + for _, match := range matches { + if s, e := os.Stat(match); e != nil || s.IsDir() { + continue + } + file, _ := os.Open(match) + uploads = append(uploads, FileUpload{ + File: file, + FileName: filepath.Base(match), + FieldName: "media", + }) + } + + return uploads +} + +type bodyJson struct { + v interface{} +} + +type bodyXml struct { + v interface{} +} + +// BodyJSON make the object be encoded in json format and set it to the request body +func BodyJSON(v interface{}) *bodyJson { + return &bodyJson{v: v} +} + +// BodyXML make the object be encoded in xml format and set it to the request body +func BodyXML(v interface{}) *bodyXml { + return &bodyXml{v: v} +} + +// Req is a convenient client for initiating requests +type Req struct { + client *http.Client + jsonEncOpts *jsonEncOpts + xmlEncOpts *xmlEncOpts + flag int +} + +// New create a new *Req +func New() *Req { + return &Req{flag: LstdFlags} +} + +type param struct { + url.Values +} + +func (p *param) getValues() url.Values { + if p.Values == nil { + p.Values = make(url.Values) + } + return p.Values +} + +func (p *param) Copy(pp param) { + if pp.Values == nil { + return + } + vs := p.getValues() + for key, values := range pp.Values { + for _, value := range values { + vs.Add(key, value) + } + } +} +func (p *param) Adds(m map[string]interface{}) { + if len(m) == 0 { + return + } + vs := p.getValues() + for k, v := range m { + vs.Add(k, fmt.Sprint(v)) + } +} + +func (p *param) Empty() bool { + return p.Values == nil +} + +// Do execute a http request with sepecify method and url, +// and it can also have some optional params, depending on your needs. +func (r *Req) Do(method, rawurl string, vs ...interface{}) (resp *Resp, err error) { + if rawurl == "" { + return nil, errors.New("req: url not specified") + } + req := &http.Request{ + Method: method, + Header: make(http.Header), + Proto: "HTTP/1.1", + ProtoMajor: 1, + ProtoMinor: 1, + } + resp = &Resp{req: req, r: r} + + var queryParam param + var formParam param + var uploads []FileUpload + var uploadProgress UploadProgress + var progress func(int64, int64) + var delayedFunc []func() + var lastFunc []func() + + for _, v := range vs { + switch vv := v.(type) { + case Header: + for key, value := range vv { + req.Header.Add(key, value) + } + case http.Header: + for key, values := range vv { + for _, value := range values { + req.Header.Add(key, value) + } + } + case *bodyJson: + fn, err := setBodyJson(req, resp, r.jsonEncOpts, vv.v) + if err != nil { + return nil, err + } + delayedFunc = append(delayedFunc, fn) + case *bodyXml: + fn, err := setBodyXml(req, resp, r.xmlEncOpts, vv.v) + if err != nil { + return nil, err + } + delayedFunc = append(delayedFunc, fn) + case url.Values: + p := param{vv} + if method == "GET" || method == "HEAD" { + queryParam.Copy(p) + } else { + formParam.Copy(p) + } + case Param: + if method == "GET" || method == "HEAD" { + queryParam.Adds(vv) + } else { + formParam.Adds(vv) + } + case QueryParam: + queryParam.Adds(vv) + case string: + setBodyBytes(req, resp, []byte(vv)) + case []byte: + setBodyBytes(req, resp, vv) + case bytes.Buffer: + setBodyBytes(req, resp, vv.Bytes()) + case *http.Client: + resp.client = vv + case FileUpload: + uploads = append(uploads, vv) + case []FileUpload: + uploads = append(uploads, vv...) + case *http.Cookie: + req.AddCookie(vv) + case Host: + req.Host = string(vv) + case io.Reader: + fn := setBodyReader(req, resp, vv) + lastFunc = append(lastFunc, fn) + case UploadProgress: + uploadProgress = vv + case DownloadProgress: + resp.downloadProgress = vv + case func(int64, int64): + progress = vv + case context.Context: + req = req.WithContext(vv) + resp.req = req + case error: + return nil, vv + } + } + + if length := req.Header.Get("Content-Length"); length != "" { + if l, err := strconv.ParseInt(length, 10, 64); err == nil { + req.ContentLength = l + } + } + + if len(uploads) > 0 && (req.Method == "POST" || req.Method == "PUT") { // multipart + var up UploadProgress + if uploadProgress != nil { + up = uploadProgress + } else if progress != nil { + up = UploadProgress(progress) + } + multipartHelper := &multipartHelper{ + form: formParam.Values, + uploads: uploads, + uploadProgress: up, + } + multipartHelper.Upload(req) + resp.multipartHelper = multipartHelper + } else { + if progress != nil { + resp.downloadProgress = DownloadProgress(progress) + } + if !formParam.Empty() { + if req.Body != nil { + queryParam.Copy(formParam) + } else { + setBodyBytes(req, resp, []byte(formParam.Encode())) + setContentType(req, "application/x-www-form-urlencoded; charset=UTF-8") + } + } + } + + if !queryParam.Empty() { + paramStr := queryParam.Encode() + if strings.IndexByte(rawurl, '?') == -1 { + rawurl = rawurl + "?" + paramStr + } else { + rawurl = rawurl + "&" + paramStr + } + } + + u, err := url.Parse(rawurl) + if err != nil { + return nil, err + } + req.URL = u + + if host := req.Header.Get("Host"); host != "" { + req.Host = host + } + + for _, fn := range delayedFunc { + fn() + } + + if resp.client == nil { + resp.client = r.Client() + } + + var response *http.Response + if r.flag&Lcost != 0 { + before := time.Now() + response, err = resp.client.Do(req) + after := time.Now() + resp.cost = after.Sub(before) + } else { + response, err = resp.client.Do(req) + } + if err != nil { + return nil, err + } + + for _, fn := range lastFunc { + fn() + } + + resp.resp = response + + if _, ok := resp.client.Transport.(*http.Transport); ok && response.Header.Get("Content-Encoding") == "gzip" && req.Header.Get("Accept-Encoding") != "" { + body, err := gzip.NewReader(response.Body) + if err != nil { + return nil, err + } + response.Body = body + } + + // output detail if Debug is enabled + if Debug { + fmt.Println(resp.Dump()) + } + return +} + +func setBodyBytes(req *http.Request, resp *Resp, data []byte) { + resp.reqBody = data + req.Body = ioutil.NopCloser(bytes.NewReader(data)) + req.ContentLength = int64(len(data)) +} + +func setBodyJson(req *http.Request, resp *Resp, opts *jsonEncOpts, v interface{}) (func(), error) { + var data []byte + switch vv := v.(type) { + case string: + data = []byte(vv) + case []byte: + data = vv + case *bytes.Buffer: + data = vv.Bytes() + default: + if opts != nil { + var buf bytes.Buffer + enc := json.NewEncoder(&buf) + enc.SetIndent(opts.indentPrefix, opts.indentValue) + enc.SetEscapeHTML(opts.escapeHTML) + err := enc.Encode(v) + if err != nil { + return nil, err + } + data = buf.Bytes() + } else { + var err error + data, err = json.Marshal(v) + if err != nil { + return nil, err + } + } + } + setBodyBytes(req, resp, data) + delayedFunc := func() { + setContentType(req, "application/json; charset=UTF-8") + } + return delayedFunc, nil +} + +func setBodyXml(req *http.Request, resp *Resp, opts *xmlEncOpts, v interface{}) (func(), error) { + var data []byte + switch vv := v.(type) { + case string: + data = []byte(vv) + case []byte: + data = vv + case *bytes.Buffer: + data = vv.Bytes() + default: + if opts != nil { + var buf bytes.Buffer + enc := xml.NewEncoder(&buf) + enc.Indent(opts.prefix, opts.indent) + err := enc.Encode(v) + if err != nil { + return nil, err + } + data = buf.Bytes() + } else { + var err error + data, err = xml.Marshal(v) + if err != nil { + return nil, err + } + } + } + setBodyBytes(req, resp, data) + delayedFunc := func() { + setContentType(req, "application/xml; charset=UTF-8") + } + return delayedFunc, nil +} + +func setContentType(req *http.Request, contentType string) { + if req.Header.Get("Content-Type") == "" { + req.Header.Set("Content-Type", contentType) + } +} + +func setBodyReader(req *http.Request, resp *Resp, rd io.Reader) func() { + var rc io.ReadCloser + switch r := rd.(type) { + case *os.File: + stat, err := r.Stat() + if err == nil { + req.ContentLength = stat.Size() + } + rc = r + + case io.ReadCloser: + rc = r + default: + rc = ioutil.NopCloser(rd) + } + bw := &bodyWrapper{ + ReadCloser: rc, + limit: 102400, + } + req.Body = bw + lastFunc := func() { + resp.reqBody = bw.buf.Bytes() + } + return lastFunc +} + +type bodyWrapper struct { + io.ReadCloser + buf bytes.Buffer + limit int +} + +func (b *bodyWrapper) Read(p []byte) (n int, err error) { + n, err = b.ReadCloser.Read(p) + if left := b.limit - b.buf.Len(); left > 0 && n > 0 { + if n <= left { + b.buf.Write(p[:n]) + } else { + b.buf.Write(p[:left]) + } + } + return +} + +type multipartHelper struct { + form url.Values + uploads []FileUpload + dump []byte + uploadProgress UploadProgress +} + +func (m *multipartHelper) Upload(req *http.Request) { + pr, pw := io.Pipe() + bodyWriter := multipart.NewWriter(pw) + go func() { + for key, values := range m.form { + for _, value := range values { + bodyWriter.WriteField(key, value) + } + } + var upload func(io.Writer, io.Reader) error + if m.uploadProgress != nil { + var total int64 + for _, up := range m.uploads { + if file, ok := up.File.(*os.File); ok { + stat, err := file.Stat() + if err != nil { + continue + } + total += stat.Size() + } + } + var current int64 + buf := make([]byte, 1024) + var lastTime time.Time + upload = func(w io.Writer, r io.Reader) error { + for { + n, err := r.Read(buf) + if n > 0 { + _, _err := w.Write(buf[:n]) + if _err != nil { + return _err + } + current += int64(n) + if now := time.Now(); now.Sub(lastTime) > 200*time.Millisecond { + lastTime = now + m.uploadProgress(current, total) + } + } + if err == io.EOF { + return nil + } + if err != nil { + return err + } + } + } + } + + i := 0 + for _, up := range m.uploads { + if up.FieldName == "" { + i++ + up.FieldName = "file" + strconv.Itoa(i) + } + fileWriter, err := bodyWriter.CreateFormFile(up.FieldName, up.FileName) + if err != nil { + continue + } + //iocopy + if upload == nil { + io.Copy(fileWriter, up.File) + } else { + if _, ok := up.File.(*os.File); ok { + upload(fileWriter, up.File) + } else { + io.Copy(fileWriter, up.File) + } + } + up.File.Close() + } + bodyWriter.Close() + pw.Close() + }() + req.Header.Set("Content-Type", bodyWriter.FormDataContentType()) + req.Body = ioutil.NopCloser(pr) +} + +func (m *multipartHelper) Dump() []byte { + if m.dump != nil { + return m.dump + } + var buf bytes.Buffer + bodyWriter := multipart.NewWriter(&buf) + for key, values := range m.form { + for _, value := range values { + m.writeField(bodyWriter, key, value) + } + } + for _, up := range m.uploads { + m.writeFile(bodyWriter, up.FieldName, up.FileName) + } + bodyWriter.Close() + m.dump = buf.Bytes() + return m.dump +} + +func (m *multipartHelper) writeField(w *multipart.Writer, fieldname, value string) error { + h := make(textproto.MIMEHeader) + h.Set("Content-Disposition", + fmt.Sprintf(`form-data; name="%s"`, fieldname)) + p, err := w.CreatePart(h) + if err != nil { + return err + } + _, err = p.Write([]byte(value)) + return err +} + +func (m *multipartHelper) writeFile(w *multipart.Writer, fieldname, filename string) error { + h := make(textproto.MIMEHeader) + h.Set("Content-Disposition", + fmt.Sprintf(`form-data; name="%s"; filename="%s"`, + fieldname, filename)) + h.Set("Content-Type", "application/octet-stream") + p, err := w.CreatePart(h) + if err != nil { + return err + } + _, err = p.Write([]byte("******")) + return err +} + +// Get execute a http GET request +func (r *Req) Get(url string, v ...interface{}) (*Resp, error) { + return r.Do("GET", url, v...) +} + +// Post execute a http POST request +func (r *Req) Post(url string, v ...interface{}) (*Resp, error) { + return r.Do("POST", url, v...) +} + +// Put execute a http PUT request +func (r *Req) Put(url string, v ...interface{}) (*Resp, error) { + return r.Do("PUT", url, v...) +} + +// Patch execute a http PATCH request +func (r *Req) Patch(url string, v ...interface{}) (*Resp, error) { + return r.Do("PATCH", url, v...) +} + +// Delete execute a http DELETE request +func (r *Req) Delete(url string, v ...interface{}) (*Resp, error) { + return r.Do("DELETE", url, v...) +} + +// Head execute a http HEAD request +func (r *Req) Head(url string, v ...interface{}) (*Resp, error) { + return r.Do("HEAD", url, v...) +} + +// Options execute a http OPTIONS request +func (r *Req) Options(url string, v ...interface{}) (*Resp, error) { + return r.Do("OPTIONS", url, v...) +} + +// Get execute a http GET request +func Get(url string, v ...interface{}) (*Resp, error) { + return std.Get(url, v...) +} + +// Post execute a http POST request +func Post(url string, v ...interface{}) (*Resp, error) { + return std.Post(url, v...) +} + +// Put execute a http PUT request +func Put(url string, v ...interface{}) (*Resp, error) { + return std.Put(url, v...) +} + +// Head execute a http HEAD request +func Head(url string, v ...interface{}) (*Resp, error) { + return std.Head(url, v...) +} + +// Options execute a http OPTIONS request +func Options(url string, v ...interface{}) (*Resp, error) { + return std.Options(url, v...) +} + +// Delete execute a http DELETE request +func Delete(url string, v ...interface{}) (*Resp, error) { + return std.Delete(url, v...) +} + +// Patch execute a http PATCH request +func Patch(url string, v ...interface{}) (*Resp, error) { + return std.Patch(url, v...) +} + +// Do execute request. +func Do(method, url string, v ...interface{}) (*Resp, error) { + return std.Do(method, url, v...) +} diff --git a/vendor/github.com/imroc/req/resp.go b/vendor/github.com/imroc/req/resp.go new file mode 100644 index 0000000..eb56b1b --- /dev/null +++ b/vendor/github.com/imroc/req/resp.go @@ -0,0 +1,215 @@ +package req + +import ( + "encoding/json" + "encoding/xml" + "fmt" + "io" + "io/ioutil" + "net/http" + "os" + "regexp" + "time" +) + +// Resp represents a request with it's response +type Resp struct { + r *Req + req *http.Request + resp *http.Response + client *http.Client + cost time.Duration + *multipartHelper + reqBody []byte + respBody []byte + downloadProgress DownloadProgress + err error // delayed error +} + +// Request returns *http.Request +func (r *Resp) Request() *http.Request { + return r.req +} + +// Response returns *http.Response +func (r *Resp) Response() *http.Response { + return r.resp +} + +// Bytes returns response body as []byte +func (r *Resp) Bytes() []byte { + data, _ := r.ToBytes() + return data +} + +// ToBytes returns response body as []byte, +// return error if error happend when reading +// the response body +func (r *Resp) ToBytes() ([]byte, error) { + if r.err != nil { + return nil, r.err + } + if r.respBody != nil { + return r.respBody, nil + } + defer r.resp.Body.Close() + respBody, err := ioutil.ReadAll(r.resp.Body) + if err != nil { + r.err = err + return nil, err + } + r.respBody = respBody + return r.respBody, nil +} + +// String returns response body as string +func (r *Resp) String() string { + data, _ := r.ToBytes() + return string(data) +} + +// ToString returns response body as string, +// return error if error happend when reading +// the response body +func (r *Resp) ToString() (string, error) { + data, err := r.ToBytes() + return string(data), err +} + +// ToJSON convert json response body to struct or map +func (r *Resp) ToJSON(v interface{}) error { + data, err := r.ToBytes() + if err != nil { + return err + } + return json.Unmarshal(data, v) +} + +// ToXML convert xml response body to struct or map +func (r *Resp) ToXML(v interface{}) error { + data, err := r.ToBytes() + if err != nil { + return err + } + return xml.Unmarshal(data, v) +} + +// ToFile download the response body to file with optional download callback +func (r *Resp) ToFile(name string) error { + //TODO set name to the suffix of url path if name == "" + file, err := os.Create(name) + if err != nil { + return err + } + defer file.Close() + + if r.respBody != nil { + _, err = file.Write(r.respBody) + return err + } + + if r.downloadProgress != nil && r.resp.ContentLength > 0 { + return r.download(file) + } + + defer r.resp.Body.Close() + _, err = io.Copy(file, r.resp.Body) + return err +} + +func (r *Resp) download(file *os.File) error { + p := make([]byte, 1024) + b := r.resp.Body + defer b.Close() + total := r.resp.ContentLength + var current int64 + var lastTime time.Time + for { + l, err := b.Read(p) + if l > 0 { + _, _err := file.Write(p[:l]) + if _err != nil { + return _err + } + current += int64(l) + if now := time.Now(); now.Sub(lastTime) > 200*time.Millisecond { + lastTime = now + r.downloadProgress(current, total) + } + } + if err != nil { + if err == io.EOF { + return nil + } + return err + } + } +} + +var regNewline = regexp.MustCompile(`\n|\r`) + +func (r *Resp) autoFormat(s fmt.State) { + req := r.req + if r.r.flag&Lcost != 0 { + fmt.Fprint(s, req.Method, " ", req.URL.String(), " ", r.cost) + } else { + fmt.Fprint(s, req.Method, " ", req.URL.String()) + } + + // test if it is should be outputed pretty + var pretty bool + var parts []string + addPart := func(part string) { + if part == "" { + return + } + parts = append(parts, part) + if !pretty && regNewline.MatchString(part) { + pretty = true + } + } + if r.r.flag&LreqBody != 0 { // request body + addPart(string(r.reqBody)) + } + if r.r.flag&LrespBody != 0 { // response body + addPart(r.String()) + } + + for _, part := range parts { + if pretty { + fmt.Fprint(s, "\n") + } + fmt.Fprint(s, " ", part) + } +} + +func (r *Resp) miniFormat(s fmt.State) { + req := r.req + if r.r.flag&Lcost != 0 { + fmt.Fprint(s, req.Method, " ", req.URL.String(), " ", r.cost) + } else { + fmt.Fprint(s, req.Method, " ", req.URL.String()) + } + if r.r.flag&LreqBody != 0 && len(r.reqBody) > 0 { // request body + str := regNewline.ReplaceAllString(string(r.reqBody), " ") + fmt.Fprint(s, " ", str) + } + if r.r.flag&LrespBody != 0 && r.String() != "" { // response body + str := regNewline.ReplaceAllString(r.String(), " ") + fmt.Fprint(s, " ", str) + } +} + +// Format fort the response +func (r *Resp) Format(s fmt.State, verb rune) { + if r == nil || r.req == nil { + return + } + if s.Flag('+') { // include header and format pretty. + fmt.Fprint(s, r.Dump()) + } else if s.Flag('-') { // keep all informations in one line. + r.miniFormat(s) + } else { // auto + r.autoFormat(s) + } +} diff --git a/vendor/github.com/imroc/req/setting.go b/vendor/github.com/imroc/req/setting.go new file mode 100644 index 0000000..74235f3 --- /dev/null +++ b/vendor/github.com/imroc/req/setting.go @@ -0,0 +1,236 @@ +package req + +import ( + "crypto/tls" + "errors" + "net" + "net/http" + "net/http/cookiejar" + "net/url" + "time" +) + +// create a default client +func newClient() *http.Client { + jar, _ := cookiejar.New(nil) + transport := &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + DualStack: true, + }).DialContext, + MaxIdleConns: 100, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + } + return &http.Client{ + Jar: jar, + Transport: transport, + Timeout: 2 * time.Minute, + } +} + +// Client return the default underlying http client +func (r *Req) Client() *http.Client { + if r.client == nil { + r.client = newClient() + } + return r.client +} + +// Client return the default underlying http client +func Client() *http.Client { + return std.Client() +} + +// SetClient sets the underlying http.Client. +func (r *Req) SetClient(client *http.Client) { + r.client = client // use default if client == nil +} + +// SetClient sets the default http.Client for requests. +func SetClient(client *http.Client) { + std.SetClient(client) +} + +// SetFlags control display format of *Resp +func (r *Req) SetFlags(flags int) { + r.flag = flags +} + +// SetFlags control display format of *Resp +func SetFlags(flags int) { + std.SetFlags(flags) +} + +// Flags return output format for the *Resp +func (r *Req) Flags() int { + return r.flag +} + +// Flags return output format for the *Resp +func Flags() int { + return std.Flags() +} + +func (r *Req) getTransport() *http.Transport { + trans, _ := r.Client().Transport.(*http.Transport) + return trans +} + +// EnableInsecureTLS allows insecure https +func (r *Req) EnableInsecureTLS(enable bool) { + trans := r.getTransport() + if trans == nil { + return + } + if trans.TLSClientConfig == nil { + trans.TLSClientConfig = &tls.Config{} + } + trans.TLSClientConfig.InsecureSkipVerify = enable +} + +func EnableInsecureTLS(enable bool) { + std.EnableInsecureTLS(enable) +} + +// EnableCookieenable or disable cookie manager +func (r *Req) EnableCookie(enable bool) { + if enable { + jar, _ := cookiejar.New(nil) + r.Client().Jar = jar + } else { + r.Client().Jar = nil + } +} + +// EnableCookieenable or disable cookie manager +func EnableCookie(enable bool) { + std.EnableCookie(enable) +} + +// SetTimeout sets the timeout for every request +func (r *Req) SetTimeout(d time.Duration) { + r.Client().Timeout = d +} + +// SetTimeout sets the timeout for every request +func SetTimeout(d time.Duration) { + std.SetTimeout(d) +} + +// SetProxyUrl set the simple proxy with fixed proxy url +func (r *Req) SetProxyUrl(rawurl string) error { + trans := r.getTransport() + if trans == nil { + return errors.New("req: no transport") + } + u, err := url.Parse(rawurl) + if err != nil { + return err + } + trans.Proxy = http.ProxyURL(u) + return nil +} + +// SetProxyUrl set the simple proxy with fixed proxy url +func SetProxyUrl(rawurl string) error { + return std.SetProxyUrl(rawurl) +} + +// SetProxy sets the proxy for every request +func (r *Req) SetProxy(proxy func(*http.Request) (*url.URL, error)) error { + trans := r.getTransport() + if trans == nil { + return errors.New("req: no transport") + } + trans.Proxy = proxy + return nil +} + +// SetProxy sets the proxy for every request +func SetProxy(proxy func(*http.Request) (*url.URL, error)) error { + return std.SetProxy(proxy) +} + +type jsonEncOpts struct { + indentPrefix string + indentValue string + escapeHTML bool +} + +func (r *Req) getJSONEncOpts() *jsonEncOpts { + if r.jsonEncOpts == nil { + r.jsonEncOpts = &jsonEncOpts{escapeHTML: true} + } + return r.jsonEncOpts +} + +// SetJSONEscapeHTML specifies whether problematic HTML characters +// should be escaped inside JSON quoted strings. +// The default behavior is to escape &, <, and > to \u0026, \u003c, and \u003e +// to avoid certain safety problems that can arise when embedding JSON in HTML. +// +// In non-HTML settings where the escaping interferes with the readability +// of the output, SetEscapeHTML(false) disables this behavior. +func (r *Req) SetJSONEscapeHTML(escape bool) { + opts := r.getJSONEncOpts() + opts.escapeHTML = escape +} + +// SetJSONEscapeHTML specifies whether problematic HTML characters +// should be escaped inside JSON quoted strings. +// The default behavior is to escape &, <, and > to \u0026, \u003c, and \u003e +// to avoid certain safety problems that can arise when embedding JSON in HTML. +// +// In non-HTML settings where the escaping interferes with the readability +// of the output, SetEscapeHTML(false) disables this behavior. +func SetJSONEscapeHTML(escape bool) { + std.SetJSONEscapeHTML(escape) +} + +// SetJSONIndent instructs the encoder to format each subsequent encoded +// value as if indented by the package-level function Indent(dst, src, prefix, indent). +// Calling SetIndent("", "") disables indentation. +func (r *Req) SetJSONIndent(prefix, indent string) { + opts := r.getJSONEncOpts() + opts.indentPrefix = prefix + opts.indentValue = indent +} + +// SetJSONIndent instructs the encoder to format each subsequent encoded +// value as if indented by the package-level function Indent(dst, src, prefix, indent). +// Calling SetIndent("", "") disables indentation. +func SetJSONIndent(prefix, indent string) { + std.SetJSONIndent(prefix, indent) +} + +type xmlEncOpts struct { + prefix string + indent string +} + +func (r *Req) getXMLEncOpts() *xmlEncOpts { + if r.xmlEncOpts == nil { + r.xmlEncOpts = &xmlEncOpts{} + } + return r.xmlEncOpts +} + +// SetXMLIndent sets the encoder to generate XML in which each element +// begins on a new indented line that starts with prefix and is followed by +// one or more copies of indent according to the nesting depth. +func (r *Req) SetXMLIndent(prefix, indent string) { + opts := r.getXMLEncOpts() + opts.prefix = prefix + opts.indent = indent +} + +// SetXMLIndent sets the encoder to generate XML in which each element +// begins on a new indented line that starts with prefix and is followed by +// one or more copies of indent according to the nesting depth. +func SetXMLIndent(prefix, indent string) { + std.SetXMLIndent(prefix, indent) +} diff --git a/vendor/modules.txt b/vendor/modules.txt new file mode 100644 index 0000000..8c94f58 --- /dev/null +++ b/vendor/modules.txt @@ -0,0 +1,2 @@ +# github.com/imroc/req v0.2.4 +github.com/imroc/req