diff --git a/client.go b/client.go index feecfaa..34dafdd 100644 --- a/client.go +++ b/client.go @@ -4,15 +4,9 @@ import ( "github.com/imroc/req" ) -func sendNewZhihuRequest(u string) ([]byte, error) { - r, err := req.Get(u, req.Header{ +func sendNewZhihuRequest(u string) (*req.Resp, error) { + return req.Get(u, req.Header{ "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.75 Safari/537.36", "Referer": "https://zhuanlan.zhihu.com/", }, nil) - - if err != nil { - return nil, err - } - - return r.ToBytes() } diff --git a/crawler.go b/crawler.go index 09134bc..c5f761c 100644 --- a/crawler.go +++ b/crawler.go @@ -1,9 +1,7 @@ package ZhihuZhuanlanCrawler import ( - "encoding/json" "fmt" - "log" ) func GetPinnedArticlePidAndAuthor(columnName string) (*PinnedArticleAndAuthor, error) { @@ -13,14 +11,12 @@ func GetPinnedArticlePidAndAuthor(columnName string) (*PinnedArticleAndAuthor, e u := fmt.Sprintf("https://zhuanlan.zhihu.com/api/columns/%s/pinned-article", columnName) res, err := sendNewZhihuRequest(u) if err != nil { - log.Println(err) return nil, err } pinnedArticleAndAuthor := PinnedArticleAndAuthor{} - err = json.Unmarshal(res, &pinnedArticleAndAuthor) + err = res.ToJSON(&pinnedArticleAndAuthor) if err != nil { - log.Println(err) return nil, err } @@ -34,14 +30,12 @@ func GetSingleArticle(pid int) (*Article, error) { u := fmt.Sprintf("https://api.zhihu.com/articles/%d", pid) res, err := sendNewZhihuRequest(u) if err != nil { - log.Println(err) return nil, err } article := Article{} - err = json.Unmarshal(res, &article) + err = res.ToJSON(&article) if err != nil { - log.Println(err) return nil, err } @@ -59,14 +53,12 @@ func GetArticlesListPids(columnName string) ([]int, error) { u := fmt.Sprintf("https://zhuanlan.zhihu.com/api/columns/%s/articles?limit=%d&offset=%d", columnName, limit, offset) res, err := sendNewZhihuRequest(u) if err != nil { - log.Println(err) return nil, err } articleList := ArticleList{} - err = json.Unmarshal(res, &articleList) + err = res.ToJSON(&articleList) if err != nil { - log.Println(err) return nil, err } @@ -80,14 +72,12 @@ func GetArticlesListPids(columnName string) ([]int, error) { u := fmt.Sprintf("https://zhuanlan.zhihu.com/api/columns/%s/articles?limit=%d&offset=%d", columnName, limit, offset) res, err := sendNewZhihuRequest(u) if err != nil { - log.Println(err) return nil, err } articleList := ArticleList{} - err = json.Unmarshal(res, &articleList) + err = res.ToJSON(&articleList) if err != nil { - log.Println(err) return nil, err } for _, entry := range articleList.Data {