updated

cmd/angel/main.go

@@ -11,7 +11,7 @@ import (
 	"github.com/urfave/cli/v2"
 )
 
-var version = "0.6.1"
+var version = "0.7.0"
 
 func main() {
 	cli.VersionFlag = &cli.BoolFlag{
@@ -49,6 +49,7 @@ func main() {
 					&ktorrent.TorrentSome{},
 					&ktorrent.KTXTorrent{},
 					&ktorrent.JuJuTorrent{},
+					&ktorrent.TorrentRJ{},
 				}
 				s = common.GetAvailableSites(s)
 				fmt.Printf("[*] Angel found %d available site(s) ...\n", len(s))

common/common.go

@@ -43,6 +43,7 @@ var (
 		"torrentgram":   "https://torrentgram29.com",
 		"torrentsome":   "https://torrentsome24.com",
 		"ktxtorrent":    "https://ktxtorrent37.com",
+		"torrentrj":     "https://torrentrj35.com",
 	}
 	UserAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 11_1_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.96 Safari/537.36"
 )
@@ -197,6 +198,7 @@ func GetAvailableSites(oldItems []Scraping) []Scraping {
 		"tshare", "torrentsir", "torrentj",
 		"torrentsee", "torrenttoast", "torrentqq",
 		"torrentsome", "ktxtorrent", "jujutorrent",
+		"torrentrj",
 	}
 	ch := make(chan int, len(items))
 	var wg sync.WaitGroup

ktorrent/torrentrj.go

@@ -0,0 +1,89 @@
+package ktorrent
+
+import (
+	"fmt"
+	"net/url"
+	"strings"
+	"sync"
+
+	"github.com/PuerkitoBio/goquery"
+	"github.com/daite/angel/common"
+)
+
+// TorrentRJ struct is for TorrentSee torrent web site
+type TorrentRJ struct {
+	Name        string
+	Keyword     string
+	SearchURL   string
+	ScrapedData *sync.Map
+}
+
+// initialize method set keyword and URL based on default url
+func (t *TorrentRJ) initialize(keyword string) {
+	t.Keyword = keyword
+	t.Name = "torrentrj"
+	t.SearchURL = common.TorrentURL[t.Name] + "/search/index?keywords=" + url.QueryEscape(t.Keyword)
+}
+
+// Crawl torrent data from web site
+func (t *TorrentRJ) Crawl(keyword string) map[string]string {
+	t.initialize(keyword)
+	data := t.getData(t.SearchURL)
+	if data == nil {
+		return nil
+	}
+	m := map[string]string{}
+	data.Range(
+		func(key, value interface{}) bool {
+			m[fmt.Sprint(key)] = fmt.Sprint(value)
+			return true
+		})
+	return m
+}
+
+// GetData method returns map(title, bbs url)
+func (t *TorrentRJ) getData(url string) *sync.Map {
+	var wg sync.WaitGroup
+	m := &sync.Map{}
+	resp, ok := common.GetResponseFromURL(url)
+	if !ok {
+		return nil
+	}
+	defer resp.Body.Close()
+	doc, err := goquery.NewDocumentFromReader(resp.Body)
+	if err != nil {
+		return nil
+	}
+	doc.Find("a.tit").Each(func(i int, s *goquery.Selection) {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			title := strings.TrimSpace(s.Text())
+			link, _ := s.Attr("href")
+			link = strings.TrimSpace(common.URLJoin(common.TorrentURL[t.Name], link))
+			magnet := t.GetMagnet(link)
+			m.Store(title, magnet)
+		}()
+	})
+	wg.Wait()
+	t.ScrapedData = m
+	return m
+}
+
+// GetMagnet method returns torrent magnet
+func (t *TorrentRJ) GetMagnet(url string) string {
+	resp, ok := common.GetResponseFromURL(url)
+	if !ok {
+		return "failed to fetch magnet"
+	}
+	defer resp.Body.Close()
+	doc, err := goquery.NewDocumentFromReader(resp.Body)
+	if err != nil {
+		return err.Error()
+	}
+	magnet, ok := doc.Find("a.ml-3").Attr("href")
+	if !ok {
+		return "no magnet"
+	}
+	return magnet
+}

ktorrent/torrentrj_test.go

@@ -0,0 +1,51 @@
+package ktorrent
+
+import (
+	"log"
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/PuerkitoBio/goquery"
+)
+
+func TestGetDataFuncForTorrentRJ(t *testing.T) {
+	f, err := os.Open("../resources/torrentrj_search.html")
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer f.Close()
+	doc, err := goquery.NewDocumentFromReader(f)
+	if err != nil {
+		log.Fatal(err)
+	}
+	got := make(map[string]string)
+	doc.Find("a.tit").Each(func(i int, s *goquery.Selection) {
+		title := strings.TrimSpace(s.Text())
+		link, _ := s.Attr("href")
+		got[title] = link
+	})
+	want := map[string]string{
+		"광서열차 2021.1080p.KOR.FHDRip.H264.AAC-JTC": "/v/106444",
+	}
+	if got["광서열차 2021.1080p.KOR.FHDRip.H264.AAC-JTC"] != "/v/106444" {
+		t.Errorf("GetData() for TorrentRJ = %q, want %q", got, want)
+	}
+}
+
+func TestGetMagnetFuncForTorrentRJ(t *testing.T) {
+	f, err := os.Open("../resources/torrentrj_bbs.html")
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer f.Close()
+	doc, err := goquery.NewDocumentFromReader(f)
+	if err != nil {
+		log.Fatal(err)
+	}
+	got, _ := doc.Find("a.ml-3").Attr("href")
+	want := "magnet:?xt=urn:btih:80788dd173e48e5eb139758c165a89c3c048d458"
+	if got != want {
+		t.Errorf("GetMagnet() for TorrentRJ = %q, want %q", got, want)
+	}
+}