diff --git a/crawl-rss/main.go b/crawl-rss/main.go index 17bc2fc..81f6b4c 100644 --- a/crawl-rss/main.go +++ b/crawl-rss/main.go @@ -9,7 +9,6 @@ import ( "time" "github.com/lib/pq" - _ "github.com/lib/pq" "github.com/mmcdole/gofeed" ) @@ -25,9 +24,13 @@ func main() { for { torrents := CrawlYts() for _, torrent := range torrents { - addTorrent(db, torrent, &crawled) + addTorrent(db, torrent, crawled) } - time.Sleep(time.Minute * 45) + torrents = CrawlEztv() + for _, torrent := range torrents { + addTorrent(db, torrent, crawled) + } + time.Sleep(time.Minute * 60) go refresh(db) } } @@ -37,15 +40,15 @@ func refresh(db *sql.DB) { db.Exec("REFRESH MATERIALIZED VIEW CONCURRENTLY search") } -func addTorrent(db *sql.DB, torr Torrent, crawled *map[string]bool) { - if !(*crawled)[string(torr.Infohash)] { +func addTorrent(db *sql.DB, torr Torrent, crawled map[string]bool) { + if !(crawled[string(torr.Infohash)]) { _, err := db.Exec("INSERT INTO torrent (infohash, name, length) VALUES ($1, $2, $3)", torr.Infohash, torr.Name, torr.Length) if err, ok := err.(*pq.Error); ok { //dark magic if err.Code != "23505" { log.Fatal(err) } } - (*crawled)[torr.Infohash] = true + crawled[torr.Infohash] = true } } @@ -69,6 +72,24 @@ func CrawlYts() []Torrent { return torrents } +func CrawlEztv() []Torrent { //maybe is there some kind of interface that this can share with CrawlYts? This function has the same signature and purpose. + fp := gofeed.NewParser() + feed, err := fp.ParseURL("https://eztv.io/ezrss.xml") + if err != nil { + log.Fatal(err) + } + var torrents []Torrent + for _, item := range feed.Items { + size, err := strconv.Atoi(item.Extensions["torrent"]["contentLength"][0].Value) + if err != nil { + log.Print(err) + continue + } + torrents = append(torrents, Torrent{item.Extensions["torrent"]["infoHash"][0].Value, item.Extensions["torrent"]["fileName"][0].Value, size}) + } + return torrents +} + // Parses torrent length from YTS description func parseSizeYts(description string) (int, error) { s := strings.Split(description, "
Size: ") diff --git a/crawl-rss/main_test.go b/crawl-rss/main_test.go index 0d2d9fa..d5e2c26 100644 --- a/crawl-rss/main_test.go +++ b/crawl-rss/main_test.go @@ -10,3 +10,13 @@ func TestCrawlYts(t *testing.T) { t.Error("no torrents crawled from yts") } } + +func TestCrawlEztv(t *testing.T) { + t.Log("t.log") + torrents := CrawlEztv() + if len(torrents) < 1 { + t.Error("no torrents crawled from eztv.io") + } + t.Log(torrents[0].Name) + t.Log(torrents[1].Name) +}