diff --git a/crawl-rss/main.go b/crawl-rss/main.go index b6149dc..cc821d4 100644 --- a/crawl-rss/main.go +++ b/crawl-rss/main.go @@ -31,6 +31,10 @@ func main() { for _, torrent := range torrents { addTorrent(db, torrent, crawled) } + torrents = CrawlTPBVideoRecent() + for _, torrent := range torrents { + addTorrent(db, torrent, crawled) + } if i%10 == 0 { torrents = CrawlTPB48hTop() for _, torrent := range torrents { @@ -41,8 +45,8 @@ func main() { } } i++ - time.Sleep(time.Minute * 60) go refresh(db) + time.Sleep(time.Minute * 60) } } @@ -53,7 +57,7 @@ func refresh(db *sql.DB) { func addTorrent(db *sql.DB, torr Torrent, crawled map[string]bool) { if !(crawled[string(torr.Infohash)]) { - _, err := db.Exec("INSERT INTO torrent (infohash, name, length) VALUES ($1, $2, $3)", torr.Infohash, torr.Name, torr.Length) + _, err := db.Exec("INSERT INTO torrent (infohash, name, length) VALUES ($1, $2, $3)", strings.ToLower(torr.Infohash), torr.Name, torr.Length) if err, ok := err.(*pq.Error); ok { //dark magic if err.Code != "23505" { log.Fatal(err) diff --git a/crawl-rss/tpb.go b/crawl-rss/tpb.go index 028e8c5..71e5a13 100644 --- a/crawl-rss/tpb.go +++ b/crawl-rss/tpb.go @@ -7,11 +7,14 @@ import ( "net/http" ) -// CrawlTPB48hTop crawls https://apibay.org/precompiled/data_top100_48h.json func CrawlTPB48hTop() []Torrent { return parseApibayJSON("https://apibay.org/precompiled/data_top100_48h.json") } +func CrawlTPBVideoRecent() []Torrent { + return parseApibayJSON("https://apibay.org/q.php?q=category%3A200") +} + func parseApibayJSON(url string) []Torrent { httpresp, err := http.Get(url) if err != nil {