package main
import (
"database/sql"
"log"
"strconv"
"sync/atomic"
"time"
"github.com/etix/goscrape"
_ "github.com/lib/pq"
)
const retryLimit = 3
const waitTime = 500 // in ms
var trackers = [ 3 ] string { "udp://tracker.coppersurfer.tk:6969" , "udp://tracker.internetwarriors.net:1337" , "udp://tracker.pirateparty.gr:6969" }
func main ( ) {
db := initDb ( )
trackerResponses := make ( chan trackerResponse , 100 )
trackerRequests := make ( map [ string ] chan [ ] string )
var counter uint64 //count of torrents scraped
quitCounter := make ( chan bool )
go func ( ) {
for {
select {
case <- quitCounter :
return
default :
log . Println ( "Torrents scraped so far: " + strconv . Itoa ( int ( atomic . LoadUint64 ( & counter ) ) ) )
time . Sleep ( 2 * time . Second )
}
}
} ( )
for _ , tracker := range trackers {
trackerRequests [ tracker ] = make ( chan [ ] string , 1000 )
go runScraper ( trackerRequests [ tracker ] , trackerResponses , tracker , waitTime , & counter )
}
datestamp := time . Now ( ) . Local ( ) . Format ( "2006-01-02" )
go runPersister ( trackerResponses , db , datestamp )
rows , err := db . Query ( "SELECT infohash FROM torrent WHERE NOT EXISTS (SELECT FROM trackerdata WHERE infohash = torrent.infohash AND scraped = '" + datestamp + "')" )
if err != nil {
log . Fatal ( err )
}
defer rows . Close ( )
var infohashes [ ] string
for rows . Next ( ) {
var infohash string
if err := rows . Scan ( & infohash ) ; err != nil {
log . Fatal ( err )
}
if len ( infohashes ) < 74 {
infohashes = append ( infohashes , infohash )
} else {
for _ , tracker := range trackers {
trackerRequests [ tracker ] <- infohashes
}
infohashes = [ ] string { }
}
}
for _ , tracker := range trackers {
trackerRequests [ tracker ] <- infohashes
}
quitCounter <- true
for len ( trackerRequests ) > 0 {
time . Sleep ( 2 * time . Second )
var left int
for i , tracker := range trackers {
left = len ( trackerRequests [ tracker ] )
if left != 0 {
log . Println ( "Tracker " + strconv . Itoa ( i ) + " requests left to send: " + strconv . Itoa ( len ( trackerRequests [ tracker ] ) ) )
}
}
}
for _ , tracker := range trackers {
close ( trackerRequests [ tracker ] )
}
for len ( trackerResponses ) > 0 {
time . Sleep ( 2 * time . Second )
log . Println ( "Tracker responses left to save: " + strconv . Itoa ( len ( trackerResponses ) ) )
}
time . Sleep ( time . Duration ( waitTime * retryLimit ) * time . Millisecond )
close ( trackerResponses )
}
//Runs a tracker that scrapes the given tracker. Takes requests from trackerRequests and sends responses to trackerResponses
//waittime is in miliseconds
func runScraper ( trackerRequests chan [ ] string , trackerResponses chan trackerResponse , tracker string , waittime int , counter * uint64 ) {
s , err := goscrape . New ( tracker )
s . SetTimeout ( time . Duration ( waitTime ) * time . Millisecond )
s . SetRetryLimit ( retryLimit )
if err != nil {
log . Fatal ( "Error:" , err )
}
for req := range trackerRequests {
infohashes := make ( [ ] [ ] byte , len ( req ) )
for i , v := range req {
if len ( v ) != 40 { //infohashes are 40 chars long in string representation.
panic ( "Infohash in trackerRequest with index " + strconv . Itoa ( i ) + " isn't 40 chars long, it's " + strconv . Itoa ( len ( v ) ) + " long." )
}
infohashes [ i ] = [ ] byte ( v )
}
res , err := s . Scrape ( infohashes ... )
if err != nil {
log . Println ( err )
} else {
atomic . AddUint64 ( counter , uint64 ( len ( infohashes ) ) )
trackerResponses <- trackerResponse { tracker , res }
}
time . Sleep ( time . Duration ( waittime ) * time . Millisecond )
}
}
func runPersister ( trackerResponses chan trackerResponse , db * sql . DB , datestamp string ) {
for res := range trackerResponses {
for _ , scrapeResult := range res . scrapeResult {
_ , err := db . Exec ( "INSERT INTO trackerdata (infohash, tracker, seeders, leechers, completed, scraped) VALUES ($1, $2, $3, $4, $5, $6)" , scrapeResult . Infohash , res . tracker , scrapeResult . Seeders , scrapeResult . Leechers , scrapeResult . Completed , datestamp )
if err != nil {
log . Fatal ( err )
}
}
}
}
func initDb ( ) * sql . DB {
connStr := "user=nextgen dbname=nextgen host=/var/run/postgresql"
db , err := sql . Open ( "postgres" , connStr )
if err != nil {
log . Fatal ( err )
}
/ * _ , err = db . Exec ( ` CREATE TYPE tracker AS ENUM ('udp://tracker.coppersurfer.tk:6969', 'udp://tracker.internetwarriors.net:1337', 'udp://tracker.opentrackr.org:1337', 'udp://tracker.pirateparty.gr:6969') ` )
if err != nil {
log . Fatal ( err )
} * /
_ , err = db . Exec ( ` CREATE TABLE IF NOT EXISTS trackerdata (
infohash char ( 40 ) ,
tracker tracker ,
seeders int NOT NULL ,
leechers int NOT NULL ,
completed int NOT NULL ,
scraped char ( 10 ) ,
PRIMARY KEY ( infohash , scraped , tracker )
) ` )
if err != nil {
log . Fatal ( err )
}
return db
}
type trackerResponse struct {
tracker string
scrapeResult [ ] * goscrape . ScrapeResult
}