diff --git a/README.md b/README.md index 562d83f..2cb8085 100644 --- a/README.md +++ b/README.md @@ -28,9 +28,9 @@ Here's what the setup looks like rn: The programs create their own tables in the DB that they need. Database name is "nextgen". -What I did first after getting the server up and running was importing the TPB dump. Download https://thepiratebay.org/static/dump/csv/torrent_dump_full.csv.gz to the import-tpb-dump directory and run `go run`. +What I did first after getting the server up and running was importing the TPB dump. This is sadly no longer possible, since TPB stopped providing the dump. -There is a complete database dump available in torrentparadise-staticbackup.torrent, so you don't have to do that. This same database dump is available on https://mega.nz/#!ddESlChb!3YBqfxG-a4fwpXzPG3QsXa-C6FeQ9AbNSGXxY7W7xm4. It contains the same data as the torrent, only .xz compressed. +There is a database dump available in torrentparadise-staticbackup.torrent. This same database dump is available on https://mega.nz/#!ddESlChb!3YBqfxG-a4fwpXzPG3QsXa-C6FeQ9AbNSGXxY7W7xm4. It contains the same data as the torrent, only .xz compressed. # Usage diff --git a/import-tpb-dump/deploy.sh b/import-tpb-dump/deploy.sh deleted file mode 100755 index 16bb4e9..0000000 --- a/import-tpb-dump/deploy.sh +++ /dev/null @@ -1,3 +0,0 @@ -go build -scp ./import-tpb-dump user@server:/home/nextgen/ -ssh user@server sudo -u nextgen /home/nextgen/import-tpb-dump \ No newline at end of file diff --git a/import-tpb-dump/main.go b/import-tpb-dump/main.go deleted file mode 100644 index 00f60bd..0000000 --- a/import-tpb-dump/main.go +++ /dev/null @@ -1,84 +0,0 @@ -package main - -import ( - "compress/gzip" - "database/sql" - "encoding/base64" - "encoding/csv" - "encoding/hex" - "io" - "log" - "os" - "time" - - "github.com/lib/pq" -) - -func main() { - f, err := os.Open("/home/nextgen/torrent_dump_full.csv.gz") - if err != nil { - log.Fatal(err) - } - defer f.Close() - gr, err := gzip.NewReader(f) - if err != nil { - log.Fatal(err) - } - defer gr.Close() - - db := initDb() - - cr := csv.NewReader(gr) - cr.LazyQuotes = true - cr.Comma = rune(';') - const layout = "2006-Jan-02 15:04:05" - if err != nil { - log.Fatal(err) - } - _, err = cr.Read() // read first line and throw it away - if err != nil { - log.Fatal(err) - } - for { - line, error := cr.Read() - if error == io.EOF { - break - } else if error != nil { - if perr, ok := err.(*csv.ParseError); ok && perr.Err == csv.ErrFieldCount { - log.Println(err) - } - } - - added, err := time.Parse(layout, line[0]) - if err != nil { - log.Println(err) - } - ihBytes, _ := base64.StdEncoding.DecodeString(line[1]) - ih := hex.EncodeToString(ihBytes) - _, err = db.Exec("INSERT INTO torrent (infohash, name, length, added) VALUES ($1, $2, $3, $4)", ih, line[2], line[3], added) - if err, ok := err.(*pq.Error); ok { //dark magic - if err.Code != "23505" { - log.Fatal(err) - } - } - } -} - -func initDb() *sql.DB { - connStr := "user=nextgen dbname=nextgen host=/var/run/postgresql" - db, err := sql.Open("postgres", connStr) - if err != nil { - log.Fatal(err) - } - - _, err = db.Exec(`CREATE TABLE IF NOT EXISTS torrent ( - infohash char(40) PRIMARY KEY NOT NULL, - name varchar NOT NULL, - length bigint, - added timestamp DEFAULT current_timestamp - )`) - if err != nil { - log.Fatal(err) - } - return db -}