Remove import TPB dump tool

The Pirate Bay no longer provides dumps AFAIK.

Commit 8fdc8d6 added crawling of top 48h TPB torrents list.
pull/27/head
Urban Guacamole 4 years ago
parent 8fdc8d6508
commit 3304722c76

@ -28,9 +28,9 @@ Here's what the setup looks like rn:
The programs create their own tables in the DB that they need. Database name is "nextgen". The programs create their own tables in the DB that they need. Database name is "nextgen".
What I did first after getting the server up and running was importing the TPB dump. Download https://thepiratebay.org/static/dump/csv/torrent_dump_full.csv.gz to the import-tpb-dump directory and run `go run`. What I did first after getting the server up and running was importing the TPB dump. This is sadly no longer possible, since TPB stopped providing the dump.
There is a complete database dump available in torrentparadise-staticbackup.torrent, so you don't have to do that. This same database dump is available on https://mega.nz/#!ddESlChb!3YBqfxG-a4fwpXzPG3QsXa-C6FeQ9AbNSGXxY7W7xm4. It contains the same data as the torrent, only .xz compressed. There is a database dump available in torrentparadise-staticbackup.torrent. This same database dump is available on https://mega.nz/#!ddESlChb!3YBqfxG-a4fwpXzPG3QsXa-C6FeQ9AbNSGXxY7W7xm4. It contains the same data as the torrent, only .xz compressed.
# Usage # Usage

@ -1,3 +0,0 @@
go build
scp ./import-tpb-dump user@server:/home/nextgen/
ssh user@server sudo -u nextgen /home/nextgen/import-tpb-dump

@ -1,84 +0,0 @@
package main
import (
"compress/gzip"
"database/sql"
"encoding/base64"
"encoding/csv"
"encoding/hex"
"io"
"log"
"os"
"time"
"github.com/lib/pq"
)
func main() {
f, err := os.Open("/home/nextgen/torrent_dump_full.csv.gz")
if err != nil {
log.Fatal(err)
}
defer f.Close()
gr, err := gzip.NewReader(f)
if err != nil {
log.Fatal(err)
}
defer gr.Close()
db := initDb()
cr := csv.NewReader(gr)
cr.LazyQuotes = true
cr.Comma = rune(';')
const layout = "2006-Jan-02 15:04:05"
if err != nil {
log.Fatal(err)
}
_, err = cr.Read() // read first line and throw it away
if err != nil {
log.Fatal(err)
}
for {
line, error := cr.Read()
if error == io.EOF {
break
} else if error != nil {
if perr, ok := err.(*csv.ParseError); ok && perr.Err == csv.ErrFieldCount {
log.Println(err)
}
}
added, err := time.Parse(layout, line[0])
if err != nil {
log.Println(err)
}
ihBytes, _ := base64.StdEncoding.DecodeString(line[1])
ih := hex.EncodeToString(ihBytes)
_, err = db.Exec("INSERT INTO torrent (infohash, name, length, added) VALUES ($1, $2, $3, $4)", ih, line[2], line[3], added)
if err, ok := err.(*pq.Error); ok { //dark magic
if err.Code != "23505" {
log.Fatal(err)
}
}
}
}
func initDb() *sql.DB {
connStr := "user=nextgen dbname=nextgen host=/var/run/postgresql"
db, err := sql.Open("postgres", connStr)
if err != nil {
log.Fatal(err)
}
_, err = db.Exec(`CREATE TABLE IF NOT EXISTS torrent (
infohash char(40) PRIMARY KEY NOT NULL,
name varchar NOT NULL,
length bigint,
added timestamp DEFAULT current_timestamp
)`)
if err != nil {
log.Fatal(err)
}
return db
}
Loading…
Cancel
Save