diff --git a/generate-top-torrents/main.go b/generate-top-torrents/main.go new file mode 100644 index 0000000..1a6a3ef --- /dev/null +++ b/generate-top-torrents/main.go @@ -0,0 +1,55 @@ +package main + +import ( + "bufio" + "encoding/csv" + "encoding/json" + "fmt" + "log" + "os" + "sort" + "strconv" +) + +type torrent struct { + ID string `json:"id"` + Text string `json:"text"` + Len int `json:"len"` + S int `json:"s"` + L int `json:"l"` + C int `json:"c"` +} + +func main() { + // https://www.dotnetperls.com/csv-go + f, err := os.Open("index-generator/dump.csv") // expects that you will run it from the parent dir + if err != nil { + log.Fatal(err) + } + records, err := csv.NewReader(bufio.NewReader(f)).ReadAll() + if err != nil { + log.Fatal(err) + } + sort.Slice(records, func(i, j int) bool { + return toInt(records[i][3]) > toInt(records[j][3]) + }) + var topResults [500]torrent + for i, record := range records { + if i < 500 { + topResults[i] = torrent{string(record[0]), string(record[1]), toInt(record[2]), toInt(record[3]), toInt(record[4]), toInt(record[5])} + } + } + out, err := json.Marshal(topResults) + if err != nil { + log.Fatal(err) + } + fmt.Println(string(out)) +} + +func toInt(s string) int { + i, err := strconv.Atoi(s) + if err != nil { + log.Fatal(err) + } + return i +} diff --git a/snippets.sql b/snippets.sql index c216d5e..9c65114 100644 --- a/snippets.sql +++ b/snippets.sql @@ -1,5 +1,5 @@ -- SELECT for the index-ready materialized view "fresh" -select distinct on (infohash) trackerdata.infohash, torrent.name, torrent.length, seeders, leechers, completed, tracker from trackerdata inner join torrent on (trackerdata.infohash = torrent.infohash) order by infohash, scraped asc, seeders desc; +select distinct on (infohash) trackerdata.infohash, torrent.name, torrent.length, trackerdata.seeders, trackerdata.leechers, trackerdata.completed, tracker from trackerdata inner join torrent on (trackerdata.infohash = torrent.infohash) where torrent.copyrighted != 't' order by infohash, scraped asc, seeders desc; -- to create this view, run: create materialized view AS (paste from above) select sum(seeders),tracker from trackerdata group by tracker; diff --git a/update-index.sh b/update-index.sh index fbb4eaa..e3091eb 100755 --- a/update-index.sh +++ b/update-index.sh @@ -7,11 +7,13 @@ ssh nextgen@server "psql -c 'REFRESH MATERIALIZED VIEW fresh'" echo "Generating index dump" rm index-generator/dump.csv -ssh nextgen@server "psql -c '\copy (select * from fresh) to stdout with format csv'" > index-generator/dump.csv +ssh nextgen@server "psql -c '\copy (select * from fresh) to stdout with (format csv)'" > index-generator/dump.csv (cd index-generator; node --max-old-space-size=10000 main.js) python3 index-generator/fix-metajson.py website/generated/inx +generate-top-torrents/generate-top-torrents > website/generated/top.json + echo "Uploading website" cd website -scp -r . root@server:/www/torrent-paradise.ml #todo use rsync https://superuser.com/a/1405020/373473 +rsync -ar ./ root@server:/www/torrent-paradise.ml # consider using --progress \ No newline at end of file diff --git a/website/adsnippet b/website/adsnippet index a61fa80..ccf403b 100644 --- a/website/adsnippet +++ b/website/adsnippet @@ -1,3 +1,3 @@
ads are here to fund the server, there are no ads if you use IPFS
- +