From 2e386bd45a647849c38ef75ea8be574d02a69f9b Mon Sep 17 00:00:00 2001 From: Urban Guacamole Date: Sat, 9 Mar 2019 12:56:11 +0100 Subject: [PATCH] Add Top torrents view by seed count --- generate-top-torrents/main.go | 55 +++++++++++++++++++++++++++++++ snippets.sql | 2 +- update-index.sh | 6 ++-- website/adsnippet | 2 +- website/head.html | 3 ++ website/index.html | 5 +++ website/top.html | 61 +++++++++++++++++++++++++++++++++++ website/view.js | 14 ++++---- 8 files changed, 138 insertions(+), 10 deletions(-) create mode 100644 generate-top-torrents/main.go create mode 100644 website/top.html diff --git a/generate-top-torrents/main.go b/generate-top-torrents/main.go new file mode 100644 index 0000000..1a6a3ef --- /dev/null +++ b/generate-top-torrents/main.go @@ -0,0 +1,55 @@ +package main + +import ( + "bufio" + "encoding/csv" + "encoding/json" + "fmt" + "log" + "os" + "sort" + "strconv" +) + +type torrent struct { + ID string `json:"id"` + Text string `json:"text"` + Len int `json:"len"` + S int `json:"s"` + L int `json:"l"` + C int `json:"c"` +} + +func main() { + // https://www.dotnetperls.com/csv-go + f, err := os.Open("index-generator/dump.csv") // expects that you will run it from the parent dir + if err != nil { + log.Fatal(err) + } + records, err := csv.NewReader(bufio.NewReader(f)).ReadAll() + if err != nil { + log.Fatal(err) + } + sort.Slice(records, func(i, j int) bool { + return toInt(records[i][3]) > toInt(records[j][3]) + }) + var topResults [500]torrent + for i, record := range records { + if i < 500 { + topResults[i] = torrent{string(record[0]), string(record[1]), toInt(record[2]), toInt(record[3]), toInt(record[4]), toInt(record[5])} + } + } + out, err := json.Marshal(topResults) + if err != nil { + log.Fatal(err) + } + fmt.Println(string(out)) +} + +func toInt(s string) int { + i, err := strconv.Atoi(s) + if err != nil { + log.Fatal(err) + } + return i +} diff --git a/snippets.sql b/snippets.sql index c216d5e..9c65114 100644 --- a/snippets.sql +++ b/snippets.sql @@ -1,5 +1,5 @@ -- SELECT for the index-ready materialized view "fresh" -select distinct on (infohash) trackerdata.infohash, torrent.name, torrent.length, seeders, leechers, completed, tracker from trackerdata inner join torrent on (trackerdata.infohash = torrent.infohash) order by infohash, scraped asc, seeders desc; +select distinct on (infohash) trackerdata.infohash, torrent.name, torrent.length, trackerdata.seeders, trackerdata.leechers, trackerdata.completed, tracker from trackerdata inner join torrent on (trackerdata.infohash = torrent.infohash) where torrent.copyrighted != 't' order by infohash, scraped asc, seeders desc; -- to create this view, run: create materialized view AS (paste from above) select sum(seeders),tracker from trackerdata group by tracker; diff --git a/update-index.sh b/update-index.sh index fbb4eaa..e3091eb 100755 --- a/update-index.sh +++ b/update-index.sh @@ -7,11 +7,13 @@ ssh nextgen@server "psql -c 'REFRESH MATERIALIZED VIEW fresh'" echo "Generating index dump" rm index-generator/dump.csv -ssh nextgen@server "psql -c '\copy (select * from fresh) to stdout with format csv'" > index-generator/dump.csv +ssh nextgen@server "psql -c '\copy (select * from fresh) to stdout with (format csv)'" > index-generator/dump.csv (cd index-generator; node --max-old-space-size=10000 main.js) python3 index-generator/fix-metajson.py website/generated/inx +generate-top-torrents/generate-top-torrents > website/generated/top.json + echo "Uploading website" cd website -scp -r . root@server:/www/torrent-paradise.ml #todo use rsync https://superuser.com/a/1405020/373473 +rsync -ar ./ root@server:/www/torrent-paradise.ml # consider using --progress \ No newline at end of file diff --git a/website/adsnippet b/website/adsnippet index a61fa80..ccf403b 100644 --- a/website/adsnippet +++ b/website/adsnippet @@ -1,3 +1,3 @@

ads are here to fund the server, there are no ads if you use IPFS

- + \ No newline at end of file diff --git a/website/head.html b/website/head.html index 0126bc9..3ed54b9 100644 --- a/website/head.html +++ b/website/head.html @@ -18,6 +18,9 @@ About + + Top torrents + IPFS diff --git a/website/index.html b/website/index.html index 73c7992..fa8041c 100644 --- a/website/index.html +++ b/website/index.html @@ -4,6 +4,7 @@ + Torrent Paradise @@ -19,6 +20,9 @@ About + + Top torrents + IPFS @@ -42,6 +46,7 @@ seamless="seamless" v-bind:src="resultPage">

Last index update: {{indexTimestamp}}

Torrents in index: {{entries}}

+ \ No newline at end of file diff --git a/website/top.html b/website/top.html new file mode 100644 index 0000000..94249b6 --- /dev/null +++ b/website/top.html @@ -0,0 +1,61 @@ + + + + + + + + Torrent Paradise + + + + + + + +
+
+

Top torrents

+ + + +
+ + + + \ No newline at end of file diff --git a/website/view.js b/website/view.js index ac4754a..2140ff7 100644 --- a/website/view.js +++ b/website/view.js @@ -1,15 +1,17 @@ app = new Vue({ el: '#app', - data: {showsearchbox: false, error: "", resultPage: "", resultPageHeight: 1, entries: -1} + data: { showsearchbox: false, error: "", resultPage: "", resultPageHeight: 1, entries: -1 } }) window.addEventListener("message", receiveMessage, false); -function receiveMessage(event){ +function receiveMessage(event) { app.resultPageHeight = event.data } - -document.getElementById('searchbox').onkeydown = function(event) { - if (event.keyCode == 13) { - searchTriggered() +searchbox = document.getElementById('searchbox') +if (searchbox != null) { + searchbox.onkeydown = function (event) { + if (event.keyCode == 13) { + searchTriggered() + } } } \ No newline at end of file