Add lightning fast server side search

More features like better ranking, more torrents, etc are coming!
pull/27/head
Urban Guacamole 4 years ago
parent 20247c3b24
commit dd341ba88b

3
.gitignore vendored

@ -6,4 +6,5 @@ about.html
copyright.html
torrent_dump_full.csv.gz
generate-top-torrents
api/api
api/api
*.txt

@ -46,6 +46,9 @@ Run `go build` in spider/ to compile and scp the binary it to the server. You ca
Run `go build` in seedleech-daemon/ to compile and scp the binary it to the server. You can use the systemd service file in `seedleech-daemon/seedleech.service`.
## IPFS vs 'static'
The directory website gets deployed to IPFS, static gets deployed to the server. Static calls the API, the IPFS version doesn't.
# Contributing

@ -2,13 +2,24 @@ package main
import (
"database/sql"
"log"
"encoding/json"
"io/ioutil"
"log"
"net/http"
_ "github.com/lib/pq"
)
type Results []Result
type Result struct {
ID string `json:"id"`
Text string `json:"text"`
Len int `json:"len"`
S int `json:"s"`
L int `json:"l"`
}
func initDb() *sql.DB {
connStr := "user=nextgen dbname=nextgen host=/var/run/postgresql"
db, err := sql.Open("postgres", connStr)
@ -29,9 +40,9 @@ func initDb() *sql.DB {
func main() {
db := initDb()
http.HandleFunc("/api/telemetry", func (w http.ResponseWriter, r *http.Request) {
http.HandleFunc("/api/telemetry", func(w http.ResponseWriter, r *http.Request) {
body, err := ioutil.ReadAll(r.Body)
if err != nil{
if err != nil {
log.Print(err)
}
_, err = db.Exec("INSERT INTO telemetry (payload) VALUES ($1)", string(body))
@ -40,6 +51,46 @@ func main() {
w.WriteHeader(http.StatusInternalServerError)
}
w.WriteHeader(http.StatusOK)
})
})
http.HandleFunc("/api/search", func(w http.ResponseWriter, r *http.Request) {
q := r.URL.Query().Get("q")
if len(q) == 0 {
log.Print("/api/search received empty q argument")
w.WriteHeader(http.StatusBadRequest)
return
}
rows, err := db.Query("select infohash, name, length, s, l from search where vect @@ plainto_tsquery($1) and copyrighted = 'f' limit 150", q)
if err != nil {
log.Print(err)
w.WriteHeader(http.StatusInternalServerError)
return
}
var results Results
for rows.Next() {
var (
infohash string
name string
length int
s int
l int
)
err := rows.Scan(&infohash, &name, &length, &s, &l)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
log.Print(err)
return
}
result := Result{infohash, name, length, s, l}
results = append(results, result)
}
w.WriteHeader(http.StatusOK)
marshaledResults, err := json.Marshal(results)
if err != nil {
log.Fatal(err)
}
w.Write(marshaledResults)
})
http.ListenAndServe(":8000", nil)
}
}

@ -23,4 +23,9 @@ SELECT pg_size_pretty(pg_total_relation_size('"<schema>"."<table>"'));
SELECT reltuples::bigint AS estimate FROM pg_class where relname='mytable';
--- create fulltext table
CREATE TABLE search AS select torrent.*, fresh.s as s, fresh.l as l, to_tsvector(torrent.name) as vect from torrent inner join fresh on fresh.infohash = torrent.infohash;
DROP TABLE search;
CREATE MATERIALIZED VIEW search AS select torrent.*, fresh.s as s, fresh.l as l, to_tsvector(replace(torrent.name, '.', ' ')) as vect from torrent inner join fresh on fresh.infohash = torrent.infohash;
create index vect_inx on search using gin(vect);
create unique index uniq_ih on search (infohash);
REFRESH MATERIALIZED VIEW fresh;
REFRESH MATERIALIZED VIEW search CONCURRENTLY;

@ -0,0 +1,17 @@
include head.html
.container.content
h1 Torrent Paradise: innovative torrent site
ul
li seed/leech counts always up-to-date
li new torrents are discovered via DHT and added automagically
li server-side search is in beta, be patient <i class="fa fa-heart" aria-hidden="true"></i>
li if you use IPFS
ul
li #[b no ads]
li no single point of failure: you can always run your own via #[a(href="ipfs.html") IPFS]
li runs in browser, so your query never leaves your computer
li based off open source code from #[a(href="https://ipfsearch.xyz") ipfsearch]
li more info at #[a(href="https://www.reddit.com/r/torrents/comments/afibhh/i_made_a_decentralized_torrent_search_site_with/") Reddit discussion]
li source code on #[a(href="https://github.com/urbanguacamole/torrent-paradise") GitHub]
li send suggestions to urban-guacamole (at) protonmail.com

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

@ -0,0 +1,46 @@
function searchTriggered() {
let searchbox = document.getElementById("searchbox");
let query = searchbox.value
searchFor(query);
}
async function searchFor(query) {
var url = new URL("https://torrent-paradise.ml/api/search")
url.searchParams.append("q",query)
const res = await fetch(url)
let results = await res.json();
if (results == null){
console.error("No results.")
results = []
}
passResultToResultpage(results)
}
function passResultToResultpage(results) {
let resultPageIframe = document.getElementById("resultPage");
resultPageIframe.contentWindow.postMessage({
type: "results",
results: JSON.stringify(results)
}, '*');
}
/**
* Sends telemetry payload, adds actionid and sessionid to it. IP is never logged.
*/
function sendTelemetry(payload){
payload.aid = actionid;
actionid = actionid + 1
if (sessionid == undefined){
sessionid = Math.round((Math.random()-0.5)*Math.pow(2,32))
payload.sid = sessionid;
}else{
payload.sid = sessionid;
}
(async (payload) => {
await fetch('https://torrent-paradise.ml/api/telemetry', {
method: 'POST',
body: JSON.stringify(payload)
})
})(payload);
}

@ -0,0 +1,6 @@
include head.html
.container.content
h1 Copyright notices
p Torrent Paradise is a decentralized search engine for torrents discovered via the distributed hash table. We are aware of the fact that much of the content on the DHT is copyrighted, just like is much of the content on the Internet copyrighted. We are fully prepared to do everything to protect the rights of rightsholders. We comply with all relevant laws. Please note that no content is stored here, only links to it. It is not in our capacity to screen the 25000 torrents that we discover every day for copyrighted content.
p If you would like to notify us about content that we link to that infringes your copyrights or the copyright of a party that you represent, you can do so via email urban-guacamole@protonmail.com.
p To accelerate the process, use the subject "Copyright notice" in your emails. Thank you.

Binary file not shown.

After

Width:  |  Height:  |  Size: 4.2 KiB

@ -0,0 +1,33 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Torrent Paradise</title>
<link rel="icon" type="image/x-icon" href="favicon.ico">
<link rel="stylesheet" href="assets/bulma.min.css">
<script defer src="assets/fa.js"></script>
</head>
<body>
<nav class="navbar is-warning" role="navigation" aria-label="main navigation">
<div class="navbar-brand">
<a class="is-light navbar-item" href="index.html">
<i class="fas fa-sun"></i>
</a>
<a class="navbar-item" href="about.html">
About
</a>
<a class="navbar-item" href="top.html">
Top torrents
</a>
<a class="navbar-item" href="ipfs.html">
IPFS
</a>
<a class="navbar-item" href="copyright.html">
Copyright
</a>
</div>
</nav>
<br />

@ -0,0 +1,60 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<meta name="description" content="Decentralized Torrent Search site">
<title>Torrent Paradise: The most innovative torrent site</title>
<link rel="icon" type="image/x-icon" href="favicon.ico">
<link rel="stylesheet" href="assets/bulma.min.css">
<script defer src="assets/fa.js"></script>
<style>
#resultPage {
width: 100%;
}
</style>
</head>
<body>
<nav class="navbar is-warning" role="navigation" aria-label="main navigation">
<div class="navbar-brand">
<a class="is-light navbar-item" href="index.html">
<i class="fas fa-sun"></i>
</a>
<a class="navbar-item" href="about.html">
About
</a>
<a class="navbar-item" href="top.html">
Top torrents
</a>
<a class="navbar-item" href="ipfs.html">
IPFS
</a>
<a class="navbar-item" href="copyright.html">
Copyright
</a>
</div>
</nav>
<br>
<div id="app" class="container">
<div class="notification">
<span style="color: hsl(348, 100%, 61%);"><i class="fa fa-gift" aria-hidden="true"></i> <i class="fa fa-heart" aria-hidden="true"></i></span><br />
Introducing blazing fast server side search. More improvements underway. If anything is broken, hit me up at urban-guacamole (at) protonmail.com. No logs are kept, but for total privacy, use IPFS.
</div>
<div class="field has-addons">
<div class="control is-expanded">
<input class="input" type='text' id="searchbox" />
</div>
<div class="control">
<button class="button is-info" type="submit" onclick="searchTriggered()">Search</button>
</div>
</div>
<iframe v-bind:style="{height: resultPageHeight + 'px'}" id="resultPage" scrolling="no" frameBorder="0" seamless="seamless" v-bind:src="resultPage"></iframe>
<p v-if="showsearchbox">Last index update: {{indexTimestamp}}</p>
<p v-if="showsearchbox">Torrents in index: {{entries}}</p>
</div>
<script src="bundle.js"></script>
<script src="assets/vue-v2.6.11.js"></script>
<script src="view.js"></script>

@ -0,0 +1,5 @@
include head.html
.container.content
h1 Remove ads and access the index even when the public website is down
p With IPFS (see the #[a(href="https://ipfs.io") official website]), a copy of the site and the whole index can be distributed just like a torrent. When you open it via IPFS, your IPFS node fetches the parts of the index and website it needs from the network.
p To use it without the public server, use #[a(href="https://ipfs.io") IPFS]. Address in IPFS is #[a(href="https://cloudflare-ipfs.com/ipns/12D3KooWB3GY1u6zMLqnf3MJ8zhX3SS1oBj7VXk3xp6sJJiFGZXp") /ipns/torrent-paradise.ml]. You can also use /ipns/12D3KooWB3GY1u6zMLqnf3MJ8zhX3SS1oBj7VXk3xp6sJJiFGZXp in case the domain doesn't work.

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

@ -0,0 +1,37 @@
<head>
<link rel="stylesheet" href="buefy.min.css">
<script defer src="../assets/fa.js"></script>
</head>
<body onload="updateSize()" style="overflow: auto;">
<div id="app">
<progress v-if="showProgress" class="progress is-info" v-bind:value="progress" max="100">0 %</progress>
<b-table v-if="resultsFound" class="is-fullwidth" :data="results" default-sort="s" default-sort-direction="desc"
:striped="true" :paginated="true" :narrowed="true">
<template slot-scope="props">
<b-table-column field="text" label="Name" sortable>
<span v-bind:title="props.row.text">{{ props.row.text }}</span>
</b-table-column>
<b-table-column field="len" label="Size">
{{ props.row.len }}
</b-table-column>
<b-table-column field="s" label="Seed" numeric sortable>
{{ props.row.s }}
</b-table-column>
<b-table-column field="l" label="Leech" numeric sortable>
{{ props.row.l }}
</b-table-column>
<b-table-column field="id" label=" " e>
<span class="icon">
<a style="color: hsl(171, 100%, 41%)" v-bind:href="'magnet:?xt=urn:btih:' + props.row.id + '&tr=udp%3A%2F%2Ftracker.coppersurfer.tk%3A6969%2Fannounce&tr=udp%3A%2F%2Ftracker.opentrackr.org%3A1337%2Fannounce&tr=udp%3A%2F%2Ftracker.internetwarriors.net%3A1337'">
<i class="fas fa-magnet"></i>
</a>
</span>
</b-table-column>
</template>
</b-table>
</div>
<script src="../assets/vue-v2.6.11.js"></script>
<script src="buefy-table.js"></script>
<script src="main.js"></script>
</body>

@ -0,0 +1,35 @@
app = new Vue({
el: '#app',
data: {
results: undefined,
resultsFound: false,
showProgress: false,
progress: 0
}
})
window.onmessage = function(e){
if (e.data.type == "results") {
let results = JSON.parse(e.data.results)
app.results = results.map((result) => {
result.len = formatBytes(result.len)
return result
})
app.resultsFound = true
setTimeout(updateSize,1)
} else if (e.data.type == "progress") {
if(e.data.progress == 1){
app.showProgress = false
}else{
app.showProgress = true
}
app.progress = e.data.progress * 100
setTimeout(updateSize,1)
}
};
function updateSize(){
window.parent.postMessage(parseInt(document.body.scrollHeight),"*")
}
function formatBytes(a,b){if(0==a)return"0 Bytes";var c=1024,d=b||2,e=["B","KB","MB","GB","TB","PB","EB","ZB","YB"],f=Math.floor(Math.log(a)/Math.log(c));return parseFloat((a/Math.pow(c,f)).toFixed(d))+" "+e[f]}

@ -0,0 +1,62 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<meta name="description" content="Decentralized Torrent Search site">
<title>Torrent Paradise</title>
<link rel="icon" type="image/x-icon" href="favicon.ico">
<link rel="stylesheet" href="assets/bulma.min.css">
<script defer src="assets/fa.js"></script>
<style>
#resultPage {
width: 100%;
}
</style>
</head>
<body>
<nav class="navbar is-warning" role="navigation" aria-label="main navigation">
<div class="navbar-brand">
<a class="is-light navbar-item" href="index.html">
<i class="fas fa-sun"></i>
</a>
<a class="navbar-item" href="about.html">
About
</a>
<a class="navbar-item" href="top.html">
Top torrents
</a>
<a class="navbar-item" href="ipfs.html">
IPFS
</a>
<a class="navbar-item" href="copyright.html">
Copyright
</a>
</div>
</nav>
<br>
<div id="app" class="container content">
<h1 class="title">Top torrents</h1>
<iframe v-bind:style="{height: resultPageHeight + 'px'}" id="resultPage" scrolling="no" frameBorder="0" seamless="seamless" v-bind:src="resultPage"></iframe>
<script src="assets/vue-v2.6.11.js"></script>
<script src="view.js"></script>
</div>
<script>app.resultPage = "resultpage"
fetch("top.json").then((s) => {
return s.text()
}).then((top) => {
let resultPageIframe = document.getElementById("resultPage");
resultPageIframe.onload = () => {
resultPageIframe.contentWindow.postMessage({
type: "results",
results: top
}, '*');
}
})</script>
</body>
</html>

File diff suppressed because one or more lines are too long

@ -0,0 +1,17 @@
app = new Vue({
el: '#app',
data: { showsearchbox: false, error: "", resultPage: "resultpage/", resultPageHeight: 1, entries: -1}
})
window.addEventListener("message", receiveMessage, false);
function receiveMessage(event) {
app.resultPageHeight = event.data
}
searchbox = document.getElementById('searchbox')
if (searchbox != null) {
searchbox.onkeydown = function (event) {
if (event.keyCode == 13) {
searchTriggered()
}
}
}
Loading…
Cancel
Save