From 3e83eb9d96d8a76b1190baa459743d29d833a48b Mon Sep 17 00:00:00 2001
From: danieleperera
Date: Sun, 5 Jul 2020 15:44:33 +0000
Subject: [PATCH] updated README.md
---
README.md | 343 +++++++++++++++++++++++++++-
onionscraper/operators/onionscan.py | 32 ++-
2 files changed, 357 insertions(+), 18 deletions(-)
diff --git a/README.md b/README.md
index 5712118..49c6e77 100644
--- a/README.md
+++ b/README.md
@@ -105,7 +105,348 @@ To run the webapp by onionscan
The output of the result is json, and in the same format it is sent to the chosen syslog.
```
-show output here
+{
+ "_index": "darkweb",
+ "_type": "_doc",
+ "_id": "ogIkEHMBlDh8pb-30_2O",
+ "_score": 1,
+ "_source": {
+ "onionscan": {
+ "hiddenService": "nzxj65x32vh2fkhk.onion",
+ "dateScanned": "2020-07-02T15:27:18.443234981Z",
+ "online": false,
+ "performedScans": [
+ "web",
+ "tls",
+ "ssh",
+ "irc",
+ "ricochet",
+ "ftp",
+ "smtp",
+ "mongodb",
+ "vnc",
+ "xmpp",
+ "bitcoin",
+ "bitcoin_test",
+ "litecoin",
+ "dogecoin"
+ ],
+ "webDetected": true,
+ "tlsDetected": false,
+ "sshDetected": false,
+ "ricochetDetected": false,
+ "ircDetected": false,
+ "ftpDetected": false,
+ "smtpDetected": false,
+ "bitcoinDetected": false,
+ "mongodbDetected": false,
+ "vncDetected": false,
+ "xmppDetected": false,
+ "skynetDetected": false,
+ "pgpKeys": null,
+ "certificates": null,
+ "bitcoinServices": {
+ "bitcoin": {
+ "detected": false,
+ "userAgent": "",
+ "prototocolVersion": 0,
+ "onionPeers": null
+ },
+ "bitcoin_test": {
+ "detected": false,
+ "userAgent": "",
+ "prototocolVersion": 0,
+ "onionPeers": null
+ },
+ "dogecoin": {
+ "detected": false,
+ "userAgent": "",
+ "prototocolVersion": 0,
+ "onionPeers": null
+ },
+ "litecoin": {
+ "detected": false,
+ "userAgent": "",
+ "prototocolVersion": 0,
+ "onionPeers": null
+ }
+ },
+ "sshKey": "",
+ "sshBanner": "",
+ "ftpFingerprint": "",
+ "ftpBanner": "",
+ "smtpFingerprint": "",
+ "smtpBanner": "",
+ "lastAction": "dogecoin",
+ "timedOut": false,
+ "error": null,
+ "identifierReport": {
+ "privateKeyDetected": false,
+ "foundApacheModStatus": false,
+ "serverVersion": "",
+ "relatedOnionServices": null,
+ "relatedOnionDomains": null,
+ "ipAddresses": null,
+ "emailAddresses": [
+ "hitman001@torbox3uiot6wchz.onion",
+ "jimmym0reno@yahoo.com",
+ "aimeerene1977@gmail.com",
+ "jennabrown15.jb@gmail.com",
+ "S.thames129@gmail.com",
+ "munira025@gmail.com",
+ "luisadavid20@gmail.com",
+ "cameron.stewart3@yahoo.com",
+ "janisea2013@gmail.com",
+ "Carinavieyra598@gmail.com",
+ "adrianmcdonald49@gmail.com",
+ "aaronjeans1@gmail.com",
+ "nsorrentino11@aol.com",
+ "amber4189@outlook.com",
+ "holliekestner@gmail.com",
+ "nattyperks01@gmail.com",
+ "dinavasa29@hotmail.com",
+ "lydiac612@gmail.com",
+ "bmduke24@gmail.com",
+ "markigharmony@gmail.com",
+ "ohdannyboy03@icloud.com",
+ "dkoontz18@gmail.com",
+ "janese_young@yahoo.com",
+ "gabssstobsss@gmail.com",
+ "thelake02@sbcglobal.net",
+ "timmyboston01@gmail.com",
+ "carloscharters1996@gmail.com",
+ "djamila28@outlook.com",
+ "heathermaeb@gmail.com",
+ "canelo2080@gmail.com",
+ "pamsanta.ps@gmail.com",
+ "horeka.mash98@gmail.com",
+ "oeh@gondtc.com",
+ "ohmygod990227@hotmail.com",
+ "marieazme@yahoo.com",
+ "shirleyteuta@gmail.com",
+ "janetcoppedge@sbcglobal.net",
+ "dimashilov30@gmail.com",
+ "benavides.kam@gmail.com",
+ "sonyainsonora@yahoo.com",
+ "benl04123@outlook.com",
+ "cmculbreath@fedex.com",
+ "antmeb@gmail.com",
+ "jrlopez61@hotmail.com",
+ "jaimie.mudge@hotmail.com",
+ "dreamworld1980@secmail.pro",
+ "tinajones@sympatico.ca",
+ "nobby@secmail.pro",
+ "twistedsun@secmail.pro",
+ "slayermodsv3@gmail.com",
+ "beastmodsv1@gmail.com",
+ "prestonkonicek@gmail.com",
+ "fnbrleaksv2@gmail.com",
+ "fnbrleaks@gmail.com",
+ "pushingeverythingyt@gmail.com",
+ "rachelkonicek@gmail.com",
+ "vsfortune@hotmail.com",
+ "dannajoywhite@gmail.com",
+ "jensenjody@gmail.com",
+ "jenniferjbisschop@gmail.com",
+ "hkbergado@gmail.com",
+ "mummifiedbabies@secmail.pro"
+ ],
+ "analyticsIDs": null,
+ "bitcoinAddresses": [
+ "1A3usPsRyCRPy9z5zpaQoj59hTATnS799d",
+ "114LbUUXsLc6QynxsTVLcAgHzEkSBB5Yww",
+ "3FMRCWEv7XqW26mqe526s4ibmEz9qJSmt6",
+ "19iqYbeATe4RxghQZJnYVFU4mjUUu76EA6",
+ "3MV9ESE7CTjGJRivnXCE4MUnzpxAYnJNLT",
+ "37VbgjzwVrM93g6zQrc4XP5RonpwnwKevj",
+ "1HtuUatKrJSR8PYs2qSxnxvPuYhf8UiCpB",
+ "1gzm7L4GNbNDUkfWZxQJdZt6b7tjoWzRb",
+ "112dznTAr3m1PHnkQYK5CpsDJboEiCScwP"
+ ],
+ "linkedOnions": [
+ "fagnojbvbqey34qd.onion",
+ "ytxmxncdn2tjyzid.onion",
+ "7rsgftiskp4xqlqd.onion",
+ "hkcr7naqdkmw76gr.onion",
+ "torbox3uiot6wchz.onion",
+ "w5oxf255cjcziyyd.onion",
+ "7g525hcd4cyowmid.onion",
+ "uaciqctyky5olsid.onion",
+ "2rdyr63onwhqp3qd.onion",
+ "dreamdogehwnde5b.onion",
+ "e5to4n5a325lvsad.onion",
+ "xwwcqj3jt4dbfeid.onion",
+ "cryptoupei2am6si.onion",
+ "o5kq76skl2em3xad.onion",
+ "k2hwrssoj7yivhid.onion",
+ "x3dphthawcqtd4id.onion"
+ ],
+ "openDirectories": null,
+ "exifImages": null
+ },
+ "crawls": [
+ "http://nzxj65x32vh2fkhk.onion/",
+ "http://nzxj65x32vh2fkhk.onion/all",
+ "http://nzxj65x32vh2fkhk.onion/all?page=1",
+ "http://nzxj65x32vh2fkhk.onion/all?page=2",
+ "http://nzxj65x32vh2fkhk.onion/all?page=3",
+ "http://nzxj65x32vh2fkhk.onion/assets/bootstrap/css/bootstrap.min.css",
+ "http://nzxj65x32vh2fkhk.onion/assets/bootstrap/css/phpdiff.css",
+ "http://nzxj65x32vh2fkhk.onion/assets/bootstrap/css/stickynotes.css",
+ "http://nzxj65x32vh2fkhk.onion/assets/bootstrap/img/favicon.png",
+ "http://nzxj65x32vh2fkhk.onion/assets/bootstrap/js/bootstrap.min.js",
+ "http://nzxj65x32vh2fkhk.onion/assets/bootstrap/js/jquery.cookie.js",
+ "http://nzxj65x32vh2fkhk.onion/assets/bootstrap/js/jquery.min.js",
+ "http://nzxj65x32vh2fkhk.onion/assets/bootstrap/js/jquery.scrollto.js",
+ "http://nzxj65x32vh2fkhk.onion/assets/bootstrap/js/stickynotes.js",
+ "http://nzxj65x32vh2fkhk.onion/diff/ptoft0c1l/pbbofkqn1",
+ "http://nzxj65x32vh2fkhk.onion/docs",
+ "http://nzxj65x32vh2fkhk.onion/feed",
+ "http://nzxj65x32vh2fkhk.onion/p0v6zvoin",
+ "http://nzxj65x32vh2fkhk.onion/p0v6zvoin/ac3zze/raw",
+ "http://nzxj65x32vh2fkhk.onion/p8nqdddzb",
+ "http://nzxj65x32vh2fkhk.onion/p8nqdddzb/wafcdu/raw",
+ "http://nzxj65x32vh2fkhk.onion/p8qorgjj2",
+ "http://nzxj65x32vh2fkhk.onion/p8qorgjj2/duswuw/raw",
+ "http://nzxj65x32vh2fkhk.onion/pawgypvjz",
+ "http://nzxj65x32vh2fkhk.onion/pawgypvjz/7artey/raw",
+ "http://nzxj65x32vh2fkhk.onion/pbbofkqn1",
+ "http://nzxj65x32vh2fkhk.onion/pbbofkqn1/j5n49p/raw",
+ "http://nzxj65x32vh2fkhk.onion/pdmtzkwup",
+ "http://nzxj65x32vh2fkhk.onion/pdmtzkwup/jfm68l/raw",
+ "http://nzxj65x32vh2fkhk.onion/pdot84cmm",
+ "http://nzxj65x32vh2fkhk.onion/pdot84cmm/cgj4qw/raw",
+ "http://nzxj65x32vh2fkhk.onion/pdubkxy1v",
+ "http://nzxj65x32vh2fkhk.onion/pdubkxy1v/i3nrxl/raw",
+ "http://nzxj65x32vh2fkhk.onion/pegkdomcx",
+ "http://nzxj65x32vh2fkhk.onion/pegkdomcx/s4yquj/raw",
+ "http://nzxj65x32vh2fkhk.onion/pfgafcrvp",
+ "http://nzxj65x32vh2fkhk.onion/pfgafcrvp/kxj5lj/raw",
+ "http://nzxj65x32vh2fkhk.onion/ph62mpvce",
+ "http://nzxj65x32vh2fkhk.onion/ph62mpvce/kmqnwb/raw",
+ "http://nzxj65x32vh2fkhk.onion/pjfn3yclj",
+ "http://nzxj65x32vh2fkhk.onion/pjfn3yclj/u6o2wm/raw",
+ "http://nzxj65x32vh2fkhk.onion/pkjhtx8tn",
+ "http://nzxj65x32vh2fkhk.onion/pkjhtx8tn/mgqope/raw",
+ "http://nzxj65x32vh2fkhk.onion/pl6wwk9go",
+ "http://nzxj65x32vh2fkhk.onion/pl6wwk9go/4n2h7m/raw",
+ "http://nzxj65x32vh2fkhk.onion/poppwa9mo",
+ "http://nzxj65x32vh2fkhk.onion/poppwa9mo/52keri/raw",
+ "http://nzxj65x32vh2fkhk.onion/ppez6ok6r",
+ "http://nzxj65x32vh2fkhk.onion/ppez6ok6r/xtgf4l/raw",
+ "http://nzxj65x32vh2fkhk.onion/pqnih2eus",
+ "http://nzxj65x32vh2fkhk.onion/pqnih2eus/mchmij/raw",
+ "http://nzxj65x32vh2fkhk.onion/private_key",
+ "http://nzxj65x32vh2fkhk.onion/psfl80ry1",
+ "http://nzxj65x32vh2fkhk.onion/psfl80ry1/fzpoil/raw",
+ "http://nzxj65x32vh2fkhk.onion/pswvhb2u2",
+ "http://nzxj65x32vh2fkhk.onion/pswvhb2u2/bwypdr/raw",
+ "http://nzxj65x32vh2fkhk.onion/psxsbptqy",
+ "http://nzxj65x32vh2fkhk.onion/psxsbptqy/ac2fwg/raw",
+ "http://nzxj65x32vh2fkhk.onion/ptoft0c1l",
+ "http://nzxj65x32vh2fkhk.onion/ptoft0c1l/jekdce/raw",
+ "http://nzxj65x32vh2fkhk.onion/pufx3aabu",
+ "http://nzxj65x32vh2fkhk.onion/pufx3aabu/jkf8k7/raw",
+ "http://nzxj65x32vh2fkhk.onion/pvxlo9hda",
+ "http://nzxj65x32vh2fkhk.onion/pvxlo9hda/tx5qns/raw",
+ "http://nzxj65x32vh2fkhk.onion/pweo3qenp",
+ "http://nzxj65x32vh2fkhk.onion/pweo3qenp/fjn8ok/raw",
+ "http://nzxj65x32vh2fkhk.onion/pwgegfzoe",
+ "http://nzxj65x32vh2fkhk.onion/pwgegfzoe/ad13am/raw",
+ "http://nzxj65x32vh2fkhk.onion/pxn7di53e",
+ "http://nzxj65x32vh2fkhk.onion/pxn7di53e/7gqdtb/raw",
+ "http://nzxj65x32vh2fkhk.onion/pxppe2cwf",
+ "http://nzxj65x32vh2fkhk.onion/pxppe2cwf/pdwog7/raw",
+ "http://nzxj65x32vh2fkhk.onion/pyb041lxb",
+ "http://nzxj65x32vh2fkhk.onion/pyb041lxb/m10i4y/raw",
+ "http://nzxj65x32vh2fkhk.onion/pyd5xhkct",
+ "http://nzxj65x32vh2fkhk.onion/pyd5xhkct/r0oupe/raw",
+ "http://nzxj65x32vh2fkhk.onion/pz7ltroub",
+ "http://nzxj65x32vh2fkhk.onion/pz7ltroub/4tfvri/raw",
+ "http://nzxj65x32vh2fkhk.onion/pzkraz18q",
+ "http://nzxj65x32vh2fkhk.onion/pzkraz18q/jhh2tn/raw",
+ "http://nzxj65x32vh2fkhk.onion/rev/p0v6zvoin",
+ "http://nzxj65x32vh2fkhk.onion/rev/p8nqdddzb",
+ "http://nzxj65x32vh2fkhk.onion/rev/p8qorgjj2",
+ "http://nzxj65x32vh2fkhk.onion/rev/pawgypvjz",
+ "http://nzxj65x32vh2fkhk.onion/rev/pbbofkqn1",
+ "http://nzxj65x32vh2fkhk.onion/rev/pdmtzkwup",
+ "http://nzxj65x32vh2fkhk.onion/rev/pdot84cmm",
+ "http://nzxj65x32vh2fkhk.onion/rev/pdubkxy1v",
+ "http://nzxj65x32vh2fkhk.onion/rev/pegkdomcx",
+ "http://nzxj65x32vh2fkhk.onion/rev/pfgafcrvp",
+ "http://nzxj65x32vh2fkhk.onion/rev/ph62mpvce",
+ "http://nzxj65x32vh2fkhk.onion/rev/pjfn3yclj",
+ "http://nzxj65x32vh2fkhk.onion/rev/pkjhtx8tn",
+ "http://nzxj65x32vh2fkhk.onion/rev/pl6wwk9go",
+ "http://nzxj65x32vh2fkhk.onion/rev/poppwa9mo",
+ "http://nzxj65x32vh2fkhk.onion/rev/ppez6ok6r",
+ "http://nzxj65x32vh2fkhk.onion/rev/pqnih2eus",
+ "http://nzxj65x32vh2fkhk.onion/rev/psfl80ry1",
+ "http://nzxj65x32vh2fkhk.onion/rev/pswvhb2u2",
+ "http://nzxj65x32vh2fkhk.onion/rev/psxsbptqy",
+ "http://nzxj65x32vh2fkhk.onion/rev/ptoft0c1l",
+ "http://nzxj65x32vh2fkhk.onion/rev/pufx3aabu",
+ "http://nzxj65x32vh2fkhk.onion/rev/pvxlo9hda",
+ "http://nzxj65x32vh2fkhk.onion/rev/pweo3qenp",
+ "http://nzxj65x32vh2fkhk.onion/rev/pwgegfzoe",
+ "http://nzxj65x32vh2fkhk.onion/rev/pxn7di53e",
+ "http://nzxj65x32vh2fkhk.onion/rev/pxppe2cwf",
+ "http://nzxj65x32vh2fkhk.onion/rev/pyb041lxb",
+ "http://nzxj65x32vh2fkhk.onion/rev/pyd5xhkct",
+ "http://nzxj65x32vh2fkhk.onion/rev/pz7ltroub",
+ "http://nzxj65x32vh2fkhk.onion/rev/pzkraz18q",
+ "http://nzxj65x32vh2fkhk.onion/server-status",
+ "http://nzxj65x32vh2fkhk.onion/trending",
+ "http://nzxj65x32vh2fkhk.onion/trending/all",
+ "http://nzxj65x32vh2fkhk.onion/trending/all?page=1",
+ "http://nzxj65x32vh2fkhk.onion/trending/all?page=2",
+ "http://nzxj65x32vh2fkhk.onion/trending/all?page=3",
+ "http://nzxj65x32vh2fkhk.onion/trending/month",
+ "http://nzxj65x32vh2fkhk.onion/trending/month?page=1",
+ "http://nzxj65x32vh2fkhk.onion/trending/month?page=2",
+ "http://nzxj65x32vh2fkhk.onion/trending/month?page=3",
+ "http://nzxj65x32vh2fkhk.onion/trending/week",
+ "http://nzxj65x32vh2fkhk.onion/trending/week?page=1",
+ "http://nzxj65x32vh2fkhk.onion/trending/week?page=2",
+ "http://nzxj65x32vh2fkhk.onion/trending/week?page=3",
+ "http://nzxj65x32vh2fkhk.onion/trending/year",
+ "http://nzxj65x32vh2fkhk.onion/trending/year?page=1",
+ "http://nzxj65x32vh2fkhk.onion/trending/year?page=2",
+ "http://nzxj65x32vh2fkhk.onion/trending/year?page=3",
+ "http://nzxj65x32vh2fkhk.onion/trending?page=1",
+ "http://nzxj65x32vh2fkhk.onion/trending?page=2",
+ "http://nzxj65x32vh2fkhk.onion/trending?page=3",
+ "http://nzxj65x32vh2fkhk.onion/user/forgot",
+ "http://nzxj65x32vh2fkhk.onion/user/login",
+ "http://nzxj65x32vh2fkhk.onion/user/register"
+ ]
+ },
+ "html": "\n\n\n\n\n\t\n\t\n\tStronghold Paste\n\n\t\n\n\t\n\t\n\t\n\n\t\n\t\n\t\n\t\n\t\n\t\n\n\t\n\n\n\n\t\n\t\t\n\t\n\tLoading\n\n\t
\n\n\t\n\n\t\n\t\t\n\t\t\t
\n\t
\n\t\t\n\t\t\n\t\t\t
\n
\n\n\t
\n\n\t\t\t
\n\n\t
\n\n\t\t\n\t\t\n\t\tSticky Notes © 2014 Sayak Banerjee.
\n\n\t\t\n\t\t\t\n\n\n\n",
+ "screenshots": {
+ "dateScreenshoted": "2020-07-02T15:28:35.722031Z",
+ "filename": "nzxj65x32vh2fkhk.onion_screenshot_a692c810.png"
+ },
+ "interestingKeywords": [
+ "rss",
+ "xml",
+ "RSS",
+ "feed",
+ "xml",
+ "XML",
+ "xml",
+ "XML"
+ ]
+ },
+ "fields": {
+ "screenshots.dateScreenshoted": [
+ "2020-07-02T15:28:35.722Z"
+ ],
+ "onionscan.dateScanned": [
+ "2020-07-02T15:27:18.443Z"
+ ]
+ }
+}
```
## Authors
diff --git a/onionscraper/operators/onionscan.py b/onionscraper/operators/onionscan.py
index fd22c61..94b0bad 100644
--- a/onionscraper/operators/onionscan.py
+++ b/onionscraper/operators/onionscan.py
@@ -230,23 +230,21 @@ class Plugin(Operator):
results = self.run_onionscan(onion)
if results['status'] == 'success' and results['data']['webDetected'] == 'true':
content = self.run_sessions(onion)
- print(content)
- #sys.exit(0)
- #if content['status'] == 'success':
- # blacklist_CONTENT = self.blacklist.search(content['data'])
- # if blacklist_CONTENT:
- # self.logger.info(f"[X] Blocked by blacklist content => matched keyword {blacklist_CONTENT.group()}")
- # else:
- # self.logger.debug("[*] CONTENT blacklist test: PASSED")
- # screenshot = self.take_screenshot(self.format_directory(self.screenshots), onion)
- # self.logger.info("Indexing!")
- # doc = {
- # 'onionscan':json.loads(results['data']),
- # 'html':content['data'],
- # 'screenshots':screenshot['data'],
- # 'interestingKeywords':self.interestingKeywords.findall(content['data'])
- # }
- # return self.parseDoc(doc)
+ if content['status'] == 'success':
+ blacklist_CONTENT = self.blacklist.search(content['data'])
+ if blacklist_CONTENT:
+ self.logger.info(f"[X] Blocked by blacklist content => matched keyword {blacklist_CONTENT.group()}")
+ else:
+ self.logger.debug("[*] CONTENT blacklist test: PASSED")
+ screenshot = self.take_screenshot(self.format_directory(self.screenshots), onion)
+ self.logger.info("Indexing!")
+ doc = {
+ 'onionscan':json.loads(results['data']),
+ 'html':content['data'],
+ 'screenshots':screenshot['data'],
+ 'interestingKeywords':self.interestingKeywords.findall(content['data'])
+ }
+ return self.parseDoc(doc)
else:
self.logger.info(f"[x] hidden service {onion} is not active")