From 3e83eb9d96d8a76b1190baa459743d29d833a48b Mon Sep 17 00:00:00 2001 From: danieleperera Date: Sun, 5 Jul 2020 15:44:33 +0000 Subject: [PATCH] updated README.md --- README.md | 343 +++++++++++++++++++++++++++- onionscraper/operators/onionscan.py | 32 ++- 2 files changed, 357 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 5712118..49c6e77 100644 --- a/README.md +++ b/README.md @@ -105,7 +105,348 @@ To run the webapp by onionscan The output of the result is json, and in the same format it is sent to the chosen syslog. ``` -show output here +{ + "_index": "darkweb", + "_type": "_doc", + "_id": "ogIkEHMBlDh8pb-30_2O", + "_score": 1, + "_source": { + "onionscan": { + "hiddenService": "nzxj65x32vh2fkhk.onion", + "dateScanned": "2020-07-02T15:27:18.443234981Z", + "online": false, + "performedScans": [ + "web", + "tls", + "ssh", + "irc", + "ricochet", + "ftp", + "smtp", + "mongodb", + "vnc", + "xmpp", + "bitcoin", + "bitcoin_test", + "litecoin", + "dogecoin" + ], + "webDetected": true, + "tlsDetected": false, + "sshDetected": false, + "ricochetDetected": false, + "ircDetected": false, + "ftpDetected": false, + "smtpDetected": false, + "bitcoinDetected": false, + "mongodbDetected": false, + "vncDetected": false, + "xmppDetected": false, + "skynetDetected": false, + "pgpKeys": null, + "certificates": null, + "bitcoinServices": { + "bitcoin": { + "detected": false, + "userAgent": "", + "prototocolVersion": 0, + "onionPeers": null + }, + "bitcoin_test": { + "detected": false, + "userAgent": "", + "prototocolVersion": 0, + "onionPeers": null + }, + "dogecoin": { + "detected": false, + "userAgent": "", + "prototocolVersion": 0, + "onionPeers": null + }, + "litecoin": { + "detected": false, + "userAgent": "", + "prototocolVersion": 0, + "onionPeers": null + } + }, + "sshKey": "", + "sshBanner": "", + "ftpFingerprint": "", + "ftpBanner": "", + "smtpFingerprint": "", + "smtpBanner": "", + "lastAction": "dogecoin", + "timedOut": false, + "error": null, + "identifierReport": { + "privateKeyDetected": false, + "foundApacheModStatus": false, + "serverVersion": "", + "relatedOnionServices": null, + "relatedOnionDomains": null, + "ipAddresses": null, + "emailAddresses": [ + "hitman001@torbox3uiot6wchz.onion", + "jimmym0reno@yahoo.com", + "aimeerene1977@gmail.com", + "jennabrown15.jb@gmail.com", + "S.thames129@gmail.com", + "munira025@gmail.com", + "luisadavid20@gmail.com", + "cameron.stewart3@yahoo.com", + "janisea2013@gmail.com", + "Carinavieyra598@gmail.com", + "adrianmcdonald49@gmail.com", + "aaronjeans1@gmail.com", + "nsorrentino11@aol.com", + "amber4189@outlook.com", + "holliekestner@gmail.com", + "nattyperks01@gmail.com", + "dinavasa29@hotmail.com", + "lydiac612@gmail.com", + "bmduke24@gmail.com", + "markigharmony@gmail.com", + "ohdannyboy03@icloud.com", + "dkoontz18@gmail.com", + "janese_young@yahoo.com", + "gabssstobsss@gmail.com", + "thelake02@sbcglobal.net", + "timmyboston01@gmail.com", + "carloscharters1996@gmail.com", + "djamila28@outlook.com", + "heathermaeb@gmail.com", + "canelo2080@gmail.com", + "pamsanta.ps@gmail.com", + "horeka.mash98@gmail.com", + "oeh@gondtc.com", + "ohmygod990227@hotmail.com", + "marieazme@yahoo.com", + "shirleyteuta@gmail.com", + "janetcoppedge@sbcglobal.net", + "dimashilov30@gmail.com", + "benavides.kam@gmail.com", + "sonyainsonora@yahoo.com", + "benl04123@outlook.com", + "cmculbreath@fedex.com", + "antmeb@gmail.com", + "jrlopez61@hotmail.com", + "jaimie.mudge@hotmail.com", + "dreamworld1980@secmail.pro", + "tinajones@sympatico.ca", + "nobby@secmail.pro", + "twistedsun@secmail.pro", + "slayermodsv3@gmail.com", + "beastmodsv1@gmail.com", + "prestonkonicek@gmail.com", + "fnbrleaksv2@gmail.com", + "fnbrleaks@gmail.com", + "pushingeverythingyt@gmail.com", + "rachelkonicek@gmail.com", + "vsfortune@hotmail.com", + "dannajoywhite@gmail.com", + "jensenjody@gmail.com", + "jenniferjbisschop@gmail.com", + "hkbergado@gmail.com", + "mummifiedbabies@secmail.pro" + ], + "analyticsIDs": null, + "bitcoinAddresses": [ + "1A3usPsRyCRPy9z5zpaQoj59hTATnS799d", + "114LbUUXsLc6QynxsTVLcAgHzEkSBB5Yww", + "3FMRCWEv7XqW26mqe526s4ibmEz9qJSmt6", + "19iqYbeATe4RxghQZJnYVFU4mjUUu76EA6", + "3MV9ESE7CTjGJRivnXCE4MUnzpxAYnJNLT", + "37VbgjzwVrM93g6zQrc4XP5RonpwnwKevj", + "1HtuUatKrJSR8PYs2qSxnxvPuYhf8UiCpB", + "1gzm7L4GNbNDUkfWZxQJdZt6b7tjoWzRb", + "112dznTAr3m1PHnkQYK5CpsDJboEiCScwP" + ], + "linkedOnions": [ + "fagnojbvbqey34qd.onion", + "ytxmxncdn2tjyzid.onion", + "7rsgftiskp4xqlqd.onion", + "hkcr7naqdkmw76gr.onion", + "torbox3uiot6wchz.onion", + "w5oxf255cjcziyyd.onion", + "7g525hcd4cyowmid.onion", + "uaciqctyky5olsid.onion", + "2rdyr63onwhqp3qd.onion", + "dreamdogehwnde5b.onion", + "e5to4n5a325lvsad.onion", + "xwwcqj3jt4dbfeid.onion", + "cryptoupei2am6si.onion", + "o5kq76skl2em3xad.onion", + "k2hwrssoj7yivhid.onion", + "x3dphthawcqtd4id.onion" + ], + "openDirectories": null, + "exifImages": null + }, + "crawls": [ + "http://nzxj65x32vh2fkhk.onion/", + "http://nzxj65x32vh2fkhk.onion/all", + "http://nzxj65x32vh2fkhk.onion/all?page=1", + "http://nzxj65x32vh2fkhk.onion/all?page=2", + "http://nzxj65x32vh2fkhk.onion/all?page=3", + "http://nzxj65x32vh2fkhk.onion/assets/bootstrap/css/bootstrap.min.css", + "http://nzxj65x32vh2fkhk.onion/assets/bootstrap/css/phpdiff.css", + "http://nzxj65x32vh2fkhk.onion/assets/bootstrap/css/stickynotes.css", + "http://nzxj65x32vh2fkhk.onion/assets/bootstrap/img/favicon.png", + "http://nzxj65x32vh2fkhk.onion/assets/bootstrap/js/bootstrap.min.js", + "http://nzxj65x32vh2fkhk.onion/assets/bootstrap/js/jquery.cookie.js", + "http://nzxj65x32vh2fkhk.onion/assets/bootstrap/js/jquery.min.js", + "http://nzxj65x32vh2fkhk.onion/assets/bootstrap/js/jquery.scrollto.js", + "http://nzxj65x32vh2fkhk.onion/assets/bootstrap/js/stickynotes.js", + "http://nzxj65x32vh2fkhk.onion/diff/ptoft0c1l/pbbofkqn1", + "http://nzxj65x32vh2fkhk.onion/docs", + "http://nzxj65x32vh2fkhk.onion/feed", + "http://nzxj65x32vh2fkhk.onion/p0v6zvoin", + "http://nzxj65x32vh2fkhk.onion/p0v6zvoin/ac3zze/raw", + "http://nzxj65x32vh2fkhk.onion/p8nqdddzb", + "http://nzxj65x32vh2fkhk.onion/p8nqdddzb/wafcdu/raw", + "http://nzxj65x32vh2fkhk.onion/p8qorgjj2", + "http://nzxj65x32vh2fkhk.onion/p8qorgjj2/duswuw/raw", + "http://nzxj65x32vh2fkhk.onion/pawgypvjz", + "http://nzxj65x32vh2fkhk.onion/pawgypvjz/7artey/raw", + "http://nzxj65x32vh2fkhk.onion/pbbofkqn1", + "http://nzxj65x32vh2fkhk.onion/pbbofkqn1/j5n49p/raw", + "http://nzxj65x32vh2fkhk.onion/pdmtzkwup", + "http://nzxj65x32vh2fkhk.onion/pdmtzkwup/jfm68l/raw", + "http://nzxj65x32vh2fkhk.onion/pdot84cmm", + "http://nzxj65x32vh2fkhk.onion/pdot84cmm/cgj4qw/raw", + "http://nzxj65x32vh2fkhk.onion/pdubkxy1v", + "http://nzxj65x32vh2fkhk.onion/pdubkxy1v/i3nrxl/raw", + "http://nzxj65x32vh2fkhk.onion/pegkdomcx", + "http://nzxj65x32vh2fkhk.onion/pegkdomcx/s4yquj/raw", + "http://nzxj65x32vh2fkhk.onion/pfgafcrvp", + "http://nzxj65x32vh2fkhk.onion/pfgafcrvp/kxj5lj/raw", + "http://nzxj65x32vh2fkhk.onion/ph62mpvce", + "http://nzxj65x32vh2fkhk.onion/ph62mpvce/kmqnwb/raw", + "http://nzxj65x32vh2fkhk.onion/pjfn3yclj", + "http://nzxj65x32vh2fkhk.onion/pjfn3yclj/u6o2wm/raw", + "http://nzxj65x32vh2fkhk.onion/pkjhtx8tn", + "http://nzxj65x32vh2fkhk.onion/pkjhtx8tn/mgqope/raw", + "http://nzxj65x32vh2fkhk.onion/pl6wwk9go", + "http://nzxj65x32vh2fkhk.onion/pl6wwk9go/4n2h7m/raw", + "http://nzxj65x32vh2fkhk.onion/poppwa9mo", + "http://nzxj65x32vh2fkhk.onion/poppwa9mo/52keri/raw", + "http://nzxj65x32vh2fkhk.onion/ppez6ok6r", + "http://nzxj65x32vh2fkhk.onion/ppez6ok6r/xtgf4l/raw", + "http://nzxj65x32vh2fkhk.onion/pqnih2eus", + "http://nzxj65x32vh2fkhk.onion/pqnih2eus/mchmij/raw", + "http://nzxj65x32vh2fkhk.onion/private_key", + "http://nzxj65x32vh2fkhk.onion/psfl80ry1", + "http://nzxj65x32vh2fkhk.onion/psfl80ry1/fzpoil/raw", + "http://nzxj65x32vh2fkhk.onion/pswvhb2u2", + "http://nzxj65x32vh2fkhk.onion/pswvhb2u2/bwypdr/raw", + "http://nzxj65x32vh2fkhk.onion/psxsbptqy", + "http://nzxj65x32vh2fkhk.onion/psxsbptqy/ac2fwg/raw", + "http://nzxj65x32vh2fkhk.onion/ptoft0c1l", + "http://nzxj65x32vh2fkhk.onion/ptoft0c1l/jekdce/raw", + "http://nzxj65x32vh2fkhk.onion/pufx3aabu", + "http://nzxj65x32vh2fkhk.onion/pufx3aabu/jkf8k7/raw", + "http://nzxj65x32vh2fkhk.onion/pvxlo9hda", + "http://nzxj65x32vh2fkhk.onion/pvxlo9hda/tx5qns/raw", + "http://nzxj65x32vh2fkhk.onion/pweo3qenp", + "http://nzxj65x32vh2fkhk.onion/pweo3qenp/fjn8ok/raw", + "http://nzxj65x32vh2fkhk.onion/pwgegfzoe", + "http://nzxj65x32vh2fkhk.onion/pwgegfzoe/ad13am/raw", + "http://nzxj65x32vh2fkhk.onion/pxn7di53e", + "http://nzxj65x32vh2fkhk.onion/pxn7di53e/7gqdtb/raw", + "http://nzxj65x32vh2fkhk.onion/pxppe2cwf", + "http://nzxj65x32vh2fkhk.onion/pxppe2cwf/pdwog7/raw", + "http://nzxj65x32vh2fkhk.onion/pyb041lxb", + "http://nzxj65x32vh2fkhk.onion/pyb041lxb/m10i4y/raw", + "http://nzxj65x32vh2fkhk.onion/pyd5xhkct", + "http://nzxj65x32vh2fkhk.onion/pyd5xhkct/r0oupe/raw", + "http://nzxj65x32vh2fkhk.onion/pz7ltroub", + "http://nzxj65x32vh2fkhk.onion/pz7ltroub/4tfvri/raw", + "http://nzxj65x32vh2fkhk.onion/pzkraz18q", + "http://nzxj65x32vh2fkhk.onion/pzkraz18q/jhh2tn/raw", + "http://nzxj65x32vh2fkhk.onion/rev/p0v6zvoin", + "http://nzxj65x32vh2fkhk.onion/rev/p8nqdddzb", + "http://nzxj65x32vh2fkhk.onion/rev/p8qorgjj2", + "http://nzxj65x32vh2fkhk.onion/rev/pawgypvjz", + "http://nzxj65x32vh2fkhk.onion/rev/pbbofkqn1", + "http://nzxj65x32vh2fkhk.onion/rev/pdmtzkwup", + "http://nzxj65x32vh2fkhk.onion/rev/pdot84cmm", + "http://nzxj65x32vh2fkhk.onion/rev/pdubkxy1v", + "http://nzxj65x32vh2fkhk.onion/rev/pegkdomcx", + "http://nzxj65x32vh2fkhk.onion/rev/pfgafcrvp", + "http://nzxj65x32vh2fkhk.onion/rev/ph62mpvce", + "http://nzxj65x32vh2fkhk.onion/rev/pjfn3yclj", + "http://nzxj65x32vh2fkhk.onion/rev/pkjhtx8tn", + "http://nzxj65x32vh2fkhk.onion/rev/pl6wwk9go", + "http://nzxj65x32vh2fkhk.onion/rev/poppwa9mo", + "http://nzxj65x32vh2fkhk.onion/rev/ppez6ok6r", + "http://nzxj65x32vh2fkhk.onion/rev/pqnih2eus", + "http://nzxj65x32vh2fkhk.onion/rev/psfl80ry1", + "http://nzxj65x32vh2fkhk.onion/rev/pswvhb2u2", + "http://nzxj65x32vh2fkhk.onion/rev/psxsbptqy", + "http://nzxj65x32vh2fkhk.onion/rev/ptoft0c1l", + "http://nzxj65x32vh2fkhk.onion/rev/pufx3aabu", + "http://nzxj65x32vh2fkhk.onion/rev/pvxlo9hda", + "http://nzxj65x32vh2fkhk.onion/rev/pweo3qenp", + "http://nzxj65x32vh2fkhk.onion/rev/pwgegfzoe", + "http://nzxj65x32vh2fkhk.onion/rev/pxn7di53e", + "http://nzxj65x32vh2fkhk.onion/rev/pxppe2cwf", + "http://nzxj65x32vh2fkhk.onion/rev/pyb041lxb", + "http://nzxj65x32vh2fkhk.onion/rev/pyd5xhkct", + "http://nzxj65x32vh2fkhk.onion/rev/pz7ltroub", + "http://nzxj65x32vh2fkhk.onion/rev/pzkraz18q", + "http://nzxj65x32vh2fkhk.onion/server-status", + "http://nzxj65x32vh2fkhk.onion/trending", + "http://nzxj65x32vh2fkhk.onion/trending/all", + "http://nzxj65x32vh2fkhk.onion/trending/all?page=1", + "http://nzxj65x32vh2fkhk.onion/trending/all?page=2", + "http://nzxj65x32vh2fkhk.onion/trending/all?page=3", + "http://nzxj65x32vh2fkhk.onion/trending/month", + "http://nzxj65x32vh2fkhk.onion/trending/month?page=1", + "http://nzxj65x32vh2fkhk.onion/trending/month?page=2", + "http://nzxj65x32vh2fkhk.onion/trending/month?page=3", + "http://nzxj65x32vh2fkhk.onion/trending/week", + "http://nzxj65x32vh2fkhk.onion/trending/week?page=1", + "http://nzxj65x32vh2fkhk.onion/trending/week?page=2", + "http://nzxj65x32vh2fkhk.onion/trending/week?page=3", + "http://nzxj65x32vh2fkhk.onion/trending/year", + "http://nzxj65x32vh2fkhk.onion/trending/year?page=1", + "http://nzxj65x32vh2fkhk.onion/trending/year?page=2", + "http://nzxj65x32vh2fkhk.onion/trending/year?page=3", + "http://nzxj65x32vh2fkhk.onion/trending?page=1", + "http://nzxj65x32vh2fkhk.onion/trending?page=2", + "http://nzxj65x32vh2fkhk.onion/trending?page=3", + "http://nzxj65x32vh2fkhk.onion/user/forgot", + "http://nzxj65x32vh2fkhk.onion/user/login", + "http://nzxj65x32vh2fkhk.onion/user/register" + ] + }, + "html": "\n\n\n\n\n\t\n\t\n\tStronghold Paste\n\n\t\n\n\t\n\t\n\t\n\n\t\n\t\n\t\n\t\n\t\n\t\n\n\t\n\n\n\n\t
\n\t\t\n\t\n\tLoading\n\n\t
\n\n\t\n\n\t
\n\t\t\n\t\t\t
\n\t
\n\t\t\n\t\t\n\t\t\t
\n
\n\n\t
\n\t\t
\n\n\t\t
\n\t\t\t
\n\t\t\t\t
\n\t\t\t\t\t\n\t\t\t\t
\n\t\t\t
\n\n\t\t\t
\n\n\t\t\t
\n\t\t\t\t
\n\t\t\t\t\t\n\t\t\t\t
\n\t\t\t
\n\t\t
\n\n\t\t
\n\t\t\t
\n\t\t\t\t
\n\t\t\t\t\t\n\t\t\t\t
\n\t\t\t
\n\t\t
\n\n\t\t\n\t\t
\n\t\t\t
\n\t\t\t\t\t\t\t\t\t
\n\t\t\t\t\t\t
\n\t\t\t\t\t\t\t
\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t
\n\n\t\t\t\t\t\t\t\n\t\t\t\t\t\t
\n\t\t\t\t\t
\n\t\t\t\t\t\t\t
\n\n\t\t\t
\n\t\t\t\t\t\t\t\t\t
\n\t\t\t\t\t\t
\n\t\t\t\t\t\t\t\n\t\t\t\t\t\t
\n\t\t\t\t\t
\n\t\t\t\t\t\t\t
\n\n\t\t\t
\n\t\t\t\t
\n\t\t\t\t\t
\n\t\t\t\t\t\t
\n\t\t\t\t\t\t\t\n\t\t\t\t\t\t
\n\n\t\t\t\t\t\t\n\n\t\t\t\t\t\t\t\t\t\t\t
\n\t\t\t\t
\n\t\t\t
\n\t\t
\n\n\t\t
\n\t
\n\n\t\t\t
\n\n\t\n\n\n\n", + "screenshots": { + "dateScreenshoted": "2020-07-02T15:28:35.722031Z", + "filename": "nzxj65x32vh2fkhk.onion_screenshot_a692c810.png" + }, + "interestingKeywords": [ + "rss", + "xml", + "RSS", + "feed", + "xml", + "XML", + "xml", + "XML" + ] + }, + "fields": { + "screenshots.dateScreenshoted": [ + "2020-07-02T15:28:35.722Z" + ], + "onionscan.dateScanned": [ + "2020-07-02T15:27:18.443Z" + ] + } +} ``` ## Authors diff --git a/onionscraper/operators/onionscan.py b/onionscraper/operators/onionscan.py index fd22c61..94b0bad 100644 --- a/onionscraper/operators/onionscan.py +++ b/onionscraper/operators/onionscan.py @@ -230,23 +230,21 @@ class Plugin(Operator): results = self.run_onionscan(onion) if results['status'] == 'success' and results['data']['webDetected'] == 'true': content = self.run_sessions(onion) - print(content) - #sys.exit(0) - #if content['status'] == 'success': - # blacklist_CONTENT = self.blacklist.search(content['data']) - # if blacklist_CONTENT: - # self.logger.info(f"[X] Blocked by blacklist content => matched keyword {blacklist_CONTENT.group()}") - # else: - # self.logger.debug("[*] CONTENT blacklist test: PASSED") - # screenshot = self.take_screenshot(self.format_directory(self.screenshots), onion) - # self.logger.info("Indexing!") - # doc = { - # 'onionscan':json.loads(results['data']), - # 'html':content['data'], - # 'screenshots':screenshot['data'], - # 'interestingKeywords':self.interestingKeywords.findall(content['data']) - # } - # return self.parseDoc(doc) + if content['status'] == 'success': + blacklist_CONTENT = self.blacklist.search(content['data']) + if blacklist_CONTENT: + self.logger.info(f"[X] Blocked by blacklist content => matched keyword {blacklist_CONTENT.group()}") + else: + self.logger.debug("[*] CONTENT blacklist test: PASSED") + screenshot = self.take_screenshot(self.format_directory(self.screenshots), onion) + self.logger.info("Indexing!") + doc = { + 'onionscan':json.loads(results['data']), + 'html':content['data'], + 'screenshots':screenshot['data'], + 'interestingKeywords':self.interestingKeywords.findall(content['data']) + } + return self.parseDoc(doc) else: self.logger.info(f"[x] hidden service {onion} is not active")