From a705daa869dbe251b24b8ae4f2a8e31e04c46e08 Mon Sep 17 00:00:00 2001
From: apurvsinghgautam <apurvsinghgautam@gmail.com>
Date: Mon, 27 Jul 2020 13:35:31 -0400
Subject: [PATCH] Added sources config to onioningestor.yml for Hunchly and
 Pystemon Added Hunchly python module Added Pystemon python module

---
 onioningestor.yml                 |  8 +++++
 onioningestor/sources/hunchly.py  | 56 +++++++++++++++++++++++++++++++
 onioningestor/sources/pystemon.py | 50 +++++++++++++++++++++++++++
 requirements/prod.txt             |  3 +-
 4 files changed, 116 insertions(+), 1 deletion(-)
 create mode 100644 onioningestor/sources/hunchly.py
 create mode 100644 onioningestor/sources/pystemon.py

diff --git a/onioningestor.yml b/onioningestor.yml
index 1350574..1dbda8c 100644
--- a/onioningestor.yml
+++ b/onioningestor.yml
@@ -22,6 +22,14 @@ sources:
       module: simplefile
       filename: onion_master_list.txt
 
+    - name: hunchly
+      module: hunchly
+      domain: https://www.dropbox.com/sh/wdleu9o7jj1kk7v/AADq2sapbxm7rVtoLOnFJ7HHa/HiddenServices.xlsx
+
+    - name: pystemon
+      module: pystemon
+      dirname: pystemon/alerts/
+
     #  - name: source-gist
     #    module: gist
     #    url: https://gist.github.com/search?l=Text&q=.onion
diff --git a/onioningestor/sources/hunchly.py b/onioningestor/sources/hunchly.py
new file mode 100644
index 0000000..3998d6b
--- /dev/null
+++ b/onioningestor/sources/hunchly.py
@@ -0,0 +1,56 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+
+__author__ = 'Apurv Singh Gautam'
+__license__ = "MIT"
+__version__ = "1.0.0"
+__maintainer__ = "Apurv Singh Gautam"
+__status__ = "Development"
+
+import xlrd
+from pathlib import Path
+from subprocess import call
+
+from onioningestor.sources import Source
+
+
+class Plugin(Source):
+
+    def __init__(self, logger, name, domain, **kwargs):
+        self.logger = logger
+        self.name = name
+        self.domain = domain
+        super().__init__(self)
+
+
+    def run(self):
+    	self.logger.info('Getting onions from Hunchly')
+    	lines = []
+    	''' 
+    	Hunchly Dropbox Link
+    	https://www.dropbox.com/sh/wdleu9o7jj1kk7v/AADq2sapbxm7rVtoLOnFJ7HHa/HiddenServices.xlsx
+    	'''
+        call(['wget', self.domain]) # Downloading Hunchly spreadsheet file
+        tmp_filename = 'HiddenServices.xlsx' # File name of Hunchly spreadsheet
+        tmp_filepath = Path(__file__).parents[0]/tmp_filename # File path of the Hunchly spreadsheet
+        workbook = xlrd.open_workbook(tmp_filepath) # Opening the Excel workbook
+        worksheet = workbook.sheet_by_name('Up') # Selecting 'Up' domains sheet
+        
+        for row_idx in range(1, worksheet.nrows): # Iterate through rows
+			lines.append(worksheet.cell(row_idx, 1).value) # Getting onion links
+        
+        for onion in lines:
+            self.onionQueue.put(
+            	(
+                    2,
+                    self.onion(
+                        url=onion,
+                        source=self.name,
+                        type='domain',
+                        status='offline',
+                        monitor=False,
+                        denylist=False)
+                    )
+                )
+
+        call(['rm', tmp_filepath]) # Deleting the temp Hunchly file
\ No newline at end of file
diff --git a/onioningestor/sources/pystemon.py b/onioningestor/sources/pystemon.py
new file mode 100644
index 0000000..69579c1
--- /dev/null
+++ b/onioningestor/sources/pystemon.py
@@ -0,0 +1,50 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+
+__author__ = 'Apurv Singh Gautam'
+__license__ = "MIT"
+__version__ = "1.0.0"
+__maintainer__ = "Apurv Singh Gautam"
+__status__ = "Development"
+
+
+import os
+from pathlib import Path
+from subprocess import call
+
+from onioningestor.sources import Source
+
+
+class Plugin(Source):
+
+    def __init__(self, logger, dirname, **kwargs):
+        self.logger = logger
+        self.dirname = dirname
+        super().__init__(self)
+
+
+    def run(self):
+    	self.logger.info('Getting onions from Pystemon')
+        dirpath = Path(__file__).parents[1]/self.dirname # Directory path of Pystemon alerts
+        for subdir, dirs, files in os.walk(dirpath) # Getting the files from pystemon alerts directory
+            for filename in files: 
+                name = subdir.split('/')[5] # Paste Source Name
+                filepath = subdir + os.sep + filename 
+
+                with open(filepath, 'r') as f:
+                    lines = f.readlines()
+
+                for onion in lines:
+                    self.onionQueue.put(
+                        (
+                            2,
+                            self.onion(
+                                url=onion,
+                                source=name,
+                                type='domain',
+                                status='offline',
+                                monitor=False,
+                                denylist=False)
+                            )
+                        )
+                call(['rm', filepath]) # Deleting the temp alert paste file
\ No newline at end of file
diff --git a/requirements/prod.txt b/requirements/prod.txt
index d7eaa75..36fbf98 100644
--- a/requirements/prod.txt
+++ b/requirements/prod.txt
@@ -13,5 +13,6 @@ schedule==0.6.0
 selenium==3.141.0
 six==1.15.0
 soupsieve==2.0.1
-stem==1.8.0
+stem==1.8.0i
 urllib3==1.25.9
+xlrd==1.2.0