import re import os import sys import json import time import random import traceback import subprocess from uuid import uuid4 from pathlib import Path from datetime import datetime as dt from json.decoder import JSONDecodeError from concurrent.futures import ProcessPoolExecutor from threading import Timer import requests from stem.control import Controller from stem import Signal from selenium import webdriver from selenium.webdriver.firefox.options import Options from selenium.webdriver.firefox.firefox_binary import FirefoxBinary from onionscraper.operators import Operator class Plugin(Operator): """OnionScraper main work logic. Handles reading the config file, calling sources, maintaining state and sending artifacts to operators. """ def __init__(self, logger, **kwargs): self.logger = logger self.logger.info('Initializing OnionScanner') screenshots = kwargs.pop('screenshots_path', None) if screenshots: self.screenshots = Path(screenshots) else: self.screenshots = Path(__file__).parents[1]/'screenshots' self.onionscan = kwargs['binpath'] self.timeout = int(kwargs['timeout']) self.proxy = kwargs['socks5'] self.torControl = kwargs['TorController'] self.retries = int(kwargs['retries']) self.headers ={ 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:72.0) Gecko/20100101 Firefox/72.0', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8', 'Accept-Language':'en-US,en;q=0.5', 'DNT': '1', 'Connection': 'keep-alive', 'Upgrade-Insecure-Requests': '1'} blacklist = kwargs['blacklist'].split(',') self.blacklist = re.compile('|'.join([re.escape(word) for word in blacklist]), re.IGNORECASE) keywords = kwargs['interestingKeywords'].split(',') self.keywords = re.compile('|'.join([re.escape(word) for word in keywords]), re.IGNORECASE) self.session = self.get_tor_session() def response(self, status, content, onion): """ status: success/failure content: dict onion: str return: dict """ return {'status': status, 'data': content, 'onion': onion} def parseDoc(self, data): data['onionscan'].pop('simpleReport', None) crawls = data['onionscan'].pop('crawls', None) hiddenService = data['onionscan'].pop('hiddenService', None) data['onionscan']['crawls'] = [*crawls] data['hiddenService'] = hiddenService for onion in crawls.keys(): print(onion) #q.enqueue(self.crawl, onion) #with open('test.json', 'w', encoding='utf-8') as f: # json.dump(data, f, ensure_ascii=False, indent=4) return data def format_directory(self, directory): d = dt.now() year = str(d.year) month = str(d.month) # prefix month and day with "0" if it is only one digit if len(month) < 2: month = "0" + month day = str(d.day) if len(day) < 2: day = "0" + day save_path = directory/year/month/day if not os.path.isdir(save_path): self.logger.info("[*] Creating directory to save screenshots") os.makedirs(save_path) return save_path def take_screenshot(self, save_path, onion): binary = FirefoxBinary('/home/tony/Projects/OnionScraper/geckodriver') fp = webdriver.FirefoxProfile() fp.set_preference('network.proxy.type', 1) fp.set_preference('network.proxy.socks', '127.0.0.1') fp.set_preference('network.proxy.socks_port', 9050) fp.set_preference('network.proxy.socks_remote_dns', True) options = Options() options.headless = True driver = webdriver.Firefox( executable_path='/home/tony/Projects/OnionScraper/geckodriver', options=options, firefox_profile=fp) url = 'http://' + onion driver.get(url) uid = str(uuid4()).split('-')[0] filename = f"{onion}_screenshot_{uid}.png" f_name = f"{save_path}/{filename}" driver.save_screenshot(f_name) driver.quit() if os.path.isfile(f_name): self.logger.info(f'[*] Screenshot was taken. {f_name}') dateScreenshoted = dt.utcnow().strftime('%Y-%m-%dT%H:%M:%S.%f')+ 'Z' result = {'dateScreenshoted':dateScreenshoted,'filename':filename} return self.response("success",result,onion) else: self.logger.error('[x] Unable to take screenshot') return self.response("failure",None,onion) def get_tor_session(self): try: s = requests.session() s.proxies = self.proxy s.headers.update(self.headers) except Exception as e: self.logger.error(e) self.logger.debug(traceback.print_exc()) return s # signal TOR for a new connection def renew_connection(self): with Controller.from_port(port = self.torControl['port']) as controller: # Now we switch TOR identities to make sure we have a good connection self.logger.info('Getting new Tor IP') # authenticate to our local TOR controller controller.authenticate(self.torControl['password']) # send the signal for a new identity controller.signal(Signal.NEWNYM) # wait for the new identity to be initialized time.sleep(controller.get_newnym_wait()) session = self.get_tor_session() self.logger.info(f"IP is {session.get('http://httpbin.org/ip').json()['origin']}") def handle_timeout(self, process, onion): # # Handle a timeout from the onionscan process. # try: # kill the onionscan process process.kill() self.logger.info("[!!!] Killed the onionscan process.") except: pass self.renew_connection() return def run_sessions(self, onion): retry = 0 result = None while True: try: url = 'http://'+onion self.logger.info(url) content = self.session.get(url) if content.status_code == 200: result = content.json() except JSONDecodeError as e: self.logger.debug(f'JSONDecodeError {e}') result = content.text except Exception as e: self.logger.error(e) self.logger.debug(traceback.print_exc()) finally: if result: return self.response("success",result,onion) else: self.logger.info('[x] No results found retrying ...') retry += 1 self.renew_connection() if retry > self.retries: self.logger.error('[x] Max retries exceeded') return self.response("failure",None, onion) def run_onionscan(self, onion): self.logger.info("[*] Running onionscan on %s", onion) # fire up onionscan process = subprocess.Popen([self.onionscan,"--webport=0","--jsonReport","--simpleReport=false",onion],stdout=subprocess.PIPE,stderr=subprocess.PIPE) # start the timer and let it run till timeout minutes process_timer = Timer(300,self.handle_timeout,args=[process,onion]) process_timer.start() # wait for the onion scan results stdout = process.communicate()[0] # we have received valid results so we can kill the timer if process_timer.is_alive(): process_timer.cancel() return self.response("success",json.loads(stdout),onion) self.logger.info("[!!!] Process timed out for %s", onion) return self.response("failure",None, onion) def handle_onion(self, onion_tuple): onion = onion_tuple.url self.logger.info(f'Processing {onion} with onionscan') try: blacklist_URL = self.blacklist.search(onion) if blacklist_URL: self.logger.info(f"[X] Blocked by blacklist => matched keyword {blacklist_URL.group()}") else: self.logger.debug("[*] URL blacklist test: PASSED") results = self.run_onionscan(onion) if results['status'] == 'success' and results['data']['webDetected'] == 'true': content = self.run_sessions(onion) print(content) #sys.exit(0) #if content['status'] == 'success': # blacklist_CONTENT = self.blacklist.search(content['data']) # if blacklist_CONTENT: # self.logger.info(f"[X] Blocked by blacklist content => matched keyword {blacklist_CONTENT.group()}") # else: # self.logger.debug("[*] CONTENT blacklist test: PASSED") # screenshot = self.take_screenshot(self.format_directory(self.screenshots), onion) # self.logger.info("Indexing!") # doc = { # 'onionscan':json.loads(results['data']), # 'html':content['data'], # 'screenshots':screenshot['data'], # 'interestingKeywords':self.interestingKeywords.findall(content['data']) # } # return self.parseDoc(doc) else: self.logger.info(f"[x] hidden service {onion} is not active") except Exception as e: self.logger.error(e) self.logger.error(traceback.print_exc()) finally: pass #sys.exit(0)