updated gitignore and operatore init

.gitignore

@@ -1,3 +1,4 @@
+temp_crawl.txt
 monitoring.*
 onion_master_list.*
 webui

onioningestor/onion.py

@@ -19,10 +19,10 @@ class Onion(object):
     @url.setter
     def url(self, domain):
         onion_pattern = re.compile(r'([a-z2-7]{16,56}\.onion)')
-        if onion_pattern.match(domain):
-            self._url = domain
+        if onion_pattern.match(domain.lower()):
+            self._url = domain.lower()
         else:
-            raise Exception("Onion domain pattern does not match")
+            raise Exception(f"'{domain.lower()}' domain pattern does not match onion pattern")
 
     def set_operator(self, response):
         self.operators.update(response)

onioningestor/operators/__init__.py

@@ -100,11 +100,13 @@ class Operator:
 
     def findCrawls(self, content, hiddenService):
         crawl = set()
+        f1 = open("temp_crawl.txt", "a")
         for onion in re.findall(r'\s?(\w+.onion)', str(content)):
             if onion != hiddenService:
                 crawl.add(onion)
         for item in crawl:
             self.logger.debug(f'crawling queue added: {item}')
+            f1.write(item+'\n')
             self.queueCrawl.put((
                 3,
                 self.onion(