Merge pull request #90 from frux/ipv6-support

Add IPv6 support

Author: jasonbosco

scraper/src/config/config_loader.py

@@ -28,6 +28,7 @@ class ConfigLoader:
     api_key = None
     app_id = None
     custom_settings = None
+    dns_resolver = None
     extra_records = []
     index_name = None
     index_name_tmp = None

scraper/src/index.py

@@ -95,7 +95,7 @@ def run_config(config):
 
     DEFAULT_REQUEST_HEADERS = headers
 
-    process = CrawlerProcess({
+    crawler_settings = {
         'LOG_ENABLED': '1',
         'LOG_LEVEL': 'ERROR',
         'USER_AGENT': config.user_agent,
@@ -106,7 +106,12 @@ def run_config(config):
         'DUPEFILTER_CLASS': DUPEFILTER_CLASS_PATH,
         'DEFAULT_REQUEST_HEADERS': DEFAULT_REQUEST_HEADERS,
         'TELNETCONSOLE_ENABLED': False
-    })
+    }
+
+    if config.dns_resolver is not None:
+        crawler_settings['DNS_RESOLVER'] = config.dns_resolver
+
+    process = CrawlerProcess(crawler_settings)
 
     process.crawl(
         DocumentationSpider,