From 16f4fcef7197a102fcce0dac9c7a1bd9fc5b1338 Mon Sep 17 00:00:00 2001
From: unknown <33389418+JadeTheZapper@users.noreply.github.com>
Date: Sun, 27 Dec 2020 22:47:22 +0900
Subject: [PATCH] Update crawling object of gather_links() from a tag to entire
 response html

---
 spider.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/spider.py b/spider.py
index 0c09f64..68523d7 100644
--- a/spider.py
+++ b/spider.py
@@ -16,7 +16,7 @@ class Spider:
 
     def __init__(self, project_name, base_url, domain_name):
         Spider.project_name = project_name
-        Spider.base_url = base_url
+        Spider.base_url = base_url if base_url[-1] != '/' else base_url[:-1] # erase last /
         Spider.domain_name = domain_name
         Spider.queue_file = Spider.project_name + '/queue.txt'
         Spider.crawled_file = Spider.project_name + '/crawled.txt'
@@ -45,18 +45,21 @@ def crawl_page(thread_name, page_url):
     # Converts raw response data into readable information and checks for proper html formatting
     @staticmethod
     def gather_links(page_url):
+        import re
         html_string = ''
         try:
             response = urlopen(page_url)
             if 'text/html' in response.getheader('Content-Type'):
                 html_bytes = response.read()
                 html_string = html_bytes.decode("utf-8")
-            finder = LinkFinder(Spider.base_url, page_url)
-            finder.feed(html_string)
+                
+            pattern = "(\"|')(\/[\w\d?\/&=#.!:_-]{1,})(\"|')"
+            matches = re.findall(pattern, html_string)
+            result_urls = [ Spider.base_url + match[1] for match in matches]
+            return result_urls
         except Exception as e:
             print(str(e))
             return set()
-        return finder.page_links()
 
     # Saves queue data to project files
     @staticmethod