RentCity/scrape.py at master · JakeStratton/RentCity · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
from bs4 import BeautifulSoup
import pandas as pd
import ast
from datetime import datetime
import os
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
import time
import sys
import numpy as np
import regex as re


# function to get rental info
def get_rental_info(urls):
     '''creates a pandas dataframe of rental info using beautiful soup to scrape
     from streeteasy.com

     Parameters
     ----------
     url: the URL of the rental page to scrape

     Returns
     -------
     rental_single_df: the dataframe of apartment info'''

     # use selenium with chromedriver to get dynamic JS page.  requests library won't wait for JS.
     chromedriver = "~/Downloads/chromedriver" # path to the chromedriver executable
     chromedriver = os.path.expanduser(chromedriver)
     print('chromedriver path: {}'.format(chromedriver))
     sys.path.append(chromedriver)
     driver = webdriver.Chrome(chromedriver)
     #rentals_info = []


     # open log file
     scrape_log = open('scrape_log.txt','a')


     for idx, url in enumerate(urls):
          try:
               driver.get(url) #get the page
               time.sleep(2)
               soup = BeautifulSoup(driver.page_source, 'html.parser') #make the soup

               result = str(datetime.now()) + ':::' + str(idx) + ':::' + str(url)
               scrape_log.write('\n%s' % (result))

               # get rental info and add raw data to list
               rental_info_raw = soup.find_all(defer=True) # gets rental info data by finding any tag that has a defer attribute
               rental_info_string = str(rental_info_raw[0])  # makes a string out of the relevant info
               rental_info = rental_info_string.split('\n') #creates a list of the raw info

               #clean data
               rental_info = [i for i in rental_info if i.startswith('googletag')]
               rental_info = [x.replace('googletag.pubads().setTargeting(\'', '"') for x in rental_info]
               rental_info = [x.replace('\',', '":') for x in rental_info]
               rental_info = [x.replace('");', '"') for x in rental_info]
               rental_info = [x.replace(');', '') for x in rental_info]
               rental_info = ','.join(rental_info)
               rental_info = '{' + rental_info + '}'
               rentals_info = [rental_info]
               #rentals_info = rentals_info + rental_info#

          except Exception as e:
               result = str(idx) + str(datetime.now()) + ':::' + str(e) + ':::' + str(url)
               scrape_log.write('\nEXCEPTION: %s' % (result))
               print('EXCEPTION: ' + str(idx) + str(e) + result)

     # convert game_info to a list of dictionaries
     rentals_info_dicts = []
     for i in rentals_info:
          rentals_info_dicts.append(ast.literal_eval(i))

     #create df
     rentals_info_df = pd.DataFrame(rentals_info_dicts)

     # save csv file from dataframe
     rentals_info_df.to_csv('rentals_info.csv')

     return rentals_info_df


def get_rental_urls(start_page, end_page):
     base_url = 'https://streeteasy.com/for-rent/nyc?page='

     # use selenium with chromedriver to get dynamic JS page.  requests library won't wait for JS.
     chromedriver = "~/Downloads/chromedriver" # path to the chromedriver executable
     chromedriver = os.path.expanduser(chromedriver)
     print('chromedriver path: {}'.format(chromedriver))
     sys.path.append(chromedriver)
     driver = webdriver.Chrome(chromedriver)

     # open log file
     scrape_log = open('url_scrape_log.txt','a')

     urls = []
     for i in range(start_page, end_page):
          try:
               driver.get(base_url + str(i))
               time.sleep(2)
               listings = driver.find_elements_by_css_selector('article.item')
               for listing in listings:
                    item = listing.find_element_by_css_selector('.details-title a').get_attribute('href')
                    urls.append(item)
          except Exception as e:
               result = str(datetime.now()) + ':::' + str(e) + ':::' + str(i)
               scrape_log.write('\nEXCEPTION: %s' % (result))
               print('EXCEPTION: ' + str(e) + result)

     #create df
     urls_df = pd.DataFrame(urls)

     # save csv file from dataframe
     urls_df.to_csv('urls.csv')

     return urls


'''

if __name__ == '__main__':
    None

    '''