Ritvik

Live Hindustan Crawlers/LiveHindustanBusiness.py

@@ -0,0 +1,23 @@
+import requests
+from bs4 import BeautifulSoup
+import urllib.request
+import time
+
+for i in range(1,900):
+    r=requests.get("http://www.livehindustan.com/business/news-"+str(i))
+    soup=BeautifulSoup(r.content,"html.parser")
+    for j in soup.find_all('div',attrs={"class":"upper-first "}):
+        for k in j.find_all('h4'):
+            for urls in k.find_all('a'):
+                print(urls.get('href'))
+                print(urls.text)
+        for img in j.find_all('a',attrs={"class":"listing-pht-blcks pull-left"}):
+            for img1 in img.find_all('img'):
+                print(img1.get('src'))
+        for q in j.find_all('div',attrs={"class":"list-time-tags tags-list"}):
+            for q1 in q.find_all('p'):
+                print(q1.text)
+            for q2 in q.find_all('span'):
+                print(q2.text)
+        print("\n\n");
+    time.sleep(5)

Live Hindustan Crawlers/LiveHindustanCricket.py

@@ -0,0 +1,23 @@
+import requests
+from bs4 import BeautifulSoup
+import urllib.request
+import time
+
+for i in range(1,1100):
+    r=requests.get("http://www.livehindustan.com/cricket/news-"+str(i))
+    soup=BeautifulSoup(r.content,"html.parser")
+    for j in soup.find_all('div',attrs={"class":"upper-first "}):
+        for k in j.find_all('h4'):
+            for urls in k.find_all('a'):
+                print(urls.get('href'))
+                print(urls.text)
+        for img in j.find_all('a',attrs={"class":"listing-pht-blcks pull-left"}):
+            for img1 in img.find_all('img'):
+                print(img1.get('src'))
+        for q in j.find_all('div',attrs={"class":"list-time-tags tags-list"}):
+            for q1 in q.find_all('p'):
+                print(q1.text)
+            for q2 in q.find_all('span'):
+                print(q2.text)
+        print("\n\n");
+    time.sleep(5)

Live Hindustan Crawlers/LiveHindustanEntertainment.py

@@ -0,0 +1,23 @@
+import requests
+from bs4 import BeautifulSoup
+import urllib.request
+import time
+
+for i in range(1,1400):
+    r=requests.get("http://www.livehindustan.com/entertainment/news-"+str(i))
+    soup=BeautifulSoup(r.content,"html.parser")
+    for j in soup.find_all('div',attrs={"class":"upper-first "}):
+        for k in j.find_all('h4'):
+            for urls in k.find_all('a'):
+                print(urls.get('href'))
+                print(urls.text)
+        for img in j.find_all('a',attrs={"class":"listing-pht-blcks pull-left"}):
+            for img1 in img.find_all('img'):
+                print(img1.get('src'))
+        for q in j.find_all('div',attrs={"class":"list-time-tags tags-list"}):
+            for q1 in q.find_all('p'):
+                print(q1.text)
+            for q2 in q.find_all('span'):
+                print(q2.text)
+        print("\n\n");
+    time.sleep(5)

Live Hindustan Crawlers/LiveHindustanGadgets.py

@@ -0,0 +1,23 @@
+import requests
+from bs4 import BeautifulSoup
+import urllib.request
+import time
+
+for i in range(1,70):
+    r=requests.get("http://www.livehindustan.com/gadgets/news-"+str(i))
+    soup=BeautifulSoup(r.content,"html.parser")
+    for j in soup.find_all('div',attrs={"class":"upper-first "}):
+        for k in j.find_all('h4'):
+            for urls in k.find_all('a'):
+                print(urls.get('href'))
+                print(urls.text)
+        for img in j.find_all('a',attrs={"class":"listing-pht-blcks pull-left"}):
+            for img1 in img.find_all('img'):
+                print(img1.get('src'))
+        for q in j.find_all('div',attrs={"class":"list-time-tags tags-list"}):
+            for q1 in q.find_all('p'):
+                print(q1.text)
+            for q2 in q.find_all('span'):
+                print(q2.text)
+        print("\n\n");
+    time.sleep(5)

Live Hindustan Crawlers/LiveHindustanInternational.py

@@ -0,0 +1,23 @@
+import requests
+from bs4 import BeautifulSoup
+import urllib.request
+import time
+
+for i in range(1,1400):
+    r=requests.get("http://www.livehindustan.com/international/news-"+str(i))
+    soup=BeautifulSoup(r.content,"html.parser")
+    for j in soup.find_all('div',attrs={"class":"upper-first "}):
+        for k in j.find_all('h4'):
+            for urls in k.find_all('a'):
+                print(urls.get('href'))
+                print(urls.text)
+        for img in j.find_all('a',attrs={"class":"listing-pht-blcks pull-left"}):
+            for img1 in img.find_all('img'):
+                print(img1.get('src'))
+        for q in j.find_all('div',attrs={"class":"list-time-tags tags-list"}):
+            for q1 in q.find_all('p'):
+                print(q1.text)
+            for q2 in q.find_all('span'):
+                print(q2.text)
+        print("\n\n");
+    time.sleep(5)

Live Hindustan Crawlers/LiveHindustanLifestyle.py

@@ -0,0 +1,23 @@
+import requests
+from bs4 import BeautifulSoup
+import urllib.request
+import time
+
+for i in range(1,670):
+    r=requests.get("http://www.livehindustan.com/lifestyle/news-"+str(i))
+    soup=BeautifulSoup(r.content,"html.parser")
+    for j in soup.find_all('div',attrs={"class":"upper-first "}):
+        for k in j.find_all('h4'):
+            for urls in k.find_all('a'):
+                print(urls.get('href'))
+                print(urls.text)
+        for img in j.find_all('a',attrs={"class":"listing-pht-blcks pull-left"}):
+            for img1 in img.find_all('img'):
+                print(img1.get('src'))
+        for q in j.find_all('div',attrs={"class":"list-time-tags tags-list"}):
+            for q1 in q.find_all('p'):
+                print(q1.text)
+            for q2 in q.find_all('span'):
+                print(q2.text)
+        print("\n\n");
+    time.sleep(5)

Live Hindustan Crawlers/LiveHindustanNational.py

@@ -0,0 +1,23 @@
+import requests
+from bs4 import BeautifulSoup
+import urllib.request
+import time
+
+for i in range(1,13120):
+    r=requests.get("http://www.livehindustan.com/national/news-"+str(i))
+    soup=BeautifulSoup(r.content,"html.parser")
+    for j in soup.find_all('div',attrs={"class":"upper-first "}):
+        for k in j.find_all('h4'):
+            for urls in k.find_all('a'):
+                print(urls.get('href'))
+                print(urls.text)
+        for img in j.find_all('a',attrs={"class":"listing-pht-blcks pull-left"}):
+            for img1 in img.find_all('img'):
+                print(img1.get('src'))
+        for q in j.find_all('div',attrs={"class":"list-time-tags tags-list"}):
+            for q1 in q.find_all('p'):
+                print(q1.text)
+            for q2 in q.find_all('span'):
+                print(q2.text)
+        print("\n\n");
+    time.sleep(5)

Live Hindustan Crawlers/LiveHindustanSports.py

@@ -0,0 +1,23 @@
+import requests
+from bs4 import BeautifulSoup
+import urllib.request
+import time
+
+for i in range(1,650):
+    r=requests.get("http://www.livehindustan.com/sports/news-"+str(i))
+    soup=BeautifulSoup(r.content,"html.parser")
+    for j in soup.find_all('div',attrs={"class":"upper-first "}):
+        for k in j.find_all('h4'):
+            for urls in k.find_all('a'):
+                print(urls.get('href'))
+                print(urls.text)
+        for img in j.find_all('a',attrs={"class":"listing-pht-blcks pull-left"}):
+            for img1 in img.find_all('img'):
+                print(img1.get('src'))
+        for q in j.find_all('div',attrs={"class":"list-time-tags tags-list"}):
+            for q1 in q.find_all('p'):
+                print(q1.text)
+            for q2 in q.find_all('span'):
+                print(q2.text)
+        print("\n\n");
+    time.sleep(5)

Punjabkesari/Punjab kesari scrapper.py

@@ -2,18 +2,28 @@
 import requests
 from bs4 import BeautifulSoup
 import urllib.request
+import time
 
 
 
-for i in range(1,1000):
+for i in range(1,5200):
     r=requests.get("http://www.punjabkesari.in/latest.aspx?pageno=" + str(i))
     soup = BeautifulSoup(r.content,"html.parser")
     for j in soup.find_all("h2"):
         for k in j.find_all("a"):
-            print(k.text)        
-    for title in soup.find_all("h2"):
-        for url in title.find_all("a"):
-            print(url.get('href'))
-    for img in soup.find_all('span'):
-        for img1 in img.find_all('img'):
-            print(img1.get('data-original'))
+            url=(k.get('href'))
+            r=requests.get(url)
+            soup=BeautifulSoup(r.content,"html.parser")
+            for title in soup.find_all('meta',attrs={'name':"description"}):
+                print(title.get('content'))
+            for key in soup.find_all('meta',attrs={"name":"keywords"}):
+                                          print(key.get("content"))
+            for link in soup.find_all("link",attrs={'rel':"amphtml"}):
+                    print(link.get('href'))
+            for img in soup.find_all("span",class_="phot"):
+                for img1 in img.find_all("img"):
+                    print(img1.get('src'))
+            for date in soup.find_all('meta',attrs={'property':"article:published_time"}):
+                print(date.get('content'))
+            print('\n')
+    time.sleep(5)