diff --git a/Live Hindustan Crawlers/LiveHindustanBusiness.py b/Live Hindustan Crawlers/LiveHindustanBusiness.py new file mode 100644 index 0000000..8c120d2 --- /dev/null +++ b/Live Hindustan Crawlers/LiveHindustanBusiness.py @@ -0,0 +1,23 @@ +import requests +from bs4 import BeautifulSoup +import urllib.request +import time + +for i in range(1,900): + r=requests.get("http://www.livehindustan.com/business/news-"+str(i)) + soup=BeautifulSoup(r.content,"html.parser") + for j in soup.find_all('div',attrs={"class":"upper-first "}): + for k in j.find_all('h4'): + for urls in k.find_all('a'): + print(urls.get('href')) + print(urls.text) + for img in j.find_all('a',attrs={"class":"listing-pht-blcks pull-left"}): + for img1 in img.find_all('img'): + print(img1.get('src')) + for q in j.find_all('div',attrs={"class":"list-time-tags tags-list"}): + for q1 in q.find_all('p'): + print(q1.text) + for q2 in q.find_all('span'): + print(q2.text) + print("\n\n"); + time.sleep(5) diff --git a/Live Hindustan Crawlers/LiveHindustanCricket.py b/Live Hindustan Crawlers/LiveHindustanCricket.py new file mode 100644 index 0000000..c0c847b --- /dev/null +++ b/Live Hindustan Crawlers/LiveHindustanCricket.py @@ -0,0 +1,23 @@ +import requests +from bs4 import BeautifulSoup +import urllib.request +import time + +for i in range(1,1100): + r=requests.get("http://www.livehindustan.com/cricket/news-"+str(i)) + soup=BeautifulSoup(r.content,"html.parser") + for j in soup.find_all('div',attrs={"class":"upper-first "}): + for k in j.find_all('h4'): + for urls in k.find_all('a'): + print(urls.get('href')) + print(urls.text) + for img in j.find_all('a',attrs={"class":"listing-pht-blcks pull-left"}): + for img1 in img.find_all('img'): + print(img1.get('src')) + for q in j.find_all('div',attrs={"class":"list-time-tags tags-list"}): + for q1 in q.find_all('p'): + print(q1.text) + for q2 in q.find_all('span'): + print(q2.text) + print("\n\n"); + time.sleep(5) diff --git a/Live Hindustan Crawlers/LiveHindustanEntertainment.py b/Live Hindustan Crawlers/LiveHindustanEntertainment.py new file mode 100644 index 0000000..289d24f --- /dev/null +++ b/Live Hindustan Crawlers/LiveHindustanEntertainment.py @@ -0,0 +1,23 @@ +import requests +from bs4 import BeautifulSoup +import urllib.request +import time + +for i in range(1,1400): + r=requests.get("http://www.livehindustan.com/entertainment/news-"+str(i)) + soup=BeautifulSoup(r.content,"html.parser") + for j in soup.find_all('div',attrs={"class":"upper-first "}): + for k in j.find_all('h4'): + for urls in k.find_all('a'): + print(urls.get('href')) + print(urls.text) + for img in j.find_all('a',attrs={"class":"listing-pht-blcks pull-left"}): + for img1 in img.find_all('img'): + print(img1.get('src')) + for q in j.find_all('div',attrs={"class":"list-time-tags tags-list"}): + for q1 in q.find_all('p'): + print(q1.text) + for q2 in q.find_all('span'): + print(q2.text) + print("\n\n"); + time.sleep(5) diff --git a/Live Hindustan Crawlers/LiveHindustanGadgets.py b/Live Hindustan Crawlers/LiveHindustanGadgets.py new file mode 100644 index 0000000..1398d11 --- /dev/null +++ b/Live Hindustan Crawlers/LiveHindustanGadgets.py @@ -0,0 +1,23 @@ +import requests +from bs4 import BeautifulSoup +import urllib.request +import time + +for i in range(1,70): + r=requests.get("http://www.livehindustan.com/gadgets/news-"+str(i)) + soup=BeautifulSoup(r.content,"html.parser") + for j in soup.find_all('div',attrs={"class":"upper-first "}): + for k in j.find_all('h4'): + for urls in k.find_all('a'): + print(urls.get('href')) + print(urls.text) + for img in j.find_all('a',attrs={"class":"listing-pht-blcks pull-left"}): + for img1 in img.find_all('img'): + print(img1.get('src')) + for q in j.find_all('div',attrs={"class":"list-time-tags tags-list"}): + for q1 in q.find_all('p'): + print(q1.text) + for q2 in q.find_all('span'): + print(q2.text) + print("\n\n"); + time.sleep(5) diff --git a/Live Hindustan Crawlers/LiveHindustanInternational.py b/Live Hindustan Crawlers/LiveHindustanInternational.py new file mode 100644 index 0000000..9785d12 --- /dev/null +++ b/Live Hindustan Crawlers/LiveHindustanInternational.py @@ -0,0 +1,23 @@ +import requests +from bs4 import BeautifulSoup +import urllib.request +import time + +for i in range(1,1400): + r=requests.get("http://www.livehindustan.com/international/news-"+str(i)) + soup=BeautifulSoup(r.content,"html.parser") + for j in soup.find_all('div',attrs={"class":"upper-first "}): + for k in j.find_all('h4'): + for urls in k.find_all('a'): + print(urls.get('href')) + print(urls.text) + for img in j.find_all('a',attrs={"class":"listing-pht-blcks pull-left"}): + for img1 in img.find_all('img'): + print(img1.get('src')) + for q in j.find_all('div',attrs={"class":"list-time-tags tags-list"}): + for q1 in q.find_all('p'): + print(q1.text) + for q2 in q.find_all('span'): + print(q2.text) + print("\n\n"); + time.sleep(5) diff --git a/Live Hindustan Crawlers/LiveHindustanLifestyle.py b/Live Hindustan Crawlers/LiveHindustanLifestyle.py new file mode 100644 index 0000000..22bbd0a --- /dev/null +++ b/Live Hindustan Crawlers/LiveHindustanLifestyle.py @@ -0,0 +1,23 @@ +import requests +from bs4 import BeautifulSoup +import urllib.request +import time + +for i in range(1,670): + r=requests.get("http://www.livehindustan.com/lifestyle/news-"+str(i)) + soup=BeautifulSoup(r.content,"html.parser") + for j in soup.find_all('div',attrs={"class":"upper-first "}): + for k in j.find_all('h4'): + for urls in k.find_all('a'): + print(urls.get('href')) + print(urls.text) + for img in j.find_all('a',attrs={"class":"listing-pht-blcks pull-left"}): + for img1 in img.find_all('img'): + print(img1.get('src')) + for q in j.find_all('div',attrs={"class":"list-time-tags tags-list"}): + for q1 in q.find_all('p'): + print(q1.text) + for q2 in q.find_all('span'): + print(q2.text) + print("\n\n"); + time.sleep(5) diff --git a/Live Hindustan Crawlers/LiveHindustanNational.py b/Live Hindustan Crawlers/LiveHindustanNational.py new file mode 100644 index 0000000..98b2f97 --- /dev/null +++ b/Live Hindustan Crawlers/LiveHindustanNational.py @@ -0,0 +1,23 @@ +import requests +from bs4 import BeautifulSoup +import urllib.request +import time + +for i in range(1,13120): + r=requests.get("http://www.livehindustan.com/national/news-"+str(i)) + soup=BeautifulSoup(r.content,"html.parser") + for j in soup.find_all('div',attrs={"class":"upper-first "}): + for k in j.find_all('h4'): + for urls in k.find_all('a'): + print(urls.get('href')) + print(urls.text) + for img in j.find_all('a',attrs={"class":"listing-pht-blcks pull-left"}): + for img1 in img.find_all('img'): + print(img1.get('src')) + for q in j.find_all('div',attrs={"class":"list-time-tags tags-list"}): + for q1 in q.find_all('p'): + print(q1.text) + for q2 in q.find_all('span'): + print(q2.text) + print("\n\n"); + time.sleep(5) diff --git a/Live Hindustan Crawlers/LiveHindustanSports.py b/Live Hindustan Crawlers/LiveHindustanSports.py new file mode 100644 index 0000000..9f5511c --- /dev/null +++ b/Live Hindustan Crawlers/LiveHindustanSports.py @@ -0,0 +1,23 @@ +import requests +from bs4 import BeautifulSoup +import urllib.request +import time + +for i in range(1,650): + r=requests.get("http://www.livehindustan.com/sports/news-"+str(i)) + soup=BeautifulSoup(r.content,"html.parser") + for j in soup.find_all('div',attrs={"class":"upper-first "}): + for k in j.find_all('h4'): + for urls in k.find_all('a'): + print(urls.get('href')) + print(urls.text) + for img in j.find_all('a',attrs={"class":"listing-pht-blcks pull-left"}): + for img1 in img.find_all('img'): + print(img1.get('src')) + for q in j.find_all('div',attrs={"class":"list-time-tags tags-list"}): + for q1 in q.find_all('p'): + print(q1.text) + for q2 in q.find_all('span'): + print(q2.text) + print("\n\n"); + time.sleep(5) diff --git a/Punjabkesari/Punjab kesari scrapper.py b/Punjabkesari/Punjab kesari scrapper.py index 1bca7b9..d8cafc0 100644 --- a/Punjabkesari/Punjab kesari scrapper.py +++ b/Punjabkesari/Punjab kesari scrapper.py @@ -2,18 +2,28 @@ import requests from bs4 import BeautifulSoup import urllib.request +import time -for i in range(1,1000): +for i in range(1,5200): r=requests.get("http://www.punjabkesari.in/latest.aspx?pageno=" + str(i)) soup = BeautifulSoup(r.content,"html.parser") for j in soup.find_all("h2"): for k in j.find_all("a"): - print(k.text) - for title in soup.find_all("h2"): - for url in title.find_all("a"): - print(url.get('href')) - for img in soup.find_all('span'): - for img1 in img.find_all('img'): - print(img1.get('data-original')) + url=(k.get('href')) + r=requests.get(url) + soup=BeautifulSoup(r.content,"html.parser") + for title in soup.find_all('meta',attrs={'name':"description"}): + print(title.get('content')) + for key in soup.find_all('meta',attrs={"name":"keywords"}): + print(key.get("content")) + for link in soup.find_all("link",attrs={'rel':"amphtml"}): + print(link.get('href')) + for img in soup.find_all("span",class_="phot"): + for img1 in img.find_all("img"): + print(img1.get('src')) + for date in soup.find_all('meta',attrs={'property':"article:published_time"}): + print(date.get('content')) + print('\n') + time.sleep(5)