Problem:
Hello everyone,
I am trying to create a webscraper which will take links from CSV file and scrape data.
but whenever I run this code it says the object of nonetype has no len().
Below is my code:
import requests
from bs4 import BeautifulSoup
import time
import os
import csv
file = {}
final_data = []
def read_file():
global file
f = open("Data.csv")
for line in f.readlines():
data = line.split(",")
file[data[0]]= data[1]
def writedata(alldata1, filename):
with open("./"+filename, "w")as csvfile:
csvfile = csv.writer(csvfile, delimiter=",")
csvfile.writerow("")
for i in range(0, len(alldata1)):
csvfile.writerow(alldata1[i])
def parse_data():
global file
global final_data
for data_obj in file.keys():
link = file[data_obj]
#print(link)
data = getbyget(link, {})
soup = BeautifulSoup(data, "html.parser")
get_details = soup.find_all(class_="box2-body")[0]
details = get_details.find_all(id="ctl00_ContentPlaceHolder1_Lblproposedenddt")
name = get_details.find_all(id="ctl00_ContentPlaceHolder1_lblprojectname")
date = ""
names = ""
sublist = []
for more in details:
date = more.text
sublist.append(date)
for nme in name:
names = nme.text
sublist.append(names)
get_state = soup.find_all(class_="box2-body")[1]
state = get_state.find_all(id="ctl00_ContentPlaceHolder1_lblState")
city = get_state.find_all(id="ctl00_ContentPlaceHolder1_lbldistrict")
cities = ""
states = ""
for statename in state:
states = statename.text
sublist.append(states)
for citys in city:
cities = citys.text
sublist.append(cities)
get_pmptername = soup.find_all(class_="box2-body")[2]
promotername = get_pmptername.find_all(id="ctl00_ContentPlaceHolder1_Lblpromotername")
geta = get_pmptername.find_all("a")
alinks = ""
promote = ""
for promoter in promotername:
promote = promoter.text
sublist.append(promote)
for linka in geta:
alinks = "http://up-rera.in/"+ linka["href"]
sublist.append(alinks)
final_data.append(sublist)
print(final_data)
def getbyget(url, values):
res = requests.get(url, data=values)
data = res.text
return data
def main():
read_file()
datas = parse_data()
writedata(datas, "UP_DATA.csv")
main()
What’s going wrong here?