python爬虫：BeautifulSoup巴乐兔租房信息爬取

找到巴乐兔上海租房网页url&＃xff1a;巴乐兔上海
通过翻页发现&＃xff1a;
第一页url&＃xff1a;‘http://sh.baletu.com/zhaofang/?entrance&＃61;14’
第二页url&＃xff1a;‘http://sh.baletu.com/zhaofang/p2o1a1/?**seachId&＃61;0&is_rec_house&＃61;0&entrance&＃61;14&solr_house_cnt&＃61;28156’
第三页url&＃xff1a;‘http://sh.baletu.com/zhaofang/p3o1a1/?**seachId&＃61;0&is_rec_house&＃61;0&entrance&＃61;14&solr_house_cnt&＃61;28159’
我们只需要关注网址的前半部分的不同之处&＃xff0c;后半部分为自动生成字段&＃xff0c;因此可以用以下代码获取每页的url

if page &＃61;&＃61; 1:url &＃61; &＃39;http://sh.baletu.com/zhaofang/?entrance&＃61;14&＃39;
else:url &＃61; &＃39;http://sh.baletu.com/zhaofang/p&＃39;&＃43;str(page)&＃43;&＃39;o1a1/&＃39;


再通过request请求网页&＃xff0c;用BeautifulSoup解析&＃xff1a;

r &＃61; requests.get(url,headers&＃61;headers)
soup &＃61; BeautifulSoup(r.content.decode(&＃39;utf-8&＃39;),&＃39;html.parser&＃39;)


通过检查网页发现&＃xff0c;需要的信息全部在class为list-center的div标签下&＃xff1a;

在该div下&＃xff0c;每个li标签就是一条租房信息&＃xff0c;因此

outer_div &＃61; soup.find(&＃39;div&＃39;,class_&＃61;"list-center")
houses &＃61; outer_div.find_all(&＃39;li&＃39;,attrs &＃61; {"class":"listUnit-date clearfix PBA_list_house"})


同样&＃xff0c;通过分析每个li标签下的子标签&＃xff0c;可以得到房屋名称&＃xff0c;区域&＃xff0c;小区等信息&＃xff0c;代码如下&＃xff1a;

houses_info &＃61; []
for house in houses:try:id &＃61; house.attrs[&＃39;num&＃39;]name &＃61; house.h3.a.attrs[&＃39;title&＃39;]address &＃61; re.search("(.*?)-(.{2})",name)area &＃61; address.group(2)community &＃61; address.group(1)url &＃61; house.find(&＃39;a&＃39;,attrs&＃61;{"target":"_blank"}).attrs[&＃39;href&＃39;]price &＃61; house.attrs[&＃39;price&＃39;]rent_type &＃61; house.attrs[&＃39;variant&＃39;]size_info &＃61; house.find(&＃39;p&＃39;,attrs&＃61;{"class":"list-pic-ps"}).find("span",attrs&＃61;{"class":False}).textsize &＃61; re.search("(\d&＃43;)",size_info).group()traffic_1 &＃61; house.find("div",attrs&＃61;{"class":"list-pic-ad"}).texttraffic_2 &＃61; re.search("距离(?:(\d&＃43;)号线)?(.*?)(?:(\d&＃43;)米)",traffic_1)if traffic_2 &＃61;&＃61; None:traffic_info &＃61; &＃39;__&＃39;else:traffic &＃61; traffic_2.groups()info_list &＃61; []for i in traffic:if i is None:info_list.append("")else:info_list.append(i)traffic_info &＃61; &＃39;_&＃39;.join(info_list)release_time &＃61; house.find("span",attrs&＃61;{"class":"room-time"}).text.replace(" 发布","")grade &＃61; house.find("span",attrs&＃61;{"class":"lan-ratedetail"}).textcomment &＃61; house.find("span",attrs&＃61;{"class":"lan-rate-people"}).textcomment_num &＃61; re.search("(\d&＃43;)",comment).group()except Exception:Nonehouses_info.append({"id":id,"name":name,"area":area,"community":community,"url":url,"price":price,"rent_type":rent_type,"size":size,"traffic_info":traffic_info,"release_time":release_time,"grade":grade,"comment_num":comment_num})


将以上代码封装成方法&＃xff0c;在定义一个保存成csv文件的方法&＃xff1a;

def randerToFile(houses_info):with open("巴乐兔_上海.csv","a",encoding&＃61;&＃39;utf-8&＃39;) as file:for house in houses_info:file.write("::".join(house.values())&＃43;"\n")


最后通过循环&＃xff0c;执行上述方法&＃xff0c;就能爬取到巴乐兔所有页的租房信息。