Python_爬虫项目

1 #-*- coding: utf-8 -*-
2 import re
3 import csv
4 import requests
5 from tqdm import tqdm
6 from urllib.parse import urlencode
7 from requests.exceptions import RequestException
8
9 def get_one_page(city, keyword, region, page):
10 &＃39;&＃39;&＃39;
11 获取网页html内容并返回
12 &＃39;&＃39;&＃39;
13 paras &＃61; {
14 &＃39;jl&＃39;: city, # 搜索城市
15 &＃39;kw&＃39;: keyword, # 搜索关键词
16 &＃39;isadv&＃39;: 0, # 是否打开更详细搜索选项
17 &＃39;isfilter&＃39;: 1, # 是否对结果过滤
18 &＃39;p&＃39;: page, # 页数
19 &＃39;re&＃39;: region # region的缩写&＃xff0c;地区&＃xff0c;2005代表海淀
20 }
21
22 headers &＃61; {
23 &＃39;User-Agent&＃39;: &＃39;Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36&＃39;,
24 &＃39;Host&＃39;: &＃39;sou.zhaopin.com&＃39;,
25 &＃39;Referer&＃39;: &＃39;https://www.zhaopin.com/&＃39;,
26 &＃39;Accept&＃39;: &＃39;text/html,application/xhtml&＃43;xml,application/xml;q&＃61;0.9,image/webp,image/apng,*/*;q&＃61;0.8&＃39;,
27 &＃39;Accept-Encoding&＃39;: &＃39;gzip, deflate, br&＃39;,
28 &＃39;Accept-Language&＃39;: &＃39;zh-CN,zh;q&＃61;0.9&＃39;
29 }
30
31 url &＃61; &＃39;https://sou.zhaopin.com/jobs/searchresult.ashx?&＃39; &＃43; urlencode(paras)
32 try:
33 # 获取网页内容&＃xff0c;返回html数据
34 response &＃61; requests.get(url, headers&＃61;headers)
35 # 通过状态码判断是否获取成功
36 if response.status_code &＃61;&＃61; 200:
37 return response.text
38 return None
39 except RequestException as e:
40 return None
41
42 def parse_one_page(html):
43 &＃39;&＃39;&＃39;
44 解析HTML代码&＃xff0c;提取有用信息并返回
45 &＃39;&＃39;&＃39;
46 # 正则表达式进行解析
47 pattern &＃61; re.compile(&＃39;(.*?).*?&＃39; # 匹配职位信息
48 &＃39;(.*?).*?&＃39; # 匹配公司网址和公司名称
49 &＃39;(.*?)&＃39;, re.S) # 匹配月薪
50
51 # 匹配所有符合条件的内容
52 items &＃61; re.findall(pattern, html)
53
54 for item in items:
55 job_name &＃61; item[0]
56 job_name &＃61; job_name.replace(&＃39;&＃39;, &＃39;&＃39;)
57 job_name &＃61; job_name.replace(&＃39;&＃39;, &＃39;&＃39;)
58 yield {
59 &＃39;job&＃39;: job_name,
60 &＃39;website&＃39;: item[1],
61 &＃39;company&＃39;: item[2],
62 &＃39;salary&＃39;: item[3]
63 }
64
65 def write_csv_file(path, headers, rows):
66 &＃39;&＃39;&＃39;
67 将表头和行写入csv文件
68 &＃39;&＃39;&＃39;
69 # 加入encoding防止中文写入报错
70 # newline参数防止每写入一行都多一个空行
71 with open(path, &＃39;a&＃39;, encoding&＃61;&＃39;gb18030&＃39;, newline&＃61;&＃39;&＃39;) as f:
72 f_csv &＃61; csv.DictWriter(f, headers)
73 f_csv.writeheader()
74 f_csv.writerows(rows)
75
76 def write_csv_headers(path, headers):
77 &＃39;&＃39;&＃39;
78 写入表头
79 &＃39;&＃39;&＃39;
80 with open(path, &＃39;a&＃39;, encoding&＃61;&＃39;gb18030&＃39;, newline&＃61;&＃39;&＃39;) as f:
81 f_csv &＃61; csv.DictWriter(f, headers)
82 f_csv.writeheader()
83
84 def write_csv_rows(path, headers, rows):
85 &＃39;&＃39;&＃39;
86 写入行
87 &＃39;&＃39;&＃39;
88 with open(path, &＃39;a&＃39;, encoding&＃61;&＃39;gb18030&＃39;, newline&＃61;&＃39;&＃39;) as f:
89 f_csv &＃61; csv.DictWriter(f, headers)
90 f_csv.writerows(rows)
91
92 def main(city, keyword, region, pages):
93 &＃39;&＃39;&＃39;
94 主函数
95 &＃39;&＃39;&＃39;
96 filename &＃61; &＃39;zl_&＃39; &＃43; city &＃43; &＃39;_&＃39; &＃43; keyword &＃43; &＃39;.csv&＃39;
97 headers &＃61; [&＃39;job&＃39;, &＃39;website&＃39;, &＃39;company&＃39;, &＃39;salary&＃39;]
98 write_csv_headers(filename, headers)
99 for i in tqdm(range(pages)):
100 &＃39;&＃39;&＃39;
101 获取该页中所有职位信息&＃xff0c;写入csv文件
102 &＃39;&＃39;&＃39;
103 jobs &＃61; []
104 html &＃61; get_one_page(city, keyword, region, i)
105 items &＃61; parse_one_page(html)
106 for item in items:
107 jobs.append(item)
108 write_csv_rows(filename, headers, jobs)
109
110 if __name__ &＃61;&＃61; &＃39;__main__&＃39;:
111 main(&＃39;北京&＃39;, &＃39;python&＃39;,2005 , 10)