爬取爱笔智能招聘职位

1 import requests
2 from urllib.parse import urlencode
3 from pyquery import PyQuery as pq
4 from pymongo import MongoClient
5 import json
6
7
8 base_url &＃61; &＃39;http://aibee.com/cn/joinus.aspx?action&＃61;jobinfo&&＃39;
9
10 headers &＃61; {
11 &＃39;Host&＃39;: &＃39;aibee.com&＃39;,
12 &＃39;Referer&＃39;: &＃39;http://aibee.com/cn/joinus.aspx&＃39;,
13 &＃39;User-Agent&＃39;: &＃39;Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36&＃39;,
14 &＃39;X-Requested-With&＃39;: &＃39;XMLHttpRequest&＃39;,
15 }
16
17 client &＃61; MongoClient()
18 db &＃61; client[&＃39;aibee&＃39;]
19 collection &＃61; db[&＃39;aibee&＃39;]
20 max_id &＃61; 50
21
22
23
24 def get_page(id):
25
26 formData &＃61; {
27 &＃39;id&＃39;: id,
28 }
29
30
31 data &＃61; urlencode(formData)
32 url &＃61; base_url &＃43; urlencode(formData)
33 try:
34 response &＃61; requests.get(url, data&＃61;data, headers&＃61;headers)
35 if response.status_code &＃61;&＃61; 200:
36
37 return response.json()
38 except requests.ConnectionError as e:
39 print(&＃39;Error&＃39;, e.args)
40
41
42 def parse_page(json_1):
43 if json_1:
44 items &＃61; json_1.get(&＃39;shuzu&＃39;)
45 for item in items:
46 if id &＃61;&＃61; 1 :
47 continue
48 else:
49
50 aibee &＃61; {}
51 aibee[&＃39;id&＃39;] &＃61; item.get(&＃39;id&＃39;)
52 aibee[&＃39;title&＃39;] &＃61; item.get(&＃39;title&＃39;)
53 aibee[&＃39;zhize&＃39;] &＃61; pq(item.get(&＃39;zhize&＃39;)).text()
54 aibee[&＃39;yaoqiu&＃39;] &＃61; pq(item.get(&＃39;yaoqiu&＃39;)).text()
55 aibee[&＃39;dtt&＃39;] &＃61; item.get(&＃39;dtt&＃39;)
56 aibee[&＃39;emailaddr&＃39;] &＃61; item.get(&＃39;emailaddr&＃39;)
57 yield aibee
58
59
60 def write_to_file(content):
61 with open(&＃39;aibee.json&＃39;,&＃39;a&＃39;,encoding&＃61;&＃39;utf-8&＃39;) as f:
62 f.write(json.dumps(content,ensure_ascii&＃61;False)&＃43;&＃39;\n&＃39;)
63 f.close()
64
65 def save_to_mongo(result):
66 if collection.insert(result):
67 print(&＃39;Saved to Mongo&＃39;)
68
69
70 if __name__ &＃61;&＃61; &＃39;__main__&＃39;:
71 for id in range(1, max_id &＃43; 1):
72 json_1 &＃61; get_page(id)
73 #print(json_1)
74
75 results &＃61; parse_page(json_1)
76 for result in results:
77 print(result)
78 write_to_file(result)
79 save_to_mongo(result)

1 import requests
2 from urllib.parse import urlencode
3 from pyquery import PyQuery as pq
4 from pymongo import MongoClient
5 import json
6
7
8 url &＃61; &＃39;http://aibee.com/cn/joinus.aspx?action&＃61;jobinfo&＃39;
9
10 headers &＃61; {
11 &＃39;Host&＃39;: &＃39;aibee.com&＃39;,
12 &＃39;Referer&＃39;: &＃39;http://aibee.com/cn/joinus.aspx&＃39;,
13 &＃39;User-Agent&＃39;: &＃39;Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36&＃39;,
14 &＃39;X-Requested-With&＃39;: &＃39;XMLHttpRequest&＃39;,
15 }
16
17 client &＃61; MongoClient()
18 db &＃61; client[&＃39;aibee&＃39;]
19 collection &＃61; db[&＃39;aibee&＃39;]
20 max_id &＃61; 50
21
22
23
24 def get_page(id):
25
26 formData &＃61; {
27 &＃39;id&＃39;: id,
28 }
29 try:
30 r &＃61; requests.post(url, data&＃61;formData, headers&＃61;headers)
31 if r.status_code &＃61;&＃61; 200:
32 return r.json()
33 except requests.ConnectionError as e:
34 print(&＃39;Error&＃39;, e.args)
35
36
37 def parse_page(json_1):
38 if json_1:
39 items &＃61; json_1.get(&＃39;shuzu&＃39;)
40 for item in items:
41 if id &＃61;&＃61; 1 :
42 continue
43 else:
44
45 aibee &＃61; {}
46 aibee[&＃39;id&＃39;] &＃61; item.get(&＃39;id&＃39;)
47 aibee[&＃39;title&＃39;] &＃61; item.get(&＃39;title&＃39;)
48 aibee[&＃39;zhize&＃39;] &＃61; pq(item.get(&＃39;zhize&＃39;)).text()
49 aibee[&＃39;yaoqiu&＃39;] &＃61; pq(item.get(&＃39;yaoqiu&＃39;)).text()
50 aibee[&＃39;dtt&＃39;] &＃61; item.get(&＃39;dtt&＃39;)
51 aibee[&＃39;emailaddr&＃39;] &＃61; item.get(&＃39;emailaddr&＃39;)
52 yield aibee
53
54
55 def write_to_file(content):
56 with open(&＃39;aibee.json&＃39;,&＃39;a&＃39;,encoding&＃61;&＃39;utf-8&＃39;) as f:
57 f.write(json.dumps(content,ensure_ascii&＃61;False)&＃43;&＃39;\n&＃39;)
58 f.close()
59
60 def save_to_mongo(result):
61 if collection.insert(result):
62 print(&＃39;Saved to Mongo&＃39;)
63
64
65 if __name__ &＃61;&＃61; &＃39;__main__&＃39;:
66 for id in range(1, max_id &＃43; 1):
67 json_1 &＃61; get_page(id)
68 #print(json_1)
69
70 results &＃61; parse_page(json_1)
71 for result in results:
72 print(result)
73 write_to_file(result)
74 save_to_mongo(result)