开发笔记:网页爬虫教程

#!/usr/bin/env python
#coding=utf-8
import urllib
import urllib2
import re
import pymongo
def getHtml(url):
page=urllib2.urlopen(url)
html=page.read()
page.close()
return html
def getContent(html):
reg=r‘

.+?src="(.+?.jpg)".+?

.+?class="title".+?
class="">(.+?).+?class="rating".+?class="subject-rate">(.+?).+?(.+?)‘
cOntentre=re.compile(reg,re.DOTALL)
cOntentlist=contentre.findall(html)
return contentlist
def getConnection(): #拿到数据库连接
cOnn=pymongo.Connection(‘localhost‘,27017)
return conn
def saveToDB(contentlist): #存储至mongodb数据库中
cOnn=getConnection()
db=conn.db
t_movie=db.t_movie
for content in contentlist:
value=dict(poster=content[0],title=content[1],rating=content[2],ticket_btn=content[3])
t_movie.save(value)def display(contentlist):
for content in contentlist:
#values=dict(poster=content[0],title=content[1],rating=content[2],ticket_btn=content[3])
print ‘poster‘,‘ ‘,content[0]
print ‘title‘,‘ ‘,content[1]
print ‘rating‘,‘ ‘,content[2]
print ‘ticket_btn‘,‘ ‘,content[3]
print‘..............................................................................‘
if __name__=="__main__":
url="http://movie.douban.com/"
html=getHtml(url)
#print html
cOntentlist=getContent(html)
print len(contentlist)
#print contentlist
display(contentlist)
saveToDB(contentlist)
print "finished"