python爬虫下载链接_python爬虫下载链接

使用requests获取html后&＃xff0c;分析html中的标签发现所需要的链接在

然后分别获却 和中的内容 &＃xff0c;使用xpath时可以写成xpath(&＃39;//table[&＃64;class&＃61;"list"]/tr[&＃64;class&＃61;"even" or "odd"]/td/span/a[1]/&＃64;href&＃39;)

import re

import requests

import urllib2

from lxml import etree

url&＃61;&＃39;https://pypi.python.org/pypi/lxml/2.3/&＃39;

head&＃61;{&＃39;User-Agent&＃39;:&＃39;Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.63 Safari/537.36&＃39;}

def gethtml(url, *args):

html &＃61; requests.get(url, *args).content

return html

def writfile(cont):

try:

fd &＃61; open(&＃39;x.txt&＃39;, &＃39;w&＃39;)

try:

fd.write(cont)

finally:

fd.close()

except IOError:

print "file not existing!"

def readfile():

try:

fd &＃61; open(&＃39;x.txt&＃39;, &＃39;r&＃39;)

try:

all_the_text &＃61; fd.read()

finally:

fd.close()

except IOError:

print "File open error !"

return all_the_text

html &＃61; gethtml(url, head)

writfile(html)

all_text &＃61; readfile()

dom &＃61; etree.HTML(all_text)

url_list &＃61; dom.xpath(&＃39;//table[&＃64;class&＃61;"list"]/tr[&＃64;class&＃61;"even" or "odd"]/td/span/a[1]/&＃64;href&＃39;)

for url in url_list:

print url

经测试&＃xff0c;可以正常获取对应的下载链接。作为初学者&＃xff0c;代码有很多不当地方&＃xff0c;还请大牛审阅之后加以指正。