python多线程下载大文件_Python多线程下载文件的方法

本文实例讲述了Python多线程下载文件的方法。分享给大家供大家参考。具体实现方法如下&＃xff1a;

import httplib

import urllib2

import time

from threading import Thread

from Queue import Queue

from time import sleep

proxy &＃61; &＃39;your proxy&＃39;;

opener &＃61; urllib2.build_opener( urllib2.ProxyHandler({&＃39;http&＃39;:proxy}) )

urllib2.install_opener( opener )

ids &＃61; {};

for i in range(1,110):

try:

listUrl &＃61; "http://www.someweb.net/sort/list_8_%d.shtml" % (i);

print listUrl;

page &＃61; urllib2.urlopen(listUrl).read();

speUrl &＃61; "http://www.someweb.net/soft/";

speUrlLen &＃61; len(speUrl);

idx &＃61; page.find(speUrl,0);

while idx!&＃61;-1:

dotIdx &＃61; page.find(".",idx &＃43; speUrlLen);

if dotIdx !&＃61; -1:

id &＃61; page[idx &＃43; speUrlLen:dotIdx];

ids[id] &＃61; 1;

idx &＃61; page.find("http://www.someweb.net/soft/",idx &＃43; speUrlLen);

except:

pass;

q &＃61; Queue()

NUM &＃61; 5

failedId &＃61; [];

def do_somthing_using(id):

try:

url &＃61; "http://www.someweb.net/download.php?softid&＃61;%s&type&＃61;dx" % (id);

h2 &＃61; httplib.HTTPConnection("your proxy", "you port");

h2.request("HEAD", url);

resp &＃61; h2.getresponse();

header &＃61; resp.getheaders();

location &＃61; header[3][1];

sContent &＃61; urllib2.urlopen(location).read();

savePath &＃61; "C:\\someweb\\%s.rar" % (id);

file&＃61;open(savePath,&＃39;wb&＃39;);

file.write(sContent);

file.close();

print savePath &＃43; " saved";

except:

pass;

def working():

while True:

arguments &＃61; q.get()

do_somthing_using(arguments)

sleep(1)

q.task_done()

for i in range(NUM):

t &＃61; Thread(target&＃61;working)

t.setDaemon(True)

t.start()

for id in ids:

q.put(id)

q.join()

希望本文所述对大家的Python程序设计有所帮助。

本文原创发布php中文网&＃xff0c;转载请注明出处&＃xff0c;感谢您的尊重&＃xff01;