python爬虫Demo

import os
import sys
import http.client      #2.7版本为httplib
import urllib.request
import formatter
import io


from html.parser import HTMLParser

class Retriever(object):
    __slots__ = (\'url\', \'file\')

    def __init__(self,url):
        self.url, self.file = self.get_url_file(url)

    def get_url_file(self, url, default = \'index.html\'):
        #\'Create usable local filename from URL\'
        parsed = urllib.request.urlparse(url)
        host = parsed.netloc.split(\'@\')[-1].split(\':\')[0]
        print(\'retriever host: %s\' %host)
        filepath = \'%s%s\' %( host, parsed.path)
        if not os.path.splitext(parsed.path)[1]:
            filepath = os.path.join(filepath, default)
        print(\'retriever filepath: %s\' % filepath)
        linkdir = os.path.dirname(filepath)
        if not os.path.isdir(linkdir):
            if os.path.exists(linkdir):
                os.unlink(linkdir)
            os.makedirs(linkdir)
        return url, filepath

    def download(self):
        #\'Download URL to specific named file\'
        try:
            retval = urllib.request.urlretrieve(self.url,self.file)
        except (IOError,http.client.InvalidURL) as e:
            retval ( (\'***ERROR: bad URL "%s" %s\') %(self.url, e))
        return retval

    def parse_links(self):
        class AnchorParser(HTMLParser):
            def handle_starttag(self, tag, attrs):
                if tag != \'a\':
                    return
                if not hasattr(self , \'data\'):
                    self.data = []
                for attr in attrs:
                    if attr[0] == \'href\':
                        self.data.append(attr[1])

        f = open(self.file, \'r\', encoding="UTF-8")
        data = f.read()
        f.close()
        parser = AnchorParser()
        parser.feed(data)
        parser.close()
        #output( (urllib.request.join(self.url, x) for x in parser.data ) )
        if not hasattr(parser, \'data\'):
            return []
        return parser.data

class Crawler(object):
    count = 0
    def __init__(self, url):
        self.q = [url]
        self.seen = set()
        parsed = urllib.request.urlparse(url)
        host = parsed.netloc.split(\'@\')[-1].split(\':\')[0]
        print("host: %s" %host)
        #self.dom = \'.\'.join(host.split(\'.\')[-2:])
        self.dom = host
        print("dom: %s" %self.dom)

    def get_page(self, url, media=False):
        r = Retriever(url)
        fname = r.download()[0]
        print(\'donwload file name: %s\' %fname)
        if fname[0] == \'*\':
            print(fname + \'...skipping parse\')
            return
        Crawler.count += 1;
        print("(%d,URL:%s, FILE:%s)" %(Crawler.count, url, fname))
        self.seen.add(url)
        ftype = os.path.splitext(fname)[1]
        if ftype not in (\'.html\', \'htm\'):
            return
        print(r.parse_links())

        for link in r.parse_links():
            if link.startswith(\'mailto:\'):
                print(\'...discarded, mailto link\')
                continue
            if not media:
                ftype = os.path.splitext(link)[1]
                if ftype in (\'.mp3\', \'.mp4\', \'.m4v\', \'.wav\'):
                    print( \'...discard, media file\')
                    continue
                if ftype in (\'.epub\'):
                    print(\'...discard, epub file\')
                    continue
            if not link.startswith(\'http://\') and not link.startswith(\'https://\'):
                link = urllib.request.urljoin(url, link)
            print(\'*\', link)
            if link not in self.seen:
                if self.dom not in link:
                    print(\'...discard,not in domain\')
                else:
                    if link not in self.q:
                        self.q.append(link)
                        print(\'...new, added to Q\')
                    else:
                        print(\'...discard, already in Q\')
            else:
                print(\'...discarded, already processed\')


    def go(self, media = False):
        #\'Process next page in queue (if any) \'
        while self.q:
            url = self.q.pop()
            self.get_page(url, media)

def main():
    if len(sys.argv) > 1:
        url = sys.argv[1]
    else:
        try:
            url = input(\'Entry starting URL: \')
        except(KeyboardInterrupt, KeyError):
            url = \'\'
    if not url:
        return
    if not url.startswith(\'http://\') and \
        not url.startswith(\'https://\'):
        url = \'http://%s/\' %url

    print(\'start url: %s\'  %url )

    robot = Crawler(url)
    robot.go()



if __name__ == \'__main__\':
    main()