高级WEB客户端：网络爬虫--crawl.py

[python]代码库

#! /usr/bin/env python

from sys import argv
from os import makedirs,unlink,sep
from os.path import dirname,exists,isdir,splitext
from string import replace,find,lower
from htmllib import HTMLParser
from urllib import urlretrieve
from urlparse import urlparse,urljoin
from formatter import DumbWriter,AbstractFormatter
from cStringIO import StringIO

class Retriever(object):    # download web page
    def __init__(self,url):
        self.url = url
	self.file = self.filename(url)

    def filename(self,url,deffile='index.html'):
        parsedurl = urlparse(url,'http',0)    # parse path
	path = parsedurl[1] + parsedurl[2]
	ext = splitext(path)
	if ext[1] == '':    # no file,use default
	    if path[-1] == '/':
	        path += deffile
	    else:
	        path += '/' + deffile
	ldir = dirname(path)    # local directory
        if sep != '/':    # os-indep. path separator.
	    ldir = repalce(ldir,'/',sep)
	if not isdir(ldir):    # create archieve dir if nec.
	    if exists(ldir): unlink(ldir)
	    makedirs(ldir)
	return path

    def download(self):    # download web page
        try:
	    retval = urlretrieve(self.url,self.file)
	except IOError:
	    retval = ('*** ERROR: invalid URL "%s"' % self.url)
	return retval

    def parseAndGetLinks(self):    # parse HTML, save links
        self.parser = HTMLParser(AbstractFormatter(\
	    DumbWriter(StringIO())))
	self.parser.feed(open(self.file).read())
	self.parser.close()
	return self.parser.anchorlist

class Crawler(object):    # manage entire crawling process
    
    count = 0    # static download page counter

    def __init__(self,url):
        self.q = [url]
	self.seen = []
	self.dom = urlparse(url)[1]

    def getPage(self,url):
        r = Retriever(url)
	retval = r.download()
	if retval[0] == '*':    # error situation, do not parse
	    print retval, '...skipping parse'
	    return
	Crawler.count += 1
	print '\n(', Crawler.count, ')'
	print 'URL:',url
	print 'FILE:',retval[0]
	self.seen.append(url)

	links = r.parseAndGetLinks()    # get and process links
	for eachlink in links:
	    if eachlink[:4] != 'http' and \
	            find(eachlink,'://') == -1:
		eachlink = urljoin(url,eachlink)
	    print '* ',eachlink

	    if find(lower(eachlink),'mailto') != -1:
	        print '...discarded, mailto link'
		continue

	    if eachlink not in self.seen:
	        if find(eachlink,self.dom) == -1:
		    print '...discarded, not in domain'
	        else:
	            if eachlink not in self.q:
		        self.q.append(eachlink)
		        print '...new, added to Q'
		    else:
	                print '...discarded, already in Q'
            else:
	        print '...discarded, already processed'

    def go(self):    # process links in queue
        while self.q:
	    url = self.q.pop()
	    self.getPage(url)

def main():
    if len(argv) > 1:
	url = argv[1]

    else:
	try:
	    url = raw_input('Enter starting URL: ')
	except (KeyboardInterrupt,EOFError):
	    url = ''
    if not url: return
    robot = Crawler(url)
    robot.go()

if __name__ == '__main__':
    main()