python之自定义爬虫脚本

## spider.py

import urllib2,httplib
import os,sys,re,time,random



#[OK]
def removeBR(site):
    result = re.search(r'\n',site)
    if result:
        return site.replace('\n','')
    return site
#[OK]
def getRequestList(fname):
    file = open(fname,"r")
    sites = file.readlines()
    list = []
    for s in sites:
        list.append(removeBR(s))
    return list
#[OK]
def inOldSet(url,set):
    if url in set:
        return True
    return False
#[]

#[OK]
def isEndWithJavascript(site):
    ret = re.search("/javascript:",site)
    if ret:
        return True
    return False

#[OK]
def ignoreJavascript(site):
    if isEndWithJavascript(site):
        s1,s2 = site.rsplit("/javascript:",1)
        return s1
    return site
#[50%_OK]
def isFile(url):
    res = re.search("(.html|.htm|.xml|.txt|.css|.js|.avi|.flv|.jpg|.gif|.bmp|.png|.xhtml|.dat|.doc|.xls|.php|.jsp|.asp)$",url)
    if res:
        return True
    return False
#[OK] http://aa.com/index.txt -> http://aa.com
def getBaseUrl(url):
    if isFile(url):
        s1,s2 = url.rsplit("/",1)
        return s1
    if url[len(url) - 1:] == "/":
        return url[:len(url) - 1]
    else:
        return url
#[OK]
def addIndexForSite(site):
    if isFile(site):
        return site
    bs = getBaseUrl(site)
    return "%s/index.html" % (bs)
#[OK]
def isError(site):
    try:
        getCntList(site)
        return False
    except:
        return True
#[OK]
def isPHPIndex(site):
    site = addIndexForSite(site)
    if isError(site):
        return True
    else:
        return False
#[OK]
def pathToFile(site):
    if isFile(site):
        return site
    bs = getBaseUrl(site)
    ext = "index.html"
    if isPHPIndex(site):
        ext = "index.php"
    return "%s/%s" % (bs,ext)
#[OK] ./a/c.txt  /a/b.txt a/v.txt -> a/a.txt
def getRelPath(sub):
    if sub[:1] == ".":
        if sub[1:2] == ".":
            return sub
        if sub[1:2] == "/":
            return sub[2:]
        return sub
    return sub
#[OK]
def getDomain(url):
    if isFullPath(url):
        s1,s2 = url.split("http://",1)
        if s2.find(r'/') != -1:
            s3,s4 = s2.split(r'/',1)
            return s3
        else:
            return s2
#[OK]
def toFullPath(sub,parent):
    if isFullPath(sub):
        return sub
    if sub == "/":
        return "http://%s" % (getDomain(parent))
    if sub[:1] == "/":
        baseUrl = getDomain(parent)
    else:
        baseUrl = getBaseUrl(parent)
    relPath = getRelPath(sub)
    return "http://%s/%s" % (baseUrl,relPath)
#[OK]
def formatUrl(url):
    url = ignoreJavascript(url)
    #if not isFile(url):
    #    url = pathToFile(url)
    return url
#[OK]
def isFullPath(url):
    if re.match("\s*http://",url):
        return True
    return False
#[OK]
def isPHPFile(site):
    ret = re.search("\.php\?",site)
    if ret:
        return True
    ret = re.search("\.php$",site)
    if ret:
        return True
    return False
#[OK?]
def isValidHTML(url):
    if isPHPFile(url) or not isFullPath(url):
        return False
    return True
def getCntList(url):
    list = []
    try:
        cnt = urllib2.urlopen(url,timeout=240).read()
        cnt = cnt.replace('\n','')
        cnt = cnt.replace('<a ','\n<a ')
        cnt = cnt.replace('<img ','\n<img ')
        sites = cnt.split('\n')
        for s in sites:
            if s:
                #print removeBR(s) + "\n\n"
                list.append(removeBR(s))
    except:
        print "load error~2"
    return list
#[OK]
def getAnchorHref(str):
    return re.finditer(r'<a\s.*href="([^<>\s\"]+)"',str)
def getImgSrc(str):
    return re.finditer(r'<img\s.*src="([^<>\s\"]+)"',str)
#[OK]
def isImage(url):
    res = re.search("(.jpg|.JPG|.gif|.GIF|.bmp|.BMP|.png|.PNG|.jpeg|.JPEG)$",url)
    if res:
        return True
    else:
        return False
#[OK]
def isHTML(url):
    res = re.search("(.html|.htm|.asp|.aspx)$",url)
    if res:
        return True
    else:
        return False
gcnt = 0
#[]
def parseHTML(url):
    global gcnt
    gcnt = gcnt + 1
    print "parsing[%d]-->%s" % (gcnt,url)
    hSet = []
    iSet = []
    res = []
    list = getCntList(url)
    for s in list:
        anchor = getAnchorHref(s)
        for a in anchor:
            print "anchor[sub]->%s" % (a.group(1))
            print "fullPath[sub]->%s" % (formatUrl(toFullPath(a.group(1),url)))
            res.append(formatUrl(toFullPath(a.group(1),url)))
        img = getImgSrc(s)
        for im in img:
            print "image[sub]->%s" % (im.group(1))
            res.append(formatUrl(toFullPath(im.group(1),url)))
    for r in res:
        if isImage(r):
            iSet.append(r)
        if isHTML(r):
            hSet.append(r)
    return iSet,hSet
#[OK]
def updateUrlSet(urlSet,htmlSet,oldSet):
    for h in htmlSet:
        if h in oldSet:
            continue
        if h in urlSet:
            continue
        urlSet.append(h)
#[OK]
def getExt(url):
    s1,s2 = url.rsplit(".",1)
    return s2
#[OK]
def getRandomName():
    return "%d_%d" % (time.time(),random.randint(0,99999))
#[OK]
def getPath(url):
    name = getRandomName()
    ext = getExt(url)
    return "%s.%s" % (name,ext)
#[OK]
def getUrl(url):
    try:
        return urllib2.urlopen(url,timeout=240).read()
    except:
        print "load error~"
        return None
#[OK]
def saveToRoot(str,path):
    newPath = os.path.join(os.getcwd(),"download")
    newPath = os.path.join(newPath,path)
    fp = open(newPath,"wb")
    fp.write(str)
    fp.close()
#[OK]
def saveImage(imgSet):
    for site in imgSet:
        print "Get-->%s" % (site)
        page = getUrl(site)
        if page == None:
            continue
        path = getPath(site)
        print "Save-->%s" % (path)
        saveToRoot(page,path)
#[Test]
def main(fname):
    urlSet = getRequestList(fname)
    oldSet = []
    for url in urlSet:
        if inOldSet(url,oldSet):
            del(url)
            continue
        newUrl = formatUrl(url)
        #print "newUrl::" + newUrl
        #newUrl = ignoreJavascript(url)
        if isValidHTML(newUrl):
            imgSet,htmlSet = parseHTML(newUrl)
            saveImage(imgSet)
            oldSet.append(url)
            oldSet.append(newUrl)
            updateUrlSet(urlSet,htmlSet,oldSet)
        else:
            del(url)

def test(set):
    for t in set:
        print formatUrl(removeBR(t))

main('list.txt')

## list.txt

http://www.gaoxiaola.com/p/gif/index_4.html

@@用法说明

spider.py list.txt

试图从网页中抓取图片