Python彻底解决采集网页乱码问题

实战十几万网页采集后的修正版：

python彻底解决网页采集乱码问题，准确率 99%

def download_page(indexurl, timeout=10):
    """
    下载网站数据并返回
    :param indexurl:
    :param timeout:
    :return:
    """
    try:
        res = urllib2.urlopen(indexurl, timeout=timeout)
        code = res.getcode()
        info = res.info()
        charset = None

        if info:
            m = re.findall(r'charset=([a-zA-Z0-9_-]+)', ' '.join(info.headers), re.I)
            if m:
                charset = str(m[0]).lower()

        if code == 200:
            html = res.read()
            if not charset and html:
                charset = pick_charset(html)

            # 如果完全采不到 charset,默认使用 gbk 反正都是乱码
            if not charset or charset == "gb2312":
                charset = 'gbk'

            if charset and charset != 'utf-8':
                try:
                    html = html.decode(charset).encode('utf-8')
                except:
                    pass
        else:
            html = ''
        return (code,
                res.geturl(),
                charset,
                html
                )
    except urllib2.URLError, e:
        return str("%r" % e)
    except socket.timeout, e:
        return str("%r" % e)
    except:
        return str(sys.exc_info())

注意引入必要的包，如果有不足的地方，欢迎指正。