正则表达式简单应用

爬取猫眼电影网TOP100信息，并将其中电影名，主演，上映时间保存到数据库中

import pymysql
import re
from urllib import request
from urllib.request import urlopen
from concurrent.futures import ThreadPoolExecutor

# 猫眼TOP100网址
url = 'http://maoyan.com/board/4'

films_info = []


# 找出TOP100全10页的所有网址
def get_all_url(url, page):
    url_li = [url]

    for i in range(page - 1):
        new_url = url + '?offset=%d' % ((i + 1) * 10)
        url_li.append(new_url)
    return url_li


# 爬取信息
def get_info(url):
    header = {'User-agent': 'FireFox/12.0'}
    req = request.Request(url, headers=header)
    with urlopen(req) as urlobj:
        return urlobj.read().decode('utf-8')


# 正则提取需要的信息
def get_content(url):
    info = get_info(url)
    pattern = r'<a href="\/films\/.*" title="(.*)" ' \
              r'data-act="boarditem-click" data-val="\{movieId:.*\}' \
              r'">.*<\/a><\/p>\s*<p class="star">\s*(主演：.*)\s*<\/p>\s*' \
              r'<p class="releasetime">(上映时间：.*)<\/p> '

    films_info.extend(re.findall(pattern, info))


def main():
    res = get_all_url(url, 10)
    # 多线程快速获得信息
    with ThreadPoolExecutor(max_workers=10) as poor:
        poor.map(get_content, res)

    # 进入homework数据库
    conn = pymysql.connect(host='localhost', user='root', passwd='wl1009', db='homework', charset='utf8')

    cur = conn.cursor()
    try:
        # 创建一个films表
        cur.execute('create table films (movie varchar(20),actor varchar(50),time varchar(40))')

        # 添加信息
        insert_sql = 'insert into films values(%s,%s,%s)'
        cur.executemany(insert_sql, films_info)

        # 显示信息
        cur.execute('select * from films')
        print(cur.fetchall())
    except Exception as e:
        print(e)
    else:
        print('信息存储成功...')

    conn.commit()
    cur.close()
    conn.close()


main()