分布式多进程phantomjs优化

1、正确设计进程的超时时间避免堵塞

2、正确放置异常的捕获位置

3、灵活使用quit()方法

4、redis哈希键分布式设计

 

from selenium import webdriver
import threading
import time
import redis
import re
import csv
import datetime
import pymysql
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
r_conn = redis.Redis(host='', port='', password='')
db = pymysql.connect(host='',port=3306,user='root',db='项目库',passwd='',charset='utf8')
def test():
    try:
        p_data = open('company_ps_info.txt', 'r')
        data1 = p_data.read()
        pat = '1\d{10}'
        phones = re.compile(pat).findall(data1)
        for phone in phones:
            try:
                pat1 = '^12\d{9}'
                test_phone1 = re.compile(pat1).findall(phone)
                pat2 = '^10\d{9}'
                test_phone2 = re.compile(pat2).findall(phone)
                pat3 = '^11\d{9}'
                test_phone3 = re.compile(pat3).findall(phone)
                if len(test_phone1) > 0 or len(test_phone2) > 0 or len(
                        test_phone3) > 0 or '1234' in phone or '0000' in phone or '1111' in phone or '2222' in phone or '3333' in phone or '4444' in phone or '5555' in phone or '6666' in phone or '7777' in phone or '8888' in phone or '9999' in phone:
                    continue
                else:
                    pass
                test_rkeys = r_conn.hget('gp_loss_lxcf', phone)
                if test_rkeys == None:
                    r_conn.hset('gp_loss_lxcf', phone, '1')
                else:
                    continue
                dcap = dict(DesiredCapabilities.PHANTOMJS)
                dcap["phantomjs.page.settings.userAgent"] = (
                    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36")
                service_args = []
                service_args.append('--load-images=no')  ##关闭图片加载
                browser = webdriver.PhantomJS(service_args=service_args, desired_capabilities=dcap)
                browser.implicitly_wait(15)
                browser.set_page_load_timeout(15)  ##设置超时时间
                try:
                    browser.get("")

                except Exception as gl:
                    print(gl)
                    print(datetime.datetime.now().strftime("%Y-%m-%d %H:%M"))
                    browser.quit()
                    time.sleep(0.1)
                browser.find_element_by_xpath('//*[@id="mobile"]').clear()
                browser.find_element_by_xpath('//*[@id="mobile"]').send_keys(phone)
                browser.find_element_by_xpath('//*[@id="password"]').clear()
                browser.find_element_by_xpath('//*[@id="password"]').send_keys('fate100938')
                time.sleep(2)
                # a=browser.get_screenshot_as_file("D:/java/0412test.jpg")
                data = browser.page_source
                browser.quit()
                print(len(data))
                print(datetime.datetime.now().strftime("%Y-%m-%d %H:%M"))
                if '' in data:
                    type = ''
                else:
                    continue
                # test_rkeys = r_conn.hget('gp_mysql_phone', phone)
                # if test_rkeys == None:
                #     r_conn.hset('gp_mysql_phone', phone, '1')
                # else:
                #     print('the hash key already exist in gp mysql!')
                #     continue
                print(type)
                print(phone)
                source = ''
                filename = ''
                with open(str(filename) + '.csv', 'a', newline='') as file:
                    writer = csv.writer(file)
                    writer.writerow([phone, '', source])

                cursor = db.cursor()
                sql = " INSERT INTO gp_app_all(phone,type,source) VALUES ('%s', '%s', '%s')" % (phone, type, source)
                # 执行sql语句
                cursor.execute(sql)
                db.commit()
            except Exception as gl:
                print(gl)
                browser.quit()
        browser.quit()
    except Exception as gl:
        print(datetime.datetime.now().strftime("%Y-%m-%d %H:%M"))
        # browser.quit()
        print(gl)
        threads_get()
def threads_get():
    for i in range(10):
        threading.Thread(target=test).start()

def create_table():
    cursor = db.cursor()
    # 如果数据表已经存在使用 execute() 方法删除表。
    #cursor.execute("DROP TABLE IF EXISTS e")

    # 创建数据表SQL语句
    sql = """CREATE TABLE gp_app(
             phone varchar(255) NOT NULL,
             type varchar(255)
             )ENGINE=InnoDB DEFAULT CHARSET=utf8;"""
    cursor.execute(sql)
if __name__=='__main__':
    # create_table()
    threads_get()