python之爬虫的入门04------爬虫代理ip、保存为CSV表格

一、爬虫伪装—使用代理ip

import urllib.request
import random

url = 'http://45.32.164.128/ip.php'                                      #URL地址

iplist = ['112.91.159.66:3128','119.31.210.170:7777','221.7.255.168:8080']    #多个代理IP地址

'''
细致的设置代理,可以用opener的open方法打开URL:
'''
proxy_support = urllib.request.ProxyHandler({'http':random.choice(iplist)})   # 参数是一个字典{'类型'：'代理IP：端口号'}

opener = urllib.request.build_opener(proxy_support)                          # 订制创建一个opener

urllib.request.install_opener(opener)                                    #安装opener


response = urllib.request.urlopen(url)
html = response.read().decode('utf8')
print(html)

二、保存为CSV表格

import csv


def write_msg(data):
    '''这个函数用来将爬取的数据写入文件'''
    with open("csvfile.csv", 'a+', newline='') as f:
        wf = csv.writer(f)
        wf.writerow(data)

if __name__ == '__main__':
    head = ['msg', 'area', 'price', 'address', 'user', 'phone']

    with open('./csvfile.csv', 'a+') as f:
        wf = csv.writer(f)
        wf.writerow(head)

    data = ['1', '2', '4', '6', 'g', 'uy']
    write_msg(data)

    data = ['3', '1', 'a', '6', 'g', 'uy']
    write_msg(data)

    data = ['w', 'wdveg546thfwer4335refveergrtbg5', '我的', 'we', 'fvr', 'df43ferrver3442ergfregbvrvebevevevfvervfvr']
    write_msg(data)