Homework6

import requests
import re
import time
headers ={
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36'
    }
url = 'https://www.baidu.com/s?wd=%E7%95%99%E4%B8%8B%E9%82%AE%E7%AE%B1'

response = requests.get(url,headers=headers)
response.enconding = 'UTF-8'
html = response.text
regex = re.compile("[a-zA-z]+://[^\s]*")
res = regex.findall(html)

new_res = []
for i in res:
    if '//cache.baiducontent.com/c?m=9' in i:
        #res.remove(i)
        new_res.append(i)
        #print(new_res)
    

        #list_1 = []
        for email in new_res:
            responses = requests.get(email,headers=headers)
            responses.enconding = 'UTF-8'
            html_1 = responses.text
            regex = re.compile("[\w!#$%&'*+/=?^_`{|}~-]+(?:\.[\w!#$%&'*+/=?^_`{|}~-]+)*@(?:[\w](?:[\w-]*[\w])?\.)+[\w](?:[\w-]*[\w])?")
            res1 = regex.findall(html_1)
            #list_1.append(res1)
            #print(list_1)
            time.sleep(2)
            lines_2 = list(set(res1))
            #print(lines_2)
            a = str(lines_2)
            with open('C:\\Users\\MacBook\\Desktop\\xiaoshuo\\email.txt',mode='a',encoding='utf-8') as f:
                 f.write(a)