人脸爬取（人脸数据集的搜集）

在进行人脸相关处理中，人脸数据集是关键，这里描述一下怎样爬取人脸数据集

1、获取艺人名称

① 获取完整url路径

在百度中搜索“中国艺人”

得到以下界面

通过分析，url的完整路径为：

"https://sp0.baidu.com/8aQDcjqpAAV3otqbppnN2DJv/api.php?resource_id=28266&from_mid=500&format=json&ie=utf-8&oe=utf-8&query=%E4%B8%AD%E5%9B%BD%E8%89%BA%E4%BA%BA&sort_key=&sort_type=1&stat0=&stat1=&stat2=&stat3=&pn="+pn+"&rn=100&_=1580457480665"

其中，pn为页码数。

② 解析全部艺人名称列表

解析上述url，通过requests获取网页内容，然后解析艺人姓名

def get_person_name():
    person_list = []
    pn_i=0
    while(True):
        pn=str(pn_i)
        pn_i+=100
        url="https://sp0.baidu.com/8aQDcjqpAAV3otqbppnN2DJv/api.php?resource_id=28266&from_mid=500&format=json&ie=utf-8&oe=utf-8&query=%E4%B8%AD%E5%9B%BD%E8%89%BA%E4%BA%BA&sort_key=&sort_type=1&stat0=&stat1=&stat2=&stat3=&pn="+pn+"&rn=100&_=1580457480665"
     
        res = requests.get(url)
        try:
            json_str=json.loads(res.text)
        except:
            continue

        figs=json_str['data'][0]['result']
     
        for i in figs:
            name=i['ename']
            print(name)
            person_list.append(name)

    return person_list

2、爬取相应艺人的相应照片

① 获取图片的url

        while pn < self.__amount:


            url = "https://image.baidu.com/search/index?tn=baiduimage&ipn=r&ct=201326592&cl=2&lm=-1&st=-1&fm=result&fr=&sf=1&fmq=1639129009987_R&pv=&ic=&nc=1&z=&hd=&latest=&copyright=&se=1&showtab=0&fb=0&width=&height=&face=0&istype=2&dyTabStr=MCwzLDEsNiwyLDQsNSw3LDgsOQ%3D%3D&ie=utf-8&sid=&word=" + word_quote

            
            try:
                time.sleep(self.time_sleep)
                req = urllib.request.Request(url=url, headers=self.headers)
                #print(req)

                page = urllib.request.urlopen(req)
                rsp = page.read()

                
                rsp = str(rsp)
                index_list = find_all_sub("objURL",rsp)
                
                img_root = []

                for i in range(0,len(index_list)):
                    temp = []
                    if i == len(index_list) -1:
                        temp = rsp[index_list[len(index_list) -1]:len(rsp)]
                    else:
                        temp = rsp[index_list[i]:index_list[i + 1]]
                    img_root.append(temp)

                
                for img_root_path in img_root:
                    temp_url = img_root_path[9:]
                    end = temp_url.find('"') 
                    image_temp_url = temp_url[:end]
                    if not find_in_list(image_url_list,image_temp_url):
                        image_url_list.append(image_temp_url)

            except UnicodeDecodeError as e:
                print(e)
                print('-----UnicodeDecodeErrorurl:', url)
            except urllib.error.URLError as e:
                print(e)
                print("-----urlErrorurl:", url)
            except socket.timeout as e:
                print(e)
                print("-----socket timout:", url)
            else:

                # 读取下一页
                print("下载下一页")
                pn += 60
            finally:
                page.close()

② 下载对应的图片

        image_root_path = "./" + word
        if not os.path.exists(image_root_path):
            os.mkdir(image_root_path)
        filepath = image_root_path + "/" + str(word) + "_" + str(number) + ".jpg"

        
        for img_url in image_url_list:
            number += 1
            filepath = image_root_path + "/" + str(word) + "_" + str(number) + ".jpg"
            print(filepath)
            count = 1
            try:
                urllib.request.urlretrieve(img_url, filepath)
            except socket.timeout:
                while count <= 3:
                    try:
                        urllib.request.urlretrieve(img_url, filepath)
                        break
                    except socket.timeout:
                        count += 1
            finally:
                # display the raw url of images
                print('\t%d\t%s' % (number, img_url))
                if count > 3:
                    print('\t%d\t%s failed' % (number, img_url))
                pass
                
        print("下载任务结束")