爬虫过程中解决html乱码和获取的文本乱码问题

爬虫过程中解决html乱码和获取的文本乱码问题
response1 = requests.get(url=detail_url, headers=headers)
responseText1 = response1.text
获取的html中有乱码，xpath解析出来的文本当然也有乱码。
解决办法：
responseText1 = response1.text.encode(‘iso-8859-1’)
utf-8也不行，用iso-8859-1

# coding=utf-8
import requests
from lxml import etree
import pandas as pd
import time
import csv

headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36',
}

#  写入表头
header = ['company', 'position', 'salary', 'address', 'experience', 'education', 'number_people', 'date', 'welfare', 'position_type']
with open('./beijing.csv', 'w', encoding='utf-8', newline='')as f:
    writer = csv.writer(f)
    writer.writerow(header)


page = 1
while 1:

    # 列表页url
    print('爬取第{}页'.format(page))
    list_url = 'https://search.51job.com/list/010000,000000,7500,38,9,99,%2B,2,{}.html'.format(str(1))
    response = requests.get(url=list_url, headers=headers)
    responseText = response.text

    # print(responseText)

    html_str = etree.HTML(responseText)
    #  得到详情页列表
    detailUrl_list = html_str.xpath("//p[@class='t1 ']/span/a/@href")
    print(detailUrl_list)

    #  请求每一个详情页url，xpath解析数据
    for detail_url in detailUrl_list:
        response1 = requests.get(url=detail_url, headers=headers)
        responseText1 = response1.text.encode('iso-8859-1')
        html_str1 = etree.HTML(responseText1)

        #  解析数据

        #  职位
        position_list = html_str1.xpath("//div[@class='cn']/h1/@title")
        position = position_list[0] if position_list else None
        print(position)

        #  公司
        company_list = html_str1.xpath("//p[@class='cname']/a/@title")
        #  处理为空的数据
        company = company_list[0] if company_list else None
        print(company)

        #  薪资
        salary_list = html_str1.xpath("//div[@class='cn']/strong/text()")
        salary = salary_list[0] if salary_list else None
        print(salary)

        #  基本信息
        try:
            other_list = html_str1.xpath("//p[@class='msg ltype']//text()")
            print(other_list)
            #  数据处理
            Other = ''.join(other_list).replace('|', '').split() if other_list else None
            print(Other)

            address = Other[0] if other_list else None
            experience = Other[1] if other_list else None
            education = Other[2] if other_list else None
            number_people = Other[3] if other_list else None
            date = Other[4] if other_list else None
            print(address, experience, education, number_people, date)
        except:
            address, experience, education, number_people, date = None, None, None, None, None

        #  福利待遇
        try:
            welfare_list = html_str1.xpath("//div[@class='t1']/span/text()")
            welfare = ','.join(welfare_list)
            print(welfare)
        except:
            welfare = '未公布福利待遇'

        try:
            position_type_list = html_str1.xpath("//p[@class='fp']/a/text()")
            position_type = ','.join(position_type_list)
            print(position_type)
        except:
            position_type = '暂无信息'

        #  将数据存入csv
        data_tuple = (company, position, salary, address, experience, education, number_people, date, welfare, position_type)

        df = pd.DataFrame(columns=data_tuple)
        df.to_csv('beijing.csv', mode='a', line_terminator='\n', sep=',', index=False)

        #  调整请求速度，可以自己调整  睡眠单位是秒
        time.sleep(1)

    #  页数+1
    page += 1