requests httpclients使用json传递多层参数爬取数据

1、背景

测试爬取广州工程行业评分数据，参数如下图。是两层的，如果用普通的键值对方式传参。会报“参数不符合规范，不能进行查询”错误。需要将参数先转成json再传递。

２、python requests方法

#!/usr/bin/env python3
#coding:utf-8

import json
import requests
def main():
    url = "http://qycx.gzcc.gov.cn:8081/eval/evalGateWay/query"
    headers = {
               "Content-Type": "application/json" # 增加该请求头
            }
    post_data1 = {
        "search": {
        "categoryCode": "101",
        "dateDate": "2022-06-16",
        "enterpriseName": "",
        "subjectTypeId": "f39465d9f92118343b93656d71437ada",
        "totalRankEnd": "",
        "totalRankStark": "",
        "totalScoreEnd": "",
        "totalScoreStark": ""
        }
    }
	#　使用json.dumps将字段转换成json格式即可
    rs = requests.post(url=url,data=json.dumps(post_data1), headers=headers)
    print(rs.headers, rs.status_code, rs.text)

if __name__ == '__main__':
    main()

3、java httpclients方法

import com.google.gson.Gson; #　导入Gson包
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;

import java.io.IOException;
import java.util.HashMap;
import java.util.Map;

public class EnterpriseCredit {
    public static void main(String[] args) throws IOException {
        // 1、创建HttpClients对象
        CloseableHttpClient httpclient = HttpClients.createDefault();

        //２、创建httpPost对象，设置请求url地址
        HttpPost httpPost = new HttpPost("http://qycx.gzcc.gov.cn:8081/eval/evalGateWay/query");
		//设置Content-Type请求头
        httpPost.setHeader("Content-Type", "application/json;charset=utf-8");

        //声明HashMap，封装表单参数search的value
        HashMap<String, String> searchValueMap = new HashMap<>();
        searchValueMap.put("categoryCode", "101");
        searchValueMap.put("dateDate", "2022-06-15");
        searchValueMap.put("enterpriseName", null);
        searchValueMap.put("subjectTypeId", "f39465d9f92118343b93656d71437ada");
        searchValueMap.put("totalRankEnd", null);
        searchValueMap.put("totalRankStark", null);
        searchValueMap.put("totalScoreEnd", null);
        searchValueMap.put("totalScoreStark", null);
        //申明一个Gson
        Gson gson = new Gson();
        //申明HashMap，封装请求表单参数
        HashMap<String, Map> searchMap = new HashMap<>();
        searchMap.put("search", searchValueMap);
        //将表单参数转换成json再转换成字符串
        String searchJson = gson.toJson(searchMap).toString();
        //创建jsonEntity，供httpPost使用
        StringEntity jsonEntity = new StringEntity(searchJson);

        //设置表单的Entity对象到Post请求中
        httpPost.setEntity(jsonEntity);

        //3、使用HttpClient发起请求，获取response
        CloseableHttpResponse response = httpclient.execute(httpPost);
        //4、解析响应
        if (response.getStatusLine().getStatusCode() == 200){
            String content = EntityUtils.toString(response.getEntity(), "utf8");
            System.out.println(content);
        }
        response.close();
        httpclient.close();
    }
}