Ubuntu32G服务器测试es数据库性能_8c32g服务器 es性能测试-优快云博客

本文链接：https://blog.youkuaiyun.com/vagatable_XX/article/details/141389407

一、环境准备

1、服务器系统信息

32G 4T x86服务器

2、es信息

版本：6.8.23

集群信息：单节点

索引：

{
    "example": {
        "mappings": {
            "docs": {
                "properties": {
                    "f1": {
                        "type": "text"
                    },
                    "f10": {
                        "type": "text"
                    },
                    "f11": {
                        "type": "text"
                    },
                    "f12": {
                        "type": "text"
                    },
                    "f13": {
                        "type": "text"
                    },
                    "f14": {
                        "type": "text"
                    },
                    "f15": {
                        "type": "text"
                    },
                    "f2": {
                        "type": "text"
                    },
                    "f3": {
                        "type": "text"
                    },
                    "f4": {
                        "type": "text"
                    },
                    "f5": {
                        "type": "text"
                    },
                    "f6": {
                        "type": "text"
                    },
                    "f7": {
                        "type": "text"
                    },
                    "f8": {
                        "type": "text"
                    },
                    "f9": {
                        "type": "text"
                    }
                }
            }
        }
    }
}

3、流量发送主机信息

二、实验步骤

1、搭建上述环境

2、日志发送脚本

public class EsTest {
    private static final String USERNAME = "es";
    private static final String PASSWORD = "password";
    private static final String INDEX_NAME = "example";
    private static final String URL = "http://IP:9200/example/docs/_bulk";
    public static void main(String[] args) {
        try {
            startUp();
        } catch (Exception e) {
            System.err.println("Exception caught in main method");
            e.printStackTrace();
        }
    }

    public static void startUp() throws Exception {
        try {
            // 创建连接池管理器
            PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager();
            // 设置最大连接数
            connManager.setMaxTotal(800);
            // 设置每个路由的最大连接数
            connManager.setDefaultMaxPerRoute(100);
            ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
            executor.setCorePoolSize(100);
            executor.setMaxPoolSize(800);
            executor.setQueueCapacity(0);
            executor.setKeepAliveSeconds(1);
            executor.setThreadNamePrefix("ES-Task-Thread-");
            executor.initialize();

            // 批量大小
            final int batchSize = 100;

            for (int i = 0; i < 30; i++) {
                executor.execute(() -> {
                    System.out.println("Thread-" + Thread.currentThread().getName() + " is running");
                    try {
                        doPost(URL,connManager);
                    } catch (Exception e) {
                        System.err.println("Exception caught in thread: " + Thread.currentThread().getName());
                        e.printStackTrace();
                    }
                });
                Thread.sleep(1500);
            }
        } catch (Exception e) {
            e.printStackTrace();
            System.out.println("restarting...");
            startUp();
        }
    }

    public static String generateBatchData(int batchSize) throws JSONException {
        StringBuilder bulkData = new StringBuilder();
        for (int i = 0; i < batchSize; i++) {
            JSONObject jsonObject = generateRandomJsonObject();
            // 操作行
            bulkData.append("{ \"index\" : { \"_index\" : \"example\" } }\n");
            // 文档行
            bulkData.append(jsonObject.toString()).append("\n");
        }
        // 最后一个换行符
        bulkData.append("\n");
        return bulkData.toString();
    }

    public static JSONObject generateRandomJsonObject() throws JSONException {
        JSONObject jsonObject = new JSONObject();
        jsonObject.put("f1",generateRandomEnglish(13));
        jsonObject.put("f2",generateRandomEnglish(13));
        jsonObject.put("f3",generateRandomEnglish(13));
        jsonObject.put("f4",generateRandomEnglish(13));
        jsonObject.put("f5",generateRandomEnglish(13));
        jsonObject.put("f6",generateRandomEnglish(13));
        jsonObject.put("f7",generateRandomEnglish(13));
        jsonObject.put("f8",generateRandomEnglish(13));
        jsonObject.put("f9",generateRandomEnglish(13));
        jsonObject.put("f10",generateRandomEnglish(13));
        jsonObject.put("f11",generateRandomEnglish(13));
        jsonObject.put("f12",generateRandomEnglish(13));
        jsonObject.put("f13",generateRandomEnglish(13));
        jsonObject.put("f14",generateRandomEnglish(13));
        jsonObject.put("f15",generateRandomEnglish(13));
        return jsonObject;
    }
    public static String generateRandomChinese() {
        int length = ThreadLocalRandom.current().nextInt(5, 11);
        StringBuilder sb = new StringBuilder();
        for (int i = 0; i < length; i++) {
            // 中文字符的 Unicode 范围从 0x4E00 到 0x9FA5
            int val = 0x4E00 + ThreadLocalRandom.current().nextInt(0x9FA5 - 0x4E00 + 1);
            sb.append((char) val);
        }
        return sb.toString();
    }

    public static String generateRandomEnglish(int length) {
        StringBuilder sb = new StringBuilder();
        for (int i = 0; i < length; i++) {
            // 生成随机的大写字母或小写字母
            boolean isUpperCase = ThreadLocalRandom.current().nextBoolean();
            int val;
            if (isUpperCase) {
                // 大写字母的 ASCII 范围从 65 到 90
                val = 65 + ThreadLocalRandom.current().nextInt(26);
            } else {
                // 小写字母的 ASCII 范围从 97 到 122
                val = 97 + ThreadLocalRandom.current().nextInt(26);
            }
            sb.append((char) val);
        }
        return sb.toString();
    }

    public static void doPost(String url, PoolingHttpClientConnectionManager connManager) {
        CloseableHttpClient httpClient = null;
        try {
            httpClient = HttpClients.custom()
                    .setConnectionManager(connManager)
                    .build();
            while (true) {
                try {
                    HttpPost httpPost = new HttpPost(url);
                    httpPost.addHeader("Content-Type", "application/json");
                    httpPost.addHeader("Authorization", getBasicAuthHeader(USERNAME, PASSWORD));
                    httpPost.setEntity(new StringEntity(generateBatchData(1000), StandardCharsets.UTF_8));
                    try (CloseableHttpResponse response = httpClient.execute(httpPost)) {

                        HttpEntity entity = response.getEntity();
                        if (entity != null) {
                            String responseContent = EntityUtils.toString(entity, "UTF-8");
                            // 可以在这里打印响应内容或其他操作

                        }
                    }
                } catch (Exception e) {
                    e.printStackTrace();
                    // 如果发生异常，关闭当前的 httpClient 并重新创建
                    closeQuietly(httpClient);
                    httpClient = HttpClients.custom()
                            .setConnectionManager(connManager)
                            .build();
                } finally {
                    Thread.sleep(1000);
                }
            }
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            closeQuietly(httpClient);
        }
    }

    private static void closeQuietly(CloseableHttpClient httpClient) {
        if (httpClient != null) {
            try {
                httpClient.close();
            } catch (IOException ignored) {
                // 忽略关闭时出现的异常
            }
        }
    }

    private static String getBasicAuthHeader(String username, String password) {
        String authString = username + ":" + password;
        byte[] authEncBytes = Base64.getEncoder().encode(authString.getBytes(StandardCharsets.UTF_8));
        return "Basic " + new String(authEncBytes);
    }
}

3、给es分配1G内存打流

4、给es分配16G内存打流