package com.duxiaoman.ent.idm.app.utils;
import com.duxiaoman.ent.idm.app.config.SparkConfig;
import com.duxiaoman.ent.idm.app.model.*;
import com.duxiaoman.ent.idm.app.pojo.ShellRequest;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
@Slf4j
@Component
public class SparkCliExecutor {
@Autowired
private SparkConfig sparkConfig;
/**
* 执行通用 Shell 命令
* @param shellRequest Shell 命令请求（包含命令列表、超时时间等）
* @return 执行结果（统一 QueryResult 格式）
*/
public QueryResult executeShellCommand(ShellRequest shellRequest) {
long startTime = System.currentTimeMillis();
List<String> output = new ArrayList<>();
int exitCode = -1;
try {
// 1. 校验命令合法性（避免空命令）
List<String> command = shellRequest.getCommand();
if (command == null || command.isEmpty()) {
String errorMsg = "Shell 命令不能为空";
log.error(errorMsg);
return createErrorResult(startTime, output, errorMsg, exitCode);
}
log.info("开始执行 Shell 命令: {}", String.join(" ", command));
// 2. 构建进程（使用命令列表，避免 Shell 注入）
ProcessBuilder processBuilder = new ProcessBuilder(command);
// 可选：设置工作目录（若命令需在指定目录执行）
if (shellRequest.getWorkDir() != null && !shellRequest.getWorkDir().isEmpty()) {
processBuilder.directory(new File(shellRequest.getWorkDir()));
log.debug("Shell 命令执行工作目录: {}", shellRequest.getWorkDir());
}
// 合并错误流到标准输出（统一捕获日志）
processBuilder.redirectErrorStream(true);
Process process = processBuilder.start();
// 3. 读取命令输出（异步读取，避免流阻塞）
try (BufferedReader reader = new BufferedReader(
new InputStreamReader(process.getInputStream(), StandardCharsets.UTF_8))) {
String line;
while ((line = reader.readLine()) != null) {
output.add(line);
log.debug("Shell 命令输出: {}", line);
}
}
// 4. 超时控制（复用原有逻辑）
if (shellRequest.getTimeoutSeconds() != null && shellRequest.getTimeoutSeconds() > 0) {
if (!process.waitFor(shellRequest.getTimeoutSeconds(), TimeUnit.SECONDS)) {
process.destroyForcibly(); // 强制终止超时进程
String errorMsg = String.format("Shell 命令超时（%d 秒），已强制终止", shellRequest.getTimeoutSeconds());
log.error(errorMsg);
return createErrorResult(startTime, output, errorMsg, -1);
}
} else {
process.waitFor(); // 无超时则等待命令执行完成
}
// 5. 检查执行结果
exitCode = process.exitValue();
if (exitCode != 0) {
String errorMsg = String.format("Shell 命令执行失败，退出码: %d，输出日志: %s",
exitCode, String.join("\n", output));
log.error(errorMsg);
return createErrorResult(startTime, output, errorMsg, exitCode);
}
// 6. 执行成功，返回结果
long endTime = System.currentTimeMillis();
log.info("Shell 命令执行成功，耗时: {} ms", (endTime - startTime));
return new QueryResult(true, output, null, null, endTime - startTime, exitCode);
} catch (IOException e) {
String errorMsg = "Shell 命令执行IO异常: " + e.getMessage();
log.error(errorMsg, e);
return createErrorResult(startTime, output, errorMsg, exitCode);
} catch (InterruptedException e) {
Thread.currentThread().interrupt(); // 恢复中断状态
String errorMsg = "Shell 命令执行被中断: " + e.getMessage();
log.error(errorMsg, e);
return createErrorResult(startTime, output, errorMsg, exitCode);
}
}
/**
* 执行Hive查询
* @param request 查询请求
* @return 执行结果
*/
public QueryResult executeHiveQueryOld(QueryRequest request) {
long startTime = System.currentTimeMillis();
List<String> output = new ArrayList<>();
int exitCode = -1;
try {
// 构建命令
List<String> command = buildCommand(request);
log.info("开始执行 hive查询操作: {}", String.join(" ", command));
// 执行命令
ProcessBuilder processBuilder = new ProcessBuilder(command);
processBuilder.redirectErrorStream(true);
Process process = processBuilder.start();
// 读取输出
try (BufferedReader reader = new BufferedReader(
new InputStreamReader(process.getInputStream()))) {
String line;
while ((line = reader.readLine()) != null) {
output.add(line);
log.debug("Spark CLI output: {}", line);
}
}
// 设置超时
if (request.getTimeoutSeconds() != null && request.getTimeoutSeconds() > 0) {
if (!process.waitFor(request.getTimeoutSeconds(), TimeUnit.SECONDS)) {
process.destroy();
String errorMsg = "hive 查询超时： " + request.getTimeoutSeconds() + " seconds";
log.error(errorMsg);
return createErrorResult(startTime, output, errorMsg, -1);
}
} else {
process.waitFor();
}
exitCode = process.exitValue();
if (exitCode != 0) {
String errorMsg = "Spark CLI execution failed with exit code: " + exitCode;
log.error(errorMsg);
return createErrorResult(startTime, output, errorMsg, exitCode);
}
long endTime = System.currentTimeMillis();
log.info("spark 执行成功 执行耗时 in {} ms", (endTime - startTime));
return new QueryResult(true, output, null,null, endTime - startTime, exitCode);
} catch (IOException | InterruptedException e) {
String errorMsg = "Spark 执行异常: " + e.getMessage();
log.error(errorMsg, e);
return createErrorResult(startTime, output, errorMsg, exitCode);
}
}
/**
* 执行Hive查询并导出结果到文件
* @param request 导出请求
* @return 执行结果
*/
public QueryResult exportHiveQuery(ExportRequest request) {
long startTime = System.currentTimeMillis();
List<String> output = new ArrayList<>();
int exitCode = -1;
try {
// 确保输出目录存在
Path outputPath = Paths.get(request.getOutputPath());
Path parentDir = outputPath.getParent();
if (parentDir != null && !Files.exists(parentDir)) {
Files.createDirectories(parentDir);
log.info("Created output directory: {}", parentDir);
}
// 构建导出查询
String exportQuery = buildExportQuery(request);
log.info("Export query: {}", exportQuery);
// 构建命令
QueryRequest queryRequest = new QueryRequest();
queryRequest.setQuery(exportQuery);
queryRequest.setDatabase(request.getDatabase());
queryRequest.setAdditionalConfigs(request.getAdditionalConfigs());
queryRequest.setAdditionalHiveConfigs(request.getAdditionalHiveConfigs());
queryRequest.setTimeoutSeconds(request.getTimeoutSeconds());
queryRequest.setQueue(request.getQueue());
queryRequest.setNumExecutors(request.getNumExecutors());
queryRequest.setExecutorMemory(request.getExecutorMemory());
List<String> command = buildCommand(queryRequest);
log.info("Executing Spark CLI export command: {}", String.join(" ", command));
// 执行命令
ProcessBuilder processBuilder = new ProcessBuilder(command);
processBuilder.redirectErrorStream(true);
Process process = processBuilder.start();
// 读取输出
try (BufferedReader reader = new BufferedReader(
new InputStreamReader(process.getInputStream()))) {
String line;
while ((line = reader.readLine()) != null) {
output.add(line);
log.debug("Spark CLI output: {}", line);
}
}
// 设置超时
if (request.getTimeoutSeconds() != null && request.getTimeoutSeconds() > 0) {
if (!process.waitFor(request.getTimeoutSeconds(), TimeUnit.SECONDS)) {
process.destroy();
String errorMsg = "Export command execution timed out after " + request.getTimeoutSeconds() + " seconds";
log.error(errorMsg);
return createErrorResult(startTime, output, errorMsg, -1);
}
} else {
process.waitFor();
}
exitCode = process.exitValue();
if (exitCode != 0) {
String errorMsg = "Spark CLI export execution failed with exit code: " + exitCode;
log.error(errorMsg);
return createErrorResult(startTime, output, errorMsg, exitCode);
}
// 检查输出文件是否存在
if (Files.exists(outputPath)) {
long fileSize = Files.size(outputPath);
log.info("Export completed successfully. File: {}, Size: {} bytes", request.getOutputPath(), fileSize);
} else {
log.warn("Export completed but output file does not exist: {}", request.getOutputPath());
}
long endTime = System.currentTimeMillis();
log.info("Spark CLI export command executed successfully in {} ms", (endTime - startTime));
return new QueryResult(true, output,null, null, endTime - startTime, exitCode);
} catch (IOException | InterruptedException e) {
String errorMsg = "Failed to execute Spark CLI export command: " + e.getMessage();
log.error(errorMsg, e);
return createErrorResult(startTime, output, errorMsg, exitCode);
}
}
/**
* 创建Hive表
* @param request 表创建请求
* @return 执行结果
*/
public QueryResult createTable(TableRequest request) {
long startTime = System.currentTimeMillis();
try {
// 构建创建表的SQL语句
String createTableSql = buildCreateTableSql(request);
log.info("Create table SQL: {}", createTableSql);
// 构建查询请求
QueryRequest queryRequest = new QueryRequest();
queryRequest.setQuery(createTableSql);
queryRequest.setDatabase(request.getDatabase());
queryRequest.setAdditionalConfigs(request.getAdditionalConfigs());
queryRequest.setAdditionalHiveConfigs(request.getAdditionalHiveConfigs());
queryRequest.setTimeoutSeconds(request.getTimeoutSeconds());
queryRequest.setQueue(request.getQueue());
queryRequest.setNumExecutors(request.getNumExecutors());
queryRequest.setExecutorMemory(request.getExecutorMemory());
return executeHiveQuery(queryRequest);
} catch (Exception e) {
String errorMsg = "Failed to create table: " + e.getMessage();
log.error(errorMsg, e);
return createErrorResult(startTime, new ArrayList<>(), errorMsg, -1);
}
}
/**
* 导入数据到Hive表
* @param request 导入请求
* @return 执行结果
*/
public QueryResult importData(ImportRequest request) {
long startTime = System.currentTimeMillis();
try {
// 构建导入数据的SQL语句
String importDataSql = buildImportDataSql(request);
log.info("Import data SQL: {}", importDataSql);
// 构建查询请求
QueryRequest queryRequest = new QueryRequest();
queryRequest.setQuery(importDataSql);
queryRequest.setDatabase(request.getDatabase());
queryRequest.setAdditionalConfigs(request.getAdditionalConfigs());
queryRequest.setAdditionalHiveConfigs(request.getAdditionalHiveConfigs());
queryRequest.setTimeoutSeconds(request.getTimeoutSeconds());
queryRequest.setQueue(request.getQueue());
queryRequest.setNumExecutors(request.getNumExecutors());
queryRequest.setExecutorMemory(request.getExecutorMemory());
return executeHiveQuery(queryRequest);
} catch (Exception e) {
String errorMsg = "Failed to import data: " + e.getMessage();
log.error(errorMsg, e);
return createErrorResult(startTime, new ArrayList<>(), errorMsg, -1);
}
}
/**
* 构建导出查询语句
*/
private String buildExportQuery(ExportRequest request) {
StringBuilder queryBuilder = new StringBuilder();
// 如果有指定数据库，先使用数据库
if (request.getDatabase() != null && !request.getDatabase().isEmpty()) {
queryBuilder.append("USE ").append(request.getDatabase()).append("; ");
}
// 构建导出语句
queryBuilder.append("INSERT OVERWRITE DIRECTORY '")
.append(request.getOutputPath())
.append("' ");
// 根据格式设置ROW FORMAT
switch (request.getFormat().toLowerCase()) {
case "csv":
queryBuilder.append("ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' ");
break;
case "tsv":
queryBuilder.append("ROW FORMAT DELIMITED FIELDS TERMINATED BY '\\t' ");
break;
case "json":
queryBuilder.append("ROW FORMAT SERDE 'org.apache.hive.hcatalog.data.JsonSerDe' ");
break;
case "parquet":
// Parquet格式不需要指定ROW FORMAT
queryBuilder.append("STORED AS PARQUET ");
break;
default:
throw new IllegalArgumentException("Unsupported export format: " + request.getFormat());
}
// 添加SELECT查询
queryBuilder.append("SELECT * FROM (")
.append(request.getQuery())
.append(") tmp");
return queryBuilder.toString();
}
/**
* 构建创建表的SQL语句
*/
private String buildCreateTableSql(TableRequest request) {
StringBuilder sqlBuilder = new StringBuilder();
// 如果有指定数据库，先使用数据库
if (request.getDatabase() != null && !request.getDatabase().isEmpty()) {
sqlBuilder.append("USE ").append(request.getDatabase()).append("; ");
}
// 添加创建表语句
sqlBuilder.append("CREATE TABLE ");
// 添加表名
if (request.getDatabase() != null && !request.getDatabase().isEmpty()) {
sqlBuilder.append(request.getDatabase()).append(".");
}
sqlBuilder.append(request.getTableName()).append(" ");
// 添加表定义
sqlBuilder.append(request.getCreateStatement()).append(" ");
// 添加表位置
if (request.getLocation() != null && !request.getLocation().isEmpty()) {
sqlBuilder.append("LOCATION '").append(request.getLocation()).append("' ");
}
// 添加表属性
if (request.getTableProperties() != null && !request.getTableProperties().isEmpty()) {
sqlBuilder.append("TBLPROPERTIES (");
boolean first = true;
for (Map.Entry<String, String> entry : request.getTableProperties().entrySet()) {
if (!first) {
sqlBuilder.append(", ");
}
sqlBuilder.append("'").append(entry.getKey()).append("'='").append(entry.getValue()).append("'");
first = false;
}
sqlBuilder.append(")");
}
return sqlBuilder.toString();
}
/**
* 构建导入数据的SQL语句
*/
private String buildImportDataSql(ImportRequest request) {
StringBuilder sqlBuilder = new StringBuilder();
// 如果有指定数据库，先使用数据库
if (request.getDatabase() != null && !request.getDatabase().isEmpty()) {
sqlBuilder.append("USE ").append(request.getDatabase()).append("; ");
}
// 添加导入数据语句
if (Boolean.TRUE.equals(request.getOverwrite())) {
sqlBuilder.append("INSERT OVERWRITE TABLE ");
} else {
sqlBuilder.append("INSERT INTO TABLE ");
}
// 添加表名
if (request.getDatabase() != null && !request.getDatabase().isEmpty()) {
sqlBuilder.append(request.getDatabase()).append(".");
}
sqlBuilder.append(request.getTableName()).append(" ");
// 根据格式选择不同的加载方式
switch (request.getFormat().toLowerCase()) {
case "csv":
sqlBuilder.append("ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' ");
break;
case "tsv":
sqlBuilder.append("ROW FORMAT DELIMITED FIELDS TERMINATED BY '\\t' ");
break;
case "json":
sqlBuilder.append("ROW FORMAT SERDE 'org.apache.hive.hcatalog.data.JsonSerDe' ");
break;
case "parquet":
// Parquet格式不需要指定ROW FORMAT
break;
default:
throw new IllegalArgumentException("Unsupported import format: " + request.getFormat());
}
sqlBuilder.append("STORED AS TEXTFILE ")
.append("LOCATION '").append(request.getInputPath()).append("'");
return sqlBuilder.toString();
}
/**
* 构建Spark CLI命令
*/
private List<String> buildCommand(QueryRequest request) {
List<String> command = new ArrayList<>();
// 添加Spark客户端路径
command.add(sparkConfig.getBeelinePath());
// 添加Spark提交参数
addParam(command, "--master", sparkConfig.getMaster());
addParam(command, "--deploy-mode", sparkConfig.getDeployMode());
// 使用请求中的队列或默认配置
String queue = request.getQueue() != null ? request.getQueue() : sparkConfig.getQueue();
addParam(command, "--queue", queue);
// 使用请求中的执行器数量或默认配置
int numExecutors = request.getNumExecutors() != null ? request.getNumExecutors() : sparkConfig.getNumExecutors();
addParam(command, "--num-executors", String.valueOf(numExecutors));
addParam(command, "--executor-cores", String.valueOf(sparkConfig.getExecutorCores()));
// 使用请求中的执行器内存或默认配置
String executorMemory = request.getExecutorMemory() != null ? request.getExecutorMemory() : sparkConfig.getExecutorMemory();
addParam(command, "--executor-memory", executorMemory);
addParam(command, "--driver-memory", sparkConfig.getDriverMemory());
// 添加Spark配置
if (sparkConfig.getConf() != null) {
for (Map.Entry<String, String> entry : sparkConfig.getConf().entrySet()) {
addConfParam(command, entry.getKey(), entry.getValue());
}
}
// 添加Hive配置
if (sparkConfig.getHiveConf() != null) {
for (Map.Entry<String, String> entry : sparkConfig.getHiveConf().entrySet()) {
addHiveConfParam(command, entry.getKey(), entry.getValue());
}
}
// 添加额外的Spark配置
if (request.getAdditionalConfigs() != null) {
for (Map.Entry<String, String> entry : request.getAdditionalConfigs().entrySet()) {
addConfParam(command, entry.getKey(), entry.getValue());
}
}
// 添加额外的Hive配置
if (request.getAdditionalHiveConfigs() != null) {
for (Map.Entry<String, String> entry : request.getAdditionalHiveConfigs().entrySet()) {
addHiveConfParam(command, entry.getKey(), entry.getValue());
}
}
// 添加Jars
if (sparkConfig.getJars() != null && !sparkConfig.getJars().isEmpty()) {
addParam(command, "--jars", sparkConfig.getJars());
}
// 添加查询
command.add("-e");
command.add(escapeQuery(request.getQuery()));
return command;
}
/**
* 添加配置参数到命令
*/
private void addParam(List<String> command, String key, String value) {
if (value != null && !value.isEmpty()) {
command.add(key);
command.add(value);
}
}
/**
* 添加Spark配置参数
*/
private void addConfParam(List<String> command, String key, String value) {
if (value != null && !value.isEmpty()) {
command.add("--conf");
command.add(key + "=" + value);
}
}
/**
* 添加Hive配置参数
*/
private void addHiveConfParam(List<String> command, String key, String value) {
if (value != null && !value.isEmpty()) {
command.add("--hiveconf");
command.add(key + "=" + value);
}
}
/**
* 转义查询语句
*/
private String escapeQuery(String query) {
// 根据需要进行转义处理
return query;
// return "\"" + query.replace("\"", "\\\"") + "\"";
}
/**
* 创建错误结果
*/
private QueryResult createErrorResult(long startTime, List<String> output, String errorMessage, int exitCode) {
return new QueryResult(false, output,null, errorMessage, System.currentTimeMillis() - startTime, exitCode);
}
public QueryResult executeHiveQuery(QueryRequest request) {
long startTime = System.currentTimeMillis();
List<String> output = new ArrayList<>();
int exitCode = -1;
try {
List<String> command = buildCommand(request);
log.info("开始执行 hive查询操作: {}", String.join(" ", command));
ProcessBuilder processBuilder = new ProcessBuilder(command);
processBuilder.redirectErrorStream(true);
// 创建ShellRequest用于重试逻辑
ShellRequest shellRequest = new ShellRequest();
shellRequest.setCommand(command);
shellRequest.setTimeoutSeconds(request.getTimeoutSeconds());
// 使用带重试的执行方法
return executeWithRetry(processBuilder, shellRequest, "Hive查询");
} catch (Exception e) {
String errorMsg = "Spark 执行异常: " + e.getMessage();
log.error(errorMsg, e);
return createErrorResult(startTime, output, errorMsg, exitCode);
}
}
/**
* 带重试的执行方法
*/
private QueryResult executeWithRetry(ProcessBuilder processBuilder, ShellRequest shellRequest,
String operationType) {
int maxRetries = 2; // 最大重试次数
int totalAttempts = maxRetries + 1; // 总尝试次数
long startTm = System.currentTimeMillis();
for (int attempt = 0; attempt < totalAttempts; attempt++) {
boolean isLastAttempt = (attempt == totalAttempts - 1);
Process process = null;
List<String> output = new ArrayList<>();
int exitCode = -1;
try {
log.info("开始执行 {} 操作，第 {} 次尝试（共 {} 次）",
operationType, attempt + 1, totalAttempts);
long attemptStartTm = System.currentTimeMillis();
// 如果是重试尝试，进行连接检查和等待
if (attempt > 0) {
// 检查连接健康状态
if (!checkConnectionHealth()) {
log.warn("连接不健康，重置连接状态");
resetConnection();
}
// 重试前等待（指数退避）
int waitTime = 2000 * attempt;
log.info("重试前等待 {} ms", waitTime);
Thread.sleep(waitTime);
}
// 启动进程
process = processBuilder.start();
// 读取输出流
try (BufferedReader reader = new BufferedReader(
new InputStreamReader(process.getInputStream(), StandardCharsets.UTF_8))) {
String line;
while ((line = reader.readLine()) != null) {
output.add(line);
if (log.isDebugEnabled()) {
log.debug("{} 输出: {}", operationType, line);
}
}
}
// 设置超时控制
boolean finished;
if (shellRequest.getTimeoutSeconds() != null && shellRequest.getTimeoutSeconds() > 0) {
finished = process.waitFor(shellRequest.getTimeoutSeconds(), TimeUnit.SECONDS);
if (!finished) {
process.destroyForcibly();
String errorMsg = String.format("%s 超时（%d 秒），已强制终止",
operationType, shellRequest.getTimeoutSeconds());
if (!isLastAttempt) {
log.warn("超时失败，准备重试");
continue; // 继续重试
} else {
return createErrorResult(startTm, output, errorMsg, -1);
}
}
} else {
process.waitFor(); // 无超时则等待命令执行完成
}
// 获取退出码
exitCode = process.exitValue();
// 检查执行结果
if (exitCode != 0) {
String errorMsg = String.format("%s 执行失败，退出码: %d", operationType, exitCode);
if (!isLastAttempt) {
log.warn("执行失败，准备重试");
continue; // 继续重试
} else {
return createErrorResult(startTm, output,
errorMsg + "，输出日志: " + String.join("\n", output), exitCode);
}
}
// 执行成功，返回结果
long endTime = System.currentTimeMillis();
log.info("{} 执行成功，第 {} 次尝试成功，耗时: {} ms",
operationType, attempt + 1, (endTime - attemptStartTm));
// 标记连接健康
connectionHealthy = true;
lastHealthCheckTime = System.currentTimeMillis();
return new QueryResult(true, output, null, null, endTime - startTm, exitCode);
} catch (IOException e) {
String errorMsg = String.format("%s IO异常: %s", operationType, e.getMessage());
if (isLastAttempt) {
return createErrorResult(startTm, output,
String.format("%s 执行失败，所有 %d 次尝试均失败: %s",
operationType, totalAttempts, errorMsg), exitCode);
}
log.warn("第 {} 次尝试IO异常，准备重试: {}", attempt + 1, e.getMessage());
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
String errorMsg = String.format("%s 执行被中断: %s", operationType, e.getMessage());
return createErrorResult(startTm, output, errorMsg, exitCode);
} finally {
// 确保进程被清理
if (process != null && process.isAlive()) {
process.destroy();
try {
// 等待进程完全终止
process.waitFor(5, TimeUnit.SECONDS);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
log.warn("进程清理等待被中断");
}
}
}
// 记录重试间隔（非最后一次尝试）
log.info("第 {} 次尝试失败，等待下一次重试", attempt + 1);
}
// 理论上不会执行到这里，但为了安全
return createErrorResult(startTm, new ArrayList<>(),
String.format("%s 执行失败，未知错误", operationType), -1);
}
/**
* 处理进程执行结果
*/
private QueryResult processResult(Process process, String operationType,
long startTime, List<String> output) throws InterruptedException {
int exitCode = process.exitValue();
if (exitCode != 0) {
String errorMsg = String.format("%s 执行失败，退出码: %d", operationType, exitCode);
log.error(errorMsg);
throw new RuntimeException(errorMsg);
}
long endTime = System.currentTimeMillis();
log.info("{} 执行成功，耗时: {} ms", operationType, (endTime - startTime));
// 标记连接健康
connectionHealthy = true;
lastHealthCheckTime = System.currentTimeMillis();
return new QueryResult(true, output, null, null, endTime - startTime, exitCode);
}
// 添加连接状态监控
private volatile boolean connectionHealthy = true;
private long lastHealthCheckTime = 0;
private static final long HEALTH_CHECK_INTERVAL = 300000; // 5分钟检查一次
/**
* 检查连接健康状态
*/
private boolean checkConnectionHealth() {
long currentTime = System.currentTimeMillis();
if (currentTime - lastHealthCheckTime > HEALTH_CHECK_INTERVAL) {
try {
// 执行简单的测试命令检查连接
List<String> testCommand = Arrays.asList(
sparkConfig.getBeelinePath(),
"-e",
"show databases;"
);
ProcessBuilder pb = new ProcessBuilder(testCommand);
Process process = pb.start();
// 读取输出避免阻塞
try (BufferedReader reader = new BufferedReader(
new InputStreamReader(process.getInputStream()))) {
while (reader.readLine() != null) {
// 消费输出
}
}
boolean success = process.waitFor(30, TimeUnit.SECONDS);
connectionHealthy = (success && process.exitValue() == 0);
lastHealthCheckTime = currentTime;
log.info("连接健康检查: {}", connectionHealthy ? "健康" : "异常");
} catch (Exception e) {
log.error("连接健康检查失败", e);
connectionHealthy = false;
}
}
return connectionHealthy;
}
/**
* 重置连接状态
*/
private void resetConnection() {
connectionHealthy = false;
lastHealthCheckTime = 0;
// 可以添加一些清理操作，比如清理临时文件等
try {
// 执行一些清理命令（可选）
List<String> cleanupCommand = Arrays.asList(
sparkConfig.getBeelinePath(),
"-e",
"!q"
);
ProcessBuilder pb = new ProcessBuilder(cleanupCommand);
Process process = pb.start();
process.waitFor(5, TimeUnit.SECONDS);
} catch (Exception e) {
log.debug("连接重置清理操作执行异常（可忽略）", e);
}
log.info("连接状态已重置");
}
}
package com.duxiaoman.ent.idm.app.config;
import lombok.Data;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Configuration;
import java.util.Map;
@Data
@Configuration
@ConfigurationProperties(prefix = "spark")
public class SparkConfig {
// Spark提交参数
private String master = "yarn";
private String deployMode = "client";
private String queue = "fintech";
private int numExecutors = 150;
private int executorCores = 2;
private String executorMemory = "8G";
private String driverMemory = "10G";
// Spark配置参数
private Map<String, String> conf;
// Hive配置参数
private Map<String, String> hiveConf;
// Jars路径
private String jars;
// Beeline路径
private String beelinePath = "/home/work/software/cloud-hadoop-client/dxm-spark-beeline/bin/beeline";
// hadoop路径
private String hadoopExecutablePath = "/home/work/software/cloud-hadoop-client/hadoop-client/bin/hadoop";
private String hdfsCmdPath = "/home/work/software/cloud-hadoop-client/hadoop-client/bin/hdfs";
// HdfsPath
private String downloadHdfsPath = "dgs://baihua/user/fsg_rcc_fintech_datamining_rd/warehouse/dxm_idm_sample";
// 默认导出目录
private String defaultExportDir = "/home/work/localfile/hive_exports";
// 默认导入目录
private String defaultImportDir = "/home/work/localfile/hive_imports/";
// 输出文件格式
private String format = "csv";
// 是否文件上传至bos地址
private boolean ifUploadToBOS = false;
// 默认数据库
private String defaultDatabase = "dxm_idm_sample";
// 临时表前缀
private String tableNameTmpPrefix = "tmp_phone_only_";
// 建表字段语句
private String createTableStatementTmp = "(serial_num BIGINT,index_col STRING, name_col STRING,idcard_col STRING,phone_col STRING,date_for STRING)";
// 查询SQL模板
private String querysqlTmp = "SELECT s.index_col AS index_col ,s.name_col AS name_col ,s.idcard_col AS idcard_col ,s.phone_col AS phone_col ,s.date_for AS date_for ,'' AS panshi_birthProvince ,'-1' AS panshi_dxm_birthCitylevel_zone ,m.work_citylevel AS panshi_dxm_workCitylevel ,'' AS panshi_unionid ,m.work_citylevel_zone AS panshi_dxm_workCitylevel_zone ,'' AS panshi_sex ,'-1' AS panshi_dxm_workcity2ndtier_age ,m.phone_isp AS panshi_dxm_phone_isp ,'-1' AS panshi_dxm_birthCitylevel ,'-1' AS panshi_dxm_workOutBirthzone_flag ,m.city_code AS panshi_dxm_citycode ,'' AS panshi_xid ,m.xid_phone AS panshi_xid_phone ,'-1' AS panshi_dxm_workOutBirthcity_flag ,'-1' AS panshi_dxm_workcity1stier_age ,'-1' AS panshi_dxm_workcity3rdtier_age ,'-1' AS panshi_dxm_workOutBirthpro_flag ,'' AS panshi_age,m.md5 AS dxm_md5_phone FROM ${dataBaseName}.${tableName} AS s LEFT JOIN dxm_idm.dim_jk_idm_phone_mapping_s AS m ON s.phone_col = m.${encryptType} ORDER BY s.serial_num ";
}
【com.dxm.inf.mmr.client.util.KyuubiManagerApi.getCluterConfig(KyuubiManagerApi.java:29) - get config is error . resp is {"code":0,"data":null,"msg":"集群: 配置不存在"}】 项目启动一段时间后，失效报错，重启java项目后，可正常， 告诉我原理是什么，解决思路，关键修改点
本文介绍了一个使用Shell脚本从输入参数中提取请求ID、标志、邮件主题及收件人地址的方法，并根据标志决定复制日志文件或输出文件至指定目录，最后通过邮件发送出去。该脚本适用于系统管理和自动化运维场景。
扫一扫
305
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
