ThreadLocal,LinkedBlockingQueue,线程池 获取数据库连接2改进

本文介绍了一种使用线程本地阻塞队列实现的数据库连接池技术,该技术能够有效管理数据库连接资源,通过动态调整连接数量来提高系统的并发处理能力。
package com.ctl.util;

import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.util.Random;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;

import com.ctl.util.ConfigUtils;
import com.ctl.util.LogUtil;

public class ThreadLocalBlockingQueueUtils {
	private static ThreadLocal<LinkedBlockingQueue<Connection>> queueHoder = new ThreadLocal<LinkedBlockingQueue<Connection>>();
	public static int num = 0;
	private static String driver;
	private static String url;
	private static String username;
	private static String password;
	private static int threadPoolMaxNum;

	private static int threadPoolMinNum;
	private static LogUtil log;
	// \u6570\u636E\u5E93\u7C7B\u578Boracle mysql db2
	private static String databasetype;
	public static int getThreadPoolMaxNum() {
		return threadPoolMaxNum;
	}
	
	public static int getThreadPoolMinNum() {
		return threadPoolMinNum;
	}

	public static String getUrl() {
		return url;
	}

	public static String getUsername() {
		return username;
	}

	public static String getPassword() {
		return password;
	}

	static {
		log = new LogUtil();
		databasetype = ConfigUtils.getType("databasetype");
		threadPoolMaxNum = Integer.parseInt(ConfigUtils
				.getType("threadPoolMaxNum"));
		threadPoolMinNum = Integer.parseInt(ConfigUtils
				.getType("threadPoolMinNum"));
		if (databasetype.equals("mysql")) {
			driver = ConfigUtils.getType("mysql.driver");
			url = ConfigUtils.getType("mysql.url");
			username = ConfigUtils.getType("mysql.username");
			password = ConfigUtils.getType("mysql.password");
		} else if (databasetype.equals("oracle")) {
			driver = ConfigUtils.getType("oracle.driver");
			url = ConfigUtils.getType("oracle.url");
			username = ConfigUtils.getType("oracle.username");
			password = ConfigUtils.getType("oracle.password");
		}
		try {
			Class.forName(driver);
		} catch (ClassNotFoundException e) {
			System.out.println(e.getMessage());
		}
		CreateConnection createConn = new CreateConnection();
		createConn.setDaemon(true);
		createConn.start();
	}

	public static synchronized LinkedBlockingQueue<Connection> getQueue() {
		LinkedBlockingQueue<Connection> queue = queueHoder.get();
		if (queue == null) {
			queue = new LinkedBlockingQueue<Connection>(threadPoolMaxNum);
			queueHoder.set(queue);
			return queue;
		}
		return queue;
	}

	// static void start() {
	// Tom tom = new Tom();
	// tom.start();
	// for(;;){
	// try {
	// Thread.sleep(200);
	// System.out.println("/*************"+getConnection()+"*************/");
	// } catch (InterruptedException e) {
	// }
	// }
	// }
	//
	// public static void main(String[] args) {
	// ThreadLocalBlockingQueueUtils.start();
	// }

	public static Connection getConnection() {
		// System.out.println("进入getConnection");
		class GetConnectionClazz extends Thread {
			LinkedBlockingQueue<Connection> queue = ThreadLocalBlockingQueueUtils
					.getQueue();
			private Connection conn;
			private int queueSize;
			public int getQueueSize() {
				return queueSize;
			}
			public Connection getConn() {
				return conn;
			}
			public synchronized void run() {
				// System.out.println("进入getConnection run()");
				try {
					// System.err.println("-----"+conn+"--------");
					while (conn == null) {// 很重要没有该while循环当按F5不断刷新时,只要有一个取出来为空后面的全为空
						conn = queue.poll(2, TimeUnit.SECONDS);
					}
					queueSize=queue.size();
					// System.err.println("*******"+conn+"*********");
					// if (conn != null) {
					// System.err.println("\u3010" + queue.size() + "\u3011"
					// + "getConnecion\u6210\u529F\uFF1A" + conn);
					// }
				} catch (InterruptedException e) {
					log.WriteLine("getConnection", e.getMessage());
				}
			}
		}
		GetConnectionClazz jj = new GetConnectionClazz();
		jj.start();
		try {
			jj.join();
		} catch (InterruptedException e) {
			log.WriteLine("getConnection()", e.getMessage());
		}
		log.WriteLine("getConnection()","\u3010" + jj.getQueueSize() + "\u3011"+ "getConnecion\u6210\u529F\uFF1A" + jj.getConn());
		return jj.getConn();
	}

}

// class GG extends Thread {
// public void run() {
// for (;;){
// try {
// ThreadLocalBlockingQueueUtils.getConnection();
// Thread.sleep(300);
// } catch (Exception e) {
// }
// }
// }
// }
/**
 * @descritpion 创建数据库连接的线程类
 * @author Administrator
 * 
 */
class CreateConnection extends Thread {
	LinkedBlockingQueue<Connection> queue = ThreadLocalBlockingQueueUtils
			.getQueue();
	LogUtil log = new LogUtil();

	public synchronized void run() {
		boolean result = false;
		while (true) {
			try {
				Random rand=new Random();
				int num=ThreadLocalBlockingQueueUtils.getThreadPoolMaxNum()-ThreadLocalBlockingQueueUtils.getThreadPoolMinNum();
				int randSize=rand.nextInt(num)+1;
				if(queue.size()>=randSize){
					Thread.sleep(100);
					continue;
				}
				Connection conn = null;
				conn = DriverManager.getConnection(
						ThreadLocalBlockingQueueUtils.getUrl(),
						ThreadLocalBlockingQueueUtils.getUsername(),
						ThreadLocalBlockingQueueUtils.getPassword());
				if (conn != null) {
					result = queue.offer(conn, 1, TimeUnit.SECONDS);
				} else {
					// System.out.println("DriverManager.getConnection is null");
					log.WriteLine("CreateConnection.run()",
							"DriverManager.getConnection()返回 null");
					continue;
				}
				if (result == false) {
					Thread.sleep(100);
					log.WriteLine("CreateConnection.run()",
							"已达到最大连接数queue.size()=" + queue.size());
					// System.out.println("已经满了size=【" + queue.size() + "】");
				} else {
					log.WriteLine("CreateConnection.run()",
							"\u3010" + queue.size() + "\u3011"
									+ "createConnection success:" + conn);
					// System.out.println("\u3010" + queue.size() + "\u3011"
					// + "createConnection success:" + conn);
				}
			} catch (InterruptedException e) {
				// e.printStackTrace();
				log.WriteLine("getConnection", e.getMessage());
				// System.err.println(e.getMessage());
			} catch (SQLException e) {
				log.WriteLine("getConnection", e.getMessage());
				// e.printStackTrace();
				// System.err.println(e.getMessage());
			}
		}
	}
}

<think>我们之前讨论了使用CompletableFuture同时执行两个SQL查询,并提到使用自定义线程池的重要性。现在用户专门询问如何创建用于此场景的自定义线程池。以下是详细解答: ### 创建自定义线程池的关键步骤 #### 1. 选择线程池类型 根据Oracle官方建议[^1],对于`CompletableFuture`的异步操作,推荐使用: - `FixedThreadPool`:当任务数量固定且需要严格限制并发数时 - `CachedThreadPool``:适合大量短生命周期的异步任务 - `WorkStealingPool`:Java 8+,利用所有可用处理器核心 **SQL查询场景建议**:由于数据库操作属于I/O密集型任务,优先选择`FixedThreadPool`并设置合适线程数(避免耗尽数据库连接) #### 2. 计算最佳线程数 根据Brian Goetz提出的公式[^2]: $$ N_{threads} = N_{cpu} \cdot U_{cpu} \cdot (1 + \frac{W}{C}) $$ 其中: - $N_{cpu}$ = CPU核心数(Runtime.getRuntime().availableProcessors()) - $U_{cpu}$ = 目标CPU利用率(0.8~1.0) - $\frac{W}{C}$ = 等待时间与计算时间的比率(数据库操作通常$W/C > 10$) **简化实践**: ```java int poolSize = Math.min( // 取较小值防止过载 MAX_DB_CONNECTIONS, // 数据库连接池最大连接数 Runtime.getRuntime().availableProcessors() * 4 // 经验值 ); ``` #### 3. 完整创建示例 ```java import java.util.concurrent.*; public class SqlThreadPool { // 推荐单例模式管理线程池 private static final ExecutorService SQL_EXECUTOR = new ThreadPoolExecutor( // 核心参数 4, // corePoolSize: 常驻线程数(保持活跃的最小线程数) 20, // maximumPoolSize: 最大线程数(突发流量时允许创建到20个线程) 30, // keepAliveTime: 空闲线程存活时间(单位需配合时间单位) TimeUnit.SECONDS, // 时间单位 new LinkedBlockingQueue<>(100), // 任务队列(有界队列避免OOM) new ThreadFactory() { // 自定义线程工厂 private final AtomicInteger counter = new AtomicInteger(0); @Override public Thread newThread(Runnable r) { return new Thread(r, "sql-worker-" + counter.incrementAndGet()); } }, new ThreadPoolExecutor.CallerRunsPolicy() // 拒绝策略(由调用线程直接运行任务) ); public static ExecutorService getExecutor() { return SQL_EXECUTOR; } } ``` #### 4. 在CompletableFuture中使用 ```java // 执行两个独立SQL查询 CompletableFuture<List<User>> future1 = CompletableFuture.supplyAsync( () -> queryUsers(connPool), SqlThreadPool.getExecutor() // 显式指定线程池 ); CompletableFuture<List<Order>> future2 = CompletableFuture.supplyAsync( () -> queryOrders(connPool), SqlThreadPool.getExecutor() ); // 组合结果(组合操作默认在调用者线程执行,可添加Async变体指定线程池) future1.thenCombineAsync(future2, (users, orders) -> combineResults(users, orders), SqlThreadPool.getExecutor() // 指定组合操作线程池 ); ``` ### 关键配置说明 | 参数 | 推荐值 | 作用说明 | |---------------------|--------------------|-------------------------------------------------------------------------| | corePoolSize | CPU核心数×1~2 | 维持常驻工作线程,减少线程创建销毁开销 | | maximumPoolSize | 数据库最大连接数×0.8 | 防止线程数超过数据库连接池承载能力[^3] | | workQueue | 有界队列(100~500) | 缓冲突发请求,队列大小需考虑系统内存 | | threadFactory | 自定义命名 | 方便线程转储(thread dump)分析 | | rejectedExecutionHandler | CallerRunsPolicy | 当任务饱和时,由调用线程直接执行(相当于降级为同步执行) | ### 生命周期管理 ```java // 应用关闭时优雅终止线程池 @PreDestroy public void shutdown() { SQL_EXECUTOR.shutdown(); try { if (!SQL_EXECUTOR.awaitTermination(5, TimeUnit.SECONDS)) { SQL_EXECUTOR.shutdownNow(); } } catch (InterruptedException e) { Thread.currentThread().interrupt(); SQL_EXECUTOR.shutdownNow(); } } ``` ### 性能监控建议 通过JMX或自定义监控: ```java ThreadPoolExecutor executor = (ThreadPoolExecutor) SQL_EXECUTOR; System.out.println("活跃线程: " + executor.getActiveCount()); System.out.println("已完成任务: " + executor.getCompletedTaskCount()); System.out.println("队列大小: " + executor.getQueue().size()); ``` > **警示**:避免的常见错误 > 1. 使用`Executors.newFixedThreadPool()`但不限制队列大小(可能导致OOM) > 2. 混合使用不同线程池(导致线程数爆炸) > 3. 忽略线程上下文切换开销(线程数不是越多越好) §§相关问题§§ 1. 如何根据数据库连接池配置动态调整线程池参数? 2. CompletableFuture组合操作应该使用默认线程池还是自定义线程池? 3. 线程池监控有哪些最佳实践? </think> ### 创建自定义线程池用于CompletableFuture执行SQL查询 以下是针对数据库查询场景创建自定义线程池的完整方案,结合CompletableFuture实现高效并发查询: --- #### 一、核心实现代码 ```java import java.util.concurrent.*; public class ConcurrentSqlExecutor { // 创建专用线程池(推荐单例模式) private static final ExecutorService SQL_EXECUTOR = new ThreadPoolExecutor( 5, // 核心线程数 = CPU核心数 × 2 (IO密集型场景) 20, // 最大线程数(根据数据库连接池配置调整) 60, // 空闲线程存活时间(秒) TimeUnit.SECONDS, new LinkedBlockingQueue<>(100), // 有界队列防OOM new CustomThreadFactory("sql-worker"), // 自定义线程命名 new ThreadPoolExecutor.CallerRunsPolicy() // 拒绝策略 ); // 执行双SQL查询 public CompletableFuture<CombinedResult> executeConcurrentQueries() { // 任务1:用户查询 CompletableFuture<List<User>> userFuture = CompletableFuture.supplyAsync(() -> executeSql("SELECT * FROM users"), SQL_EXECUTOR // 显式指定线程池 ); // 任务2:订单查询 CompletableFuture<List<Order>> orderFuture = CompletableFuture.supplyAsync(() -> executeSql("SELECT * FROM orders"), SQL_EXECUTOR ); // 合并结果 return userFuture.thenCombine(orderFuture, CombinedResult::new); } // SQL执行方法示例 private <T> List<T> executeSql(String sql) { try (Connection conn = dataSource.getConnection(); PreparedStatement ps = conn.prepareStatement(sql)) { // 执行查询并转换结果集 } catch (SQLException e) { throw new CompletionException(e); } } // 自定义线程工厂(便于问题排查) static class CustomThreadFactory implements ThreadFactory { private final String namePrefix; private final AtomicInteger counter = new AtomicInteger(1); CustomThreadFactory(String namePrefix) { this.namePrefix = "pool-" + namePrefix + "-thread-"; } public Thread newThread(Runnable r) { return new Thread(r, namePrefix + counter.getAndIncrement()); } } // 实体类省略... } ``` --- #### 二、关键配置说明 1. **线程池参数优化**: - **核心线程数**:建议设置为$N_{cpu} \times 2$(IO密集型场景) - **最大线程数**:不超过数据库连接池的`maxPoolSize`(如HikariCP配置) - **队列容量**:使用有界队列`LinkedBlockingQueue`防止OOM - **拒绝策略**: - `CallerRunsPolicy`:主线程直接执行(避免任务丢失) - 生产环境可替换为日志记录+告警策略 2. **线程命名规范**: ```java new Thread(r, "sql-worker-" + counter.getAndIncrement()) ``` - 作用:线程转储(thread dump)时可快速识别线程用途 3. **资源隔离**: - 为不同类型的操作创建独立线程池: ```java // 查询专用池 ExecutorService queryExecutor = ... // 写入专用池 ExecutorService writeExecutor = ... ``` --- #### 三、执行流程示意图 ```mermaid sequenceDiagram participant 主线程 participant SQL_EXECUTOR as 自定义线程池 participant 数据库 主线程->>SQL_EXECUTOR: supplyAsync(用户查询) 主线程->>SQL_EXECUTOR: supplyAsync(订单查询) SQL_EXECUTOR->>数据库: 执行SQL1 SQL_EXECUTOR->>数据库: 执行SQL2 数据库-->>SQL_EXECUTOR: 返回结果集1 数据库-->>SQL_EXECUTOR: 返回结果集2 SQL_EXECUTOR-->>主线程: 组合结果(CombinedResult) ``` --- #### 四、性能优化建议 1. **连接池配合**: ```java // 使用HikariCP连接池(避免DriverManager) HikariDataSource dataSource = new HikariDataSource(); dataSource.setMaximumPoolSize(20); // 与线程池maxSize匹配 ``` 2. **超时控制**: ```java combinedResult.get(5, TimeUnit.SECONDS); // 查询总超时 ``` 3. **上下文传递**: ```java // 需要传递ThreadLocal时(如分库分表路由) CompletableFuture.supplyAsync( asyncContext.wrap(() -> executeSql(...)), SQL_EXECUTOR ); ``` --- #### 五、注意事项 1. **避免使用默认线程池**: - `ForkJoinPool.commonPool()` 不适合阻塞IO操作[^2] - 默认池无队列限制可能引起资源耗尽 2. **线程池生命周期**: ```java // 应用关闭时执行 @PreDestroy public void shutdown() { SQL_EXECUTOR.shutdownNow(); } ``` 3. **监控指标**: - 使用Micrometer监控: ```java new ThreadPoolExecutorMetrics(SQL_EXECUTOR, "sql_pool").bindTo(registry); ``` > **最佳实践**:生产环境建议线程数满足$N_{threads} \leq \frac{N_{connections}}{2}$($N_{connections}$为数据库连接数)
评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值