从Jetty源码分析线程池配置(8.1.15)
Jetty8.1.15 源码分析 Jetty 线程模型
问题描述
- Jetty启动后,如果线程数过少导致请求阻塞,造成大量的close_wait状态的连接
大量close_wait的排查可以参考 https://www.jianshu.com/p/b3d45ce92cbd
Jetty所用版本 8.1.15.v20140411
代码示例
相关依赖
<dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-server</artifactId>
<version>8.1.15.v20140411</version>
</dependency>
<dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-util</artifactId>
<version>8.1.15.v20140411</version>
</dependency>
<dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-http</artifactId>
<version>8.1.15.v20140411</version>
</dependency>
<dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-io</artifactId>
<version>8.1.15.v20140411</version>
</dependency>
Demo示例
import org.eclipse.jetty.server.Connector;
import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.server.handler.HandlerCollection;
import org.eclipse.jetty.server.nio.SelectChannelConnector;
import org.eclipse.jetty.util.thread.ExecutorThreadPool;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.SynchronousQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
public class JettyDemo {
public static void main( String[] args ) throws Exception {
// The Server
Server server = new Server();
// 当在4核心的电脑上启动时,new ExecutorThreadPool()的第一个参数核心线程数设置为2造成服务阻塞,为3可正常运行
ExecutorThreadPool executorThreadPool = new ExecutorThreadPool(3, 100, 3);
ThreadPoolExecutor executor = new ThreadPoolExecutor(2,100,10, TimeUnit.SECONDS,new ArrayBlockingQueue(1));
//executor.prestartAllCoreThreads();
ExecutorThreadPool executorThreadPool2 = new ExecutorThreadPool(executor);
server.setThreadPool(executorThreadPool);
// HTTP connector
SelectChannelConnector connector = new SelectChannelConnector();// 非阻塞
String ip = "127.0.0.1";
int port = 8000;
if (ip != null && ip.trim().length() > 0) {
// The network interface this connector binds to as an IP
// address or a hostname. If null or 0.0.0.0, then bind to
// all interfaces.
connector.setHost(ip);
}
connector.setPort(port);
connector.setMaxIdleTime(10000);
// connector.setServer(server);
server.setConnectors(new Connector[] { connector });
// Set a handler
HandlerCollection handlerc = new HandlerCollection();
handlerc.setHandlers(new HelloHandler[]{new HelloHandler()});
server.setHandler(handlerc);
try {
// 服务器启动
server.start();
server.join(); // 阻塞到线程中止
} catch (Exception e) {
} finally {
}
}
}
HelloHandler的源码
import org.eclipse.jetty.server.Request;
import org.eclipse.jetty.server.handler.AbstractHandler;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.IOException;
import java.io.PrintWriter;
public class HelloHandler extends AbstractHandler
{
final String greeting;
String body;
public HelloHandler()
{
this("Hello World");
}
public HelloHandler( String greeting )
{
this(greeting, null);
}
public HelloHandler( String greeting, String body )
{
this.greeting = greeting;
this.body = body;
}
public void handle( String target,
Request baseRequest,
HttpServletRequest request,
HttpServletResponse response ) throws IOException,
ServletException
{
response.setContentType("text/html; charset=utf-8");
response.setStatus(HttpServletResponse.SC_OK);
PrintWriter out = response.getWriter();
body = target;
out.println("<h1>" + greeting + "</h1>");
if (body != null) {
out.println(body);
}
baseRequest.setHandled(true);
}
}
出现问题
- 当项目启动后,通过浏览器访问时出现请求一直处于阻塞状态,无法得到返回结果
解决详细分析
Jetty线程模型
具体可以参看知乎大佬老钱的这篇文章 Jetty基础之线程模型
Jetty的线程架构模型非常简单,分为acceptors,selectors和workers三个线程池。acceptors负责接受新连接,然后交给selectors处理HTTP消息协议的解包,最后由workers处理请求。前两个线程池采用非阻塞模型,一个线程可以处理很多socket的读写,所以线程池数量较小。
问题溯源
Jetty的线程模型分为acceptors,selectors和workers三个线程池,请求处于阻塞状态主要是Jetty没有办法完成对请求的处理,那么就是这三个部分的某个环节没有分配到线程,导致请求无法进行流转或处理,所以需要追溯源码来看Jetty对这三个部分是如何进行分配线程的
源码追溯
说明:acceptors,selectors和worker三个部分共用 Server 中配置的线程池,这三个部分的线程都是由这个线程池来进行管理和创建
acceptor 线程分配源码
SelectChannelConnector中源码,可以看到线程数是取1 和 (核心数+3)/4 的最大值
// org.eclipse.jetty.server.nio.SelectChannelConnector
public SelectChannelConnector()
{
_manager.setMaxIdleTime(getMaxIdleTime());
addBean(_manager,true);
setAcceptors(Math.max(1,(Runtime.getRuntime().availableProcessors()+3)/4));
}
AbstractConnector源码(SelectChannelConnector的父类),可以看到 doStart() 就是通过线程池来分配 acceptor 线程
// org.eclipse.jetty.server.AbstractConnector
public void setAcceptors(int acceptors)
{
if (acceptors > 2 * Runtime.getRuntime().availableProcessors())
LOG.warn("Acceptors should be <=2*availableProcessors: " + this);
_acceptors = acceptors;
}
@Override
protected void doStart() throws Exception
{
if (_server == null)
throw new IllegalStateException("No server");
// open listener port
open();
// 获取 Server 设置的线程池
if (_threadPool == null)
{
_threadPool = _server.getThreadPool();
addBean(_threadPool,false);
}
super.doStart();
// 启动 selector 部分的线程,_threadPool.dispatch(new Acceptor(i)) 内部就是执行线程池的 execute() 方法
// Start selector thread
synchronized (this)
{
_acceptorThreads = new Thread[getAcceptors()];
for (int i = 0; i < _acceptorThreads.length; i++)
if (!_threadPool.dispatch(new Acceptor(i)))
throw new IllegalStateException("!accepting");
if (_threadPool.isLowOnThreads())
LOG.warn("insufficient threads configured for {}",this);
}
LOG.info("Started {}",this);
}
_threadPool.dispatch(new Acceptor(i)) 的具体实现
// org.eclipse.jetty.util.thread.ExecutorThreadPool#dispatch
public boolean dispatch(Runnable job)
{
try
{
_executor.execute(job);
return true;
}
catch(RejectedExecutionException e)
{
LOG.warn(e);
return false;
}
}
总结:acceptor模块的线程数是取1 和 (核心数+3)/4 的最大值,并且这些线程是由 Server 中配置的线程池来进行分配的
selectors 线程分配源码
selectors 线程数设置,可以看到这里设置 select 的线程数 和 acceptor 的线程数相同
// org.eclipse.jetty.server.nio.SelectChannelConnector#doStart
@Override
protected void doStart() throws Exception
{
// 可以看到这里设置 select 的线程数 和 acceptor 的线程数相同
_manager.setSelectSets(getAcceptors());
_manager.setMaxIdleTime(getMaxIdleTime());
_manager.setLowResourcesConnections(getLowResourcesConnections());
_manager.setLowResourcesMaxIdleTime(getLowResourcesMaxIdleTime());
super.doStart();
}
selectors 模块线程启动
// org.eclipse.jetty.io.nio.SelectorManager#doStart
@Override
protected void doStart() throws Exception
{
...
// 根据 selectSets 配置的线程数 启动 select 模块的线程
// start a thread to Select
for (int i=0;i<getSelectSets();i++) {
final int id=i;
// 重点 dispatch 线程分配
boolean selecting=dispatch(new Runnable() {
public void run() {
...
}
});
...
}
}
dispatch 线程分配,可以看到先获取到线程池,这个线程池还是 Server 线程池
// org.eclipse.jetty.server.nio.SelectChannelConnector.ConnectorSelectorManager#dispatch
// ConnectorSelectorManager 是 SelectChannelConnector 的内部类
@Override
public boolean dispatch(Runnable task)
{
ThreadPool pool=getThreadPool();
if (pool==null)
pool=getServer().getThreadPool();
return pool.dispatch(task);
}
pool.dispatch(task) 的具体实现
// org.eclipse.jetty.util.thread.ExecutorThreadPool#dispatch
public boolean dispatch(Runnable job)
{
try
{
_executor.execute(job);
return true;
}
catch(RejectedExecutionException e)
{
LOG.warn(e);
return false;
}
}
总结:selector模块的线程数是取 acceptor 模块的线程数,并且这些线程也是由 Server 中配置的线程池来进行分配的
worker 模块分配源码
worker 并不是真实存在的模块,而是 selector 会从其内部 ConcurrentLinkedQueue 变量中获取任务来调用 ThreadPool 的 execute() 执行
SelectSet 部分源码,可以看到有 ConcurrentLinkedQueue 用来存取对象(重点关注 _changes 对象)
public class SelectSet implements Dumpable {
// Acceptor 会将请求放置在里面,SelectSet 的 doSelect 方法会循环的读取
private final ConcurrentLinkedQueue<Object> _changes = new ConcurrentLinkedQueue<Object>();
private ConcurrentMap<SelectChannelEndPoint,Object> _endPoints = new ConcurrentHashMap<SelectChannelEndPoint, Object>();
}
Acceptor 往队列中放置对象
Acceptor 操作的入口 run() 方法
// org.eclipse.jetty.server.AbstractConnector.Acceptor
private class Acceptor implements Runnable
{
...
public void run() {
...
try {
// 当状态为正在运行且获取到有请求连接时,进行 accept
while (isRunning() && getConnection() != null)
{
try
{
accept(_acceptor);
}
...
}
}
...
}
}
接下来进入 SelectChannelConnector#accept(id) 的方法中
// org.eclipse.jetty.server.nio.SelectChannelConnector#accept
@Override
public void accept(int acceptorID) throws IOException {
...
if (server!=null && server.isOpen() && _manager.isStarted()) {
SocketChannel channel = server.accept();
channel.configureBlocking(false);
Socket socket = channel.socket();
configure(socket);
// 通过SelectorManager#register(java.nio.channels.SocketChannel) 来进行注册
_manager.register(channel);
}
}
接下来进入 SelectorManager#register(java.nio.channels.SocketChannel) 的方法中
public void register(SocketChannel channel)
{
// The ++ increment here is not atomic, but it does not matter.
// so long as the value changes sometimes, then connections will
// be distributed over the available sets.
int s=_set++;
if (s<0)
s=-s;
s=s%_selectSets;
SelectSet[] sets=_selectSet;
if (sets!=null) {
SelectSet set=sets[s];
// 这里很重要,对其进行了添加 change,所以 SelectSet 的 ConcurrentLinkedQueue 中才有了对象,SelectSet 在 doSelect 操作时就能获取到了
set.addChange(channel);
set.wakeup();
}
}
SelectSet 的 doSelect() 获取对象执行操作
SelectSet 的 doSelect() 进行 ConcurrentLinkedQueue 的 poll() 操作,获取对象执行任务
// org.eclipse.jetty.io.nio.SelectorManager.SelectSet#doSelect
/**
* Select and dispatch tasks found from changes and the selector.
*
* @throws IOException
*/
public void doSelect() throws IOException
{
try
{
...
// Make any key changes required
Object change;
int changes=_changes.size();
while (changes-->0 && (change=_changes.poll())!=null)
{
Channel ch=null;
SelectionKey key=null;
try
{
if (change instanceof EndPoint) {
// Update the operations for a key.
SelectChannelEndPoint endpoint = (SelectChannelEndPoint)change;
ch=endpoint.getChannel();
endpoint.doUpdateKey();
}
...
// acceptor 放入的change 是 NioSocketChannel ,所以第一次会进入这里
else if (change instanceof SocketChannel) {
// Newly registered channel
final SocketChannel channel=(SocketChannel)change;
ch=channel;
key = channel.register(selector,SelectionKey.OP_READ,null);
// 这里会创建出一个 SelectChannelEndPoint
SelectChannelEndPoint endpoint = createEndPoint(channel,key);
key.attach(endpoint);
// 封装完成之后,分派线程用来执行 SelectChannelEndPoint 中的 run 方法
endpoint.schedule();
}
...
}
...
}
...
}
private SelectChannelEndPoint createEndPoint(SocketChannel channel, SelectionKey sKey) throws IOException {
SelectChannelEndPoint endp = newEndPoint(channel,this,sKey);
LOG.debug("created {}",endp);
endPointOpened(endp);
_endPoints.put(endp,this);
return endp;
}
}
newEndPoint() 这个方法会在后来讲,先看下 endpoint.schedule();
newEndPoint() 非常重要,它创建出来的对象就是用来处理 channel 中的数据
// org.eclipse.jetty.io.nio.SelectChannelEndPoint#schedule
public void schedule() {
synchronized (this) {
...
else {
// 用来执行任务分派
dispatch();
...
}
}
}
public void dispatch() {
synchronized(this) {
if (_state<=STATE_UNDISPATCHED) {
if (_onIdle)
_state = STATE_NEEDS_DISPATCH;
else {
_state = STATE_DISPATCHED;
// 重要,用来让线程池执行任务
boolean dispatched = _manager.dispatch(_handler);
if(!dispatched) {
_state = STATE_NEEDS_DISPATCH;
LOG.warn("Dispatched Failed! "+this+" to "+_manager);
updateKey();
}
}
}
}
}
_manager.dispatch(_handler) 的具体实现
// org.eclipse.jetty.server.nio.SelectChannelConnector.ConnectorSelectorManager
private final class extends SelectorManager
{
@Override
public boolean dispatch(Runnable task)
{
// 这个的 ThreadPool 还是Jetty 提供的,仍然是 Server 设置的那个线程池
ThreadPool pool=getThreadPool();
if (pool==null)
pool=getServer().getThreadPool();
return pool.dispatch(task);
}
pool.dispatch(task) 用来执行任务
// org.eclipse.jetty.util.thread.ExecutorThreadPool#dispatch
public boolean dispatch(Runnable job) {
try {
_executor.execute(job);
return true;
} catch(RejectedExecutionException e) {
LOG.warn(e);
return false;
}
}
总结:worker 模块是直接从 Server 中配置的线程池来进行分配线程执行任务的
总结
- Jetty 线程模型分为 Acceptor、Selector、Worker 三个部分,三个部分的线程都是由 Server 设置的 ThreadPool 来进行分配的,其中 Worker 不是实际存在,当 Selector 将请求任务分派后,交给 ThreadPool 中的线程执行
- Jetty 在启动时会先分配 Acceptor、Selector 这两个部分的线程,所以如果线程池最大线程数小于两者设定之和 (maxThreads <= Acceptor + Selector),就会造成 Worker 分配不到线程,从而造成阻塞
- Acceptor :通过 SelectorManager#register(java.nio.channels.SocketChannel) 来给 SelectSet 中的 ConcurrentLinkedQueue 增加元素
- Selector :通过 SelectSet 的 doSelect() 方法从 ConcurrentLinkedQueue 中 poll() 元素,并进行对元素分派
- Worker :通过 SelectSet 的 doSelect() 方法获取元素后(该元素实现了 Runnable 接口),调用 ConnectorSelectorManager 的 dispatch() 来执行任务,即使用线程池来执行给任务
扩展:newEndPoint() 创建 SelectChannelEndPoint 用来处理请求
// org.eclipse.jetty.server.nio.SelectChannelConnector#newEndPoint
protected SelectChannelEndPoint newEndPoint(SocketChannel channel, SelectSet selectSet, SelectionKey key) throws IOException {
SelectChannelEndPoint endp= new SelectChannelEndPoint(channel,selectSet,key, SelectChannelConnector.this._maxIdleTime);
endp.setConnection(selectSet.getManager().newConnection(channel,endp, key.attachment()));
return endp;
}
我们从 SelectChannelEndPoint 源码看到,其内部有 SelectSet,SelectorManager,_handler等,其中处理请求的在 _handler 属性的 run() 方法中,而 run() 的内部其实是调用了 handle() 用来处理请求
public class SelectChannelEndPoint extends ChannelEndPoint implements AsyncEndPoint, ConnectedEndPoint {
private final SelectorManager.SelectSet _selectSet;
private final SelectorManager _manager;
private SelectionKey _key;
private final Runnable _handler = new Runnable() {
public void run() { handle(); }
};
protected void handle() {
boolean dispatched=true;
try {
while(dispatched) {
try {
while(true) {
// 重点就这一句话,用来将连接和你的 handler 做关联
final AsyncConnection next = (AsyncConnection)_connection.handle();
if (next!=_connection) {
LOG.debug("{} replaced {}",next,_connection);
Connection old=_connection;
_connection=next;
_manager.endPointUpgraded(this,old);
continue;
}
break;
}
}
catch...finally {}
}
}
finally {}
}
异步连接 AsyncHttpConnection 的 handle() 处理请求
// org.eclipse.jetty.server.AsyncHttpConnection#handle
@Override
public Connection handle() throws IOException {
...
try
{
...
// While progress and the connection has not changed
while (progress && connection==this) {
progress=false;
try
{
// Handle resumed request
if (_request._async.isAsync())
{
if (_request._async.isDispatchable())
// 重点,真实处理请求
handleRequest();
}
...
}
catch ... finally {}
}
}
finally {}
return connection;
}
handleRequest() 的具体实现
// org.eclipse.jetty.server.AbstractHttpConnection#handleRequest
protected void handleRequest() throws IOException {
...
try {
...
while (handling) {
_request.setHandled(false);
String info=null;
try {
...
if (_request._async.isInitial())
{
...
// 重点,
server.handle(this);
}
else
{
...
server.handleAsync(this);
}
}
catch ... finally { }
}
}
finally { }
}
server.handle(this); 处理请求
// org.eclipse.jetty.server.Server#handle
public void handle(AbstractHttpConnection connection) throws IOException, ServletException {
final String target=connection.getRequest().getPathInfo();
final Request request=connection.getRequest();
final Response response=connection.getResponse();
if (LOG.isDebugEnabled()) {
LOG.debug("REQUEST "+target+" on "+connection);
// 重点:这里用来真实处理请求
handle(target, request, request, response);
LOG.debug("RESPONSE "+target+" "+connection.getResponse().getStatus()+" handled="+request.isHandled());
} else
// 重点:这里用来真实处理请求
handle(target, request, request, response);
}
HandlerWrapper 包装了自定义的Handler,这里的 _handler 就是自定义的 HelloHandler,然后调用其 handle() 方法执行相关逻辑
// org.eclipse.jetty.server.handler.HandlerWrapper#handle
public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException
{
if (_handler!=null && isStarted())
{
_handler.handle(target,baseRequest, request, response);
}
}