Nginx+Tomcat+Memcached共享session集群配置-优快云博客

本文介绍了一个使用 Nginx 负载均衡、Memcached 进行 session 共享和 Tomcat 集群配置的具体方案。通过详细展示配置过程，包括 nginx.conf 和 server.xml 文件的具体设置，帮助读者理解如何搭建高可用的 Web 服务器集群。同时也提出了存在的两个主要问题：基于 IP 的 session 共享导致的问题和 Memcached 数据丢失的情况。

1、采用Nginx负载均衡
2、memcached共享session
3、tomcat集群配置(3台CentOS 6)
(172.18.188.64): 操作系统CentOS 6; 安装nginx、memcached和tomcat 6
(172.18.188.76): 操作系统CentOS 6; 安装tomcat 6
(172.18.188.78): 操作系统CentOS 6; 安装tomcat 6
nginx、memcached、tomcat 6安装省略.
nginx配置如下:
nginx.conf如下:
#运行nginx所在的用户名和用户组
#user root root;
#启动进程数
worker_processes 8;
#全局错误日志及PID文件
error_log /usr/local/webserver/nginx/logs/nginx_error.log crit;
pid /usr/local/webserver/nginx/nginx.pid;
#Specifies the value for maximum file descriptors that can be opened by this process.
worker_rlimit_nofile 65535;
#工作模式及连接数上限
events
{
use epoll;
worker_connections 65535;
}
#设定http服务器，利用它的反向代理功能提供负载均衡支持
http
{
#设定mime类型
include mime.types;
default_type application/octet-stream;
include /usr/local/webserver/nginx/conf/proxy.conf;
#charset gb2312;
#设定请求缓冲
server_names_hash_bucket_size 128;
client_header_buffer_size 32k;
large_client_header_buffers 4 32k;
#client_max_body_size 8m;
sendfile on;
tcp_nopush on;
keepalive_timeout 60;
tcp_nodelay on;
# fastcgi_connect_timeout 300;
# fastcgi_send_timeout 300;
# fastcgi_read_timeout 300;
# fastcgi_buffer_size 64k;
# fastcgi_buffers 4 64k;
# fastcgi_busy_buffers_size 128k;
# fastcgi_temp_file_write_size 128k;
gzip on;
gzip_min_length 1k;
gzip_buffers 4 16k;
gzip_http_version 1.0;
gzip_comp_level 2;
gzip_types text/plain application/x-javascript text/css application/xml;
gzip_vary on;
#limit_zone crawler $binary_remote_addr 10m;
###禁止通过ip访问站点
# server{
# server_name _;
# return 404;
# }
upstream tserver {
server 172.18.188.64:8080 weight=1;
server 172.18.188.76:8080 weight=1;
server 172.18.188.78:8080 weight=1;
}
server
{
listen 80;
server_name vmwarehost;
index index.html index.htm index.jsp;
root /home/www/web/ROOT;
#limit_conn crawler 20;
location /
{
proxy_pass http://tserver;
}
location /NginxStatus
{
stub_status on;
access_log off;
}
location ~ .*\.(htm|html|gif|jpg|jpeg|png|bmp|swf|ico)$
{
expires 30d;
}
location ~ .*\.(js|css)?$
{
expires 1h;
}
#定义访问日志的写入格式
log_format access '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" $http_x_forwarded_for';
access_log /usr/local/webserver/nginx/logs/localhost.log access;
}
}
proxy.confi配置如下
#!nginx (-)
# proxy.conf
proxy_redirect off;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
client_max_body_size 10m;
client_body_buffer_size 128k;
proxy_connect_timeout 90;
proxy_send_timeout 90;
proxy_read_timeout 90;
proxy_buffer_size 4k;
proxy_buffers 4 32k;
proxy_busy_buffers_size 64k;
proxy_temp_file_write_size 64k;
tomcat 6 server.xml配置如下:
<?xml version='1.0' encoding='utf-8'?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<!-- Note: A "Server" is not itself a "Container", so you may not
define subcomponents such as "Valves" at this level.
Documentation at /docs/config/server.html
-->
<Server port="8005" shutdown="SHUTDOWN">
<Listener className="org.apache.catalina.core.AprLifecycleListener" SSLEngine="on" />
<Listener className="org.apache.catalina.core.JasperListener" />
<Listener className="org.apache.catalina.core.JreMemoryLeakPreventionListener" />
<Listener className="org.apache.catalina.mbeans.ServerLifecycleListener" />
<Listener className="org.apache.catalina.mbeans.GlobalResourcesLifecycleListener" />
<!-- Global JNDI resources
Documentation at /docs/jndi-resources-howto.html
-->
<GlobalNamingResources>
<!-- Editable user database that can also be used by
UserDatabaseRealm to authenticate users
-->
<Resource name="UserDatabase" auth="Container"
type="org.apache.catalina.UserDatabase"
description="User database that can be updated and saved"
factory="org.apache.catalina.users.MemoryUserDatabaseFactory"
pathname="conf/tomcat-users.xml" />
</GlobalNamingResources>
<!-- A "Service" is a collection of one or more "Connectors" that share
a single "Container" Note: A "Service" is not itself a "Container",
so you may not define subcomponents such as "Valves" at this level.
Documentation at /docs/config/service.html
-->
<Service name="Catalina">
<!--
<Executor name="tomcatThreadPool" namePrefix="catalina-exec-"
maxThreads="150" minSpareThreads="4"/>
-->
<!-- A "Connector" represents an endpoint by which requests are received
and responses are returned. Documentation at :
Java HTTP Connector: /docs/config/http.html (blocking & non-blocking)
Java AJP Connector: /docs/config/ajp.html
APR (HTTP/AJP) Connector: /docs/apr.html
Define a non-SSL HTTP/1.1 Connector on port 8080
-->
<Connector port="8080" protocol="HTTP/1.1"
maxHttpHeaderSize="8192" connectionTimeout="20000"
disableUploadTimeout="true" enableLookups="false"
redirectPort="8443" maxThreads="600"
minSpareThreads="25" maxSpareThreads="75" acceptCount="100" />
<!--
<Connector executor="tomcatThreadPool"
port="8080" protocol="HTTP/1.1"
connectionTimeout="20000"
redirectPort="8443" />
-->
<!-- Define a SSL HTTP/1.1 Connector on port 8443
This connector uses the JSSE configuration, when using APR, the
connector should be using the OpenSSL style configuration
described in the APR documentation -->
<!--
<Connector port="8443" protocol="HTTP/1.1" SSLEnabled="true"
maxThreads="150" scheme="https" secure="true"
clientAuth="false" sslProtocol="TLS" />
-->
<Connector port="8009" protocol="AJP/1.3"
maxHttpHeaderSize="8192" connectionTimeout="20000"
disableUploadTimeout="true" maxThreads="600"
minSpareThreads="25" maxSpareThreads="75"
enableLookups="false" redirectPort="8443" />
<!-- An Engine represents the entry point (within Catalina) that processes
every request. The Engine implementation for Tomcat stand alone
analyzes the HTTP headers included with the request, and passes them
on to the appropriate Host (virtual host).
Documentation at /docs/config/engine.html -->
<!-- You should set jvmRoute to support load-balancing via AJP ie :
<Engine name="Catalina" defaultHost="localhost" jvmRoute="jvm1">
-->
<Engine name="Catalina" defaultHost="localhost" jvmRoute="tomcat1">
<!--For clustering, please take a look at documentation at:
/docs/cluster-howto.html (simple how to)
/docs/config/cluster.html (reference documentation) -->
<!--
<Cluster className="org.apache.catalina.ha.tcp.SimpleTcpCluster"/>
-->
<Cluster className="org.apache.catalina.ha.tcp.SimpleTcpCluster"
channelSendOptions="6">
<!--
<Manager className="org.apache.catalina.ha.session.BackupManager"
expireSessionsOnShutdown="false"
notifyListenersOnReplication="true"
mapSendOptions="6"/>
-->
<!--
<Manager className="org.apache.catalina.ha.session.DeltaManager"
expireSessionsOnShutdown="false"
notifyListenersOnReplication="true"/>
-->
<Channel className="org.apache.catalina.tribes.group.GroupChannel">
<Membership className="org.apache.catalina.tribes.membership.McastService"
address="228.0.0.4"
port="45564"
frequency="500"
dropTime="3000"/>
<Receiver className="org.apache.catalina.tribes.transport.nio.NioReceiver"
address="172.18.188.64"
port="4001"
autoBind="100"
selectorTimeout="5000"
maxThreads="6"/>
<Sender className="org.apache.catalina.tribes.transport.ReplicationTransmitter">
<Transport className="org.apache.catalina.tribes.transport.nio.PooledParallelSender"/>
</Sender>
<Interceptor className="org.apache.catalina.tribes.group.interceptors.TcpFailureDetector"/>
<Interceptor className="org.apache.catalina.tribes.group.interceptors.MessageDispatch15Interceptor"/>
<Interceptor className="org.apache.catalina.tribes.group.interceptors.ThroughputInterceptor"/>
</Channel>
<Valve className="org.apache.catalina.ha.tcp.ReplicationValve"
filter=".*\.gif;.*\.js;.*\.jpg;.*\.png;.*\.htm;.*\.html;.*\.css;.*\.txt;" />
<Deployer className="org.apache.catalina.ha.deploy.FarmWarDeployer"
tempDir="/tmp/war-temp/"
deployDir="/tmp/war-deploy/"
watchDir="/tmp/war-listen/"
watchEnabled="false"/>
<ClusterListener className="org.apache.catalina.ha.session.ClusterSessionListener"/>
</Cluster>
<!-- The request dumper valve dumps useful debugging information about
the request and response data received and sent by Tomcat.
Documentation at: /docs/config/valve.html -->
<!--
<Valve className="org.apache.catalina.valves.RequestDumperValve"/>
-->
<!-- This Realm uses the UserDatabase configured in the global JNDI
resources under the key "UserDatabase". Any edits
that are performed against this UserDatabase are immediately
available for use by the Realm. -->
<Realm className="org.apache.catalina.realm.UserDatabaseRealm"
resourceName="UserDatabase"/>
<!-- Define the default virtual host
Note: XML Schema validation will not work with Xerces 2.2.
-->
<Host name="localhost" appBase="/home/www/web"
unpackWARs="true" autoDeploy="true"
xmlValidation="false" xmlNamespaceAware="false">
<!-- SingleSignOn valve, share authentication between web applications
Documentation at: /docs/config/valve.html -->
<!--
<Valve className="org.apache.catalina.authenticator.SingleSignOn" />
-->
<!-- Access log processes all example.
Documentation at: /docs/config/valve.html -->
<!--
<Valve className="org.apache.catalina.valves.AccessLogValve" directory="logs"
prefix="localhost_access_log." suffix=".txt" pattern="common" resolveHosts="false"/>
-->
</Host>
</Engine>
</Service>
</Server>
content.xml配置如下
<Context>
<WatchedResource>WEB-INF/web.xml</WatchedResource>
<!--
<Manager pathname="" />
-->
<!-- Uncomment this to enable Comet connection tacking (provides events
on session expiration as well as webapp lifecycle) -->
<!--
<Valve className="org.apache.catalina.valves.CometConnectionManagerValve" />
-->
<Manager className="de.javakaffee.web.msm.MemcachedBackupSessionManager"
memcachedNodes="n1:172.18.188.64:11211"
requestUriIgnorePattern=".*\.(png|gif|jpg|css|js)$"
sessionBackupAsync="false"
sessionBackupTimeout="100"
transcoderFactoryClass="de.javakaffee.web.msm.serializer.javolution.JavolutionTranscoderFactory"
copyCollectionsForSerialization="false"/>
</Context>
memcached启动命令:
useradd -s /sbin/nologin memcached
memcached -d -m 2048 -l 172.18.188.64 -p 11211 -u memcached
ngingx启动命令
sbin/nginx
tomcat启动命令(startup.sh)
分别启动memcached nginx tomcat能实现session的简单共享.
按照上面配置Nginx Memcached Tomcat启动运行都没有问题，并且可以实现Session的共享.但是有2个问题
一、Session的共享是基于访问IP的，即在同一台电脑上开2个IE窗口时，获取到Session中内容是相同的，也就是sessionid除了最后面的jvmrout不一样，其他都一样，内容也一样，这样就造成如果2个用户先后在同一台电脑上登录形成session混乱，有没有可能配置成同一ip在不同的ie窗口中不共享session，一个ie窗口对应一个session，而不是一个ip共享一个session.后台的访问依然是由nginx根据weight做分发而不是固定到一台固定的tomcat机器?
二、Memcached 启动是用-m 512发现会有数据丢失而且丢失几率很大，在一个页面上连续不断的刷新时就会发现session中的内容会清空.