CentOS安装FastDFS集群

网上大多配置不够准确，自己花两天时间在虚拟机中配置了一个FastDFS小型集群

安装环境：

    服务器：CentOS 7
    集群配置：Tracker(192.168.0.125) Storage1(192.168.0.126) Storage2(192.168.0.127)
    Nginx:  1.10.3
    FastDFS: 5.0.5
    libfastcommon: 1.0.7
    fastdfs-nginx-module: 1.16


1. Linux安装FastDFS,需要准备五个前置环境 
GCC yum install gcc-c++
libevent库 yum -y install libevent
pcre yum -y install pcre pcre-devel  
zlib yum -y install zlib zlib-devel  
openssl yum -y install openssl openssl-devel

2. CentOS7关闭防火墙
systemctl stop firewalld.service #停止firewall


systemctl disable firewalld.service #禁止firewall开机启动


3. 配置Tracker
①：将Nginx、FastDFS、libfastcommon、fastdfs-nginx-module安装包上传到Tracker路径/usr/local/  并解压
②：进入libfastcommon文件夹 执行命令 ./make.sh  和./make.sh install 将生成到lib64中的文件拷贝到lib cp /usr/lib64/libfastcommon.so /usr/lib/
③：进入到FastDFS文件夹 执行命令 ./make.sh  和./make.sh install 安装成功后将 conf文件夹下client.conf、http.conf、mime.types、storage.conf、storage_ids.conf、


tracker.conf 拷贝到/etc/fdfs/下    cp ./conf/client.conf http.conf mime.types storage.conf storage_ids.conf tracker.conf /etc/fdfs/  修改tracker.conf  vi tracker.conf 


将base_path=/home/yuqing/FastDFS 改为当前FastDFS安装目录/usr/local/FastDFS   修改  http.server_port=80  启动测试 /usr/bin/fdfs_trackerd /etc/fdfs/tracker.conf restart  


使用命令netstat -natp 查看 tracker服务是否有开启
4. 配置Storage
前面两步与Tracker的配置相同，拷贝文件到/etc/fdfs/后修改配置文件storage.conf   vi storage.conf  修改 group_name=group1 base_path=/usr/local/FastDFS  


store_path0=/usr/local/FastDFS/storage(如果有多个挂载磁盘定义多个store_path【store_path1=....】)  tracker_server=192.168.0.125:22122 启动Storage 并测试  


/usr/bin/fdfs_storaged /etc/fdfs/storage.conf restart  （注：如果两个Storage一起配置在一个group,但是没有安装配置Nginx，测试上传数据会报错，在/usr/local/logs/storage.log


中有记录从一个storage备份到另一个storage失败） 此时可以使用管理工具查看集群状态 首先配置/etc/fdfs/client.conf  将base_path和tracker_server修改 然后使用命令 


/usr/bin/fdfs_monitor /etc/fdfs/client.conf  会出现下面信息


[2017-10-16 00:17:36] DEBUG - base_path=/usr/local/FastDFS, connect_timeout=30, network_timeout=60, tracker_server_count=1, anti_steal_token=0, anti_steal_secret_key 


length=0, use_connection_pool=0, g_connection_pool_max_idle_time=3600s, use_storage_id=0, storage server id count: 0


server_count=1, server_index=0


tracker server is 192.168.0.125:22122


group count: 1


Group 1:
group name = group1
disk total space = 18121 MB
disk free space = 13340 MB
trunk free space = 0 MB
storage server count = 2
active server count = 2
storage server port = 23000
storage HTTP port = 8888
store path count = 1
subdir count per path = 256
current write server index = 0
current trunk file id = 0


        Storage 1:
                id = 192.168.0.126
                ip_addr = 192.168.0.126  ACTIVE
                http domain = 
                version = 5.05
                join time = 2017-10-15 15:17:11
                up time = 2017-10-16 08:13:29
                total storage = 18121 MB
                free storage = 13378 MB
                upload priority = 10
                store_path_count = 1
                subdir_count_per_path = 256
                storage_port = 23000
                storage_http_port = 8888
                current_write_path = 0
                source storage id = 192.168.0.127
                if_trunk_server = 0
                connection.alloc_count = 256
                connection.current_count = 1
                connection.max_count = 1
                total_upload_count = 4
                success_upload_count = 4
                total_append_count = 0
                success_append_count = 0
                total_modify_count = 0
                success_modify_count = 0
                total_truncate_count = 0
                success_truncate_count = 0
                total_set_meta_count = 4
                success_set_meta_count = 4
                total_delete_count = 0
                success_delete_count = 0
                total_download_count = 4
                success_download_count = 4
                total_get_meta_count = 0
                success_get_meta_count = 0
                total_create_link_count = 0
                success_create_link_count = 0
                total_delete_link_count = 0
                success_delete_link_count = 0
                total_upload_bytes = 2716586
                success_upload_bytes = 2716586
                total_append_bytes = 0
                success_append_bytes = 0
                total_modify_bytes = 0
                success_modify_bytes = 0
                stotal_download_bytes = 1823116
                success_download_bytes = 1823116
                total_sync_in_bytes = 0
                success_sync_in_bytes = 0
                total_sync_out_bytes = 0
                success_sync_out_bytes = 0
                total_file_open_count = 8
                success_file_open_count = 8
                total_file_read_count = 10
                success_file_read_count = 10
                total_file_write_count = 12
                success_file_write_count = 12
                last_heart_beat_time = 2017-10-16 00:17:26
                last_source_update = 2017-10-16 07:59:55
                last_sync_update = 1969-12-31 16:00:00
                last_synced_timestamp = 1969-12-31 16:00:00 
        Storage 2:
                id = 192.168.0.127
                ip_addr = 192.168.0.127  ACTIVE
                http domain = 
                version = 5.05
                join time = 2017-10-15 07:17:37
                up time = 2017-10-16 00:17:12
                total storage = 18121 MB
                free storage = 13340 MB
                upload priority = 10
                store_path_count = 1
                subdir_count_per_path = 256
                storage_port = 23000
                storage_http_port = 8888
                current_write_path = 0
                source storage id = 
                if_trunk_server = 0
                connection.alloc_count = 256
                connection.current_count = 0
                connection.max_count = 0
                total_upload_count = 0
                success_upload_count = 0
                total_append_count = 0
                success_append_count = 0
                total_modify_count = 0
                success_modify_count = 0
                total_truncate_count = 0
                success_truncate_count = 0
                total_set_meta_count = 0
                success_set_meta_count = 0
                total_delete_count = 0
                success_delete_count = 0
                total_download_count = 0
                success_download_count = 0
                total_get_meta_count = 0
                success_get_meta_count = 0
                total_create_link_count = 0
                success_create_link_count = 0
                total_delete_link_count = 0
                success_delete_link_count = 0
                total_upload_bytes = 0
                success_upload_bytes = 0
                total_append_bytes = 0
                success_append_bytes = 0
                total_modify_bytes = 0
                success_modify_bytes = 0
                stotal_download_bytes = 0
                success_download_bytes = 0
                total_sync_in_bytes = 2716782
                success_sync_in_bytes = 2716782
                total_sync_out_bytes = 0
                success_sync_out_bytes = 0
                total_file_open_count = 8
                success_file_open_count = 8
                total_file_read_count = 0
                success_file_read_count = 0
                total_file_write_count = 16
                success_file_write_count = 16
                last_heart_beat_time = 2017-10-16 00:17:13
                last_source_update = 1969-12-31 16:00:00
                last_sync_update = 2017-10-16 00:01:57
                last_synced_timestamp = 2017-10-16 07:59:55 (0s delay)
5. 整合Nginx
Tracker:
①. 修改fastdfs-nginx-module配置  cd /usr/local/fastdfs-nginx-module/src/ 修改config文件将/usr/local/改为/usr/  vi config   将 mod_FastDFS.conf 拷贝


到/etc/fdfs/下 cp mod_FastDFS.conf /etc/fdfs/ 修改mod_FastDFS.conf  vi /etc/fdfs/mod_FastDFS.conf   修改 base_path=/usr/local/FastDFS  


tracker_server=192.168.0.125:22122  url_have_group_name=true（url中包含group名称）   将libfdfsclient.so 拷贝到/usr/lib/ cp /usr/lib64/libfdfsclient.so /usr/lib/   创建


nginx/client目录 mkdir -p /var/temp/nginx/client 
②. 进入到Nginx文件夹 配置 cd /usr/local/nginx/   使用命令 
./configure \
--prefix=/usr/local/nginx \
--pid-path=/var/run/nginx/nginx.pid \
--lock-path=/var/lock/nginx.lock \
--error-log-path=/var/log/nginx/error.log \
--http-log-path=/var/log/nginx/access.log \
--with-http_gzip_static_module \
--http-client-body-temp-path=/var/temp/nginx/client \
--http-proxy-temp-path=/var/temp/nginx/proxy \
--http-fastcgi-temp-path=/var/temp/nginx/fastcgi \
--http-uwsgi-temp-path=/var/temp/nginx/uwsgi \
--http-scgi-temp-path=/var/temp/nginx/scgi \
--add-module=/usr/local/fastdfs-nginx-module/src


执行完成不报错  后执行 make 和 make install
③. 修改nginx.conf  cd /usr/local/nginx/conf/  vi nginx.conf 在http中加入storage组并且修改server 和location 
upstream storage_server_group1{
    server 192.168.0.126:80 weight=10;
server 192.168.0.127:80 weight=10;
    }


   server {
listen 80;
server_name 192.168.0.125;

location /group1{
proxy_redirect off;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_pass http://storage_server_group1;
}
④. 启动nginx cd /usr/local/nginx/sbin/  使用命令 ./nginx  netstap -natp 测试nginx是否开启成功
Storage:
①. Nginx安装和配置与Tracker一样，mod_FastDFS.conf中将base_path 和 store_path0分别配置
②. 修改nginx.conf 中server为
server {
        listen       80;
        server_name  本机IP(192.168.0.126);


        location /group1/M00/{
                root /home/fastdfs/fdfs_storage/data;
                ngx_fastdfs_module;
        }
}
6. 将所有的Nginx服务、Tracker服务和Storage服务重启 (如果nginx服务已经启动 使用 netstat -natp 查询nginx服务的pid  使用命令 kill pid)
7. 测试 使用/usr/bin/fdfs_test /etc/fdfs/client.conf upload /usr/test/abc.jpg 会出现下面信息
This is FastDFS client test program v5.05
# connect timeout in seconds
# default value is 30s
connect_timeout=2


# network recv and send timeout in seconds
# default value is 30s
network_timeout=30


# the base path to store log files
base_path=/usr/local/FastDFS


# if load FastDFS parameters from tracker server
# since V1.12
# default value is false
load_fdfs_parameters_from_tracker=true


# storage sync file max delay seconds
# same as tracker.conf
# valid only when load_fdfs_parameters_from_tracker is false
# since V1.12
# default value is 86400 seconds (one day)
storage_sync_file_max_delay = 86400


# if use storage ID instead of IP address
# same as tracker.conf
# valid only when load_fdfs_parameters_from_tracker is false
# default value is false
# since V1.13
use_storage_id = false


# specify storage ids filename, can use relative or absolute path
# same as tracker.conf
# valid only when load_fdfs_parameters_from_tracker is false
# since V1.13
storage_ids_filename = storage_ids.conf


# FastDFS tracker_server can ocur more than once, and tracker_server format is
#  "host:port", host can be hostname or ip address
# valid only when load_fdfs_parameters_from_tracker is true
tracker_server=192.168.0.125:22122


# the port of the local storage server
# the default value is 23000
storage_server_port=23000


# the group name of the local storage server
group_name=group1


# if the url / uri including the group name
# set to false when uri like /M00/00/00/xxx
# set to true when uri like ${group_name}/M00/00/00/xxx, such as group1/M00/xxx
# default value is false
url_have_group_name = true


# path(disk or mount point) count, default value is 1
# must same as storage.conf
store_path_count=1


# store_path#, based 0, if store_path0 not exists, it's value is base_path
# the paths must be exist
# must same as storage.conf
store_path0=/usr/local/FastDFS/tracker
#store_path1=/home/yuqing/fastdfs1


# standard log level as syslog, case insensitive, value list:
### emerg for emergency
### alert


Copyright (C) 2008, Happy Fish / YuQing


FastDFS may be copied only under the terms of the GNU General
Public License V3, which may be found in the FastDFS source kit.
Please visit the FastDFS Home Page http://www.csource.org/ 
for more detail.


[2017-10-16 00:19:01] DEBUG - base_path=/usr/local/FastDFS, connect_timeout=30, network_timeout=60, tracker_server_count=1, anti_steal_token=0, anti_steal_secret_key 


length=0, use_connection_pool=0, g_connection_pool_max_idle_time=3600s, use_storage_id=0, storage server id count: 0


tracker_query_storage_store_list_without_group: 
        server 1. group_name=, ip_addr=192.168.0.126, port=23000
        server 2. group_name=, ip_addr=192.168.0.127, port=23000


group_name=group1, ip_addr=192.168.0.126, port=23000
storage_upload_by_filename
group_name=group1, remote_filename=M00/00/00/wKgAflnkzXGAftxEAAbRQFPsBpc874.jpg
source ip address: 192.168.0.126
file timestamp=2017-10-16 08:17:05
file size=446784
file crc32=1407977111
example file url: http://192.168.0.126/group1/M00/00/00/wKgAflnkzXGAftxEAAbRQFPsBpc874.jpg
storage_upload_slave_by_filename
group_name=group1, remote_filename=M00/00/00/wKgAflnkzXGAftxEAAbRQFPsBpc874_big.jpg
source ip address: 192.168.0.126
file timestamp=2017-10-16 08:17:05
file size=446784
file crc32=1407977111
example file url: http://192.168.0.126/group1/M00/00/00/wKgAflnkzXGAftxEAAbRQFPsBpc874_big.jpg
查看group1中的两个storage服务器中都有了该图片
因为Tracker的Nginx安装在Tracker服务器上  所以使用三台服务器的IP加上路径都可以访问