7.1.4 智慧物流【实时数据采集,Kafka与nginx配置、java模拟车载传感采集数据发送到Kafka】

最新推荐文章于 2024-07-26 02:47:45 发布

weixin_47134119

最新推荐文章于 2024-07-26 02:47:45 发布

阅读量1k

点赞数

文章标签： kafka nginx java

本文链接：https://blog.youkuaiyun.com/weixin_47134119/article/details/120340231

版权

实时数据采集

文章目录

实时数据采集
第一节配置Nginx
第二节车载客户端

架构流程
在这里插入图片描述
车辆会装有各种传感器以及与后台通信的客户端，通过客户端把相关数据传送到后台。

第一节配置Nginx

linux123上

1、安装git工具，安装wget下载工具

参照 https://blog.youkuaiyun.com/weixin_47134119/article/details/112253597

yum install wget git -y 
yum install gcc-c++ -y

2、切换到/usr/local/src目录，然后将kafka的客户端源码使用git clone到本地

cd /usr/local/src 
git clone git://github.com/edenhill/librdkafka

在这里插入图片描述
注意：要保证下载成功！！不能出现failed字样

3、进入librdkafka目录，对kafka客户端源码进行编译

cd /usr/local/src/librdkafka 
yum install -y gcc gcc-c++ pcre-devel zlib-devel 
./configure 
make && make install

注意：gcc编译的时候出现如下问题：
collect2 cannot find ‘ld’
解决方式：yum reinstall binutils -y

4、安装nginx 整合kafka的插件，进入到/usr/local/src目录下，使用git clone nginx整合kafka的源码

cd /usr/local/src 
git clone https://github.com/brg-liuwei/ngx_kafka_module

在这里插入图片描述

5、下载nginx源码包

cd /usr/local/src
wget http://nginx.org/download/nginx-1.17.8.tar.gz
tar -zxvf nginx-1.17.8.tar.gz
cd nginx-1.17.8
yum install gcc zlib zlib-devel openssl openssl-devel pcre pcre-devel -y

6、进入到nginx的源码目录下(编译nginx，包含与kafka整合的插件)

cd /usr/local/src/nginx-1.17.8
./configure --add-module=/usr/local/src/ngx_kafka_module/
make && make install

此时nginx安装目录在：/usr/local/nginx/conf

7、修改nginx配置文件

设置一个location和kafka的topic信息
注意：

1、配置文件路径是：/usr/local/nginx/conf下的nginx.conf文件，不要与之前下载的nginx安装包目录混淆！！
2、内容都是添加到http标签内

#添加kafka集群
kafka;
kafka_broker_list linux121:9092,linux122:9092,linux123:9092;

#server 里添加
server {
   
		location /log/lg_bus_info {
   
            kafka_topic lg_bus_info;
        }
}

完整文件参考


#user  nobody;
worker_processes  1;

#error_log  logs/error.log;
#error_log  logs/error.log  notice;
#error_log  logs/error.log  info;

#pid        logs/nginx.pid;


events {
   
    worker_connections  1024;
}


http {
   
    include       mime.types;
    default_type  application/octet-stream;

    #log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
    #                  '$status $body_bytes_sent "$http_referer" '
    #                  '"$http_user_agent" "$http_x_forwarded_for"';

    #access_log  logs/access.log  main;

    sendfile        on;
    #tcp_nopush     on;

    #keepalive_timeout  0;
    keepalive_timeout  65;
	kafka;
	kafka_broker_list linux121:9092,linux122:9092,linux123:9092;

    #gzip  on;

    server {
   
        listen       80;
        server_name  localhost;

        #charset koi8-r;

        #access_log  logs/host.access.log  main;

        location / {
   
            root   html;
            index  index.html index.htm;
        }
		
		location /log/lg_bus_info {
   
            kafka_topic lg_bus_info;
        }

        #error_page  404              /404.html;

        # redirect server error pages to the static page /50x.html
        #
        error_page   500 502 503 504  /50x.html;
        location = /50x.html {
   
            root   html;
        }

        # proxy the PHP scripts to Apache listening on 127.0.0.1:80
        #
        #location ~ \.php$ {
   
        #    proxy_pass   http://127.0.0.1;
        #}

        # pass the PHP scripts to FastCGI server listening on 127.0.0.1:9000
        #
        #location ~ \.php$ {
   
        #    root           html;
        #    fastcgi_pass   127.0.0.1:9000;
        #    fastcgi_index  index.php;
        #    fastcgi_param  SCRIPT_FILENAME  /scripts$fastcgi_script_name;
        #    include        fastcgi_params;
        #}