大数据项目之ip条数统计 nginx flume kafka hive

一.项目目的

对日志中ip条数进行统计统计

二.项目流程

图片出自：https://blog.youkuaiyun.com/lucasmaluping/article/details/102685434

三.项目操作

1.frame打成jar包上传到linux集群

通过java -jar，测试是否 运行成功：hdp-1:8889

2.修改配置文件，自定义ngnix输出日志的格式   

修改配置文件 cd /usr/local/nginx/conf     vi nginx.conf

#user  nobody;
worker_processes  1;

#error_log  logs/error.log;
#error_log  logs/error.log  notice;
#error_log  logs/error.log  info;

#pid        logs/nginx.pid;


events {
    worker_connections  1024;
}


http {
    include       mime.types;
    default_type  application/octet-stream;
##ngnix输出日志的格式 
    log_format main '{ time:"$time_iso8601", '
                'ip:$remote_addr", '
                'referer":"$http_referer", '
                'request:"$request", '
                'status": $status, '
                'bytes":$body_bytes_sent, '
                'agent": "$http_user_agent", '
                'x_forwarded:"$http_x_forwarded_for", }';

    #access_log  logs/access.log  main;

    sendfile        on;
    #tcp_nopush     on;

    #keepalive_timeout  0;
    keepalive_timeout  65;

    #gzip  on;
##指明nginx转发地址
upstream frame-tomcat {