EMQ与Kafka插件emqx_plugin_kafka修改使用实现EMQ上行消息转发kafaka供服务消费 服务消息存入kafka EMQ主动消费

emqx和kafka消息通信插件实现应用

插件地址：https://github.com/ULTRAKID/emqx_plugin_kafka

1、拉取插件代码导入自己仓库

拉取插件代码

git clone https://github.com/ULTRAKID/emqx_plugin_kafka.git

之后通过git传入自己仓库或者直接fork一份到自己github仓库

2、EMQX编译

拉取EMQX源码（先配置ssh免密）

git clone git@github.com:emqx/emqx.git

修改EMQX主目录下Makefile文件，添加如下行

export EMQX_EXTRA_PLUGINS = emqx_plugin_kafka

修改EMQX目录下lib-extra/plugins文件，在erlang_plugins中添加如下行打入emqx_plugin_kafka插件

, {
   emqx_plugin_kafka, {
   git, "git@git.talkweb.com.cn:iot3.0/cloud/iot/emqx_plugin_kafka.git", {
   branch, "main"}}} #这里仓库为自己仓库地址 方便之后代码修改提交

编译

make

此过程可能会有点长 甚至有些依赖包下不下来 无法访问github

这是由于访问github网速慢的原因 推荐SwitchHosts这个软件来替换Hosts文件加快访问github

具体操作见链接：https://zhuanlan.zhihu.com/p/443847295

3、运行

配置kafka地址，修改emqx配置文件emqx_plugin_kafka.conf，具体目录在_build/emqx/rel/emqx/etc/plugins

注意：_bulid目录需要在编译后才会生成

vim _build/emqx/rel/emqx/etc/plugins/emqx_plugin_kafka.conf 

kafka.host = 192.168.141.77 #这里修改为自己kafka所在服务器的ip

kafka.port = 9092 

开启emqx日志级别为info 方便之后日志查看

vim _build/emqx/rel/emqx/etc/emqx.conf

log.level = info

运行emqx

_build/emqx/rel/emqx/bin/emqx console

此时应该没什么问题

访问emqx启动服务器ip:18083，找到插件（plugins）菜单，找到插件emqx_plugin_kafka选择启动

通过MQTTX工具连接emqx，ip选择emqx安装所在服务器ip，名字随便起，就可以连接

连接成功后就可以往emqx指定topic发送消息

4、Kafka工具

这里我们选择KafkaTools，连接到Kafka服务器所在ip的Kafka之后，因为我们还没有修改插件中Kafka默认topic，我们在topic – emqx-topic就会收到上述消息

同时emqx所在服务器也会打印info信息

5、插件代码修改

注意：此部分只是按照个人需求修改

修改配置文件emqx_plugin_kafka.conf

kafka.host = 192.168.141.77 #kafka的服务器地址 这里配置就不用每次重新编译emqx时去改kafka的配置文件的host
kafka.payloadtopic = tlink_device_ #kafka的topic前缀

修改rebar.config.script变量GitDescribe为master

EMQX_MGMT_DEP = {
   emqx_management, {
   git, UrlPrefix ++ "emqx-management", "master"}},

修改主要逻辑代码emqx_plugin_kafka.erl

添加按mqtt的topic分发消息到kafka不同的topic

get_kafka_topic_produce(Topic, Message) ->
  ?LOG_INFO("[KAFKA PLUGIN]Kafka topic = -s-n", [Topic]),
  TopicPrefix = string:left(binary_to_list(Topic),6),
  TlinkFlag = string:equal(TopicPrefix, <<"tlink/">>),
  if
    TlinkFlag == true ->
      TopicStr = binary_to_list(Topic),
      OtaIndex = string:str(TopicStr,"ota"),
      SubRegisterIndex = string:str(TopicStr,"sub/register"),
      SubLogin = string:str(TopicStr,"sub/login"),
      if
        OtaIndex /= 0 ->
          TopicKafka = list_to_binary([ekaf_get_topic(), <<"ota">>]);
        SubRegisterIndex /= 0 ->
          TopicKafka = list_to_binary([ekaf_get_topic(), <<"sub_register">>]);
        SubLogin /= 0 ->
          TopicKafka = list_to_binary([ekaf_get_topic(), <<"sub_status">>]);
        OtaIndex + SubRegisterIndex + SubLogin == 0 ->
          TopicKafka = list_to_binary([ekaf_get_topic(), <<"msg">>])
      end,
      produce_kafka_payload(TopicKafka, Topic, Message);
    TlinkFlag == false ->
      ?LOG_INFO("[KAFKA PLUGIN]MQTT topic prefix is not tlink = ~s~n",[Topic])
  end,
  ok.

完整代码如下

%%--------------------------------------------------------------------
%% Copyright (c) 2015-2017 Feng Lee <feng@emqtt.io>.
%%
%% Modified by Ramez Hanna <rhanna@iotblue.net>
%% 
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%%     http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------

-module(emqx_plugin_kafka).

% -include("emqx_plugin_kafka.hrl").

% -include_lib("emqx/include/emqx.hrl").

-include("emqx.hrl").
-include_lib("kernel/include/logger.hrl").

-export([load/1, unload/0]).

%% Client Lifecircle Hooks
-export([on_client_connect/3
  , on_client_connack/4
  , on_client_connected/3
  , on_client_disconnected/4
  , on_client_authenticate/3
  , on_client_check_acl/5
  , on_client_subscribe/4
  , on_client_unsubscribe/4
]).

%% Session Lifecircle Hooks
-export([on_session_created/3
  , on_session_subscribed/4
  , on_session_unsubscribed/4
  , on_session_resumed/3
  , on_session_discarded/3
  , on_session_takeovered/3
  , on_session_terminated/4
]).

%% Message Pubsub Hooks
-export([on_message_publish/2
  , on_message_delivered/3
  , on_message_acked/3
  , on_message_dropped/4
]).


%% Called when the plugin application start
load(Env) ->
  ekaf_init([Env]),
  emqx:hook('client.connect', {
   ?MODULE, on_client_connect, [Env]}),
  emqx:hook('client.connack', {
   ?MODULE, on_client_connack, [Env]}),
  emqx:hook('client.connected', {
   ?MODULE, on_client_connected, [Env]}),
  emqx:hook('client.disconnected', {
   ?MODULE, on_client_disconnected, [Env]}),
  emqx:hook('client.authenticate', {
   ?MODULE, on_client_authenticate, [Env]}),
  emqx:hook('client.check_acl', {
   ?MODULE, on_client_check_acl, [Env]}),
  emqx:hook('client.subscribe', {
   ?MODULE, on_client_subscribe, [Env]}),
  emqx:hook('client.unsubscribe', {
   ?MODULE, on_client_unsubscribe, [Env]}),
  emqx:hook('session.created', {
   ?MODULE, on_session_created, [Env]}),
  emqx:hook('session.subscribed', {
   ?MODULE, on_session_subscribed, [Env]}),
  emqx:hook('session.unsubscribed', {
   ?MODULE, on_session_unsubscribed, [Env]}),
  emqx:hook('session.resumed', {
   ?MODULE, on_session_resumed, [Env]}),
  emqx:hook('session.discarded', {
   ?MODULE, on_session_discarded, [Env]}),
  emqx:hook('session.takeovered', {
   ?MODULE, on_session_takeovered, [Env]}),
  emqx:hook('session.terminated', {
   ?MODULE, on_session_terminated, [Env]}),
  emqx:hook('message.publish', {
   ?MODULE, on_message_publish,