使用nodejs生产消费kafka

使用nodejs生产消费kafka

cnpm i node-kafka

producer.js

const kafka = require('kafka-node');

let conn = {'kafkaHost':'10.66.28.25:9092'};

var MQ = function (){
    this.mq_producers = {};
}

MQ.prototype.AddProducer = function (conn, handler){
    console.log('增加生产者',conn, this);
    let client = new kafka.KafkaClient(conn);
    let producer = new kafka.Producer(client);

    producer.on('ready', function(){
        if(!!handler){
            handler(producer);
        }
    });

    producer.on('error', function(err){
        console.error('producer error ',err.stack);
    });

    this.mq_producers['common'] = producer;
    return producer;
}
console.log(MQ);
var mq = new MQ();

mq.AddProducer(conn, function (producer){
    producer.createTopics(['broadcast'], function (){
        setInterval(function(){
            mq.mq_producers['common'].send([{topic:['mike_test'], 
            messages:[JSON.stringify({"cmd":"testRpc","value":"Hello World2"})]}], function (){
                console.log("..... ");
            })
        }, 2000);
    })
});

consumer.js

const Kafka = require('kafka-node');
    const Consumer = Kafka.Consumer;
    const Offset = Kafka.Offset;
    const kafkaUrl = '10.66.28.23:9092,10.66.28.24:9092,10.66.28.25:9092';
    const client = new Kafka.KafkaClient({ kafkaHost: kafkaUrl });
    let topics = [];
    topics.push('mike_test');
    let offset = new Offset(client);
    offset.fetchLatestOffsets(topics, (event, offsetValue) => {
        if (offsetValue == undefined) return;
        let topics_Partitions = [];
        for (var t in topics) {
            let offsetNums = offsetValue[topics[t]];
            let jsonStr = JSON.stringify(offsetNums).split(',');
            let offsetNum = offsetNums[0];
            for (var i = 0; i < jsonStr.length; i++) {
                let topic1 = { topic: topics[t], offset: offsetNums[i], partition: i }
                console.log(topic1);
                topics_Partitions.push(topic1);
            }
        }
        let options = {
            groupId: 'group-node' + (new Date()).getTime(),
            autoCommit: false,
            fetchMaxWaitMs: 5000,
            fetchMaxBytes: 1024 * 1024,
            fromOffset: true
        };
        let consumer = new Consumer(
            client, topics_Partitions, options
        );
        consumer.on('message', function (message) {
            console.log(message);
        });
    });

运行之后结果：


学习链接：http://x7j.net/budzt
官方文档：https://www.npmjs.com/package/kafka-node