使用librdkafka的C++接口实现Kafka生产者和消费者客户端

1. librdkafka简介：

librdkafka 是 Apache Kafka 的 C/C++ 开发包，提供 生产者、消费者 和 管理客户端。

设计理念是可靠以及高性能的消息传输，当前可支持每秒超过100万的消息生产和300万每秒的消息消费。

官方README 文档对librdkafka的介绍：
“librdkafka — the Apache Kafka c/C++ client library”

librdkafka/INTRODUCTION.md
https://github.com/edenhill/librdkafka/blob/master/INTRODUCTION.md

librdkafka/examples/
https://github.com/edenhill/librdkafka/tree/master/examples

Usage：
使用时，需要在源程序中包含包含 "rdkafka.h" 头文件

2. librdkafka的C++接口：

2.1 RdKafka::Conf::create()：

创建Conf配置实例，用于填充用户指定的各配置项：

//namespace RdKafka;

//brief Create configuration object:
//RdKafka::Conf ---> 配置接口类，用来设置对生产者、消费者、broker的各配置项的值
static Conf *create(ConfType type);	

enum ConfType {
   
   
	CONF_GLOBAL,	//Global configuration
	CONF_TOPIC		//Topic specific configuration
};

使用举例：

RdKafka::Conf *m_config = RdKafka::Conf::create(RdKafka::Conf::CONF_GLOBAL);
if(m_config == nullptr) {
   
   
	std::cout << "Create Rdkafka Global Conf Failed." << std::endl;
}

RdKafka::Conf *m_topicConfig = RdKafka::Conf::create(RdKafka::Conf::CONF_TOPIC);
if(m_config == nullptr) {
   
   
	std::cout << "Create Rdkafka Topic Conf Failed." << std::endl;
}

2.2 Conf::ConfResult set()：

Conf类中的多个set成员函数，用于对不同的配置项进行赋值：

class Conf {
   
   
public:
	virtual Conf::ConfResult set(const std::string &name, const std::string &value, std::string &errstr);
	virtual Conf::ConfResult set(const std::string &name, DeliveryReportCb *dr_cb, std::string &errstr);
	virtual Conf::ConfResult set(const std::string &name, EventCb *event_cb, std::string &errstr);
	virtual Conf::ConfResult set(const std::string &name, PartitionerCb *partitioner_cb, std::string &errstr);
	//...
	//...
};

enum ConfResult {
   
   
	CONF_UNKNOWN = -2,	//Unknown configuration property
	CONF_INVALID = -1,	//Invalid configuration value
	CONF_OK = 0			//Configuration property was succesfully set
};

使用举例：

RdKafka::Conf::ConfResult  	result;
std::string					error_str;

RdKafka::Conf *m_config;
//设置 "booststrap::servers" 配置项：
result = m_config->set("bootstrap.servers", "127.0.0.`:9092", error_str);
if(result != RdKafka::Conf::CONF_OK) {
   
   
    std::cout << "Global Conf set 'booststrap.servers' failed: " << error_str << std::endl;
}

//设置 "event_cb" 配置项：
RdKafka::EventCb* m_event_cb = new ProducerEventCb;
result = m_config->set("event_cb", m_event_cb, error_str);
if(result != RdKafka::Conf::CONF_OK) {
   
   
    std::cout << "Global Conf set 'event_cb' failed: " << error_str << std::endl;
}

2.3 RdKafka::Producer::create()：

创建Producer生产者客户端：

class Producer : public virtual Handle {
   
   
public:
	static Producer *create(Conf *conf, std::string &errstr);
};

使用举例：

RdKfka::Producer *m_producer;

m_producer = RdKafka::Producer::create(m_config, error_str);
if(m_producer == nullptr) {
   
   
    std::cout << "Create Topic failed: " << error_str << std::endl;
}

2.4 RdKafka::Topic::create()：

创建Topic主题对象：

class Topic {
   
   
public:
	static Topic *create(Handle *base, const std::string &tipic_str, const Conf *conf, std::string &errstr);
};

使用举例：

RdKafka::Topic *m_topic;

m_topic = RdKafka::Topic::create(m_producer, m_topicStr, m_topicConfig, error_str);
if(m_topic == nullptr) {
   
   
    std::cout << "Create Topic failed: " << error_str << std::endl;
}

2.5 RdKafka::Producer::produce()：

class Producer : public virtual Handle {
   
   
public:
	virtual ErrorCode produce(Topic *topic, int32_t partition, int msgflags, 
						void *payload, size_t len, const std::string *key, void *msg_opaque);

	virtual ErrorCode produce();
};


//Use RdKafka::err2str() to translate an error code a human readable string
enum ErrorCode {
   
   
	//Internal errors to rdkafka:
	ERR_BEGIN = -200,		//Begin internal error codes
	ERR_BAD_MSG = -199,		//Received message is incorrect
	//...
	ERR_END = -100,			//End interval error codes

	//Kafka broker errors: 
	ERROR_UNKNOWN = -1,		//Unknown broker error
	ERROR_NO_ERROR = 0,		//Success
	//...
};

使用举例：

RdKafka::ErrorCode error_code = m_producer->produce(m_topic, RdKafka::Topic::PARTITION_UA, RdKafka::Producer::RK_MSG_COPY, 
										payload, len, key, NULL);

m_producer->pool(0);	//poll()参数为0意味着不阻塞，poll(0)主要是为了触发应用程序提供的回调函数

if(error_code != ERROR_NO_ERROR) {
   
   
	std::cerr << "Produce failed: " << RdKafka::err2str(error_code) << std::endl;
	if(error_code == ERR_QUEUE_FULL) {
   
   
		m_producer->poll(1000);		//如果发送失败的原始是队列正满，则阻塞等待一段时间
	}
	else if(error_code == ERR_MSG_SIZE_TOO_LARGE) {
   
   
		//如果消息过大超过了max_size，则需要对消息做裁剪后重新发送
	}
	else {
   
   
		std::cerr << "ERR_UNKNOWN_PARTITION or ERR_UNKNOWN_TOPIC" << std::endl;
	}
}

2.6 RdKafka::KafkaConsumer::create()：

创建Consumer消费者客户端：

class KafkaConsumer : public virtual Handle {
   
   
public:
	static KafkaConsumer *create(const Conf *conf, std::string &errstr);
};

使用举例：

RdKafka::KafkaConsume