背景
因为我们的业务场景是需要不断接入kafka消费的，为了避免不改动代码，做到灵活配置的目的，将topic存储到数据库中，启动过程中加载这些topic，所以启动过程中需要采用字节码生成class，然后class在实例化一个对象，返回给spring，交给spring管理。
KafkaListenerAnnotationBeanPostProcessor实现了BeanPostProcessor接口，并重写了postProcessAfterInitialization接口，当我们的bean交给sparing管理的时候，就会命中这个接口，进行@KafkaListener 注解解析，并启动监听topic
三个字节码工具
-
asm 易用性不高
-
javassist，javassit不支持泛型，
-
bytebuddy 我们采用这个，因为@KafkaListener注解的方法需要接受一个泛型的参数，
public void test(List<ConsumerRecord<String, String>> records, Acknowledgment ack) { doSome() }
bytebuddy 泛型文档
butebuddy泛型操作文档连接https://www.jianshu.com/p/628b7128904a https://www.jianshu.com/p/89050a744178
bytebuddy返回值 https://blog.youkuaiyun.com/CaptHua/article/details/123195024
添加注解 https://www.jianshu.com/p/e07d1f84b8d7
Bytebuddy代码实操
@Configuration
public class BinlogKafkaConsumerConfig {
// 这里定义一些常量，关于类名，方法名
public final static String GEN_CLASS_NAME = "com.consumer.BinlogByGenCodeConsumer";
public final static String METHOD_GET_ID_KEY_DATA_REQ_VO_MAP = "getIdKeyDataReqVOMap";
public final static String SUPER_METHOD_GET_ID_KEY_DATA_REQ_VO_MAP = "getIdKeyDataReqVOMapByBinlog";
public final static String SUPER_METHOD_RECEIVE_COMMON_MSG_BY_GEN_CODE = "receiveCommonMsgByGenCode";
public final static String METHOD_FORMAT_GET_ID_KEY_DATA_REQ_VO_MAP = "receiveMsgByGenCode%s";
@Bean
public Object binlogKafkaConsumerGenObj() {
//启动随系统注册数据源
try {
return doRun();
} catch (Exception e) {
AssertUtils.throwError(e.getMessage());
}
}
public BaseDataConsumer doRun() throws Exception {
// 查出数据库中维护的topic信息
List<Topic> list= dao.list();
if (CollectionUtils.isEmpty(list)) {
return null;
}
// 这里我们生成一个泛型的参数
TypeDescription.Generic listConsumerRecordGeneric = TypeDescription.Generic.Builder
.parameterizedType(TypeDescription.ForLoadedType.of(List.class), TypeDescription.Generic.Builder
.parameterizedType(ConsumerRecord.class, String.class, String.class).build()).build();
// 继承一个父类，用于将本类 的方法转调父类
DynamicType.Builder<BaseDataConsumer> definition = new ByteBuddy()
.subclass(BaseConsumer.class)
.name(GEN_CLASS_NAME)
// 定义一个方法，当调用这个方法时转调父类方法
.method(named(METHOD_GET_ID_KEY_DATA_REQ_VO_MAP))
.intercept(MethodCall.invoke(named(SUPER_METHOD_GET_ID_KEY_DATA_REQ_VO_MAP)).withArgument(0, 1));
int i = 0;
for (Topic topic: list) {
// 循环定义一个方法名字，拼接i来实现不同方法名
String methodName = String.format(METHOD_FORMAT_GET_ID_KEY_DATA_REQ_VO_MAP, topic.getId() + i);
// 定义方法是无返回值的，并且是public公共的
definition = definition.defineMethod(methodName, void.class, Modifier.PUBLIC)
// 声明参数，一个是我们定义的泛型，批量接收消息内容的，一个是kafka需要ack的对象
.withParameters(listConsumerRecordGeneric, TypeDescription.ForLoadedType.of(Acknowledgment.class))
// 转调 父类方法，因为@KafkaListener的方法固定两个入参都是kafka给我们的，我们转调父类方法需要三个参数
// 前两个和本方法参数一致，我们直接通过索引 0 和 1 直接传给父类，第三个需要传递一个固定值，我们声明一个常量传递过去
.intercept(MethodCall.invoke(named(SUPER_METHOD_RECEIVE_COMMON_MSG_BY_GEN_CODE)).with(
new MethodCall.ArgumentLoader.ForMethodParameter.Factory(0),
new MethodCall.ArgumentLoader.ForMethodParameter.Factory(1),
new MethodCall.ArgumentLoader.ForStackManipulation(systemIdConstantValue.toStackManipulation(), systemIdConstantValue.getTypeDescription()))
)
// 这里我们定义注解，注解类通过方法getKafkaListenerParam获取
.visit(new MemberAttributeExtension.ForMethod()
.annotateMethod(getKafkaListenerParam(fcpTopicConfigInfo))
.on(named(methodName)));
i++;
}
//
try (DynamicType.Unloaded<BaseDataConsumer> make = definition.make()) {
DynamicType.Loaded<BaseDataConsumer> load = make.load(getClass().getClassLoader());
if (SystemUtils.isWindows() && StrUtil.isNotBlank(gitLabConfig.getByteBuddyGenPath())) {
// 保存到当前目录
File file = new File(gitLabConfig.getByteBuddyGenPath());
// 通过dynamicType 将字节码保存到文件
load.saveIn(file);
}
BaseDataConsumer baseDataConsumer = load.getLoaded().getDeclaredConstructor().newInstance();
log.info("加载topic完毕，本次共加载size:{}", fcpTopicConfigInfoList.size());
return baseDataConsumer;
}
}
public AnnotationDescription getKafkaListenerParam(FcpTopicConfigInfo fcpTopicConfigInfo) {
String binlogId = fcpTopicConfigInfo.getBinlogId();
String topics = fcpTopicConfigInfo.getTopics();
String groupId = fcpTopicConfigInfo.getGroupId();
AssertUtils.isTrue(Objects.nonNull(fcpTopicConfigInfo.getAutoStartup()), "autoStartup 不能为空");
boolean autoStartup = BooleanUtil.toBoolean(fcpTopicConfigInfo.getAutoStartup().toString());
int concurrency = fcpTopicConfigInfo.getPartitionNum() / gitLabConfig.getMachineNum();
if (concurrency < 1) {
concurrency = 1;
}
String colonyName = fcpTopicConfigInfo.getColonyName();
AssertUtils.isTrue(StrUtil.isNotBlank(binlogId), "binlogId 不能为空");
AssertUtils.isTrue(StrUtil.isNotBlank(topics), "topics 不能为空");
AssertUtils.isTrue(StrUtil.isNotBlank(groupId), "groupId 不能为空");
AssertUtils.isTrue(StrUtil.isNotBlank(colonyName), "colonyName 不能为空");
log.info("kafka配置topic:{},concurrency:{},binlogId:{},colonyName:{}", fcpTopicConfigInfo.getTopics(), concurrency, binlogId, colonyName);
return AnnotationDescription
.Builder.ofType(KafkaListener.class)
.define("id", binlogId)
.defineArray("topics", topics.split(","))
.define("groupId", groupId)
.define("concurrency", String.valueOf(concurrency))
.define("autoStartup", String.valueOf(autoStartup))
.define("containerFactory", colonyName + "KafkaListenerContainerFactory")
.build();
}
}
Javassist
目前的缺陷就是方法参数不能是泛型，导致kafka接受消息，List<comsuer<String,String>>报错
@Configuration
public class BinlogKafkaConsumerConfig {
// 获取类池：生成class的
static ClassPool pool = ClassPool.getDefault();
// 制造类(需要告诉javassist类名)
static CtClass ctClass;
@Autowired
private FcpTopicConfigInfoDao fcpTopicConfigInfoDao;
static {
try {
pool.importPackage("com.common.enums.SyncTypeEnum");
ctClass = pool.get("com.consumer.BinlogByGenCodeConsumer");
} catch (Exception e) {
log.warn("获取类失败", e);
}
}
@Bean
public Object binlogKafkaConsumerGenObj() {
//启动随系统注册数据源
log.info("开始加载表中所有配置信息");
//try {
// doRun();
// Class<?> aClass = ctClass.toClass();
// return aClass.newInstance();
//} catch (Exception e) {
// log.warn("加载表中所有配置信息，报错", e);
// AssertUtils.throwError(e.getMessage());
//}
return new Object();
}
public void doRun() throws Exception {
List<FcpTopicConfigInfo> fcpTopicConfigInfoList = fcpTopicConfigInfoDao.listAllValidTopicConfigInfo();
if (CollectionUtils.isEmpty(fcpTopicConfigInfoList)) {
log.info("没有要加载的配置信息，return");
return;
}
log.info("开始加载topic，本次共需加载size:{}", fcpTopicConfigInfoList.size());
for (FcpTopicConfigInfo fcpTopicConfigInfo : fcpTopicConfigInfoList) {
log.info("开始加载topic,fcpTopicConfigInfo:{}", fcpTopicConfigInfo);
generateAddMethod(fcpTopicConfigInfo.getBinlogId(),
fcpTopicConfigInfo.getTopic(),
fcpTopicConfigInfo.getGroupId(),
fcpTopicConfigInfo.getConcurrency(),
BooleanUtil.toBoolean(fcpTopicConfigInfo.getAutoStartup().toString()),
fcpTopicConfigInfo.getColonyName());
}
log.info("加载topic完毕，本次共加载size:{}", fcpTopicConfigInfoList.size());
}
public static void main(String[] args) throws Exception {
generateAddMethod("id1", "1", "1", 1, false, "1");
}
public static void generateAddMethod(
String binlogId,
String topics,
String groupId,
Integer concurrency,
Boolean autoStartup,
String colonyName
) throws Exception {
AssertUtils.isTrue(StrUtil.isNotBlank(binlogId), "binlogId 不能为空");
AssertUtils.isTrue(StrUtil.isNotBlank(topics), "topics 不能为空");
AssertUtils.isTrue(StrUtil.isNotBlank(groupId), "groupId 不能为空");
AssertUtils.isTrue(Objects.nonNull(concurrency), "concurrency 不能为空");
AssertUtils.isTrue(Objects.nonNull(autoStartup), "autoStartup 不能为空");
AssertUtils.isTrue(StrUtil.isNotBlank(colonyName), "colonyName 不能为空");
String methodName = "receive";
CtMethod declaredMethod = ctClass.getDeclaredMethod(methodName);
ConstPool cp = declaredMethod.getMethodInfo().getConstPool();
ClassMap classMap = new ClassMap();
CtMethod copy = CtNewMethod.copy(declaredMethod, ctClass, classMap);
copy.setName(methodName + binlogId);
String body = "{" +
" log.info(\"========== BinlogDataConsumer ==========拉取数: {}\", String.valueOf($1.size()));" +
" long start = System.currentTimeMillis();" +
" this.dealKafkaData($1, SyncTypeEnum.BINLOG.getCode(), String.valueOf(\"" + binlogId + "\"));" +
" $2.acknowledge();" +
" long costTime = System.currentTimeMillis() - start;" +
" log.info(\"消费时间: {} ms, 平均每笔耗时: {} ms\", String.valueOf(costTime), String.valueOf(costTime / $1.size())); " +
" }";
copy.setBody(body);
MethodInfo methodInfo = copy.getMethodInfo();
// 方法添加注解
AnnotationsAttribute methodAnnotationsAttribute = new AnnotationsAttribute(cp, AnnotationsAttribute.visibleTag);
Annotation methodAnnotation = new Annotation("org/springframework/kafka/annotation/KafkaListener", cp);
methodAnnotation.addMemberValue("id", new StringMemberValue(binlogId, cp));
methodAnnotation.addMemberValue("topics", getTopicArrayMemberValue(topics, cp));
methodAnnotation.addMemberValue("groupId", new StringMemberValue(groupId, cp));
methodAnnotation.addMemberValue("concurrency", new StringMemberValue(concurrency.toString(), cp));
methodAnnotation.addMemberValue("autoStartup", new StringMemberValue(autoStartup.toString(), cp));
methodAnnotation.addMemberValue("containerFactory", new StringMemberValue(colonyName + "KafkaListenerContainerFactory", cp));
methodAnnotationsAttribute.addAnnotation(methodAnnotation);
methodInfo.addAttribute(methodAnnotationsAttribute);
// 将方法添加到类中
ctClass.addMethod(copy);
ctClass.writeFile("D:\\GitProject\\devops\\consumer\\src\\main\\java\\com\\consumer\\runner");
}
private static ArrayMemberValue getTopicArrayMemberValue(String topics, ConstPool cp) {
ArrayMemberValue arrayMemberValue = new ArrayMemberValue(cp);
String[] topicArray = topics.split(",");
MemberValue[] memberValue = new MemberValue[topicArray.length];
for (int i = 0; i < topicArray.length; i++) {
memberValue[i] = new StringMemberValue(topicArray[i], cp);
}
arrayMemberValue.setValue(memberValue);
return arrayMemberValue;
}
}
661
扫一扫
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
