Protocol Buffers动态解析自描述文件和Any类型

kevin luan

已于 2022-04-03 16:55:27 修改

阅读量1.3k

点赞数

分类专栏： java 文章标签： java

于 2022-03-16 23:22:13 首次发布

java 专栏收录该内容

60 篇文章

订阅专栏

本文介绍了Google的ProtocolBuffers数据序列化工具，详细解析了如何生成和处理.proto自描述文件，并展示了Java代码生成及解析过程。同时，文章通过实例演示了如何使用Any类型进行消息编码和解码，强调了其在不同消息类型转换中的作用。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

Protocol Buffers，是Google公司开发的一种数据描述语言，类似于XML能够将结构化数据序列化，可用于数据存储、通信协议等方面。

解析proto自描述文件

定义proto文件定义：

syntax = "proto3";

package model;
option java_package = "com.lyh.proto.entity";
option java_outer_classname = "ModelOutter";
message Entry {
  int64 id = 1;
  Meta meta = 3;
};
message Meta {
  int32 id = 1;
  string name = 2;
}

根据proto定义生成Java代码和自描述文件

#生成描述文件（提示：执行该命令需要本地安装proto程序）
protoc -I=./ --descriptor_set_out=proto.desc --java_out=../java/ model.proto

解析字描述文件处理

import static java.lang.invoke.MethodType.methodType;
import com.google.protobuf.DescriptorProtos.FileDescriptorProto;
import com.google.protobuf.DescriptorProtos.FileDescriptorSet;
import com.google.protobuf.Descriptors.Descriptor;
import com.google.protobuf.Descriptors.FileDescriptor;
import com.google.protobuf.Message;
import java.lang.invoke.MethodHandle;
import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;

/**
 * google定义自描述文件
 * <pre>
 * https://developers.google.com/protocol-buffers/docs/techniques?hl=zh-CN#self-description
 * </pre>
 */
@SuppressWarnings("unchecked")
public class ProtoDescParserFactory {

    //class name in proto file
    private static Map<String, MethodHandle> PARSE_METHODS_PROTO           = new HashMap<>();
    // class name in java file
    private static Map<String, MethodHandle> PARSE_METHODS_JAVA            = new HashMap<>();
    //class name in java file
    private static Map<String, MethodHandle> DEFAULT_INSTANCE_METHODS_JAVA = new HashMap<>();
    private static Map<String, Message>      DEFAULT_MESSAGE_INSTANCE      = new ConcurrentHashMap<>();

    static {
        try {
            final FileDescriptorSet descriptorSet = FileDescriptorSet.parseFrom(ProtoDescFile.getProtoDescFile());
            final List<FileDescriptor> resolveFDs = new ArrayList<>();
            for (final FileDescriptorProto fdp : descriptorSet.getFileList()) {
                final FileDescriptor[] dependencies = new FileDescriptor[resolveFDs.size()];
                resolveFDs.toArray(dependencies);
                final FileDescriptor fd = FileDescriptor.buildFrom(fdp, dependencies);
                resolveFDs.add(fd);
                for (final Descriptor descriptor : fd.getMessageTypes()) {
                    final String className = getClassName(fdp, fd, descriptor);
                    final Class<?> clazz = Class.forName(className);
                    final MethodHandle parseFromHandler = MethodHandles.lookup().findStatic(clazz, "parseFrom",
                        methodType(clazz, byte[].class));
                    final MethodHandle getInstanceHandler = MethodHandles.lookup().findStatic(clazz,
                        "getDefaultInstance", methodType(clazz));
                    System.out.println(String.format("注册MessageProtoName：%s", descriptor.getFullName()));
                    PARSE_METHODS_PROTO.put(descriptor.getFullName(), parseFromHandler);
                    System.out.println(String.format("注册MessageJavaName：%s", className));
                    PARSE_METHODS_JAVA.put(className, parseFromHandler);
                    System.out.println(String.format("注册getDefaultInstance：%s", className));
                    DEFAULT_INSTANCE_METHODS_JAVA.put(className, getInstanceHandler);
                    registerProtobufSerializer(className, (Message) getInstanceHandler.invoke());
                }
            }
        } catch (final Throwable t) {
            t.printStackTrace(); // NOPMD
        }
    }

    private static String getClassName(FileDescriptorProto fdp, FileDescriptor fd, Descriptor descriptor) {
        final String className;
        String packageName = fd.getOptions().getJavaPackage();
        if (fdp.getOptions().hasJavaOuterClassname()) {
            className = packageName + "." + fdp.getOptions().getJavaOuterClassname() + "$" + descriptor.getName();
        } else if (fdp.getOptions().hasJavaMultipleFiles()) {
            className = packageName + "." + descriptor.getName();
        } else {
            String protoFileName = fdp.getName().substring(0, fdp.getName().length() - 6);
            char[] chats = protoFileName.toCharArray();
            if (Character.isLowerCase(chats[0])) {
                chats[0] = Character.toUpperCase(chats[0]);
            }//TODO 这里只是简单处理了首字母大写转化处理
            className = packageName + "." + new String(chats) + "$" + descriptor.getName();
        }
        return className;
    }

    private static void registerProtobufSerializer(String className, Message message) {
        DEFAULT_MESSAGE_INSTANCE.putIfAbsent(className, message);
    }

    public static void load() {
        if (PARSE_METHODS_JAVA.isEmpty() || PARSE_METHODS_PROTO.isEmpty() || DEFAULT_INSTANCE_METHODS_JAVA.isEmpty()) {
            throw new IllegalStateException("Parse protocol file failed.");
        }
    }

    public static <T extends Message> T getDefaultInstance(final String className) {
        final MethodHandle handle = DEFAULT_INSTANCE_METHODS_JAVA.get(className);
        if (handle == null) {
            throw new MessageClassNotFoundException(className + " not found");
        }
        try {
            return (T) handle.invoke();
        } catch (Throwable t) {
            throw new SerializationException("getDefaultInstance(" + className + ") ERROR", t);
        }
    }

    public static <T extends Message> T newMessageByJavaClassName(final String className, final byte[] bs) {
        final MethodHandle handle = PARSE_METHODS_JAVA.get(className);
        if (handle == null) {
            throw new MessageClassNotFoundException(className + " not found");
        }
        try {
            return (T) handle.invoke(bs);
        } catch (Throwable t) {
            throw new SerializationException("newMessageByJavaClassName(className:" + className + ") ERROR", t);
        }
    }

    public static <T extends Message> T newMessageByProtoClassName(final String className, final byte[] bs) {
        final MethodHandle handle = PARSE_METHODS_PROTO.get(className);
        if (handle == null) {
            throw new MessageClassNotFoundException(className + " not found");
        }
        try {
            return (T) handle.invoke(bs);
        } catch (Throwable t) {
            throw new SerializationException("newMessageByProtoClassName(protoName:" + className + ")", t);
        }
    }
}

public class ProtoDescFile {
    public static InputStream getProtoDescFile() {
        //提示：生成的自描述文件可以随项目一起发布也可以配置在远程服务器
        return ProtoDescFile.class.getResourceAsStream("/proto.desc");
    }
}

测试用例

    @Test
    public void testMeta() {
        String protoClassName = ModelOutter.Meta.class.getName();
        Message metaMsg = ProtoDescParserFactory.getDefaultInstance(protoClassName);
        Assert.assertTrue(metaMsg == ModelOutter.Meta.getDefaultInstance());
        Meta meta = Meta.newBuilder().setId(100).build();
        //根据Java类名解析
        Message message1 = ProtoDescParserFactory.newMessageByJavaClassName(protoClassName, meta.toByteArray());
        Assert.assertEquals(message1.toString(), meta.toString());
        //根据proto文件描述解析
        Message message2 = ProtoDescParserFactory.newMessageByProtoClassName("model.Meta", meta.toByteArray());
        Assert.assertEquals(message2.toString(), meta.toString());
    }

Any类型编码&解码

定义proto 文件

syntax = "proto3";

package model;

import "google/protobuf/any.proto";
option java_package = "com.lyh.proto.msg";
option java_outer_classname = "MsgOuter";
option java_multiple_files = true;

message MsgDatagram {
  string topic = 1;
  int64 send_time = 2;
  google.protobuf.Any datagram = 3;
};
message OrderMsg{
  int64 order_id = 1;
  int64 create_time = 2;
  uint32 status = 3;
}
message Goods{
  int64 id = 1;
  string name = 2;
  int64 spu = 3;
  int64 sku = 4;
}

#生成JAVA代码
protoc -I=./ --java_out=../java/ msg.proto

调用示例

@Test
    public void testGoods() throws InvalidProtocolBufferException {
        MsgDatagram msgDatagram = MsgDatagram.newBuilder().setTopic("topic://a.b.c")
                .setSendTime(System.currentTimeMillis()).build();
        Goods rawGoods = Goods.newBuilder().setId(200).setSku(1).setSpu(2).build();
        //将Goods类型对象编码到Any类型
        Any any = Any.pack(rawGoods, "www.china.com");
        msgDatagram = msgDatagram.toBuilder().setDatagram(any).build();
        System.out.println(msgDatagram.toString());
        byte[] data = msgDatagram.toByteArray();
        msgDatagram = MsgDatagram.parseFrom(data);
        //将Any类型解码到目标类型
        Goods goods = msgDatagram.getDatagram().unpack(Goods.class);
        Assert.assertEquals(rawGoods.toString(), goods.toString());
    }

其实Any类型本身也就是普通Message，查看Any proto定义

message Any {
  //类型
  string type_url = 1;
  // Must be a valid serialized protocol buffer of the above specified type.
  bytes value = 2;
}

完整代码地址

proto-test/src/test/java/com/lyh/proto/api/TestAnyMessage.java · 在路上/lyh-api - Gitee.comhttps://gitee.com/kevinLuan/lyh-api/blob/master/proto-test/src/test/java/com/lyh/proto/api/TestAnyMessage.java