hive添加IP解析的udf函数_java.lang.NoSuchMethodError: com.fasterxml.jackson.databind.node.ObjectNode.<init>...

最新推荐文章于 2024-03-21 21:00:31 发布

chuzhanjiao8628

最新推荐文章于 2024-03-21 21:00:31 发布

阅读量1k

点赞数

CC 4.0 BY-SA版权

文章标签： java 大数据

原文链接：https://my.oschina.net/u/2489618/blog/1624980

本文详细记录了在Hive中使用UDF函数解析IP地址遇到的问题及解决方案，包括代码编写、错误排查、依赖冲突处理及最终版本调整。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

这个问题着实是个大坑啊，使用mr解析用户ip没有问题，然后，想直接使用相同代码在hive中添加udf函数，不断地掉坑啊，踩坑踩了两天，总算是解决了。

1、编写代码

public class IP extends UDF{
    
    private static Logger logger = Logger.getLogger(IP.class);
    public static boolean enableFileWatch = false;
    private static int offset;
    private static int[] index = new int[256];
    private static ByteBuffer dataBuffer;
    private static ByteBuffer indexBuffer;
    private static Path ipFile = new Path("/file/IpParse.dat");
    private static Path maxMindFile = new Path("/hive_udf/GeoIP2-City.mmdb");
    private static DatabaseReader reader;
    private static InputStream in = null;
    private static ReentrantLock lock = new ReentrantLock();
    
//    public static void main(String[] args){
//        try {
//            IP ip = new IP();
//            System.out.println(ip.evaluate("121.12.12.12"));
//        } catch (Exception e) {
//            // TODO Auto-generated catch block
//            e.printStackTrace();
//        }
//    }
    
    
    public String evaluate(String ip)  {
        try{
            Configuration conf = new Configuration();
            InputStream in = FileSystem.getLocal(conf).open(maxMindFile);
            reader = new DatabaseReader.Builder(in).build();
            FSDataInputStream fin = null;
            fin = ipFile.getFileSystem(conf).open(ipFile);
            dataBuffer = ByteBuffer.allocate(Long.valueOf(fin.available()).intValue());
           
            int readBytesLength;
            byte[] chunk = new byte[4096];
            while (fin.available() > 0) {
                readBytesLength = fin.read(chunk);
                dataBuffer.put(chunk, 0, readBytesLength);
            }
            dataBuffer.position(0);
            int indexLength = dataBuffer.getInt();
            byte[] indexBytes = new byte[indexLength];
            dataBuffer.get(indexBytes, 0, indexLength - 4);
            indexBuffer = ByteBuffer.wrap(indexBytes);
            indexBuffer.order(ByteOrder.LITTLE_ENDIAN);
            offset = indexLength;

            int loop = 0;
            while (loop++ < 256) {
                index[loop - 1] = indexBuffer.getInt();
            }
            indexBuffer.order(ByteOrder.BIG_ENDIAN);
          
            //解析IP
            if (ip == null) {
                ip = "0.0.0.0";
            } else {
                String[] ips = ip.split("\\.", -1);
                if (ips.length != 4) {
                    ip = "0.0.0.0";
                }
            }
            int ip_prefix_value = new Integer(ip.substring(0, ip.indexOf(".")));
            long ip2long_value = ip2long(ip);
            int start = index[ip_prefix_value];
            int max_comp_len = offset - 1028;
            long index_offset = -1;
            int index_length = -1;
            byte b = 0;
            for (start = start * 8 + 1024; start < max_comp_len; start += 8) {
                if (int2long(indexBuffer.getInt(start)) >= ip2long_value) {
                    index_offset = bytesToLong(b, indexBuffer.get(start + 6), indexBuffer.get(start + 5), indexBuffer.get(start + 4));
                    index_length = 0xFF & indexBuffer.get(start + 7);
                    break;
                }
            }

            byte[] areaBytes;            
            dataBuffer.position(offset + (int) index_offset - 1024);
            areaBytes = new byte[index_length];
            dataBuffer.get(areaBytes, 0, index_length);
            

            String local = new String(areaBytes);
            String[] locals = local.split("\t", -1);
            if (locals.length < 5) {
                local += "\t";
            }
            if (!locals[0].equals("中国")) {
                try {
                    StringBuffer sb = new StringBuffer();
                    InetAddress addr = InetAddress.getByName(ip);
                    CityResponse city = reader.city(addr);
//                    logger.info(Thread.currentThread().getId());
                    sb.append(city.getCountry().getNames().get("zh-CN")).append("\t");
                    sb.append(city.getMostSpecificSubdivision().getName()).append("\t");
                    sb.append(city.getCity().getName()).append("\t");
                    sb.append("\t");
                    local = sb.toString();
                } catch (Exception e) {
                    e.printStackTrace();
                }
            }
            return local;
        }catch(Exception ex){
            ex.printStackTrace();
            return ex.getMessage();
        }        
    }


    private static long bytesToLong(byte a, byte b, byte c, byte d) {
        return int2long((((a & 0xff) << 24) | ((b & 0xff) << 16) | ((c & 0xff) << 8) | (d & 0xff)));
    }

    private static int str2Ip(String ip) {
        String[] ss = ip.split("\\.");
        int a, b, c, d;
        a = Integer.parseInt(ss[0]);
        b = Integer.parseInt(ss[1]);
        c = Integer.parseInt(ss[2]);
        d = Integer.parseInt(ss[3]);
        return (a << 24) | (b << 16) | (c << 8) | d;
    }

    private static long ip2long(String ip) {
        return int2long(str2Ip(ip));
    }

    private static long int2long(int i) {
        long l = i & 0x7fffffffL;
        if (i < 0) {
            l |= 0x080000000L;
        }
        return l;
    }
}

2、代码写完之后，使用本地运行和在linux下运行，都没有问题，但是放到hive中，就会报错。

Caused by: java.lang.NoSuchMethodError: com.fasterxml.jackson.databind.node.ObjectNode.<init>(Lcom/fasterxml/jackson/databind/node/JsonNodeFactory;Ljava/util/Map;)V
        at com.maxmind.db.Decoder.decodeMap(Decoder.java:285)
        at com.maxmind.db.Decoder.decodeByType(Decoder.java:154)
        at com.maxmind.db.Decoder.decode(Decoder.java:147)
        at com.maxmind.db.Decoder.decodeMap(Decoder.java:281)
        at com.maxmind.db.Decoder.decodeByType(Decoder.java:154)
        at com.maxmind.db.Decoder.decode(Decoder.java:147)
        at com.maxmind.db.Decoder.decode(Decoder.java:87)
        at com.maxmind.db.Reader.<init>(Reader.java:132)
        at com.maxmind.db.Reader.<init>(Reader.java:116)
        at com.maxmind.geoip2.DatabaseReader.<init>(DatabaseReader.java:35)
        at com.maxmind.geoip2.DatabaseReader.<init>(DatabaseReader.java:23)
        at com.maxmind.geoip2.DatabaseReader$Builder.build(DatabaseReader.java:129)
        at com.bankofamerica.gisds.ExtractEnterpriseData.ExtractEnterpriseDB(ExtractEnterpriseData.java:27)
        at com.package.name.App.evaluate(App.java:73)

3、解决问题

看到这个错误，考虑是jar冲突，因为在解析过程中需要使用hadoop的jar包引用，因此在所有引用中删除掉jackson_databind,无济于事，还是顽强报相同的错误。

<exclusions>
     <exclusion>
     <groupId>com.fasterxml.jackson.core</groupId>
     <artifactId>jackson-databind</artifactId>
     </exclusion>
     <exclusion>
     <groupId>com.fasterxml.jackson.core</groupId>
     <artifactId>jackson-annotations</artifactId>
     </exclusion>
</exclusions>

4、我引用的GeoIP2为2.9.0版本，这是问题的所在

<dependency>
      <groupId>com.maxmind.geoip2</groupId>
      <artifactId>geoip2</artifactId>
      <version>2.9.0</version>
      <exclusions>
                <exclusion>
                    <groupId>com.fasterxml.jackson.core</groupId>
                    <artifactId>jackson-databind</artifactId>
                </exclusion>
                <exclusion>
                    <groupId>com.fasterxml.jackson.core</groupId>
                    <artifactId>jackson-annotations</artifactId>
                </exclusion>
            </exclusions>
    </dependency>

5、将版本改为2.5.0，问题解决。

6、导入hive

delete jar /hive_udf/tt.jar;

add jar /hive_udf/tt.jar;

add file /hive_udf/GeoIP2-City.mmdb;

add file /hive_udf/IpParse.dat;

CREATE TEMPORARY FUNCTION pars_IP AS 'com.dewmobile.ParsIP.IP';

SELECT pars_IP('121.12.12.12');

7、运行：

8、pom.xml

<dependencies>		
		<dependency>
			<groupId>org.apache.hadoop</groupId>
			<artifactId>hadoop-mapreduce-client-core</artifactId>
			<version>2.3.0</version>
		</dependency>
		<dependency>
			<groupId>org.apache.hadoop</groupId>
			<artifactId>hadoop-common</artifactId>
			<version>2.3.0</version>
		</dependency>
		<dependency>
			<groupId>com.maxmind.geoip2</groupId>
			<artifactId>geoip2</artifactId>
			<version>2.5.0</version>
		</dependency>

		<dependency>
			<groupId>org.apache.hive</groupId>
			<artifactId>hive-exec</artifactId>
			<version>1.1.1</version>
		</dependency>
	</dependencies>
	<build>
		<plugins>			
			<plugin>
				<artifactId> maven-assembly-plugin </artifactId>
				<configuration>
					<descriptorRefs>
						<descriptorRef>jar-with-dependencies</descriptorRef>
					</descriptorRefs>
					<archive>
					</archive>
				</configuration>
				<executions>
					<execution>
						<id>make-assembly</id>
						<phase>package</phase>
						<goals>
							<goal>single</goal>
						</goals>
					</execution>
				</executions>
			</plugin>
		</plugins>
	</build>

转载于:https://my.oschina.net/u/2489618/blog/1624980