Hive+GenericUDF示例一

最新推荐文章于 2023-10-31 10:19:28 发布

yangxiangpao

最新推荐文章于 2023-10-31 10:19:28 发布

阅读量855

点赞数

分类专栏： Hive 文章标签： hive udf GenericUDF

Hive 专栏收录该内容

5 篇文章

订阅专栏

本文介绍如何使用通用GDF（GenericUDF）在Hive中处理包含复杂类型（如List，struct等）的数据，以实现行程记录的高效查询与结果整理。通过示例展示了如何编写通用UDF，对行程数据进行日期筛选与时间跨度整合，最终得到清晰的行程汇总结果。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

转载自 http://blog.youkuaiyun.com/wisgood/article/details/26169313

和UDF相比，通用GDF（GenericUDF）支持复杂类型（比如List，struct等）的输入和输出。

下面来看一个小示例。

Hive中whereme表中包含若干人的行程如下：

[plain]view plaincopy 
    
 
    
 A       2013-10-10 8:00:00      home  
 A       2013-10-10 10:00:00     Super Market  
 A       2013-10-10 12:00:00     KFC  
 A       2013-10-10 15:00:00     school  
 A       2013-10-10 20:00:00     home  
 A       2013-10-15 8:00:00      home  
 A       2013-10-15 10:00:00     park  
 A       2013-10-15 12:00:00     home  
 A       2013-10-15 15:30:00     bank  
 A       2013-10-15 19:00:00     home  

通过查询我们要得到如下结果：

[plain]view plaincopy 
    
 
    
 A   2013-10-10  08:00:00    home    10:00:00    Super Market  
 A   2013-10-10  10:00:00    Super Market    12:00:00    KFC  
 A   2013-10-10  12:00:00    KFC 15:00:00    school  
 A   2013-10-10  15:00:00    school  20:00:00    home  
 A   2013-10-15  08:00:00    home    10:00:00    park  
 A   2013-10-15  10:00:00    park    12:00:00    home  
 A   2013-10-15  12:00:00    home    15:30:00    bank  
 A   2013-10-15  15:30:00    bank    19:00:00    home  

1.编写GenericUDF.

[java]view plaincopy 
    
 
    
 package com.wz.udf;  
 import org.apache.hadoop.io.Text;  
 import org.apache.hadoop.io.LongWritable;  
 import org.apache.hadoop.io.IntWritable;  
 import org.apache.hadoop.io.FloatWritable;  
 import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;  
 import org.apache.hadoop.hive.ql.exec.UDFArgumentException;  
 import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException;  
 import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException;  
 import org.apache.hadoop.hive.ql.metadata.HiveException;  
 import org.apache.hadoop.hive.serde2.lazy.LazyString;  
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;  
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;  
 import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector;  
 import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;  
 import org.apache.hadoop.hive.serde2.objectinspector.StandardListObjectInspector;  
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;  
 import org.apache.hadoop.hive.serde2.objectinspector.StructField;  
 import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;  
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;  
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector;  
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.IntObjectInspector;  
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.FloatObjectInspector;  
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector;  
 import java.text.DateFormat;  
 import java.text.SimpleDateFormat;  
 import java.util.Date;   
 import java.util.Calendar;  
 import java.util.ArrayList;  
    
 public class helloGenericUDF extends GenericUDF {  
      输入变量定义  
      private ObjectInspector peopleObj;  
      private ObjectInspector timeObj;  
      private ObjectInspector placeObj;  
      //之前记录保存  
      String strPreTime = "";  
      String strPrePlace = "";   
      String strPrePeople = "";  
    
      @Override  
      //1.确认输入类型是否正确  
      //2.输出类型的定义  
      public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {  
           peopleObj = (ObjectInspector)arguments[0];  
           timeObj = (ObjectInspector)arguments[1];  
           placeObj = (ObjectInspector)arguments[2];  
           //输出结构体定义  
           ArrayList structFieldNames = new ArrayList();  
           ArrayList structFieldObjectInspectors = new ArrayList();  
           structFieldNames.add("people");  
       structFieldNames.add("day");  
           structFieldNames.add("from_time");  
           structFieldNames.add("from_place");  
           structFieldNames.add("to_time");  
           structFieldNames.add("to_place");  
    
           structFieldObjectInspectors.add( PrimitiveObjectInspectorFactory.writableStringObjectInspector );  
           structFieldObjectInspectors.add( PrimitiveObjectInspectorFactory.writableStringObjectInspector );  
           structFieldObjectInspectors.add( PrimitiveObjectInspectorFactory.writableStringObjectInspector );  
           structFieldObjectInspectors.add( PrimitiveObjectInspectorFactory.writableStringObjectInspector );  
       structFieldObjectInspectors.add( PrimitiveObjectInspectorFactory.writableStringObjectInspector );  
       structFieldObjectInspectors.add( PrimitiveObjectInspectorFactory.writableStringObjectInspector );  
   
           StructObjectInspector si2;  
           si2 = ObjectInspectorFactory.getStandardStructObjectInspector(structFieldNames, structFieldObjectInspectors);   
           return si2;  
      }  
    
      //遍历每条记录  
      @Override  
      public Object evaluate(DeferredObject[] arguments) throws HiveException{  
       LazyString LPeople = (LazyString)(arguments[0].get());  
       String strPeople = ((StringObjectInspector)peopleObj).getPrimitiveJavaObject( LPeople );  
   
       LazyString LTime = (LazyString)(arguments[1].get());  
       String strTime = ((StringObjectInspector)timeObj).getPrimitiveJavaObject( LTime );  
   
       LazyString LPlace = (LazyString)(arguments[2].get());  
       String strPlace = ((StringObjectInspector)placeObj).getPrimitiveJavaObject( LPlace );  
       
       Object[] e;     
       e = new Object[6];  
   
           try  
       {  
                 //如果是同一个人，同一天  
         if(strPrePeople.equals(strPeople) && IsSameDay(strTime) )  
         {  
                 e[0] = new Text(strPeople);  
                         e[1] = new Text(GetYearMonthDay(strTime));  
                 e[2] = new Text(GetTime(strPreTime));  
                 e[3] = new Text(strPrePlace);  
                 e[4] = new Text(GetTime(strTime));  
                 e[5] = new Text(strPlace);  
         }  
                 else  
                 {  
                 e[0] = new Text(strPeople);  
             e[1] = new Text(GetYearMonthDay(strTime));  
                 e[2] = new Text("null");  
                 e[3] = new Text("null");  
                 e[4] = new Text(GetTime(strTime));  
                 e[5] = new Text(strPlace);  
                 }  
           }  
           catch(java.text.ParseException ex)  
           {  
           }  
              
       strPrePeople = new String(strPeople);  
       strPreTime= new String(strTime);  
       strPrePlace = new String(strPlace);  
   
           return e;  
      }  
    
      @Override  
      public String getDisplayString(String[] children) {  
           assert( children.length>0 );  
    
           StringBuilder sb = new StringBuilder();  
           sb.append("helloGenericUDF(");  
           sb.append(children[0]);  
           sb.append(")");  
    
           return sb.toString();  
      }  
   
      //比较相邻两个时间段是否在同一天  
      private boolean IsSameDay(String strTime) throws java.text.ParseException{     
      if(strPreTime.isEmpty()){  
          return false;  
          }  
          String curDay = GetYearMonthDay(strTime);  
          String preDay = GetYearMonthDay(strPreTime);  
      return curDay.equals(preDay);  
      }  
   
      //获取年月日  
      private String GetYearMonthDay(String strTime)  throws java.text.ParseException{  
          DateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");  
          Date curDate = df.parse(strTime);  
      df = new SimpleDateFormat("yyyy-MM-dd");  
          return df.format(curDate);  
      }  
   
      //获取时间  
      private String GetTime(String strTime)  throws java.text.ParseException{  
          DateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");  
          Date curDate = df.parse(strTime);  
          df = new SimpleDateFormat("HH:mm:ss");  
          return df.format(curDate);  
      }  
 }  

2.在Hive里面创建两张表，一张包含结构体的表保存执行GenericUDF查询后的结果，另外一张用于保存最终结果.

[plain]view plaincopy 
    
 
    
 hive> create table whereresult(people string,day string,from_time string,from_place string,to_time string,to_place string);  
 OK  
 Time taken: 0.287 seconds  
 hive> create table tmpResult(info struct<people:string,day:string,from_time:str>ing,from_place:string,to_time:string,to_place:string>);  
 OK  
 Time taken: 0.074 seconds  

3.执行GenericUDF查询,得到最终结果。

[plain]view plaincopy 
    
 
    
 hive> insert overwrite table tmpResult select hellogenericudf(whereme.people,whereme.time,whereme.place) from whereme;  
 hive> insert overwrite table whereresult select info.people,info.day,info.from_time,info.from_place,info.to_time,info.to_place from tmpResult where info.from_time<>'null';  
 Total MapReduce jobs = 2  
 Launching Job 1 out of 2  
 Number of reduce tasks is set to 0 since there's no reduce operator  
 Starting Job = job_201312022129_0006, Tracking URL = http://localhost:50030/jobdetails.jsp?jobid=job_201312022129_0006  
 Kill Command = /home/wangzhun/hadoop/hadoop-0.20.2/bin/../bin/hadoop job  -Dmapred.job.tracker=localhost:9001 -kill job_201312022129_0006  
 Hadoop job information for Stage-1: number of mappers: 1; number of reducers: 0  
 2013-12-02 22:48:40,733 Stage-1 map = 0%,  reduce = 0%  
 2013-12-02 22:48:49,825 Stage-1 map = 100%,  reduce = 0%  
 2013-12-02 22:48:52,869 Stage-1 map = 100%,  reduce = 100%  
 Ended Job = job_201312022129_0006  
 Ended Job = -383357832, job is filtered out (removed at runtime).  
 Moving data to: hdfs://localhost:9000/tmp/hive-root/hive_2013-12-02_22-48-24_406_2701579121398466034/-ext-10000  
 Loading data to table default.whereresult  
 Deleted hdfs://localhost:9000/user/hive/warehouse/whereresult  
 Table default.whereresult stats: [num_partitions: 0, num_files: 1, num_rows: 0, total_size: 346, raw_data_size: 0]  
 8 Rows loaded to whereresult  
 MapReduce Jobs Launched:   
 Job 0: Map: 1   HDFS Read: 420 HDFS Write: 346 SUCESS  
 Total MapReduce CPU Time Spent: 0 msec  
 OK  
 Time taken: 29.098 seconds  
 hive> select * from whereresult;  
 OK  
 A   2013-10-10  08:00:00    home    10:00:00    Super Market  
 A   2013-10-10  10:00:00    Super Market    12:00:00    KFC  
 A   2013-10-10  12:00:00    KFC 15:00:00    school  
 A   2013-10-10  15:00:00    school  20:00:00    home  
 A   2013-10-15  08:00:00    home    10:00:00    park  
 A   2013-10-15  10:00:00    park    12:00:00    home  
 A   2013-10-15  12:00:00    home    15:30:00    bank  
 A   2013-10-15  15:30:00    bank    19:00:00    home  
 Time taken: 0.105 seconds  