Spark读取Hbase

object WriteHbase {
  def main(args: Array[String]): Unit = {
    //获取conf
    val conf=HBaseConfiguration.create()
    val tablenamein = args(0)
    val tablenameout = args(1)
    //设置读取的表
    conf.set(TableInputFormat.INPUT_TABLE,tablenamein)
    //设置写入的表
    conf.set(TableOutputFormat.OUTPUT_TABLE,tablenameout)

    val sparkConf=new SparkConf()
    sparkConf.setAppName("read and write for hbase ")
    sparkConf.setMaster("local[3]")

    val sc=new SparkContext(sparkConf)

    //为job指定输出格式和输出表名

    val newAPIJobConfiguration1 = Job.getInstance(conf)
    newAPIJobConfiguration1.getConfiguration().set(TableOutputFormat.OUTPUT_TABLE, tablenameout)
    newAPIJobConfiguration1.setOutputFormatClass(classOf[TableOutputFormat[ImmutableBytesWritable]])


    //全量读取hbase表
    val rdd=sc.newAPIHadoopRDD(conf,classOf[TableInputFormat]
      ,classOf[ImmutableBytesWritable]
      ,classOf[Result]
    )

    //过滤空数据，然后对每一个记录做更新，并转换成写入的格式
    val final_rdd= rdd.map(forDatas)

    //转换后的结果，再次做过滤
    val save_rdd=final_rdd.filter(checkNull)

    //最终在写回hbase表
    final_rdd.saveAsNewAPIHadoopDataset(newAPIJobConfiguration1.getConfiguration)
    sc.stop()
  }
  //作用：过滤掉空列簇的数据
  def checkNotEmptyKs(f:((ImmutableBytesWritable,Result))):Boolean={
    val r=f._2
    val rowkey=Bytes.toString(r.getRow)
    val map:scala.collection.mutable.Map[Array[Byte],Array[Byte]]= r.getFamilyMap(Bytes.toBytes("f")).asScala
    val map1:scala.collection.mutable.Map[Array[Byte],Array[Byte]]= r.getFamilyMap(Bytes.toBytes("h")).asScala
    val map2:scala.collection.mutable.Map[Array[Byte],Array[Byte]]= r.getFamilyMap(Bytes.toBytes("mtdt")).asScala



    if(map.isEmpty || map1.isEmpty || map2.isEmpty)  false else true
  }
  //作用：读取每一条数据，做update后，在转化成写入操作
  def forDatas(f: (ImmutableBytesWritable,Result)): (ImmutableBytesWritable,Put)={
    val r=f._2 //获取Result
    val put:Put=new Put(r.getRow)
    val ks=Bytes.toBytes("f")
    val ks1=Bytes.toBytes("h")
    val ks2=Bytes.toBytes("mtdt")

    val map:scala.collection.mutable.Map[Array[Byte],Array[Byte]]= r.getFamilyMap(ks).asScala
    map.foreach(kv=>{
    val kid= Bytes.toString(kv._1)
    var value=Bytes.toString(kv._2)
      put.add(ks,kv._1,Bytes.toBytes(value))
    }
    )
    val map1:scala.collection.mutable.Map[Array[Byte],Array[Byte]]= r.getFamilyMap(ks2).asScala
    map1.foreach(kv=>{
    val kid= Bytes.toString(kv._1)
      var value=Bytes.toString(kv._2)
      put.add(ks2,kv._1,Bytes.toBytes(value))
    }
    )
    val map2:scala.collection.mutable.Map[Array[Byte],Array[Byte]]= r.getFamilyMap(ks1).asScala
    map2.foreach(kv=>{
    val kid= Bytes.toString(kv._1)
      var value=Bytes.toString(kv._2)
      put.add(ks2,kv._1,Bytes.toBytes(value))
    }
    )
    if(put.isEmpty)  null  else (new ImmutableBytesWritable(),put)

  }
  //checkNull 作用：过滤最终结果里面的null数据
  def checkNull(f:((ImmutableBytesWritable,Put))):Boolean={
    if(f==null)  false  else true
  }

}