spark读取Hbase

最新推荐文章于 2024-12-26 21:24:47 发布

原创最新推荐文章于 2024-12-26 21:24:47 发布 · 498 阅读

0 ·

CC 4.0 BY-SA版权

spark 同时被 2 个专栏收录

18 篇文章

订阅专栏

hadoop

1 篇文章

订阅专栏

本文介绍了两种使用Apache Spark从HBase中读取数据的方法。第一种方法通过配置HBase表名并直接读取所有数据；第二种方法则利用Scan对象设定起始和结束行键，实现更精确的数据读取范围，提高了数据处理效率。

方式一

package com.bupt.spark.hbase.readhbase

import org.apache.hadoop.hbase.HBaseConfiguration
import org.apache.hadoop.hbase.io.ImmutableBytesWritable
import org.apache.hadoop.hbase.mapreduce.{TableInputFormat, TableOutputFormat}
import org.apache.hadoop.hbase.client.Result
import org.apache.hadoop.hbase.util.Bytes
import org.apache.spark.{SparkConf, SparkContext}

object SparkHbaseScan {
  def main(args: Array[String]): Unit = {
    val conf = new SparkConf().setAppName("aa").setMaster("local[*]")
    val sc = new SparkContext(conf)
    val configuration = HBaseConfiguration.create()
    configuration.set(TableInputFormat.INPUT_TABLE, "student")
    val hbaseRDD = sc.newAPIHadoopRDD(configuration,classOf[TableInputFormat],classOf[ImmutableBytesWritable],classOf[Result])
    println(hbaseRDD.getNumPartitions)
    hbaseRDD.foreach(t => {
      val cf = Bytes.toBytes("info")
      val cn = Bytes.toBytes("count")
      val rowkey = Bytes.toString(t._1.get())
      val count = Bytes.toString(t._2.getValue(cf, cn))
      println(s"rowKey:${rowkey},count:${count}")

    })

  }

}

方式二：

package com.bupt.spark.hbase.readhbase

import org.apache.hadoop.hbase.HBaseConfiguration
import org.apache.hadoop.hbase.client.{Result, Scan}
import org.apache.hadoop.hbase.io.ImmutableBytesWritable
import org.apache.hadoop.hbase.mapreduce.{TableInputFormat, TableMapReduceUtil}
import org.apache.hadoop.hbase.util.Bytes
import org.apache.spark.{SparkConf, SparkContext}

object SparkHbaseScanStartEnd {
  def main(args: Array[String]): Unit = {
    val conf = new SparkConf().setAppName("aa").setMaster("local[*]")
    val sc = new SparkContext(conf)
    val configuration = HBaseConfiguration.create()
    //设置scan对象 限制起止行
    val scan = new Scan()
    scan.addFamily(Bytes.toBytes("info"))
    scan.setBatch(1000)
    scan.setCacheBlocks(false)
    scan.setStartRow(Bytes.toBytes("spark_part"))
    scan.setStopRow(Bytes.toBytes("spark_part|"))
    configuration.set(TableInputFormat.INPUT_TABLE, "student")
    configuration.set(TableInputFormat.SCAN,TableMapReduceUtil.convertScanToString(scan))
    val hbaseRDD = sc.newAPIHadoopRDD(configuration, classOf[TableInputFormat], classOf[ImmutableBytesWritable], classOf[Result])
    println(hbaseRDD.getNumPartitions)
    hbaseRDD.foreach(t => {
      val cf = Bytes.toBytes("info")
      val cn = Bytes.toBytes("count")
      val rowkey = Bytes.toString(t._1.get())
      val count = Bytes.toString(t._2.getValue(cf, cn))
      println(s"rowKey:${rowkey},count:${count}")

    })
  }
}