Spark 连接 Hbase 配置

最新推荐文章于 2025-05-22 11:11:17 发布

power0405hf

最新推荐文章于 2025-05-22 11:11:17 发布

阅读量3.2k

点赞数

CC 4.0 BY-SA版权

本文链接：https://blog.youkuaiyun.com/power0405hf/article/details/50118163

scala 同时被 3 个专栏收录

48 篇文章

订阅专栏

spark

34 篇文章

订阅专栏

HBase

14 篇文章

订阅专栏

这篇博客介绍了如何使用Spark的Scala API连接并操作HBase数据库，具体示例是对terminal_data_file1表进行计数操作。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

1.count

对表terminal_data_file1进行count：

package qyTest3

import org.apache.hadoop.hbase.HBaseConfiguration
import org.apache.hadoop.hbase.mapreduce.TableInputFormat
import org.apache.spark.{SparkContext, SparkConf}

/**
 * Created by root on 11/30/15.
 */
object hbaseRead {
  def main(args: Array[String]) {
    val conf = new SparkConf().setAppName("hbaseRead")
    conf.setMaster("spark://10.6.3.11:7077")
    conf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
    val sc = new SparkContext(conf)
    sc.addJar("/usr/qy/IdeaProjects/qyTest3/out/artifacts/qyTest3/qyTest3.jar")

    println("hello world")
    println("aa")
    val myConf = HBaseConfiguration.create()
    /* myConf.set("hbase.zookeeper.quorum", "master3,slave31,slave32")
     myConf.set("hbase.master", "master3:60000")*/
    myConf.set("hbase.zookeeper.quorum",
      "compute000,compute001,compute002,compute003,compute004," +
        "compute005,compute006,compute007,compute008,compute009,compute010," +
        "compute011,compute012,compute013,compute014,compute015,compute016," +
        "compute017,compute018,compute019,compute020,compute021,compute022," +
        "compute023,compute024,compute025,compute026,compute027,compute028," +
        "compute029,compute030,compute031,compute032,compute033,compute034," +
        "compute035,compute036,compute037,compute038")
    myConf.set("hbase.master", "10.10.10.10:60000")
    myConf.set("hbase.zookeeper.property.clientPort", "2181")
    myConf.set("hbase.defaults.for.version.skip", "true")
    myConf.set(TableInputFormat.INPUT_TABLE, "terminal_data_file1")
    myConf.set(TableInputFormat.SCAN_COLUMNS, "cf:auth_id") // cf:2Column

    val hbaseRDD = sc.newAPIHadoopRDD(myConf, classOf[TableInputFormat],
      classOf[org.apache.hadoop.hbase.io.ImmutableBytesWritable],
      classOf[org.apache.hadoop.hbase.client.Result])


    val count = hbaseRDD.count()

    println("hbase RDD Count:" + count)

    sc.stop()
  }
}