Spark读取hdfs文件并写入hive表中

瞎胡侃

于 2020-07-15 19:18:39 发布

阅读量3.1k

点赞数 1

分类专栏：大数据文章标签： spark hive hdfs scala

本文链接：https://blog.youkuaiyun.com/zl18310999566/article/details/107368467

版权

package com.job

import org.apache.commons.cli.{BasicParser, CommandLine, Options}
import org.apache.spark.sql.{Row, SparkSession}
import org.apache.spark.sql.types.{StringType, StructType}

case class Transfer2HiveConfig(query_day: String)

object Transfer2HiveJob {
  val QUERY_DAY = "query-day"

  def parseAsConfig(commandLine: CommandLine): Option[Transfer2HiveConfig] = {
    val queryDay = commandLine.getOptionValue(QUERY_DAY)
    Some(Transfer2HiveConfig(queryDay))
  }

  def main(args: Array[String]): Unit = {
    val parser = new BasicParser()
    val options = new Options()
    options.addOption("qd", QUERY_DAY, true, "query day")
    val commandLine = parser.parse(options, args)

    parseAsConfig(commandLine) match {
      case Some(config) =>
        val spark = SparkSession.builder()
          .appName(s"app-name")
          .config("spark.serializer", "org.ap