spark scala code demo for def class etc.

Spark大数据处理实践

最新推荐文章于 2023-12-14 18:02:27 发布

原创最新推荐文章于 2023-12-14 18:02:27 发布 · 457 阅读

0 ·

CC 4.0 BY-SA版权

spark 专栏收录该内容

4 篇文章

订阅专栏

本文介绍了一个使用Apache Spark进行大数据处理的应用实例。通过定义特定的数据处理类NF和对象NAnalysis，实现了对枚举列表数据的并行处理。该过程包括初始化SparkContext、数据映射、缓存、收集和打印结果等步骤。

a demo for code reference:

import org.apache.spark.SparkContext
import java.util.HashMap
import org.apache.spark.rdd.RDD

class NF(hv: ClassA) extends Serializable {

  var powerMap: HashMap[String, Integer] = null
  var pf2: HashMap[String, Integer] = null;
  var sheddingMap: HashMap[String, java.lang.Double] = null;

  def nf(str: String): (HashMap[String, Integer]) =
    {
      val retmap = new HashMap[String, Integer]()

      

      retmap
    }

  def shedNf(str: String): (HashMap[String, Object]) =
    {
      val arr = str.split("`")
      val singalMap = new HashMap[String, Array[Int]]()
      val arrstrInt = arr(1).split("&")
      val arrInt: Array[Int] = new Array[Int](arrstrInt.length)
      val shedMap: HashMap[String, Object] = method()
      shedMap
    }

  def indexNf(str: String): (HashMap[String, java.lang.Double]) =
    {
     
     indexMap
    }

}

object NAnalysis {

  def main(args: Array[String]): Unit = {
    val start = System.currentTimeMillis();
    val sc = new SparkContext("spark://ip:7077", "ntest", System.getenv("SPARK_HOME"), SparkContext.jarOfClass(this.getClass))
    hv.init(args(0).toInt)
    val enumList = hv.getElementEnumList();
    enumList.size()
    val arr = new Array[String](enumList.size());

    val keyset = enumList.keySet();
    val keyIter = keyset.iterator();
    var i = 0;
    while (keyIter.hasNext()) {
      val strkey = keyIter.next();
      val strvalue = enumList.get(strkey);

      var temp = ""
      for (j <- 0 to strvalue.length - 1) {
        if (j == (strvalue.length - 1)) {
          temp = temp + strvalue(j)
        } else {
          temp = temp + strvalue(j) + "&"
        }
      }
      arr(i) = strkey + "`" + temp
      i = i + 1
    }

    val nf: NF = new NF(hv);
    val dataset1: RDD[(String)] = sc.parallelize(arr, 8);
    dataset1.cache()
    val powerRdd = dataset1;
    val shedRdd = dataset1;
    val indexRddC = dataset1;

   
    val dataset2 = powerRdd.map(nf.nf)
    val powerarr = dataset2.collect();
    val powerMap: HashMap[String, Integer] = new HashMap[String, Integer]();
    for (z <- 0 to (powerarr.size - 1)) {
      powerMap.putAll(powerarr(z))
    }
    
       val powerset=powerMap.keySet();
       val powerIt=powerset.iterator()
       while(powerIt.hasNext())
        {
         val powerkey=powerIt.next()
         val powervalue=powerMap.get(powerkey)
         println("power:\t"+powerkey+"\t"+powervalue)
     }
     val end = System.currentTimeMillis();

    println("total time is:" + (end - start) / 1000)
  }

}