ZZJXP的博客

checkpoint

缓存cache

mapPartitions

<dependency> <groupId>org.apache.spark</groupId> <artifactId>spark-mllib_2.11</artifactId> <version>2.4.5</version></dependency>package com.shujia.mllibimport org.apache.spark.ml.{featur..

package com.shujia.streamingimport org.apache.spark.sql.streaming.OutputModeimport org.apache.spark.sql.{DataFrame, SaveMode, SparkSession}object Demo05StructuredStreaming { def main(args: Array[String]): Unit = { //创建SparkSession val spar.

package com.shujia.streamingimport org.apache.spark.broadcast.Broadcastimport org.apache.spark.sql.SparkSessionimport org.apache.spark.streaming.dstream.{DStream, ReceiverInputDStream}import org.apache.spark.streaming.{Durations, StreamingContext}...

package com.shujia.streamingimport org.apache.spark.sql.SparkSessionimport org.apache.spark.streaming.dstream.DStreamimport org.apache.spark.streaming.{Durations, StreamingContext}object Demo03Window { def main(args: Array[String]): Unit = { /.

package com.shujia.coreimport com.shujia.core.Demo10Join.Studentimport org.apache.spark.{SparkConf, SparkContext}import org.apache.spark.rdd.RDDobject Demo16Action { def main(args: Array[String]): Unit = { //常见的Action算子 //foreach take col.

package com.shujia.streamingimport org.apache.spark.sql.SparkSessionimport org.apache.spark.streaming.dstream.{DStream, ReceiverInputDStream}import org.apache.spark.streaming.{Durations, StreamingContext}object Demo01WordCountOnStreaming { d...

<dependency> <groupId>org.apache.spark</groupId> <artifactId>spark-streaming_2.11</artifactId> <version>2.4.5</version></dependency>package com.shujia.streamingimport org.apache.spa...

aggregateByKey

package com.shujia.coreimport org.apache.spark.rdd.RDDimport org.apache.spark.{SparkConf, SparkContext}object Demo11Cartesian { def main(args: Array[String]): Unit = { //创建Spark Context val conf: SparkConf = new SparkConf() conf.setAppN.

package com.shujia.coreimport org.apache.spark.{SparkConf, SparkContext}import org.apache.spark.rdd.RDDobject Demo09Union { def main(args: Array[String]): Unit = { //创建Spark Context val conf: SparkConf = new SparkConf() conf.setAppName(.

package com.shujia.sqlimport org.apache.spark.sql.expressions.Windowimport org.apache.spark.sql.{DataFrame, SparkSession}object Demo06SparkOnHive { def main(args: Array[String]): Unit = { /** * 通过enableHiveSupport()可以开启Hive的支持 * 需要在po.

package com.shujia.sqlimport org.apache.spark.sql.expressions.Windowimport org.apache.spark.sql.{DataFrame, Dataset, Row, SparkSession}object Demo04DSL { def main(args: Array[String]): Unit = { val spark: SparkSession = SparkSession .buil.

公司代码,年度,1月-------------------------12月的收入金额burk,year,tsl01,tsl02,tsl03,tsl04,tsl05,tsl06,tsl07,tsl08,tsl09,tsl10,tsl11,tsl12853101,2010,100200,25002,19440,20550,14990,17227,40990,28778,19088,29889,10990,20990853101,2011,19446,20556,14996,17233,40996,2..

package com.shujia.sqlimport org.apache.spark.sql.{DataFrame, Dataset, Row, SparkSession}object Demo04DSL { def main(args: Array[String]): Unit = { val spark: SparkSession = SparkSession .builder() .appName("Demo04DSL") .master.

package com.shujia.sqlimport org.apache.spark.sql.{DataFrame, SparkSession}object Demo04DSL { def main(args: Array[String]): Unit = { val spark: SparkSession = SparkSession .builder() .appName("Demo04DSL") .master("local") .

package com.shujia.sqlimport org.apache.spark.sql.{DataFrame, SaveMode, SparkSession}object Demo03SourceAPI { //Spark SQL中常见的DataSourceAPI def main(args: Array[String]): Unit = { val spark: SparkSession = SparkSession .builder() .a.

<dependency> <groupId>org.apache.spark</groupId> <artifactId>spark-sql_2.11</artifactId> <version>2.4.5</version></dependency>package com.shujia.sqlimport org.apache.spark.SparkContext...

package com.shujia.coreimport org.apache.spark.rdd.RDDimport org.apache.spark.{SparkConf, SparkContext}object Demo04FlatMap { def main(args: Array[String]): Unit = { //创建Spark Context val conf: SparkConf = new SparkConf() conf.setAppNam.

[root@master spark-2.4.5]# cd conf/[root@master conf]# lsdocker.properties.template slaves.templatefairscheduler.xml.template spark-defaults.conf.templatelog4j.properties.template spark-env.shmetrics.properties.template spark-env.sh.template..

package com.shujia.coreimport org.apache.spark.rdd.RDDimport org.apache.spark.{SparkConf, SparkContext}object Demo02WordCountOnYarn { /** * 将程序提交到Yarn上的流程： * 1、setMaster不能指定为local本地方式运行 * 2、将输入输出路径换成HDFS的路径 * 3、将程序打成jar包(如果运行时出现依赖问题，可以添.

q

package com.shujia.coreimport org.apache.spark.rdd.RDDimport org.apache.spark.{SparkConf, SparkContext}object Demo01WordCount { def main(args: Array[String]): Unit = { //初始化Spark环境 //创建Spark配置对象 val conf: SparkConf = new SparkConf() .

q

<dependencies> <dependency> <groupId>org.scala-lang</groupId> <artifactId>scala-library</artifactId> <version>2.11.12</version> </dependency> <dependency> ..