ZZJXP的博客

违法行为分析

事故指标统计

数据准备工作

checkpoint

缓存cache

mapPartitions

Kafka的使用

Kafka的搭建

常用的算子

自定义MySQL Source

Spark跟Flink的区别、常见的Source

Flink开发环境搭建及WordCount

位置数据融合表

在时间轴上进行聚类

位置数据融合表

位置数据融合表

开发环境搭建

DataX安装

CRM数据采集

[root@master soft]# vim /etc/profilealias soft='cd /usr/local/soft/'[root@master soft]# source /etc/profile[root@master soft]# soft[root@master soft]# cd ~[root@master ~]# pwd/root[root@master ~]# soft[root@master soft]# pwd/usr/local/soft[.

[ods@master ~]$ mkdir ctyun[ods@master ~]$ lsctyun students.txt[ods@master ~]$ cd ctyun/[ods@master ctyun]$ ls[ods@master ctyun]$ pwd/home/ods/ctyun[ods@master ctyun]$ mkdir oidd[ods@master ctyun]$ cd oidd/[ods@master oidd]$ ls[ods@master oidd].

[root@master ~]# cd /usr/local/soft/hadoop-2.7.6/[root@master hadoop-2.7.6]# lsbin include libexec logs README.txt shareetc lib LICENSE.txt NOTICE.txt sbin tmp[root@master hadoop-2.7.6]# cd etc/[root@master etc]# lshado.

大数据项目流程

<dependency> <groupId>org.apache.spark</groupId> <artifactId>spark-mllib_2.11</artifactId> <version>2.4.5</version></dependency>package com.shujia.mllibimport org.apache.spark.ml.{featur..

package com.shujia.streamingimport org.apache.spark.sql.streaming.OutputModeimport org.apache.spark.sql.{DataFrame, SaveMode, SparkSession}object Demo05StructuredStreaming { def main(args: Array[String]): Unit = { //创建SparkSession val spar.

package com.shujia.streamingimport org.apache.spark.broadcast.Broadcastimport org.apache.spark.sql.SparkSessionimport org.apache.spark.streaming.dstream.{DStream, ReceiverInputDStream}import org.apache.spark.streaming.{Durations, StreamingContext}...

package com.shujia.streamingimport org.apache.spark.sql.SparkSessionimport org.apache.spark.streaming.dstream.DStreamimport org.apache.spark.streaming.{Durations, StreamingContext}object Demo03Window { def main(args: Array[String]): Unit = { /.

package com.shujia.coreimport com.shujia.core.Demo10Join.Studentimport org.apache.spark.{SparkConf, SparkContext}import org.apache.spark.rdd.RDDobject Demo16Action { def main(args: Array[String]): Unit = { //常见的Action算子 //foreach take col.

package com.shujia.streamingimport org.apache.spark.sql.SparkSessionimport org.apache.spark.streaming.dstream.{DStream, ReceiverInputDStream}import org.apache.spark.streaming.{Durations, StreamingContext}object Demo01WordCountOnStreaming { d...

<dependency> <groupId>org.apache.spark</groupId> <artifactId>spark-streaming_2.11</artifactId> <version>2.4.5</version></dependency>package com.shujia.streamingimport org.apache.spa...

aggregateByKey

package com.shujia.coreimport org.apache.spark.rdd.RDDimport org.apache.spark.{SparkConf, SparkContext}object Demo11Cartesian { def main(args: Array[String]): Unit = { //创建Spark Context val conf: SparkConf = new SparkConf() conf.setAppN.

package com.shujia.coreimport org.apache.spark.{SparkConf, SparkContext}import org.apache.spark.rdd.RDDobject Demo09Union { def main(args: Array[String]): Unit = { //创建Spark Context val conf: SparkConf = new SparkConf() conf.setAppName(.

package com.shujia.sqlimport org.apache.spark.sql.expressions.Windowimport org.apache.spark.sql.{DataFrame, SparkSession}object Demo06SparkOnHive { def main(args: Array[String]): Unit = { /** * 通过enableHiveSupport()可以开启Hive的支持 * 需要在po.

package com.shujia.sqlimport org.apache.spark.sql.expressions.Windowimport org.apache.spark.sql.{DataFrame, Dataset, Row, SparkSession}object Demo04DSL { def main(args: Array[String]): Unit = { val spark: SparkSession = SparkSession .buil.

公司代码,年度,1月-------------------------12月的收入金额burk,year,tsl01,tsl02,tsl03,tsl04,tsl05,tsl06,tsl07,tsl08,tsl09,tsl10,tsl11,tsl12853101,2010,100200,25002,19440,20550,14990,17227,40990,28778,19088,29889,10990,20990853101,2011,19446,20556,14996,17233,40996,2..

package com.shujia.sqlimport org.apache.spark.sql.{DataFrame, Dataset, Row, SparkSession}object Demo04DSL { def main(args: Array[String]): Unit = { val spark: SparkSession = SparkSession .builder() .appName("Demo04DSL") .master.

package com.shujia.sqlimport org.apache.spark.sql.{DataFrame, SparkSession}object Demo04DSL { def main(args: Array[String]): Unit = { val spark: SparkSession = SparkSession .builder() .appName("Demo04DSL") .master("local") .

package com.shujia.sqlimport org.apache.spark.sql.{DataFrame, SaveMode, SparkSession}object Demo03SourceAPI { //Spark SQL中常见的DataSourceAPI def main(args: Array[String]): Unit = { val spark: SparkSession = SparkSession .builder() .a.

<dependency> <groupId>org.apache.spark</groupId> <artifactId>spark-sql_2.11</artifactId> <version>2.4.5</version></dependency>package com.shujia.sqlimport org.apache.spark.SparkContext...