tangshiweibbs的专栏

package com.uplooking.bigdata.core.p2;  import org.apache.spark.SparkConf; import org.apache.spark.api.java.JavaPairRDD; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api

package com.java;import org.apache.spark.SparkConf;import org.apache.spark.api.java.JavaPairRDD;import org.apache.spark.api.java.JavaRDD;import org.apache.spark.api.java.JavaSparkContext;i

public static void main(String[] args) { SparkConf conf = new SparkConf(); conf.set("spark.app.name", "JavaSparkSQLSave"); conf.set("spark.master", "local"); JavaSparkContext sc = new

def main(args: Array[String]): Unit = { //1、注册驱动 classOf[org.apache.hive.jdbc.HiveDriver] //2、获得连接Connection val url = "jdbc:hive2://master:10000/default" val username = "root" val passwo

package com.uplooking.bigdata.sql.p3import com.uplooking.bigdata.utils.MySparkUtilimport org.apache.spark.SparkConfimport org.apache.spark.sql.{Column, SQLContext}/** * SparkSQL函数的操作 */obje

package com.uplooking.bigdata.sql.p2import org.apache.spark.{SparkConf, SparkContext}import org.apache.spark.sql.hive.HiveContext/**  * sparksql集成hive的基本操作  * 需求：  * 把在jdbc操作过程使用hive来一

public static void main(String[] args) { SparkConf conf = new SparkConf().setMaster("local[2]").setAppName("JavaSparkStreamingHDFS"); JavaSparkContext sc = new JavaSparkContext(conf); Java

public static void main(String[] args) { SparkConf conf = new SparkConf().setMaster("local[2]").setAppName("JavaSparkStreamingNC"); JavaStreamingContext jsc = new JavaStreamingContext(conf, Du

package com.uplooking.bigdata.streaming.p2;import org.apache.kafka.common.serialization.StringDeserializer;import org.apache.spark.SparkConf;import org.apache.spark.api.java.JavaSparkContext;

package com.uplooking.bigdata.streaming.p2;import kafka.serializer.StringDecoder;import org.apache.spark.SparkConf;import org.apache.spark.streaming.Durations;import org.apache.spark.streami

def main(args: Array[String]): Unit = { val conf = new SparkConf().setMaster("local").setAppName("ScalaSparkElasticSearch") /** * 根据es官网的描述，集成需要设置： * es.index.auto.create--->true *

package com.uplooking.bigdata.core.p3;  import org.apache.spark.SparkConf; import org.apache.spark.api.java.JavaPairRDD; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api

package com.uplooking.bigdata.core.p3;  import org.apache.spark.SparkConf; import org.apache.spark.SparkContext; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.Ja

package com.uplooking.bigdata.core.p3;  import com.uplooking.bigdata.domain.SecondSort; import org.apache.spark.SparkConf; import org.apache.spark.api.java.JavaPairRDD; import org.apache.spa

package com.uplooking.bigdata.core.p1;  import org.apache.spark.SparkConf; import org.apache.spark.api.java.JavaPairRDD; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api

package com.uplooking.bigdata.core.p2;  import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; import org

val conf = new SparkConf().setMaster("local").setAppName("ScalaDataFrameOps") val sc = new SparkContext(conf) val sqlContext = new SQLContext(sc) val listRDD = sc.parallelize(List("zhangsan 13 1

///定义一个RDDJavaRDD listRDD = sc.parallelize(Arrays.asList( new Person("张三", 13, 168.8), new Person("李四", 14, 169.8), new Person("王五", 15, 175.8), new Person("赵六", 16, 1

private static void broadCastOps(JavaSparkContext sc) { //加载user表到rdd JavaRDD linesRDD = sc.parallelize(Arrays.asList("1,3,张三,河北", "2,1,李四,北京", "3,0,王五,天津", "4,1,赵六,广东")); JavaRDD sexLineR

def main(args: Array[String]): Unit = { val conf = new SparkConf().setAppName("ScalaSparkSQLJDBCOps").setMaster("local") conf.set("spark.sql.shuffle.partitions", "1") val sc = new SparkContext(c

def main(args: Array[String]): Unit = { val conf = new SparkConf().setAppName("ScalaSparkSQLJson").setMaster("local") conf.set("spark.sql.shuffle.partitions", "1") val sc = new SparkContext(conf

def main(args: Array[String]): Unit = { val conf = new SparkConf().setMaster("local").setAppName("ScalaSparkDataFrameOps") val sc = new SparkContext(conf) val sqlContext = new SQLContext(sc)

def main(args: Array[String]): Unit = { val conf = new SparkConf().setMaster("local").setAppName("ScalaSparkSQL") val sc = new SparkContext(conf) val sqlContext = new SQLContext(sc) //1.读取文件格

package scalaimport org.apache.spark.{SparkConf, SparkContext}/** * Created by root on 17-4-11. */object SparkAPI extends App { val conf = new SparkConf().setAppName("SparkTransformationTes

1.Spark启动集群后，Master节点会获取各个Work节点的资源信息，内存和CPU数量信息。2.Word节点要通过心跳机制去连接master节点，同时要定时向master节点汇报自身的资源情况。3.master收到work的汇报消息后给work简单反馈。4.driver节点向master节点提交spark作业，发送注册通知，需要master为application预留相应的资源。