spark cogroup算子

最新推荐文章于 2024-11-18 10:40:04 发布

weixin_30613727

最新推荐文章于 2024-11-18 10:40:04 发布

阅读量180

点赞数

CC 4.0 BY-SA版权

文章标签：大数据 scala java

原文链接：http://www.cnblogs.com/tele-share/p/10268576.html

本文通过一个具体的示例，详细对比了Spark中Cogroup与Join算子的不同之处，特别是在处理具有多个值的键时，Cogroup如何返回键与值的可迭代集合。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

java

 1 /** 
 2  *cogroup与join算子不同的是如果rdd中的一个key,对应多个value,则返回<Iterable<key>,Iterable<value>>
 3  *@author Tele
 4  */
 5 public class CogroupDemo {
 6     private static SparkConf conf = new SparkConf().setMaster("local").setAppName("congroupdemo");
 7     private static JavaSparkContext jsc = new JavaSparkContext(conf);
 8     public static void main(String[] args) {
 9         //每个学生有多门成绩
10         List<Tuple2<Integer,String>> studentList = Arrays.asList(
11                                                     new Tuple2<Integer,String>(1,"tele"), 
12                                                     new Tuple2<Integer,String>(1,"xx"), 
13                                                     new Tuple2<Integer,String>(2,"yeye"), 
14                                                     new Tuple2<Integer,String>(3,"wyc")
15                                                    );
16 
17         List<Tuple2<Integer,Integer>> scoreList = Arrays.asList(
18                                                   new Tuple2<Integer,Integer>(1,100),
19                                                   new Tuple2<Integer,Integer>(1,110),
20                                                   new Tuple2<Integer,Integer>(1,120),
21                                                   new Tuple2<Integer,Integer>(2,90),
22                                                   new Tuple2<Integer,Integer>(2,60),
23                                                   new Tuple2<Integer,Integer>(2,50),
24                                                   new Tuple2<Integer,Integer>(3,70),
25                                                   new Tuple2<Integer,Integer>(3,70)
26                                                   );
27         
28         JavaPairRDD<Integer, String> studentRDD = jsc.parallelizePairs(studentList);
29         JavaPairRDD<Integer, Integer> scoreRDD = jsc.parallelizePairs(scoreList);
30         
31         JavaPairRDD<Integer, Tuple2<Iterable<String>, Iterable<Integer>>> result = studentRDD.cogroup(scoreRDD);
32         result.foreach(new VoidFunction<Tuple2<Integer,Tuple2<Iterable<String>,Iterable<Integer>>>>() {
33             
34             private static final long serialVersionUID = 1L;
35 
36             @Override
37             public void call(Tuple2<Integer, Tuple2<Iterable<String>, Iterable<Integer>>> t) throws Exception {
38                 System.out.println("学号:" + t._1);
39                 System.out.println("姓名:" + t._2._1);
40                 System.out.println("成绩:" + t._2._2);
41                 
42             /*    System.out.print("成绩:[");
43                 t._2._2.forEach(i->System.out.print(i + ","));
44                 System.out.println("]");
45                 System.out.println("====================");*/
46                 
47             }
48         });
49         
50         jsc.close();
51     }
52 }

scala

 1 object CogroupDemo {
 2     def main(args: Array[String]): Unit = {
 3         val conf = new SparkConf().setMaster("local").setAppName("cogroupdemo");
 4         val sc = new SparkContext(conf);
 5         
 6         val studentArr = Array((1,"tele"),(2,"yeye"),(3,"wyc"));
 7         val scoreArr = Array((1,100),(1,200),(2,80),(2,300),(3,100));
 8         
 9         val studentRDD = sc.parallelize(studentArr,1);
10         val scoreRDD = sc.parallelize(scoreArr,1);
11         
12         val result = studentRDD.cogroup(scoreRDD);
13         result.foreach(t=>{
14           println("学号:" + t._1);
15           println("姓名:" + t._2._1.mkString(" "));
16           println("成绩:" + t._2._2.mkString(","));
17           println("============");
18         })
19     }
20 }