Spark Java 二次排序

1.输入

1 5
2 4
3 5
1 8
2 1
3 2
5 1
6 2

2.代码实现

2.1自定义比较器

import scala.math.Ordered;

import java.io.Serializable;

/**
 * User:leen
 * Date:2017/10/18 0018
 * Time:11:23
 *
 *
 * 比较器：
 * 1. 实现 Ordered<T> 和 Serializable 接口
 * 2. 重写 equals 和 hashCode 方法
 * 3. 重写 $less ，$greater ，$less$eq ，$greater$eq 方法
 * 4. 重写 compare ， compareTo 方法
 * 5. 构造函数
 */

public class SecondarySortKey implements Ordered<SecondarySortKey>, Serializable {

    int first;
    int second;

    public SecondarySortKey() {
    }
    /**
     * 构造方法
     * @param first
     * @param second
     */
    public SecondarySortKey(int first, int second) {
        this.first = first;
        this.second = second;
    }

    public boolean $less(SecondarySortKey that) {
        if (this.first < that.getFirst()) {
            return true;
        } else if (this.first == that.getFirst() && this.second < that.getSecond()) {
            return true;
        }
        return false;
    }

    public boolean $greater(SecondarySortKey that) {
        if (this.first > that.getFirst()) {
            return true;
        } else if (this.first == that.getFirst() && this.second > that.getSecond()) {
            return true;
        }
        return false;
    }

    public boolean $less$eq(SecondarySortKey that) {
        if (this.$less(that)) {
            return true;
        } else if (this.first == that.getFirst() && this.second == that.getSecond()) {
            return true;
        }
        return false;
    }

    public boolean $greater$eq(SecondarySortKey that) {
        if (this.$greater(that)) {
            return true;
        } else if (this.first == that.getFirst() && this.second == that.getSecond()) {
            return true;
        }
        return false;
    }

    public int compare(SecondarySortKey that) {
        if (this.first - that.getFirst() != 0) {
            return this.first - that.getFirst();
        } else {
            return this.second - that.getSecond();
        }
    }

    public int compareTo(SecondarySortKey that) {
        if (this.first - that.getFirst() != 0) {
            return this.first - that.getFirst();
        } else {
            return this.second - that.getSecond();
        }
    }

    public int getFirst() {
        return first;
    }

    public void setFirst(int first) {
        this.first = first;
    }

    public int getSecond() {
        return second;
    }

    public void setSecond(int second) {
        this.second = second;
    }

    @Override
    public boolean equals(Object o) {
        if (this == o) return true;
        if (o == null || getClass() != o.getClass()) return false;

        SecondarySortKey that = (SecondarySortKey) o;

        if (first != that.first) return false;
        return second == that.second;
    }

    @Override
    public int hashCode() {
        int result = first;
        result = 31 * result + second;
        return result;
    }
}

2.2调用比较器实现二次排序

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.PairFunction;
import org.apache.spark.api.java.function.VoidFunction;
import scala.Tuple2;

/**
 * User:leen
 * Date:2017/10/18 0018
 * Time:12:00
 */
public class SecondarySort {
    public static void main(String[] args) {
        SparkConf conf = new SparkConf().setAppName("SecondarySort").setMaster("local");
        JavaSparkContext sc = new JavaSparkContext(conf);
        JavaRDD<String> lines = sc.textFile("C:\\Users\\leen\\Desktop\\sort.txt");

        JavaPairRDD<SecondarySortKey,String> pairs = lines.mapToPair(new PairFunction<String, SecondarySortKey, String>() {
            public Tuple2<SecondarySortKey, String> call(String line) throws Exception {
                String[] lineSplits = line.split(" ");
                SecondarySortKey key = new SecondarySortKey(Integer.valueOf(lineSplits[0]), Integer.valueOf(lineSplits[1]));
                return new Tuple2<SecondarySortKey, String>(key, line);
            }
        });

        JavaPairRDD<SecondarySortKey,String> sortedPairs = pairs.sortByKey();
        sortedPairs.foreach(new VoidFunction<Tuple2<SecondarySortKey, String>>() {
            public void call(Tuple2<SecondarySortKey, String> res) throws Exception {
                System.out.println(res._1().hashCode()+" : "+ res._2());
            }
        });


    }
}

3.结果输出

36  : 1 5
39  : 1 8
63  : 2 1
66  : 2 4
95  : 3 2
98  : 3 5
156 : 5 1
188 : 6 2