Java Spark自定义累加器的实现（需要注意的细节！！！）

最新推荐文章于 2025-05-29 08:00:00 发布

置顶 Purplme

最新推荐文章于 2025-05-29 08:00:00 发布

阅读量1.4k

点赞数

CC 4.0 BY-SA版权

分类专栏： spark 文章标签： spark spark自定义累加器

本文链接：https://blog.youkuaiyun.com/Purplme/article/details/84064802

spark 专栏收录该内容

1 篇文章

订阅专栏

本文详细介绍了如何在Apache Spark中实现自定义累加器，通过具体代码示例展示了AccumulatorParam接口的实现方法，包括addAccumulator、addInPlace和zero方法的具体逻辑。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

Spark自定义累加器需要实现 AccumulatorParam

！！！！！！

需要注意的是，源码中给出

也就是说两个方法的实现是不一样的。

下面是我的实现

TimeAccumulator.java


import constant.Constants;
import org.apache.spark.AccumulatorParam;
import util.StringUtils;

public class TimeAccumulator implements AccumulatorParam<String> {

    @Override
    public String addAccumulator(String s1, String s2) {
        //校验：s2位空的话，直接返回s1
        if(StringUtils.isEmpty(s2)) {
            return s1;
        }
        // 使用StringUtils工具类，从s1中，提取s2对应的值，并累加
        String oldValue = StringUtils.getFieldFromConcatString(s1, "\\|", s2);
        if(oldValue != null) {
            int newValue = Integer.valueOf(oldValue) + 1;
            return StringUtils.setFieldInConcatString(s1, "\\|", s2, String.valueOf(newValue));
        }
        return s1;
    }

    @Override
    public String addInPlace(String r1, String r2) {
        //校验：r1位空的话，直接返回r2
        if(StringUtils.isEmpty(r1)) {
            return r2;
        }
        // 使用StringUtils工具类，从r1 r2对应的值，并累加
        int oldValue1 = Integer.parseInt(StringUtils.getFieldFromConcatString(r1,"\\|",Constants.S_1s_3s));
        int oldValue2 = Integer.parseInt(StringUtils.getFieldFromConcatString(r1,"\\|",Constants.S_4s_6s));
        int oldValue3 = Integer.parseInt(StringUtils.getFieldFromConcatString(r1,"\\|",Constants.S_7s_9s));
        int oldValue4 = Integer.parseInt(StringUtils.getFieldFromConcatString(r1,"\\|",Constants.S_10s_30s));
        int oldValue5 = Integer.parseInt(StringUtils.getFieldFromConcatString(r1,"\\|",Constants.S_30s_60s));
        int oldValue6 = Integer.parseInt(StringUtils.getFieldFromConcatString(r1,"\\|",Constants.S_1m_3m));
        int oldValue7 = Integer.parseInt(StringUtils.getFieldFromConcatString(r1,"\\|",Constants.S_3m_10m));
        int oldValue8 = Integer.parseInt(StringUtils.getFieldFromConcatString(r1,"\\|",Constants.S_10m_30m));
        int oldValue9 = Integer.parseInt(StringUtils.getFieldFromConcatString(r1,"\\|",Constants.S_30m));

        int newValue1 = Integer.parseInt(StringUtils.getFieldFromConcatString(r2,"\\|",Constants.S_1s_3s));
        int newValue2 = Integer.parseInt(StringUtils.getFieldFromConcatString(r2,"\\|",Constants.S_4s_6s));
        int newValue3 = Integer.parseInt(StringUtils.getFieldFromConcatString(r2,"\\|",Constants.S_7s_9s));
        int newValue4 = Integer.parseInt(StringUtils.getFieldFromConcatString(r2,"\\|",Constants.S_10s_30s));
        int newValue5 = Integer.parseInt(StringUtils.getFieldFromConcatString(r2,"\\|",Constants.S_30s_60s));
        int newValue6 = Integer.parseInt(StringUtils.getFieldFromConcatString(r2,"\\|",Constants.S_1m_3m));
        int newValue7 = Integer.parseInt(StringUtils.getFieldFromConcatString(r2,"\\|",Constants.S_3m_10m));
        int newValue8 = Integer.parseInt(StringUtils.getFieldFromConcatString(r2,"\\|",Constants.S_10m_30m));
        int newValue9 = Integer.parseInt(StringUtils.getFieldFromConcatString(r2,"\\|",Constants.S_30m));

        return Constants.S_1s_3s+"="+(oldValue1+newValue1)+"|"+Constants.S_4s_6s+"="+(oldValue2+newValue2)+"|"+Constants.S_7s_9s+"="+(oldValue3+newValue3)+"|"+
                Constants.S_10s_30s+"="+(oldValue4+newValue4)+"|"+Constants.S_30s_60s+"="+(oldValue5+newValue5)+"|"+Constants.S_1m_3m+"="+(oldValue6+newValue6)+"|"+
                Constants.S_3m_10m+"="+(oldValue7+newValue7)+"|"+Constants.S_10m_30m+"="+(oldValue8+newValue8)+"|"+Constants.S_30m+"="+(oldValue9+newValue9);
    }

    @Override
    public String zero(String initialValue) {
        return Constants.S_1s_3s+"=0|"+Constants.S_4s_6s+"=0|"+Constants.S_7s_9s+"=0|"+
                Constants.S_10s_30s+"=0|"+Constants.S_30s_60s+"=0|"+Constants.S_1m_3m+"=0|"+
                Constants.S_3m_10m+"=0|"+Constants.S_10m_30m+"=0|"+Constants.S_30m+"=0";
    }
}

StringUtils.java

package util;

public class StringUtils {
    /**
     * 判断字符串是否为空
     *
     * @param str 字符串
     * @return 是否为空
     */
    public static boolean isEmpty(String str) {
        return "".equals(str) || str == null;
    }

    /**
     * 判断字符串是否不为空
     *
     * @param str 字符串
     * @return 是否不为空
     */
    public static boolean isNotEmpty(String str) {
        return str != null && str.length() > 0;
    }

    /**
     * 从拼接的字符串中提取字段
     * searchKeywords=abc|clickCategoryIds=1,2,3
     *
     * @param str       字符串   searchKeywords=abc|clickCategoryIds=1,2,3
     * @param delimiter 分隔符   |
     * @param field     字段   clickCategoryIds
     * @return 字段值   1,2,3
     */
    public static String getFieldFromConcatString(String str, String delimiter, String field) {
        String[] split = str.split(delimiter);
        String value = null;
        for (String s : split) {
            String[] split1 = s.split("=");
            if(field.equals(split1[0])){
                value = split1[1];
            }
        }
        return value;
    }

    /**
     * 从拼接的字符串中给字段设置值
     * searchKeywords=abc|clickCategoryIds=1,2,3
     *
     * @param str           字符串  searchKeywords=abc|clickCategoryIds=1,2,3
     * @param delimiter     分隔符  |
     * @param field         字段名 searchKeywords
     * @param newFieldValue 新的field值  efg
     * @return 字段值  searchKeywords=efg|clickCategoryIds=1,2,3
     */
    public static String setFieldInConcatString(String str, String delimiter, String field, String newFieldValue) {
        String[] split = str.split(delimiter);
        StringBuilder sb = new StringBuilder();
        for (int i = 0; i < split.length; i++) {
            String[] split1 = split[i].split("=");
            if(field.equals(split1[0])){
                sb.append(split1[0]);
                sb.append("=");
                sb.append(newFieldValue);
                sb.append("|");
                continue;
            }
            sb.append(split[i]);sb.append("|");
        }
        return sb.substring(0,sb.lastIndexOf("|"));
    }
}