spark更新广播变量实例

最新推荐文章于 2023-03-02 23:43:49 发布

原创最新推荐文章于 2023-03-02 23:43:49 发布 · 4.1k 阅读

4 ·

CC 4.0 BY-SA版权

文章标签：

#spark更新广播变量 #广播变量 spark

Spark 专栏收录该内容

2 篇文章

订阅专栏

本文介绍了一个用于Spark环境中广播变量的更新和获取机制的具体实现。该机制通过判断Redis中的黑名单和白名单数据是否已更新来决定是否更新Spark的广播变量。如果数据发生变更，则从Redis获取最新数据并应用到Spark作业中。

实际项目中需要使用更新广播变量的方式来满足需求，其代码具体如下：

@NotProguard
public class WbBroadcastWrapper {
	
	Broadcast<List<String>> broadcastBlackWhiteList = null;
	private static WbBroadcastWrapper obj = new WbBroadcastWrapper();

    private WbBroadcastWrapper(){}

    public static WbBroadcastWrapper getInstance() {
        return obj;
    }
    
    public JavaSparkContext getSparkContext(SparkContext sc) {
        JavaSparkContext jsc = JavaSparkContext.fromSparkContext(sc);
        return jsc;
    }
    
    public Broadcast<List<String>> updateAndGetWbBroadCast(SparkContext sparkContext,Map<String,String> params) {
    	if(broadcastBlackWhiteList == null || judgeBlackAndWhiteIsUpdate(params)) {
    		JRedisClusterHelper instance = JRedisClusterHelper.getInstance();
    		String path = params.get("redis.jedisPoolConfig.urls");
    		JedisCluster jedisCluster = instance.getJedisCluster(path);
			String blackWhiteSql = jedisCluster.hget("black_white_list", "1");
			DbFetcher dbFetcher = new DbFetcher();
			List<String> fetchSqlList = dbFetcher.fetchSqlList(blackWhiteSql,params);
			if(broadcastBlackWhiteList != null) {
				broadcastBlackWhiteList.unpersist();
			}
			broadcastBlackWhiteList = getSparkContext(sparkContext).broadcast(fetchSqlList);
    	}
    	return broadcastBlackWhiteList;
    }
    
    
	public static boolean judgeBlackAndWhiteIsUpdate(Map<String,String> parmas) {
		JRedisClusterHelper instance = JRedisClusterHelper.getInstance();
		String path = parmas.get("redis.jedisPoolConfig.urls");
		JedisCluster jedisCluster = instance.getJedisCluster(path);
		String modified = jedisCluster.get(FinalParameterHelper.BLACK_AND_WHITE_MODIFIED);
		if(modified != null && Integer.parseInt(modified.trim()) == 1) {
			jedisCluster.set(FinalParameterHelper.BLACK_AND_WHITE_MODIFIED, "0");
			return true;
		}
		return false;
	}
    
}