hive,redis杂记

陆续归纳，整理

1.hive优化

1.1建表优化

建表语句demo,存储格式为ORC,并且设置索引，*“orc.bloom.filter.columns”=“distinct_id”,*这一句的意思是 把 distinct_id这一列作为索引。分隔符为 \27，有效防止导入数据的时候，发生数据错位。
参考链接：ORC原理及查询优化

create table if not exists  test.cust_events(
event string,
user_id bigint,
distinct_id string,
record_time timestamp,
credit_billing_source string,
transfer_account string,
)
partitioned by ( load_dt VARCHAR(10))
ROW FORMAT DELIMITED FIELDS TERMINATED BY '\27'
STORED AS ORCFile TBLPROPERTIES
('orc.compress'='SNAPPY',
'orc.create.index'='true',
"orc.bloom.filter.columns"="distinct_id",
'orc.bloom.filter.fpp'='0.05',
'orc.stripe.size'='10485760',
'orc.row.index.stride'='10000');

2.使用java连接redis哨兵集群

连接哨兵,下面代码中的ip1,ip2,ip3,port1,port2,port3，请根据自己的环境进行配置

import org.apache.commons.pool2.impl.GenericObjectPoolConfig;
import redis.clients.jedis.Jedis;
import redis.clients.jedis.JedisSentinelPool;

import java.util.HashSet;
import java.util.Set;


public class SentinelPool {

    private static JedisSentinelPool pool;
    private static Set<String> sentinel = new HashSet<>();

    static {
        sentinel.add("ip1:port1");
        sentinel.add("ip2:port2");
        sentinel.add("ip3:port3");
        initPool();
    }

    private static void initPool() {
        pool = new JedisSentinelPool("mymaster",
                sentinel,
                new GenericObjectPoolConfig(),
                3000);
    }

    public static Jedis getJedis() {
        return pool.getResource();
    }

    public static void returnBrokenResource(Jedis jedis) {
        pool.returnBrokenResource(jedis);
    }

    public static void returnResource(Jedis jedis) {
        pool.returnResource(jedis);
    }
}

对redis进行操作的工具类

import redis.clients.jedis.Jedis;

public class JedisSentinelUtil {


    /**
     * 设置 key
     * @param key   键
     * @param value 值
     * @return
     */
    public static String set(String key, String value) {
        Jedis jedis = null;
        String result = null;
        try {
            jedis = SentinelPool.getJedis();
            result= jedis.set(key, value);
        } catch (Exception e) {
            SentinelPool.returnBrokenResource(jedis);
            return result;
        }
        SentinelPool.returnResource(jedis);
        return result;
    }


    /**
     * 设置一个有过期时间的 kv
     * @param key
     * @param value
     * @param exTime
     * @return
     */
    public static String setEx(String key, String value,int exTime) {
        Jedis jedis = null;
        String result = null;
        try {
            jedis = SentinelPool.getJedis();
            result = jedis.setex(key, exTime, value);
        } catch (Exception e) {
            SentinelPool.returnBrokenResource(jedis);
            return result;
        }
        SentinelPool.returnResource(jedis);
        return result;
    }

    /**
     * key 不存在就设置
     * @param key
     * @param value
     * @return
     */
    public static Long setNx(String key, String value) {
        Jedis jedis = null;
        Long result = null;
        try {
            jedis = SentinelPool.getJedis();
            result= jedis.setnx(key, value);
        } catch (Exception e) {
            SentinelPool.returnBrokenResource(jedis);
            return result;
        }
        SentinelPool.returnResource(jedis);
        return result;
    }
    /**
     * 删除 key
     * @param key
     * @return
     */
    public static Long del(String key) {
        Jedis jedis = null;
        Long result = null;
        try {
            jedis = SentinelPool.getJedis();
            result = jedis.del(key);
        } catch (Exception e) {
            SentinelPool.returnBrokenResource(jedis);
            return result;
        }
        SentinelPool.returnResource(jedis);
        return result;
    }

    /**
     * 获取 Key 对应的值
     * @param key
     * @return
     */
    public static String get(String key) {
        Jedis jedis = null;
        String result = null;
        try {
            jedis = SentinelPool.getJedis();
            result = jedis.get(key);
        } catch (Exception e) {
            SentinelPool.returnBrokenResource(jedis);
            return result;
        }
        SentinelPool.returnResource(jedis);
        return result;
    }


    /**
     * 设置一个新的值，并返回旧的值
     * @param key
     * @param value
     * @return
     */
    public static String geSet(String key,String value) {
        Jedis jedis = null;
        String result = null;
        try {
            jedis = SentinelPool.getJedis();
            result = jedis.getSet(key, value);
        } catch (Exception e) {
            SentinelPool.returnBrokenResource(jedis);
            return result;
        }
        SentinelPool.returnResource(jedis);
        return result;
    }


    /**
     * 设置过期时间
     * @param key
     * @param exTime
     * @return
     */
    public static Long expire(String key, int exTime) {
        Jedis jedis = null;
        Long result = null;
        try {
            jedis = SentinelPool.getJedis();
            result= jedis.expire(key, exTime);
        } catch (Exception e) {
            SentinelPool.returnBrokenResource(jedis);
            return result;
        }
        SentinelPool.returnResource(jedis);
        return result;
    }
    /**
     * 获取 dbsize
     * @return
     */
    public static Long getDbSize() {
        Jedis jedis = null;
        Long result = null;
        try {
            jedis = SentinelPool.getJedis();
            result = jedis.dbSize();
        } catch (Exception e) {
            SentinelPool.returnBrokenResource(jedis);
            return result;
        }
        SentinelPool.returnResource(jedis);
        return result;
    }
}

参考链接：JedisSentinelPool 连接Redis 主节点工具类