Java Generate/Merge Files(1)Redis Content Fetch

Java Generate/Merge Files(1)Redis Content Fetch

Fetch Data from Redis
https://github.com/xetorthio/jedis
https://github.com/lettuce-io/lettuce-core

I first use lettuce, it is not that well document as JEDIS and I tried with uncompress the data I gzcompress in PHP. I can not uncompress that. So I finally use JEDIS instead.

Some Lettuce-core codes for future references.
package com.j2c.feeds2g.services;

import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;

import com.lambdaworks.redis.RedisFuture;
import com.lambdaworks.redis.RedisURI;
import com.lambdaworks.redis.ValueScanCursor;
import com.lambdaworks.redis.api.async.RedisSetAsyncCommands;
import com.lambdaworks.redis.api.async.RedisStringAsyncCommands;
import com.lambdaworks.redis.cluster.RedisClusterClient;
import com.lambdaworks.redis.cluster.api.StatefulRedisClusterConnection;

public class RedisLettuceServiceImpl implements RedisService{


RedisClusterClient redisClusterClient;

public void processJobsBySource(Integer sourceID) {
StatefulRedisClusterConnection<String, String> connection = redisClusterClient.connect();

RedisStringAsyncCommands<String, String> async = connection.async();


RedisFuture<String> smembers = async.getrange("source_referencenumbers_1299", 0, 10);

try {

smembers.get(10, TimeUnit.SECONDS);


} catch (InterruptedException e) {

e.printStackTrace();


} catch (ExecutionException e) {

e.printStackTrace();


} catch (TimeoutException e) {

e.printStackTrace();


}

}

public void processJobsByBucket(String bucketJobIDs) {


}

public void setRedisClusterClient(RedisClusterClient redisClusterClient) {
this.redisClusterClient = redisClusterClient;

}


public static void main(String[] args) {
RedisURI redisUri = RedisURI.Builder.redis("stage-jobs-c.cache.amazonaws.com").build();


RedisClusterClient clusterClient = RedisClusterClient.create(redisUri);
StatefulRedisClusterConnection<String, String> connection = clusterClient.connect();

RedisSetAsyncCommands<String, String> setAsync = connection.async();

RedisFuture<ValueScanCursor<String>> results = setAsync.sscan("source_referencenumbers_1299");

RedisStringAsyncCommands<String, String> async = connection.async();

try {

ValueScanCursor<String> contents = results.get(10, TimeUnit.SECONDS);


List<String> jobIDs = contents.getValues();


System.out.println(jobIDs);


String jobID = "jobinfo_1299_" + jobIDs.get(0);


RedisFuture<String> results2 = async.get(jobID);


System.out.println("key = " + jobID);


String jobInfo = results2.get(10, TimeUnit.SECONDS);


System.out.println("raw job info = " + jobInfo);


} catch (InterruptedException e) {

e.printStackTrace();


} catch (ExecutionException e) {

e.printStackTrace();


} catch (TimeoutException e) {

e.printStackTrace();


}

}


}

<dependency>
<groupId>biz.paluch.redis</groupId>
<artifactId>lettuce</artifactId>
<version>4.3.1.Final</version>
</dependency>

Try to format the unit tests the JEDIS codes. For JEDIS, I am using JedisCluster.
And I want to use JedisPool as well, but it seems that JedisPool does not support JedisCluster, I am implement one myself with object common pool.

Some JedisPool codes

package com.j2c.feeds2g.services;

import java.util.HashSet;
import java.util.Set;

import com.j2c.feeds2g.commons.utils.CompressUtil;

import redis.clients.jedis.HostAndPort;
import redis.clients.jedis.JedisCluster;

public class RedisJRedisServiceImpl implements RedisService {

public void processJobsBySource(Integer sourceID) {

}

public void processJobsByBucket(String bucketJobIDs) {

}

public static void main(String[] args) {

Set<HostAndPort> jedisClusterNodes = new HashSet<HostAndPort>();
// Jedis Cluster will attempt to discover cluster nodes automatically
jedisClusterNodes.add(new HostAndPort("stage-jobs-c.pnuura.clustercfg.use1.cache.amazonaws.com", 6379));
JedisCluster jedis = new JedisCluster(jedisClusterNodes);

// JedisPoolConfig poolConfig = new JedisPoolConfig();
// poolConfig.setMaxTotal(1000);
// poolConfig.setMaxIdle(10);
// poolConfig.setMinIdle(1);
// poolConfig.setMaxWaitMillis(30000);
// JedisPool jedisPool = new JedisPool(poolConfig,
// "stage-jobs-c.pnuura.com",
// 6379,
// 1000
// );
//
// Jedis jedis = jedisPool.getResource();

// ScanParams scanParams = new ScanParams().count(100);
// String cursor = "0";
// int count = 0;
// do {
// count = count + 100;
// ScanResult<String> scanResult =
// jredisCluster.sscan("source_referencenumbers_1299", cursor,
// scanParams);
// cursor = scanResult.getStringCursor();
// List<String> jobIDs = scanResult.getResult();
// if (!jobIDs.isEmpty()) {
// System.out.println(count + " " + jobIDs.get(0));
// }
// } while (!"0".equals(cursor));

String jobID = "jobinfo_1299_1679_5e36e4a78e5cc158cd48de067ef085e0";
byte[] jobinfo = jedis.get(jobID.getBytes());
byte[] jobinfo2 = CompressUtil.gzuncompress(jobinfo);

String jobinfo3 = new String(jobinfo2);
System.out.println(jobinfo3);
}

}

References:
http://www.mkyong.com/spring-batch/spring-batch-and-spring-taskscheduler-example/

compress java
http://type.so/java/php-gzuncompress-in-java.html

spring with JEDIS
http://docs.spring.io/spring-data/redis/docs/1.8.1.RELEASE/reference/html/

http://projects.spring.io/spring-data-redis/#quick-start

logging set up
https://wiki.base22.com/display/btg/How+to+setup+SLF4J+and+LOGBack+in+a+web+app+-+fast