记录是一种精神，是加深理解最好的方式之一。
最近深入研究了Jedis的源码，对Jedis的使用进行深入理解，提笔记录。
曹金桂 cao_jingui@163.com(如有遗漏之处还请指教)
时间：2016年12月06日15:00
概述
分片允许数据存放在不同的物理机器上，以适应数据量过大的场景，克服单台机器内存或者磁盘空间的限制。Redis在3.0之前是没有集群的，就算现在3.0推出3.0集群版本，也没有几家公司用于生产。Jedis作为Java的Redis客户端工具，其内部也提供了客户的分片的实现。具体可以参考我的文章《Jedis使用教程完整版》，讲解了Jedis客户端使用，包括了Jedis的客户端分片的使用方法。
Jedis客户端分片存在问题
Jedis的客户端分片采用一致性hash算法，实现数据的分配存储。但当某一个分片服务挂了之后，客户端需要修改服务的指向才能动态的切换到当前分片的从服务去。其实完全可以采用和JedisSentinel一样的方法，利用Jedis的Sentinel构建高可用的服务。
实现
个人参照JedisSentinelPool的实现，完成了对构建Jedis分片的高可用的修改。源码如下：
package redis.clients.jedis;
import org.apache.commons.pool2.PooledObject;
import org.apache.commons.pool2.PooledObjectFactory;
import org.apache.commons.pool2.impl.DefaultPooledObject;
import org.apache.commons.pool2.impl.GenericObjectPoolConfig;
import redis.clients.jedis.exceptions.JedisConnectionException;
import redis.clients.jedis.exceptions.JedisException;
import redis.clients.util.Hashing;
import redis.clients.util.Pool;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.regex.Pattern;
public class SentinelShardedJedisPool extends Pool<SentinelShardedJedis> {
protected Logger log = Logger.getLogger(getClass().getName());
private final Map<String, JedisShardInfo> shardInfoMap = new ConcurrentHashMap<>(); // key=masterName
private final Map<String, String> sentinels = new ConcurrentHashMap<>(); // key=host+port value=masterName
private SentinelShardedJedisFactory factory;
private Set<MasterListener> masterListeners = new HashSet<>();
public SentinelShardedJedisPool(Set<SentinelInfo> sentinelInfos, final GenericObjectPoolConfig poolConfig, Hashing algo, Pattern keyTagPattern) {
initShards(sentinelInfos);
factory = new SentinelShardedJedisFactory(shardInfoMap.values(), algo, keyTagPattern);
super.initPool(poolConfig, factory);
}
private void initShards(Set<SentinelInfo> sentinelInfos) {
for (SentinelInfo info : sentinelInfos) {
JedisShardInfo shardInfo = getJedisSharedInfoFromSentinelInfo(info);
sentinels.put(shardInfo.getHost() + ":" + shardInfo.getPort(), info.getMasterName());
shardInfoMap.put(info.getMasterName(), shardInfo);
for (String sentinel : info.getSentinels()) {
HostAndPort hap = HostAndPort.parseString(sentinel);
MasterListener masterListener = new MasterListener(info.getMasterName(), hap.getHost(), hap.getPort());
masterListeners.add(masterListener);
masterListener.setDaemon(true);
masterListener.start();
}
}
}
private JedisShardInfo getJedisSharedInfoFromSentinelInfo(SentinelInfo info) {
log.info("Trying to find master from available Sentinels...");
for (String sentinel : info.getSentinels()) {
final HostAndPort hap = HostAndPort.parseString(sentinel);
log.fine("Connecting to Sentinel " + hap);
Jedis jedis = null;
try {
jedis = new Jedis(hap.getHost(), hap.getPort());
List<String> masterAddr = jedis.sentinelGetMasterAddrByName(info.getMasterName());
if (masterAddr == null || masterAddr.size() != 2) {
log.warning("Can not get master addr, master name: " + info.getMasterName() + ". Sentinel: " + hap + ".");
continue;
}
String host = masterAddr.get(0);
int port = Integer.parseInt(masterAddr.get(1));
JedisShardInfo jedisShardInfo = new JedisShardInfo(host, port);
log.fine("Found Redis master at " + masterAddr);
return jedisShardInfo;
} catch (JedisException e) {
log.warning("Cannot get master address from sentinel running @ " + hap + ". Reason: " + e + ". Trying next one.");
} finally {
if (jedis != null) {
jedis.close();
}
}
}
throw new JedisConnectionException("All sentinels down, cannot determine where is " + info.getMasterName() + " master is running...");
}
private void reInitPool() {
internalPool.clear();
}
@Override
public void destroy() {
for (MasterListener m : masterListeners) {
m.shutdown();
}
super.destroy();
}
@Override
public SentinelShardedJedis getResource() {
while (true) {
SentinelShardedJedis jedis = super.getResource();
for (Jedis shard : jedis.getAllShards()) {
String host = shard.getClient().getHost();
int port = shard.getClient().getPort();
if (sentinels.containsKey(host + ":" + port)) {
jedis.setDataSource(this);
return jedis;
}
}
returnBrokenResource(jedis);
}
}
@Override
protected void returnBrokenResource(final SentinelShardedJedis resource) {
if (resource != null) {
returnBrokenResourceObject(resource);
}
}
@Override
protected void returnResource(final SentinelShardedJedis resource) {
if (resource != null) {
resource.resetState();
returnResourceObject(resource);
}
}
private class MasterListener extends Thread {
private String masterName;
private String sentinelHost;
private int sentinelPort;
private volatile Jedis j;
private AtomicBoolean running = new AtomicBoolean(false);
public MasterListener(String masterName, String host, int port) {
this.masterName = masterName;
this.sentinelHost = host;
this.sentinelPort = port;
this.setName(String.format("MasterListener-%s-[%s:%d]", masterName, host, port));
}
@Override
public void run() {
running.set(true);
while (running.get()) {
j = new Jedis(sentinelHost, sentinelPort);
try {
if (!running.get()) {
break;
}
j.subscribe(new JedisPubSub() {
@Override
public void onMessage(String channel, String message) {
log.fine("Sentinel " + sentinelHost + ":" + sentinelPort + " published: " + message + ".");
String[] switchMasterMsg = message.split(" ");
if (switchMasterMsg.length > 3) {
if (masterName.equals(switchMasterMsg[0])) {
JedisShardInfo oldInfo = shardInfoMap.get(masterName);
sentinels.remove(oldInfo.getHost() + ":" + oldInfo.getPort());
JedisShardInfo newInfo = new JedisShardInfo(switchMasterMsg[3], switchMasterMsg[4]);
newInfo.setConnectionTimeout(oldInfo.getConnectionTimeout());
newInfo.setPassword(oldInfo.getPassword());
newInfo.setSoTimeout(oldInfo.getSoTimeout());
shardInfoMap.put(masterName, newInfo);
sentinels.put(newInfo.getHost() + ":" + newInfo.getPort(), masterName);
reInitPool();
} else {
log.fine("Ignoring message on +switch-master for master name " + switchMasterMsg[0] + ", our master name is " + masterName);
}
} else {
log.severe("Invalid message received on Sentinel " + sentinelHost + ":" + sentinelPort + " on channel +switch-master: " + message);
}
}
}, "+switch-master");
} catch (JedisConnectionException e) {
if (running.get()) {
log.log(Level.SEVERE, "Lost connection to Sentinel at " + sentinelHost + ":" + sentinelPort + ". Sleeping 5000ms and retrying.", e);
try {
Thread.sleep(5000);
} catch (InterruptedException e1) {
log.log(Level.SEVERE, "Sleep interrupted: ", e1);
}
} else {
log.fine("Unsubscribing from Sentinel at " + sentinelHost + ":" + sentinelPort);
}
} finally {
j.close();
}
}
}
public void shutdown() {
try {
log.fine("Shutting down listener on " + sentinelHost + ":" + sentinelPort);
running.set(false);
if (j != null) {
j.disconnect();
}
} catch (Exception e) {
log.log(Level.SEVERE, "Caught exception while shutting down: ", e);
}
}
}
/**
* PoolableObjectFactory custom impl.
*/
private class SentinelShardedJedisFactory implements PooledObjectFactory<SentinelShardedJedis> {
private Collection<JedisShardInfo> shards;
private Hashing algo;
private Pattern keyTagPattern;
public SentinelShardedJedisFactory(Collection<JedisShardInfo> shards, Hashing algo, Pattern keyTagPattern) {
this.shards = shards;
this.algo = algo;
this.keyTagPattern = keyTagPattern;
}
@Override
public PooledObject<SentinelShardedJedis> makeObject() throws Exception {
SentinelShardedJedis jedis = new SentinelShardedJedis(new ArrayList<>(shards), algo, keyTagPattern);
return new DefaultPooledObject<SentinelShardedJedis>(jedis);
}
@Override
public void destroyObject(PooledObject<SentinelShardedJedis> pooledShardedJedis) throws Exception {
final SentinelShardedJedis shardedJedis = pooledShardedJedis.getObject();
for (Jedis jedis : shardedJedis.getAllShards()) {
try {
try {
jedis.quit();
} catch (Exception e) {
}
jedis.disconnect();
} catch (Exception e) {
}
}
}
@Override
public boolean validateObject(PooledObject<SentinelShardedJedis> pooledShardedJedis) {
try {
SentinelShardedJedis jedis = pooledShardedJedis.getObject();
for (Jedis shard : jedis.getAllShards()) {
String host = shard.getClient().getHost();
int port = shard.getClient().getPort();
if (!shardInfoMap.containsKey(host + port)) {
return false;
}
if (!shard.ping().equals("PONG")) {
return false;
}
}
return true;
} catch (Exception ex) {
return false;
}
}
@Override
public void activateObject(PooledObject<SentinelShardedJedis> p) throws Exception {
}
@Override
public void passivateObject(PooledObject<SentinelShardedJedis> p) throws Exception {
}
}
public static class SentinelInfo {
private String masterName;
private String clientName;
private Set<String> sentinels;
public SentinelInfo(String masterName, String clientName, Set<String> sentinels) {
this.masterName = masterName;
this.clientName = clientName;
this.sentinels = sentinels;
}
public String getMasterName() {
return masterName;
}
public String getClientName() {
return clientName;
}
public Set<String> getSentinels() {
return sentinels;
}
}
}
package redis.clients.jedis;
import redis.clients.util.Hashing;
import java.util.List;
import java.util.regex.Pattern;
public class SentinelShardedJedis extends ShardedJedis {
protected SentinelShardedJedisPool dataSource = null;
public SentinelShardedJedis(List<JedisShardInfo> shards) {
super(shards);
}
public SentinelShardedJedis(List<JedisShardInfo> shards, Hashing algo) {
super(shards, algo);
}
public SentinelShardedJedis(List<JedisShardInfo> shards, Pattern keyTagPattern) {
super(shards, keyTagPattern);
}
public SentinelShardedJedis(List<JedisShardInfo> shards, Hashing algo, Pattern keyTagPattern) {
super(shards, algo, keyTagPattern);
}
public void setDataSource(SentinelShardedJedisPool sentinelShardedJedisPool) {
this.dataSource = sentinelShardedJedisPool;
}
}
使用方法
JedisPoolConfig poolConfig = new JedisPoolConfig();
poolConfig.setMinIdle(4);
poolConfig.setMaxTotal(100);
poolConfig.setMaxWaitMillis(30000);
Set<String> sentinels = new HashSet<>();
sentinels.add("192.168.2.122:26379");
sentinels.add("192.168.2.128:26379");
Set<SentinelShardedJedisPool.SentinelInfo> sentinelInfos = new HashSet<>();
sentinelInfos.add(new SentinelShardedJedisPool.SentinelInfo("mymaster1", "m1", sentinels));
sentinelInfos.add(new SentinelShardedJedisPool.SentinelInfo("mymaster2", "m2", sentinels));
SentinelShardedJedisPool pool = new SentinelShardedJedisPool(sentinelInfos, poolConfig, Hashing.MURMUR_HASH, null);
SentinelShardedJedis jedis = pool.getResource();
jedis.set("key", "value");
jedis.close();
pool.close();
小结
本实现个人已经测试通过。但若采用本方案必须要经过一定的性能测试。如果问题欢迎一起讨论。
