使用redis的有序集合实现前缀的autocomplete。
1.首先把数据存入redis有续集,
原文件格式如下:
存入redis中格式,完整的名字有*结尾:
11609) "y"
11610) "ya"
11611) "yal"
11612) "yalo"
11613) "yalon"
11614) "yalond"
11615) "yalonda*"
11616) "yas"
11617) "yasm"
11618) "yasme"
11619) "yasmee"
11620) "yasmeen*"
11621) "yasmi"
11622) "yasmin*"
11623) "ye"
11624) "yel"
11625) "yele"
11626) "yelen"
11627) "yelena*"
11628) "yet"
11629) "yett"
11630) "yetta*"
11631) "yetti"
11632) "yettie*"
11633) "yetty*"
2.根据输入的前缀,设置开始查找的起始位置
127.0.0.1:6379> zrank test y
(integer) 11608
127.0.0.1:6379>
如y开头,从11608开始查找。
3.从起始位置分批(循环)向后查询(zrange),将符合前缀且*结尾的词放入返回列表,直至当列表size大于所需要的数量(maxNeeded)或不符合所输入的前缀时,返回列表。
用jedis实现:
package com.zzhblh.sample;
import redis.clients.jedis.Jedis;
import redis.clients.jedis.JedisPool;
import redis.clients.jedis.JedisPoolConfig;
import redis.clients.jedis.Pipeline;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.URL;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Set;
public class testJedisAutocomplete {
private final Jedis redis;
private final Pipeline p;
private final String redisKey;
private long start,end;
testJedisAutocomplete(final Jedis redis, final Pipeline p, final String redisKey, final URL dictionaryUrl, final boolean populate,
final boolean clear) throws IOException {
this.redis = redis;
this.p = p;
this.redisKey = redisKey;
if(clear) {
// Truncate existing Dictionary
clearDictionary();
}
if(populate) {
start = System.currentTimeMillis();
// Load the entries from dictionaryUrl
loadDictionary(dictionaryUrl);
end = System.currentTimeMillis();
System.out.println("add with pipeline used [" + (end - start) / 1000 + "] seconds ..");
p.close();
}
}
public List<String> complete(final String prefix, final int count) {
// 1. Find the position of the prefix in the sorted set in Redis
if(null == prefix) {
return Collections.emptyList();
}
int prefixLength = prefix.length();
int start = redis.zrank(redisKey, prefix).intValue();
if(start < 0 || prefixLength == 0) {
return Collections.emptyList();
}
List<String> results = new ArrayList<String>();
int found = 0, rangeLength = 50, maxNeeded = count;
while(found < maxNeeded) {
Set<String> rangeResults = redis.zrange(redisKey, start, start + rangeLength - 1);
start += rangeLength;
if(rangeResults.isEmpty()) {
break;
}
for(final String entry : rangeResults) {
int minLength = Math.min(entry.length(), prefixLength);
if(!entry.substring(0, minLength).equalsIgnoreCase(prefix.substring(0, minLength))) {
return results;
}
if(entry.endsWith("*") && found < maxNeeded) {
results.add(entry.substring(0, entry.length() - 1));
found = results.size();
}
}
}
return results;
}
private void clearDictionary() {
redis.del(redisKey);
}
private void loadDictionary(final URL dictionaryUrl) throws IOException {
InputStreamReader inputStreamReader = new InputStreamReader(dictionaryUrl.openStream());
BufferedReader reader = new BufferedReader(inputStreamReader);
String word;
while((word = reader.readLine()) != null) {
word = word.trim();
// Add the word if the word does not start with #
if(!word.isEmpty() && !word.startsWith("#")) {
addWord(word);
}
}
reader.close();
inputStreamReader.close();
}
private void addWord(final String word) {
// Add all the possible prefixes of the given word and also the given
// word with a * suffix.
p.zadd(redisKey, 0, word + "*");
for(int index = 1, total = word.length(); index < total; index++) {
p.zadd(redisKey, 0, word.substring(0, index));
}
}
public static class Builder {
private final Jedis redis;
private final Pipeline p;
private final String redisKey;
private final URL dictionaryUrl;
private boolean clear = true;
private boolean populate = true;
public Builder(final Jedis redis, final Pipeline p, final String redisKey, final URL dictionaryUrl) {
this.redis = redis;
this.p = p;
this.redisKey = redisKey;
this.dictionaryUrl = dictionaryUrl;
}
public Builder clear(final boolean clear) {
this.clear = clear;
this.populate = true;
return this;
}
public Builder populate(final boolean populate) {
this.populate = populate;
return this;
}
public testJedisAutocomplete build() throws IOException {
return new testJedisAutocomplete(redis, p, redisKey, dictionaryUrl, populate, clear);
}
}
public static void main(final String[] args) throws Exception {
JedisPoolConfig config = new JedisPoolConfig();
JedisPool pool = new JedisPool(config, "107.170.248.158", 6379);
Jedis redis = pool.getResource();
redis.auth("123");
Pipeline p = redis.pipelined();
URL url = new URL("http://antirez.com/misc/female-names.txt");
testJedisAutocomplete autoComplete = new testJedisAutocomplete.Builder(redis, p, "test", url).clear(true).populate(true).build();
System.out.println(autoComplete.complete("y", 80));
}
}