From 18d96c73d6e94b0dd3dbc3710e29acd9aee986d0 Mon Sep 17 00:00:00 2001 From: LamGC Date: Mon, 30 Mar 2020 10:10:53 +0800 Subject: [PATCH] =?UTF-8?q?[Add]=20CacheStore=20=E5=A2=9E=E5=8A=A0supporte?= =?UTF-8?q?dPersistence=E4=BB=A5=E8=AE=A9=E4=BD=BF=E7=94=A8=E6=96=B9?= =?UTF-8?q?=E7=9F=A5=E6=99=93=E5=AE=9E=E7=8E=B0=E6=98=AF=E5=90=A6=E6=94=AF?= =?UTF-8?q?=E6=8C=81=E6=8C=81=E4=B9=85=E5=8C=96;=20[Change]=20CQProcess=20?= =?UTF-8?q?=E4=BD=BF=E7=94=A8RedisPoolCacheStore;?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pom.xml | 5 + src/main/java/net/lamgc/cgj/CQProcess.java | 103 ++++-------------- .../java/net/lamgc/cgj/cache/CacheStore.java | 6 + .../cgj/cache/JsonRedisCacheDatabase.java | 26 +++++ .../lamgc/cgj/cache/LocalHashCacheStore.java | 27 ++++- .../lamgc/cgj/cache/RedisCacheDatabase.java | 95 ++++++++++++++++ .../lamgc/cgj/cache/RedisPoolCacheStore.java | 90 +++++++++++++++ 7 files changed, 271 insertions(+), 81 deletions(-) create mode 100644 src/main/java/net/lamgc/cgj/cache/JsonRedisCacheDatabase.java create mode 100644 src/main/java/net/lamgc/cgj/cache/RedisCacheDatabase.java create mode 100644 src/main/java/net/lamgc/cgj/cache/RedisPoolCacheStore.java diff --git a/pom.xml b/pom.xml index ccd3c22..7cbb0e3 100644 --- a/pom.xml +++ b/pom.xml @@ -145,6 +145,11 @@ + + redis.clients + jedis + 3.2.0 + \ No newline at end of file diff --git a/src/main/java/net/lamgc/cgj/CQProcess.java b/src/main/java/net/lamgc/cgj/CQProcess.java index 00c0c5f..fdd2dfd 100644 --- a/src/main/java/net/lamgc/cgj/CQProcess.java +++ b/src/main/java/net/lamgc/cgj/CQProcess.java @@ -21,8 +21,6 @@ import org.slf4j.LoggerFactory; import java.io.*; import java.net.URI; import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.StandardOpenOption; import java.text.SimpleDateFormat; import java.util.*; import java.util.concurrent.ArrayBlockingQueue; @@ -41,17 +39,19 @@ public class CQProcess { .serializeNulls() .create(); + private final static URI redisServerUri = URI.create("redis://192.168.1.17"); + private final static Hashtable imageCache = new Hashtable<>(); - private final static CacheStore illustInfoCache = new LocalHashCacheStore<>(); + private final static JsonRedisCacheDatabase illustInfoCache = new JsonRedisCacheDatabase(redisServerUri, "illustInfo", gson); - private final static CacheStore illustPreLoadDataCache = new LocalHashCacheStore<>(); + private final static JsonRedisCacheDatabase illustPreLoadDataCache = new JsonRedisCacheDatabase(redisServerUri, "illustPreLoadData", gson); - private final static CacheStore searchBodyCache = new LocalHashCacheStore<>(); + private final static JsonRedisCacheDatabase searchBodyCache = new JsonRedisCacheDatabase(redisServerUri, "searchBody", gson); private final static CacheStore> pagesCache = new LocalHashCacheStore<>(); - private final static CacheStore rankingCache = new LocalHashCacheStore<>(); + private final static JsonRedisCacheDatabase rankingCache = new JsonRedisCacheDatabase(redisServerUri, "ranking", gson); private final static EventExecutor imageCacheExecutor = new EventExecutor(new ThreadPoolExecutor( 1, @@ -226,7 +226,7 @@ public class CQProcess { log.info("搜索缓存命中."); } - JsonObject resultBody = searchBodyCache.getCache(requestUrl).getAsJsonObject("body"); + JsonObject resultBody = searchBodyCache.getCache(requestUrl).getAsJsonObject().getAsJsonObject("body"); StringBuilder result = new StringBuilder("内容 " + content + " 的搜索结果：\n"); log.info("正在处理信息..."); @@ -448,26 +448,12 @@ public class CQProcess { if (!illustInfoCache.exists(illustIdStr)) { synchronized (illustInfoLock) { if (!illustInfoCache.exists(illustIdStr)) { - File cacheFile = new File(getImageStoreDir(), illustId + ".illustInfo.json"); - log.info("IllustInfoFileName: {}", cacheFile.getName()); - JsonObject illustInfoObj; - if (!cacheFile.exists()) { - try { - cacheFile.createNewFile(); - illustInfoObj = pixivDownload.getIllustInfoByIllustId(illustId); - Files.write(cacheFile.toPath(), gson.toJson(illustInfoObj).getBytes(StandardCharsets.UTF_8), StandardOpenOption.CREATE); - } catch (IOException e) { - cacheFile.delete(); - throw e; - } - } else { - illustInfoObj = gson.fromJson(new FileReader(cacheFile), JsonObject.class); - } + JsonObject illustInfoObj = pixivDownload.getIllustInfoByIllustId(illustId); illustInfoCache.update(illustIdStr, illustInfoObj, null); } } } - return illustInfoCache.getCache(illustIdStr); + return illustInfoCache.getCache(illustIdStr).getAsJsonObject(); } private final static Object illustPreLoadDataLock = new Object(); @@ -476,24 +462,10 @@ public class CQProcess { if (!illustPreLoadDataCache.exists(illustIdStr)) { synchronized (illustPreLoadDataLock) { if (!illustPreLoadDataCache.exists(illustIdStr)) { - File cacheFile = new File(getImageStoreDir(), illustId + ".illustPreLoadData.json"); log.info("缓存失效, 正在更新..."); - log.info("illustPreLoadDataFileName: {}", cacheFile.getName()); - JsonObject preLoadDataObj; - if (!cacheFile.exists()) { - try { - cacheFile.createNewFile(); - preLoadDataObj = pixivDownload.getIllustPreLoadDataById(illustId) - .getAsJsonObject("illust") - .getAsJsonObject(Integer.toString(illustId)); - Files.write(cacheFile.toPath(), gson.toJson(preLoadDataObj).getBytes(StandardCharsets.UTF_8), StandardOpenOption.CREATE); - } catch(IOException e) { - cacheFile.delete(); - throw e; - } - } else { - preLoadDataObj = gson.fromJson(new FileReader(cacheFile), JsonObject.class); - } + JsonObject preLoadDataObj = pixivDownload.getIllustPreLoadDataById(illustId) + .getAsJsonObject("illust") + .getAsJsonObject(Integer.toString(illustId)); long expire = 7200 * 1000; String propValue = CQPluginMain.globalProp.getProperty("cache.illustPreLoadData.expire", "7200000"); @@ -511,7 +483,7 @@ public class CQProcess { } } } - return illustPreLoadDataCache.getCache(illustIdStr); + return illustPreLoadDataCache.getCache(illustIdStr).getAsJsonObject(); } private final static Object illustPagesLock = new Object(); @@ -520,25 +492,7 @@ public class CQProcess { if (!pagesCache.exists(pagesSign)) { synchronized (illustPagesLock) { if (!pagesCache.exists(pagesSign)) { - File cacheFile = new File(getImageStoreDir(), illustId + "." + quality.name() + ".illustPages.json"); - log.info("illustPagesFileName: {}", cacheFile.getName()); - List linkList; - if (!cacheFile.exists()) { - try { - cacheFile.createNewFile(); - linkList = PixivDownload.getIllustAllPageDownload(pixivDownload.getHttpClient(), pixivDownload.getCookieStore(), illustId, quality); - JsonArray jsonArray = new JsonArray(linkList.size()); - linkList.forEach(jsonArray::add); - Files.write(cacheFile.toPath(), gson.toJson(jsonArray).getBytes(StandardCharsets.UTF_8), StandardOpenOption.CREATE); - } catch (IOException e) { - cacheFile.delete(); - throw e; - } - } else { - JsonArray jsonArray = gson.fromJson(new FileReader(cacheFile), JsonArray.class); - linkList = new ArrayList<>(jsonArray.size()); - jsonArray.forEach(jsonElement -> linkList.add(jsonElement.getAsString())); - } + List linkList = PixivDownload.getIllustAllPageDownload(pixivDownload.getHttpClient(), pixivDownload.getCookieStore(), illustId, quality); pagesCache.update(pagesSign, linkList, null); } } @@ -564,31 +518,20 @@ public class CQProcess { synchronized(rankingLock) { if(!rankingCache.exists(requestSign)) { log.info("Ranking缓存失效, 正在更新...(RequestSign: {})", requestSign); - File cacheFile = new File(getImageStoreDir(), date + "." + contentType.name() + "." + mode.modeParam + ".ranking.json"); - JsonArray rankingArr; - if(!cacheFile.exists()) { - List rankingResult = pixivDownload.getRanking(contentType, mode, queryDate, 1, 500); - rankingArr = new JsonArray(rankingResult.size()); - rankingResult.forEach(rankingArr::add); - JsonObject cacheBody = new JsonObject(); - cacheBody.addProperty("updateTimestamp", new Date().getTime()); - cacheBody.addProperty("ContentType", contentType.name()); - cacheBody.addProperty("RankingMode", mode.modeParam); - cacheBody.add("ranking", rankingArr); - Files.write(cacheFile.toPath(), gson.toJson(cacheBody).getBytes(StandardCharsets.UTF_8), StandardOpenOption.CREATE); - log.info("已从Pixiv获取数据并缓存到文件."); - } else { - JsonObject cacheBody = gson.fromJson(new FileReader(cacheFile), JsonObject.class); - rankingArr = cacheBody.getAsJsonArray("ranking"); - log.info("已从文件获取缓存数据."); - } - + List rankingResult = pixivDownload.getRanking(contentType, mode, queryDate, 1, 500); + JsonArray rankingArr = new JsonArray(rankingResult.size()); + rankingResult.forEach(rankingArr::add); + JsonObject cacheBody = new JsonObject(); + cacheBody.addProperty("updateTimestamp", new Date().getTime()); + cacheBody.addProperty("ContentType", contentType.name()); + cacheBody.addProperty("RankingMode", mode.modeParam); + cacheBody.add("ranking", rankingArr); rankingCache.update(requestSign, rankingArr, null); } } } - return PixivDownload.getRanking(rankingCache.getCache(requestSign), start, range); + return PixivDownload.getRanking(rankingCache.getCache(requestSign).getAsJsonArray(), start, range); } } diff --git a/src/main/java/net/lamgc/cgj/cache/CacheStore.java b/src/main/java/net/lamgc/cgj/cache/CacheStore.java index 2844d81..7f7e98c 100644 --- a/src/main/java/net/lamgc/cgj/cache/CacheStore.java +++ b/src/main/java/net/lamgc/cgj/cache/CacheStore.java @@ -40,4 +40,10 @@ public interface CacheStore { */ boolean clear(); + /** + * 是否支持持久化 + * @return 如果支持返回true + */ + boolean supportedPersistence(); + } diff --git a/src/main/java/net/lamgc/cgj/cache/JsonRedisCacheDatabase.java b/src/main/java/net/lamgc/cgj/cache/JsonRedisCacheDatabase.java new file mode 100644 index 0000000..fa43cf9 --- /dev/null +++ b/src/main/java/net/lamgc/cgj/cache/JsonRedisCacheDatabase.java @@ -0,0 +1,26 @@ +package net.lamgc.cgj.cache; + +import com.google.gson.Gson; +import com.google.gson.JsonElement; + +import java.net.URI; + +public class JsonRedisCacheDatabase extends RedisPoolCacheStore { + + private final Gson gson; + + public JsonRedisCacheDatabase(URI redisServerUri, String prefix, Gson gson) { + super(redisServerUri, prefix); + this.gson = gson; + } + + @Override + protected String parse(JsonElement data) { + return this.gson.toJson(data); + } + + @Override + protected JsonElement analysis(String dataStr) { + return this.gson.fromJson(dataStr, JsonElement.class); + } +} diff --git a/src/main/java/net/lamgc/cgj/cache/LocalHashCacheStore.java b/src/main/java/net/lamgc/cgj/cache/LocalHashCacheStore.java index 5d01e36..fcc59a7 100644 --- a/src/main/java/net/lamgc/cgj/cache/LocalHashCacheStore.java +++ b/src/main/java/net/lamgc/cgj/cache/LocalHashCacheStore.java @@ -6,7 +6,27 @@ import java.util.Objects; public class LocalHashCacheStore implements CacheStore { - private final Hashtable> cache = new Hashtable<>(); + private final Hashtable> cache; + + public LocalHashCacheStore() { + this(0); + } + + public LocalHashCacheStore(int initialCapacity) { + this(initialCapacity, 0F); + } + + public LocalHashCacheStore(int initialCapacity, float loadFactor) { + if(initialCapacity != 0) { + if(loadFactor <= 0F) { + cache = new Hashtable<>(initialCapacity); + } else { + cache = new Hashtable<>(initialCapacity, loadFactor); + } + } else { + cache = new Hashtable<>(); + } + } @Override public void update(String key, T value, Date expire) { @@ -54,4 +74,9 @@ public class LocalHashCacheStore implements CacheStore { cache.clear(); return true; } + + @Override + public boolean supportedPersistence() { + return false; + } } diff --git a/src/main/java/net/lamgc/cgj/cache/RedisCacheDatabase.java b/src/main/java/net/lamgc/cgj/cache/RedisCacheDatabase.java new file mode 100644 index 0000000..a5a5d98 --- /dev/null +++ b/src/main/java/net/lamgc/cgj/cache/RedisCacheDatabase.java @@ -0,0 +1,95 @@ +package net.lamgc.cgj.cache; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import redis.clients.jedis.Jedis; +import redis.clients.jedis.Transaction; +import redis.clients.jedis.exceptions.JedisConnectionException; + +import java.net.URI; +import java.util.Date; + +public abstract class RedisCacheDatabase implements CacheStore { + + private final Jedis jedis; + private final Logger log; + private final String keyPrefix; + + public RedisCacheDatabase(URI redisServerUri, String prefix) { + this(redisServerUri, null, prefix); + } + + /** + * 创建一个Redis缓存数据库对象 + * @param redisServerUri 数据库链接 + * @param password 登录密码(如果有) + * @throws JedisConnectionException 当连接失败时抛出 + */ + public RedisCacheDatabase(URI redisServerUri, String password, String prefix) throws JedisConnectionException { + this.jedis = new Jedis(redisServerUri.getHost(), redisServerUri.getPort() <= 0 ? 6379 : redisServerUri.getPort()); + log = LoggerFactory.getLogger("RedisCacheDatabase@" + Integer.toHexString(jedis.hashCode())); + log.info("Redis数据库连接状态: {}", jedis.ping()); + if(password != null) { + this.jedis.auth(password); + } + keyPrefix = prefix.endsWith(".") ? prefix : prefix + "."; + } + + public void connect() { + if(!"PONE".equals(jedis.ping())) { + jedis.connect(); + } + } + + @Override + public void update(String key, T value, Date expire) { + Transaction multi = jedis.multi(); + multi.set(keyPrefix + key, parse(value)); + if(expire != null) { + multi.expireAt(key, expire.getTime()); + log.debug("已设置Key {} 的过期时间(Expire: {})", key, expire.getTime()); + } + multi.exec(); + } + + @Override + public T getCache(String key) { + return analysis(jedis.get(keyPrefix + key)); + } + + @Override + public boolean exists(String key) { + return jedis.exists(keyPrefix + key); + } + + @Override + public boolean exists(String key, Date date) { + return exists(key); + } + + @Override + public boolean clear() { + String result = jedis.flushDB(); + log.info("flushDB返回结果: {}", result); + return true; + } + + /** + * 转换方法 + * @param dataObj 原数据 + * @return 文本型数据 + */ + protected abstract String parse(T dataObj); + + /** + * 将String数据转换成指定类型的对象 + * @param dataStr String数据 + * @return 泛型指定类型的对象 + */ + protected abstract T analysis(String dataStr); + + @Override + public boolean supportedPersistence() { + return true; + } +} diff --git a/src/main/java/net/lamgc/cgj/cache/RedisPoolCacheStore.java b/src/main/java/net/lamgc/cgj/cache/RedisPoolCacheStore.java new file mode 100644 index 0000000..f7b8dbb --- /dev/null +++ b/src/main/java/net/lamgc/cgj/cache/RedisPoolCacheStore.java @@ -0,0 +1,90 @@ +package net.lamgc.cgj.cache; + +import org.apache.commons.pool2.impl.GenericObjectPoolConfig; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import redis.clients.jedis.*; + +import java.net.URI; +import java.util.Date; + +public abstract class RedisPoolCacheStore implements CacheStore { + + private final JedisPool jedisPool; + private final String keyPrefix; + private final Logger log; + + public RedisPoolCacheStore(URI redisServerUri, String prefix) { + this(redisServerUri, null, 0, null, prefix); + } + + public RedisPoolCacheStore(URI redisServerUri, JedisPoolConfig config, int timeout, String password, String prefix) { + jedisPool = new JedisPool(config == null ? new GenericObjectPoolConfig() : config, redisServerUri.getHost(), + redisServerUri.getPort() <= 0 ? 6379 : redisServerUri.getPort(), + timeout <= 0 ? Protocol.DEFAULT_TIMEOUT : timeout, password); + log = LoggerFactory.getLogger("RedisPoolCacheStore@" + Integer.toHexString(jedisPool.hashCode())); + this.keyPrefix = prefix; + } + + @Override + public void update(String key, T value, Date expire) { + Jedis jedis = jedisPool.getResource(); + Transaction multi = jedis.multi(); + multi.set(keyPrefix + key, parse(value)); + if(expire != null) { + multi.expireAt(key, expire.getTime()); + log.debug("已设置Key {} 的过期时间(Expire: {})", key, expire.getTime()); + } + multi.exec(); + jedis.close(); + } + + @Override + public T getCache(String key) { + Jedis jedis = jedisPool.getResource(); + T result = analysis(jedis.get(keyPrefix + key)); + jedis.close(); + return result; + } + + @Override + public boolean exists(String key) { + Jedis jedis = jedisPool.getResource(); + boolean result = jedis.exists(keyPrefix + key); + jedis.close(); + return result; + } + + @Override + public boolean exists(String key, Date date) { + return exists(key); + } + + @Override + public boolean clear() { + Jedis jedis = jedisPool.getResource(); + String result = jedis.flushDB(); + jedis.close(); + log.info("flushDB返回结果: {}", result); + return true; + } + + /** + * 转换方法 + * @param dataObj 原数据 + * @return 文本型数据 + */ + protected abstract String parse(T dataObj); + + /** + * 将String数据转换成指定类型的对象 + * @param dataStr String数据 + * @return 泛型指定类型的对象 + */ + protected abstract T analysis(String dataStr); + + @Override + public boolean supportedPersistence() { + return true; + } +}