From 056d9740b97a5a53f0d28b552d8d9c5b5ee74cdb Mon Sep 17 00:00:00 2001 From: LamGC Date: Mon, 30 Mar 2020 02:17:25 +0800 Subject: [PATCH] =?UTF-8?q?[Add]=20=E8=B0=83=E6=95=B4=E7=BC=93=E5=AD=98?= =?UTF-8?q?=E7=9B=B8=E5=85=B3=E7=B1=BB=E7=9A=84=E7=BB=93=E6=9E=84,=20?= =?UTF-8?q?=E5=A2=9E=E5=8A=A0CacheStore=E6=8E=A5=E5=8F=A3;?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/main/java/net/lamgc/cgj/CQProcess.java | 86 +++++++++---------- .../java/net/lamgc/cgj/cache/CacheStore.java | 43 ++++++++++ .../lamgc/cgj/cache/LocalHashCacheStore.java | 57 ++++++++++++ 3 files changed, 140 insertions(+), 46 deletions(-) create mode 100644 src/main/java/net/lamgc/cgj/cache/CacheStore.java create mode 100644 src/main/java/net/lamgc/cgj/cache/LocalHashCacheStore.java diff --git a/src/main/java/net/lamgc/cgj/CQProcess.java b/src/main/java/net/lamgc/cgj/CQProcess.java index 6f53aca..00c0c5f 100644 --- a/src/main/java/net/lamgc/cgj/CQProcess.java +++ b/src/main/java/net/lamgc/cgj/CQProcess.java @@ -3,9 +3,7 @@ package net.lamgc.cgj; import com.google.common.base.Strings; import com.google.gson.*; import io.netty.handler.codec.http.HttpHeaderNames; -import net.lamgc.cgj.cache.CacheObject; -import net.lamgc.cgj.cache.ImageCacheHandler; -import net.lamgc.cgj.cache.ImageCacheObject; +import net.lamgc.cgj.cache.*; import net.lamgc.cgj.pixiv.PixivDownload; import net.lamgc.cgj.pixiv.PixivSearchBuilder; import net.lamgc.cgj.pixiv.PixivURL; @@ -21,6 +19,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.*; +import java.net.URI; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.StandardOpenOption; @@ -38,24 +37,22 @@ public class CQProcess { private final static File imageStoreDir = new File(System.getProperty("cgj.cqRootDir"), "data/image/cgj/"); - private final static Hashtable imageCache = new Hashtable<>(); - - private final static Hashtable illustInfoCache = new Hashtable<>(); - - private final static Hashtable> illustPreLoadDataCache = new Hashtable<>(); - - private final static Hashtable> searchBodyCache = new Hashtable<>(); - - private final static Hashtable> pagesCache = new Hashtable<>(); - - private final static Hashtable rankingCache = new Hashtable<>(); - - private final static Object searchCacheLock = new Object(); - private final static Gson gson = new GsonBuilder() .serializeNulls() .create(); + private final static Hashtable imageCache = new Hashtable<>(); + + private final static CacheStore illustInfoCache = new LocalHashCacheStore<>(); + + private final static CacheStore illustPreLoadDataCache = new LocalHashCacheStore<>(); + + private final static CacheStore searchBodyCache = new LocalHashCacheStore<>(); + + private final static CacheStore> pagesCache = new LocalHashCacheStore<>(); + + private final static CacheStore rankingCache = new LocalHashCacheStore<>(); + private final static EventExecutor imageCacheExecutor = new EventExecutor(new ThreadPoolExecutor( 1, (int) Math.ceil(Runtime.getRuntime().availableProcessors() / 2F), @@ -141,6 +138,7 @@ public class CQProcess { return "功能未完成"; } + private final static Object searchCacheLock = new Object(); @Command public static String search(@Argument(name = "content") String content, @Argument(name = "type", force = false) String type, @@ -193,11 +191,9 @@ public class CQProcess { String requestUrl = searchBuilder.buildURL(); log.info("RequestUrl: {}", requestUrl); - CacheObject cacheObject = new CacheObject<>(); - int requestUrlSign = requestUrl.hashCode(); - if(!searchBodyCache.containsKey(requestUrlSign) || (cacheObject = searchBodyCache.get(requestUrlSign)).isExpire(new Date())) { + if(!searchBodyCache.exists(requestUrl)) { synchronized (searchCacheLock) { - if (!searchBodyCache.containsKey(requestUrlSign) || (cacheObject = searchBodyCache.get(requestUrlSign)).isExpire(new Date())) { + if (!searchBodyCache.exists(requestUrl)) { log.info("searchBody缓存失效, 正在更新..."); JsonObject jsonObject; HttpGet httpGetRequest = pixivDownload.createHttpGetRequest(requestUrl); @@ -222,8 +218,7 @@ public class CQProcess { } newExpireDate.setTime(newExpireDate.getTime() + expire); - cacheObject.update(jsonObject, newExpireDate); - searchBodyCache.put(requestUrlSign, cacheObject); + searchBodyCache.update(requestUrl, jsonObject, newExpireDate); log.info("searchBody缓存已更新(到期时间: {})", newExpireDate); } } @@ -231,7 +226,7 @@ public class CQProcess { log.info("搜索缓存命中."); } - JsonObject resultBody = searchBodyCache.get(requestUrlSign).get().getAsJsonObject("body"); + JsonObject resultBody = searchBodyCache.getCache(requestUrl).getAsJsonObject("body"); StringBuilder result = new StringBuilder("内容 " + content + " 的搜索结果：\n"); log.info("正在处理信息..."); @@ -449,9 +444,10 @@ public class CQProcess { private final static Object illustInfoLock = new Object(); private static JsonObject getIllustInfo(int illustId) throws IOException { - if (!illustInfoCache.containsKey(illustId)) { + String illustIdStr = Integer.toString(illustId); + if (!illustInfoCache.exists(illustIdStr)) { synchronized (illustInfoLock) { - if (!illustInfoCache.containsKey(illustId)) { + if (!illustInfoCache.exists(illustIdStr)) { File cacheFile = new File(getImageStoreDir(), illustId + ".illustInfo.json"); log.info("IllustInfoFileName: {}", cacheFile.getName()); JsonObject illustInfoObj; @@ -467,22 +463,20 @@ public class CQProcess { } else { illustInfoObj = gson.fromJson(new FileReader(cacheFile), JsonObject.class); } - illustInfoCache.put(illustId, illustInfoObj); + illustInfoCache.update(illustIdStr, illustInfoObj, null); } } } - return illustInfoCache.get(illustId); + return illustInfoCache.getCache(illustIdStr); } private final static Object illustPreLoadDataLock = new Object(); public static JsonObject getIllustPreLoadData(int illustId) throws IOException { - CacheObject cacheObject = new CacheObject<>(); - Date currentDate = new Date(); - if (!illustPreLoadDataCache.containsKey(illustId) || (cacheObject = illustPreLoadDataCache.get(illustId)).isExpire(currentDate)) { + String illustIdStr = Integer.toString(illustId); + if (!illustPreLoadDataCache.exists(illustIdStr)) { synchronized (illustPreLoadDataLock) { - if (!illustPreLoadDataCache.containsKey(illustId) || (cacheObject = illustPreLoadDataCache.get(illustId)).isExpire(currentDate)) { + if (!illustPreLoadDataCache.exists(illustIdStr)) { File cacheFile = new File(getImageStoreDir(), illustId + ".illustPreLoadData.json"); - log.info("因为到期而失效: {}", cacheObject.isExpire(new Date())); log.info("缓存失效, 正在更新..."); log.info("illustPreLoadDataFileName: {}", cacheFile.getName()); JsonObject preLoadDataObj; @@ -512,21 +506,20 @@ public class CQProcess { Date newExpire = new Date(); newExpire.setTime(newExpire.getTime() + expire); - cacheObject.update(preLoadDataObj, newExpire); - illustPreLoadDataCache.put(illustId, cacheObject); + illustPreLoadDataCache.update(illustIdStr, preLoadDataObj, newExpire); log.info("作品Id {} preLoadData缓存已更新(到期时间: {})", illustId, newExpire); } } } - return illustPreLoadDataCache.get(illustId).get(); + return illustPreLoadDataCache.getCache(illustIdStr); } private final static Object illustPagesLock = new Object(); public static List getIllustPages(int illustId, PixivDownload.PageQuality quality) throws IOException { - int pagesSign = (illustId + "." + quality.name()).hashCode(); - if (!pagesCache.containsKey(pagesSign)) { + String pagesSign = illustId + "." + quality.name(); + if (!pagesCache.exists(pagesSign)) { synchronized (illustPagesLock) { - if (!pagesCache.containsKey(pagesSign)) { + if (!pagesCache.exists(pagesSign)) { File cacheFile = new File(getImageStoreDir(), illustId + "." + quality.name() + ".illustPages.json"); log.info("illustPagesFileName: {}", cacheFile.getName()); List linkList; @@ -546,12 +539,12 @@ public class CQProcess { linkList = new ArrayList<>(jsonArray.size()); jsonArray.forEach(jsonElement -> linkList.add(jsonElement.getAsString())); } - pagesCache.put(pagesSign, linkList); + pagesCache.update(pagesSign, linkList, null); } } } - return pagesCache.get(pagesSign); + return pagesCache.getCache(pagesSign); } private static File getImageStoreDir() { @@ -565,10 +558,11 @@ public class CQProcess { private final static Object rankingLock = new Object(); private static List getRankingInfoByCache(PixivURL.RankingContentType contentType, PixivURL.RankingMode mode, Date queryDate, int start, int range) throws IOException { String date = new SimpleDateFormat("yyyyMMdd").format(queryDate); - int requestSign = ("Ranking." + contentType.name() + "." + mode.name() + "." + date).hashCode(); - if(!rankingCache.containsKey(requestSign)) { + //int requestSign = ("Ranking." + contentType.name() + "." + mode.name() + "." + date).hashCode(); + String requestSign = "Ranking." + contentType.name() + "." + mode.name() + "." + date; + if(!rankingCache.exists(requestSign)) { synchronized(rankingLock) { - if(!rankingCache.containsKey(requestSign)) { + if(!rankingCache.exists(requestSign)) { log.info("Ranking缓存失效, 正在更新...(RequestSign: {})", requestSign); File cacheFile = new File(getImageStoreDir(), date + "." + contentType.name() + "." + mode.modeParam + ".ranking.json"); JsonArray rankingArr; @@ -589,12 +583,12 @@ public class CQProcess { log.info("已从文件获取缓存数据."); } - rankingCache.put(requestSign, rankingArr); + rankingCache.update(requestSign, rankingArr, null); } } } - return PixivDownload.getRanking(rankingCache.get(requestSign), start, range); + return PixivDownload.getRanking(rankingCache.getCache(requestSign), start, range); } } diff --git a/src/main/java/net/lamgc/cgj/cache/CacheStore.java b/src/main/java/net/lamgc/cgj/cache/CacheStore.java new file mode 100644 index 0000000..2844d81 --- /dev/null +++ b/src/main/java/net/lamgc/cgj/cache/CacheStore.java @@ -0,0 +1,43 @@ +package net.lamgc.cgj.cache; + +import java.util.Date; + +public interface CacheStore { + + /** + * 更新或添加缓存项 + * @param key 缓存键名 + * @param value 缓存值 + * @param expire 过期时间, 如不过期传入null + */ + void update(String key, T value, Date expire); + + /** + * 获取缓存数据 + * @param key 键名 + * @return 如果存在, 返回对象, 不存在或获取失败则返回null + */ + T getCache(String key); + + /** + * 如果该键存在且未过期则返回true. + * @param key 要查询的键 + * @return 如果存在且未过期则返回true + */ + boolean exists(String key); + + /** + * 如果该键存在且未过期则返回true + * @param key 要查询的键 + * @param date 检查的时间 + * @return 如果存在且未过期则返回true + */ + boolean exists(String key, Date date); + + /** + * 清空缓存 + * @return 如果清空成功, 返回true + */ + boolean clear(); + +} diff --git a/src/main/java/net/lamgc/cgj/cache/LocalHashCacheStore.java b/src/main/java/net/lamgc/cgj/cache/LocalHashCacheStore.java new file mode 100644 index 0000000..5d01e36 --- /dev/null +++ b/src/main/java/net/lamgc/cgj/cache/LocalHashCacheStore.java @@ -0,0 +1,57 @@ +package net.lamgc.cgj.cache; + +import java.util.Date; +import java.util.Hashtable; +import java.util.Objects; + +public class LocalHashCacheStore implements CacheStore { + + private final Hashtable> cache = new Hashtable<>(); + + @Override + public void update(String key, T value, Date expire) { + if(cache.containsKey(key)) { + cache.get(key).update(value, expire); + } else { + CacheObject cacheObject = new CacheObject<>(value, expire); + cache.put(key, cacheObject); + } + } + + @Override + public T getCache(String key) { + if(!cache.containsKey(key)) { + return null; + } + CacheObject cacheObject = cache.get(key); + if(cacheObject.isExpire(new Date())) { + cache.remove(key); + return null; + } + return cacheObject.get(); + } + + @Override + public boolean exists(String key) { + return exists(key, null); + } + + @Override + public boolean exists(String key, Date date) { + if(!cache.containsKey(key)) { + return false; + } + CacheObject cacheObject = cache.get(key); + if(cacheObject.isExpire(Objects.isNull(date) ? new Date() : date)) { + cache.remove(key); + return false; + } + return true; + } + + @Override + public boolean clear() { + cache.clear(); + return true; + } +}