Merge branch 'optimize-memory-cache'

src/main/java/net/lamgc/cgj/bot/cache/CacheStoreCentral.java

@@ -12,6 +12,8 @@ import net.lamgc.cgj.exception.HttpRequestException;
 import net.lamgc.cgj.pixiv.PixivDownload;
 import net.lamgc.cgj.pixiv.PixivSearchLinkBuilder;
 import net.lamgc.cgj.pixiv.PixivURL;
+import net.lamgc.cgj.util.Locker;
+import net.lamgc.cgj.util.LockerMap;
 import net.lamgc.cgj.util.URLs;
 import net.lamgc.utils.encrypt.MessageDigestUtils;
 import net.lz1998.cq.utils.CQCode;
@@ -51,6 +53,8 @@ public final class CacheStoreCentral {
         central = new CacheStoreCentral();
     }
 
+    private final LockerMap<String> lockerMap = new LockerMap<>();
+
     private CacheStoreCentral() {}
 
     private final Hashtable<String, File> imageCache = new Hashtable<>();
@@ -59,6 +63,13 @@ public final class CacheStoreCentral {
             new JsonRedisCacheStore(BotGlobal.getGlobal().getRedisServer(),
                     "imageChecksum", BotGlobal.getGlobal().getGson());
 
+    /*
+     * 注意：
+     *      在启用了远端缓存的情况下, 不允许滥用本地缓存
+     *      只有在处理命令中需要短时间大量存取的缓存项才能进行本地缓存(例如PreLoadData需要在排序中大量获取);
+     *      如果没有短时间大量存取的需要, 切勿使用本地缓存
+     */
+
     /**
      * 作品信息缓存 - 不过期
      */
@@ -67,13 +78,13 @@ public final class CacheStoreCentral {
                     "illustInfo", BotGlobal.getGlobal().getGson());
 
     /**
-     * 作品信息预加载数据 - 有效期 2 小时, 本地缓存有效期1 ± 0.25
+     * 作品信息预加载数据 - 有效期 2 小时, 本地缓存有效期 0.5 ± 0.25 小时
      */
     private final CacheStore<JsonElement> illustPreLoadDataCache =
             CacheStoreUtils.hashLocalHotDataStore(
                     new JsonRedisCacheStore(BotGlobal.getGlobal().getRedisServer(),
                             "illustPreLoadData", BotGlobal.getGlobal().getGson()),
-                    3600000, 900000);
+                    1800000, 900000);
     /**
      * 搜索内容缓存, 有效期 2 小时
      */
@@ -228,14 +239,20 @@ public final class CacheStoreCentral {
      */
     public JsonObject getIllustInfo(int illustId, boolean flushCache)
             throws IOException, NoSuchElementException {
-        String illustIdStr = buildSyncKey(Integer.toString(illustId));
+        Locker<String> locker = buildSyncKey(Integer.toString(illustId));
+        String illustIdStr = locker.getKey();
         JsonObject illustInfoObj = null;
         if (!illustInfoCache.exists(illustIdStr) || flushCache) {
-            synchronized (illustIdStr) {
+            try {
-                if (!illustInfoCache.exists(illustIdStr) || flushCache) {
+                locker.lock();
-                    illustInfoObj = BotGlobal.getGlobal().getPixivDownload().getIllustInfoByIllustId(illustId);
+                synchronized (locker) {
-                    illustInfoCache.update(illustIdStr, illustInfoObj, null);
+                    if (!illustInfoCache.exists(illustIdStr) || flushCache) {
+                        illustInfoObj = BotGlobal.getGlobal().getPixivDownload().getIllustInfoByIllustId(illustId);
+                        illustInfoCache.update(illustIdStr, illustInfoObj, null);
+                    }
                 }
+            } finally {
+                locker.unlock();
             }
         }
 
@@ -255,31 +272,37 @@ public final class CacheStoreCentral {
      * @throws IOException 当Http请求处理发生异常时抛出
      */
     public JsonObject getIllustPreLoadData(int illustId, boolean flushCache) throws IOException {
-        String illustIdStr = buildSyncKey(Integer.toString(illustId));
+        Locker<String> locker = buildSyncKey(Integer.toString(illustId));
+        String illustIdStr = locker.getKey();
         JsonObject result = null;
         if (!illustPreLoadDataCache.exists(illustIdStr) || flushCache) {
-            synchronized (illustIdStr) {
+            try {
-                if (!illustPreLoadDataCache.exists(illustIdStr) || flushCache) {
+                locker.lock();
-                    log.trace("IllustId {} 缓存失效, 正在更新...", illustId);
+                synchronized (locker) {
-                    JsonObject preLoadDataObj = BotGlobal.getGlobal().getPixivDownload()
+                    if (!illustPreLoadDataCache.exists(illustIdStr) || flushCache) {
-                            .getIllustPreLoadDataById(illustId)
+                        log.trace("IllustId {} 缓存失效, 正在更新...", illustId);
-                            .getAsJsonObject("illust")
+                        JsonObject preLoadDataObj = BotGlobal.getGlobal().getPixivDownload()
-                            .getAsJsonObject(Integer.toString(illustId));
+                                .getIllustPreLoadDataById(illustId)
+                                .getAsJsonObject("illust")
+                                .getAsJsonObject(Integer.toString(illustId));
 
-                    long expire = 7200 * 1000;
+                        long expire = 7200 * 1000;
-                    String propValue = SettingProperties.
+                        String propValue = SettingProperties.
-                            getProperty(SettingProperties.GLOBAL, "cache.illustPreLoadData.expire", "7200000");
+                                getProperty(SettingProperties.GLOBAL, "cache.illustPreLoadData.expire", "7200000");
-                    log.debug("PreLoadData有效时间设定: {}", propValue);
+                        log.debug("PreLoadData有效时间设定: {}", propValue);
-                    try {
+                        try {
-                        expire = Long.parseLong(propValue);
+                            expire = Long.parseLong(propValue);
-                    } catch (Exception e) {
+                        } catch (Exception e) {
-                        log.warn("全局配置项 \"{}\" 值非法, 已使用默认值: {}", propValue, expire);
+                            log.warn("全局配置项 \"{}\" 值非法, 已使用默认值: {}", propValue, expire);
+                        }
+
+                        result = preLoadDataObj;
+                        illustPreLoadDataCache.update(illustIdStr, preLoadDataObj, expire);
+                        log.trace("作品Id {} preLoadData缓存已更新(有效时间: {})", illustId, expire);
                     }
-
-                    result = preLoadDataObj;
-                    illustPreLoadDataCache.update(illustIdStr, preLoadDataObj, expire);
-                    log.trace("作品Id {} preLoadData缓存已更新(有效时间: {})", illustId, expire);
                 }
+            } finally {
+                locker.unlock();
             }
         }
 
@@ -292,17 +315,24 @@ public final class CacheStoreCentral {
 
     public List<String> getIllustPages(int illustId, PixivDownload.PageQuality quality, boolean flushCache)
             throws IOException {
-        String pagesSign = buildSyncKey(Integer.toString(illustId), ".", quality.name());
+        Locker<String> locker
+                = buildSyncKey(Integer.toString(illustId), ".", quality.name());
+        String pagesSign = locker.getKey();
         List<String> result = null;
         if (!pagesCache.exists(pagesSign) || flushCache) {
-            synchronized (pagesSign) {
+            try {
-                if (!pagesCache.exists(pagesSign) || flushCache) {
+                locker.lock();
-                    List<String> linkList = PixivDownload
+                synchronized (locker) {
-                            .getIllustAllPageDownload(BotGlobal.getGlobal().getPixivDownload().getHttpClient(),
+                    if (!pagesCache.exists(pagesSign) || flushCache) {
-                                    BotGlobal.getGlobal().getPixivDownload().getCookieStore(), illustId, quality);
+                        List<String> linkList = PixivDownload
-                    result = linkList;
+                                .getIllustAllPageDownload(BotGlobal.getGlobal().getPixivDownload().getHttpClient(),
-                    pagesCache.update(pagesSign, linkList, null);
+                                        BotGlobal.getGlobal().getPixivDownload().getCookieStore(), illustId, quality);
+                        result = linkList;
+                        pagesCache.update(pagesSign, linkList, null);
+                    }
                 }
+            } finally {
+                locker.unlock();
             }
         }
 
@@ -342,23 +372,30 @@ public final class CacheStoreCentral {
         }
 
         String date = new SimpleDateFormat("yyyyMMdd").format(queryDate);
-        String requestSign = buildSyncKey(contentType.name(), ".", mode.name(), ".", date);
+        Locker<String> locker
+                = buildSyncKey(contentType.name(), ".", mode.name(), ".", date);
+        String requestSign = locker.getKey();
         List<JsonObject> result = null;
         if(!rankingCache.exists(requestSign) || flushCache) {
-            synchronized(requestSign) {
+            try {
-                if(!rankingCache.exists(requestSign) || flushCache) {
+                locker.lock();
-                    log.trace("Ranking缓存失效, 正在更新...(RequestSign: {})", requestSign);
+                synchronized (locker) {
-                    List<JsonObject> rankingResult = BotGlobal.getGlobal().getPixivDownload()
+                    if (!rankingCache.exists(requestSign) || flushCache) {
-                            .getRanking(contentType, mode, queryDate, 1, 500);
+                        log.trace("Ranking缓存失效, 正在更新...(RequestSign: {})", requestSign);
-                    long expireTime = 0;
+                        List<JsonObject> rankingResult = BotGlobal.getGlobal().getPixivDownload()
-                    if(rankingResult.size() == 0) {
+                                .getRanking(contentType, mode, queryDate, 1, 500);
-                        expireTime = 5400000 + expireTimeFloatRandom.nextInt(1800000);
+                        long expireTime = 0;
-                        log.warn("数据获取失败, 将设置浮动有效时间以准备下次更新. (ExpireTime: {}ms)", expireTime);
+                        if (rankingResult.size() == 0) {
+                            expireTime = 5400000 + expireTimeFloatRandom.nextInt(1800000);
+                            log.warn("数据获取失败, 将设置浮动有效时间以准备下次更新. (ExpireTime: {}ms)", expireTime);
+                        }
+                        result = new ArrayList<>(rankingResult).subList(start - 1, start + range - 1);
+                        rankingCache.update(requestSign, rankingResult, expireTime);
+                        log.trace("Ranking缓存更新完成.(RequestSign: {})", requestSign);
                     }
-                    result = new ArrayList<>(rankingResult).subList(start - 1, start + range - 1);
-                    rankingCache.update(requestSign, rankingResult, expireTime);
-                    log.trace("Ranking缓存更新完成.(RequestSign: {})", requestSign);
                 }
+            } finally {
+                locker.unlock();
             }
         }
 
@@ -428,42 +465,49 @@ public final class CacheStoreCentral {
 
         log.debug("正在搜索作品, 条件: {}", searchBuilder.getSearchCondition());
 
-        String requestUrl = searchBuilder.buildURL().intern();
+        Locker<String> locker
+                = buildSyncKey(searchBuilder.buildURL());
+        String requestUrl = locker.getKey();
         log.debug("RequestUrl: {}", requestUrl);
         JsonObject resultBody = null;
         if(!searchBodyCache.exists(requestUrl)) {
-            synchronized (requestUrl) {
+            try {
-                if (!searchBodyCache.exists(requestUrl)) {
+                locker.lock();
-                    log.trace("searchBody缓存失效, 正在更新...");
+                synchronized (locker) {
-                    JsonObject jsonObject;
+                    if (!searchBodyCache.exists(requestUrl)) {
-                    HttpGet httpGetRequest = BotGlobal.getGlobal().getPixivDownload().
+                        log.trace("searchBody缓存失效, 正在更新...");
-                            createHttpGetRequest(requestUrl);
+                        JsonObject jsonObject;
-                    HttpResponse response = BotGlobal.getGlobal().getPixivDownload().
+                        HttpGet httpGetRequest = BotGlobal.getGlobal().getPixivDownload().
-                            getHttpClient().execute(httpGetRequest);
+                                createHttpGetRequest(requestUrl);
+                        HttpResponse response = BotGlobal.getGlobal().getPixivDownload().
+                                getHttpClient().execute(httpGetRequest);
 
-                    String responseBody = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8);
+                        String responseBody = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8);
-                    log.trace("ResponseBody: {}", responseBody);
+                        log.trace("ResponseBody: {}", responseBody);
-                    jsonObject = BotGlobal.getGlobal().getGson().fromJson(responseBody, JsonObject.class);
+                        jsonObject = BotGlobal.getGlobal().getGson().fromJson(responseBody, JsonObject.class);
 
-                    if (jsonObject.get("error").getAsBoolean()) {
+                        if (jsonObject.get("error").getAsBoolean()) {
-                        log.error("接口请求错误, 错误信息: {}", jsonObject.get("message").getAsString());
+                            log.error("接口请求错误, 错误信息: {}", jsonObject.get("message").getAsString());
-                        throw new HttpRequestException(response.getStatusLine(), responseBody);
+                            throw new HttpRequestException(response.getStatusLine(), responseBody);
+                        }
+
+                        long expire = 7200 * 1000;
+                        String propValue = SettingProperties
+                                .getProperty(SettingProperties.GLOBAL, "cache.searchBody.expire", "7200000");
+                        try {
+                            expire = Long.parseLong(propValue);
+                        } catch (Exception e) {
+                            log.warn("全局配置项 \"{}\" 值非法, 已使用默认值: {}", propValue, expire);
+                        }
+                        resultBody = jsonObject.getAsJsonObject().getAsJsonObject("body");
+                        searchBodyCache.update(requestUrl, jsonObject, expire);
+                        log.trace("searchBody缓存已更新(有效时间: {})", expire);
+                    } else {
+                        log.trace("搜索缓存命中.");
                     }
-
-                    long expire = 7200 * 1000;
-                    String propValue = SettingProperties
-                            .getProperty(SettingProperties.GLOBAL, "cache.searchBody.expire", "7200000");
-                    try {
-                        expire = Long.parseLong(propValue);
-                    } catch (Exception e) {
-                        log.warn("全局配置项 \"{}\" 值非法, 已使用默认值: {}", propValue, expire);
-                    }
-                    resultBody = jsonObject.getAsJsonObject().getAsJsonObject("body");
-                    searchBodyCache.update(requestUrl, jsonObject, expire);
-                    log.trace("searchBody缓存已更新(有效时间: {})", expire);
-                } else {
-                    log.trace("搜索缓存命中.");
                 }
+            } finally {
+                locker.unlock();
             }
         } else {
             log.trace("搜索缓存命中.");
@@ -494,12 +538,12 @@ public final class CacheStoreCentral {
      * @param keys String对象
      * @return 合并后, 如果常量池存在合并后的结果, 则返回常量池中的对象, 否则存入常量池后返回.
      */
-    private static String buildSyncKey(String... keys) {
+    private Locker<String> buildSyncKey(String... keys) {
         StringBuilder sb = new StringBuilder();
         for (String string : keys) {
             sb.append(string);
         }
-        return sb.toString().intern();
+        return lockerMap.createLocker(sb.toString(), true);
     }
 
     /**

src/main/java/net/lamgc/cgj/util/Locker.java

@@ -0,0 +1,75 @@
+package net.lamgc.cgj.util;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.atomic.AtomicInteger;
+
+public final class Locker<K> {
+
+    private final static Logger log = LoggerFactory.getLogger(Locker.class);
+
+    private final LockerMap<K> fromMap;
+
+    private final K key;
+
+    private final boolean autoDestroy;
+
+    private final AtomicInteger lockCount = new AtomicInteger(0);
+
+    /**
+     * 构造一个锁对象
+     * @param map 所属LockerMap
+     * @param key 所属Key
+     */
+    Locker(LockerMap<K> map, K key, boolean autoDestroy) {
+        this.fromMap = map;
+        this.key = key;
+        this.autoDestroy = autoDestroy;
+    }
+
+    /**
+     * 上锁
+     */
+    public void lock() {
+        lockCount.incrementAndGet();
+    }
+
+    /**
+     * 解锁
+     */
+    public void unlock() {
+        int newValue = lockCount.decrementAndGet();
+        if(newValue <= 0 && autoDestroy) {
+            destroy();
+        }
+    }
+
+    /**
+     * 获取锁对象所属Key
+     */
+    public K getKey() {
+        return key;
+    }
+
+    /**
+     * 销毁锁对象
+     */
+    public void destroy() {
+        fromMap.destroyLocker(this);
+    }
+
+    @Override
+    public String toString() {
+        return "Locker@" + this.hashCode() + "{" +
+                "fromMap=" + fromMap +
+                ", key=" + key +
+                '}';
+    }
+
+    @Override
+    protected void finalize() throws Throwable {
+        super.finalize();
+        log.trace("{} 已销毁.", this.toString());
+    }
+}

src/main/java/net/lamgc/cgj/util/LockerMap.java

@@ -0,0 +1,31 @@
+package net.lamgc.cgj.util;
+
+import java.util.HashMap;
+
+public class LockerMap<K> {
+
+    private final HashMap<K, Locker<K>> lockerHashMap = new HashMap<>();
+
+    /**
+     * 创建锁
+     * @param key Key
+     * @return 如果Key所属锁存在, 则返回对应锁, 否则返回新锁
+     */
+    public Locker<K> createLocker(K key, boolean autoDestroy) {
+        if(lockerHashMap.containsKey(key)) {
+            return lockerHashMap.get(key);
+        }
+        Locker<K> newLocker = new Locker<>(this, key, autoDestroy);
+        lockerHashMap.put(key, newLocker);
+        return newLocker;
+    }
+
+    /**
+     * 销毁锁
+     * @param locker 锁对象
+     */
+    public void destroyLocker(Locker<K> locker) {
+        lockerHashMap.remove(locker.getKey());
+    }
+
+}

src/test/java/net/lamgc/cgj/util/LockerMapTest.java

@@ -0,0 +1,18 @@
+package net.lamgc.cgj.util;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+public class LockerMapTest {
+
+    @Test
+    public void createAndFinalizeTest() {
+        LockerMap<String> map = new LockerMap<>();
+        Locker<String> locker = map.createLocker("Test", true);
+        Assert.assertEquals(locker, map.createLocker("Test", true));
+        locker.lock();
+        locker.unlock();
+        System.gc();
+    }
+
+}