Merge branch 'optimize-memory-cache'

src/main/java/net/lamgc/cgj/bot/cache/CacheStoreCentral.java

@@ -12,6 +12,8 @@ import net.lamgc.cgj.exception.HttpRequestException;
 import net.lamgc.cgj.pixiv.PixivDownload;
 import net.lamgc.cgj.pixiv.PixivSearchLinkBuilder;
 import net.lamgc.cgj.pixiv.PixivURL;
+import net.lamgc.cgj.util.Locker;
+import net.lamgc.cgj.util.LockerMap;
 import net.lamgc.cgj.util.URLs;
 import net.lamgc.utils.encrypt.MessageDigestUtils;
 import net.lz1998.cq.utils.CQCode;
@@ -51,6 +53,8 @@ public final class CacheStoreCentral {
         central = new CacheStoreCentral();
     }
 
+    private final LockerMap<String> lockerMap = new LockerMap<>();
+
     private CacheStoreCentral() {}
 
     private final Hashtable<String, File> imageCache = new Hashtable<>();
@@ -59,6 +63,13 @@ public final class CacheStoreCentral {
             new JsonRedisCacheStore(BotGlobal.getGlobal().getRedisServer(),
                     "imageChecksum", BotGlobal.getGlobal().getGson());
 
+    /*
+     * 注意：
+     *      在启用了远端缓存的情况下, 不允许滥用本地缓存
+     *      只有在处理命令中需要短时间大量存取的缓存项才能进行本地缓存(例如PreLoadData需要在排序中大量获取);
+     *      如果没有短时间大量存取的需要, 切勿使用本地缓存
+     */
+
     /**
      * 作品信息缓存 - 不过期
      */
@@ -67,13 +78,13 @@ public final class CacheStoreCentral {
                     "illustInfo", BotGlobal.getGlobal().getGson());
 
     /**
-     * 作品信息预加载数据 - 有效期 2 小时, 本地缓存有效期1 ± 0.25
+     * 作品信息预加载数据 - 有效期 2 小时, 本地缓存有效期 0.5 ± 0.25 小时
      */
     private final CacheStore<JsonElement> illustPreLoadDataCache =
             CacheStoreUtils.hashLocalHotDataStore(
                     new JsonRedisCacheStore(BotGlobal.getGlobal().getRedisServer(),
                             "illustPreLoadData", BotGlobal.getGlobal().getGson()),
-                    3600000, 900000);
+                    1800000, 900000);
     /**
      * 搜索内容缓存, 有效期 2 小时
      */
@@ -228,15 +239,21 @@ public final class CacheStoreCentral {
      */
     public JsonObject getIllustInfo(int illustId, boolean flushCache)
             throws IOException, NoSuchElementException {
-        String illustIdStr = buildSyncKey(Integer.toString(illustId));
+        Locker<String> locker = buildSyncKey(Integer.toString(illustId));
+        String illustIdStr = locker.getKey();
         JsonObject illustInfoObj = null;
         if (!illustInfoCache.exists(illustIdStr) || flushCache) {
-            synchronized (illustIdStr) {
+            try {
+                locker.lock();
+                synchronized (locker) {
                     if (!illustInfoCache.exists(illustIdStr) || flushCache) {
                         illustInfoObj = BotGlobal.getGlobal().getPixivDownload().getIllustInfoByIllustId(illustId);
                         illustInfoCache.update(illustIdStr, illustInfoObj, null);
                     }
                 }
+            } finally {
+                locker.unlock();
+            }
         }
 
         if(Objects.isNull(illustInfoObj)) {
@@ -255,10 +272,13 @@ public final class CacheStoreCentral {
      * @throws IOException 当Http请求处理发生异常时抛出
      */
     public JsonObject getIllustPreLoadData(int illustId, boolean flushCache) throws IOException {
-        String illustIdStr = buildSyncKey(Integer.toString(illustId));
+        Locker<String> locker = buildSyncKey(Integer.toString(illustId));
+        String illustIdStr = locker.getKey();
         JsonObject result = null;
         if (!illustPreLoadDataCache.exists(illustIdStr) || flushCache) {
-            synchronized (illustIdStr) {
+            try {
+                locker.lock();
+                synchronized (locker) {
                     if (!illustPreLoadDataCache.exists(illustIdStr) || flushCache) {
                         log.trace("IllustId {} 缓存失效, 正在更新...", illustId);
                         JsonObject preLoadDataObj = BotGlobal.getGlobal().getPixivDownload()
@@ -281,6 +301,9 @@ public final class CacheStoreCentral {
                         log.trace("作品Id {} preLoadData缓存已更新(有效时间: {})", illustId, expire);
                     }
                 }
+            } finally {
+                locker.unlock();
+            }
         }
 
         if(Objects.isNull(result)) {
@@ -292,10 +315,14 @@ public final class CacheStoreCentral {
 
     public List<String> getIllustPages(int illustId, PixivDownload.PageQuality quality, boolean flushCache)
             throws IOException {
-        String pagesSign = buildSyncKey(Integer.toString(illustId), ".", quality.name());
+        Locker<String> locker
+                = buildSyncKey(Integer.toString(illustId), ".", quality.name());
+        String pagesSign = locker.getKey();
         List<String> result = null;
         if (!pagesCache.exists(pagesSign) || flushCache) {
-            synchronized (pagesSign) {
+            try {
+                locker.lock();
+                synchronized (locker) {
                     if (!pagesCache.exists(pagesSign) || flushCache) {
                         List<String> linkList = PixivDownload
                                 .getIllustAllPageDownload(BotGlobal.getGlobal().getPixivDownload().getHttpClient(),
@@ -304,6 +331,9 @@ public final class CacheStoreCentral {
                         pagesCache.update(pagesSign, linkList, null);
                     }
                 }
+            } finally {
+                locker.unlock();
+            }
         }
 
         if(Objects.isNull(result)) {
@@ -342,16 +372,20 @@ public final class CacheStoreCentral {
         }
 
         String date = new SimpleDateFormat("yyyyMMdd").format(queryDate);
-        String requestSign = buildSyncKey(contentType.name(), ".", mode.name(), ".", date);
+        Locker<String> locker
+                = buildSyncKey(contentType.name(), ".", mode.name(), ".", date);
+        String requestSign = locker.getKey();
         List<JsonObject> result = null;
         if(!rankingCache.exists(requestSign) || flushCache) {
-            synchronized(requestSign) {
-                if(!rankingCache.exists(requestSign) || flushCache) {
+            try {
+                locker.lock();
+                synchronized (locker) {
+                    if (!rankingCache.exists(requestSign) || flushCache) {
                         log.trace("Ranking缓存失效, 正在更新...(RequestSign: {})", requestSign);
                         List<JsonObject> rankingResult = BotGlobal.getGlobal().getPixivDownload()
                                 .getRanking(contentType, mode, queryDate, 1, 500);
                         long expireTime = 0;
-                    if(rankingResult.size() == 0) {
+                        if (rankingResult.size() == 0) {
                             expireTime = 5400000 + expireTimeFloatRandom.nextInt(1800000);
                             log.warn("数据获取失败, 将设置浮动有效时间以准备下次更新. (ExpireTime: {}ms)", expireTime);
                         }
@@ -360,6 +394,9 @@ public final class CacheStoreCentral {
                         log.trace("Ranking缓存更新完成.(RequestSign: {})", requestSign);
                     }
                 }
+            } finally {
+                locker.unlock();
+            }
         }
 
         if (Objects.isNull(result)) {
@@ -428,11 +465,15 @@ public final class CacheStoreCentral {
 
         log.debug("正在搜索作品, 条件: {}", searchBuilder.getSearchCondition());
 
-        String requestUrl = searchBuilder.buildURL().intern();
+        Locker<String> locker
+                = buildSyncKey(searchBuilder.buildURL());
+        String requestUrl = locker.getKey();
         log.debug("RequestUrl: {}", requestUrl);
         JsonObject resultBody = null;
         if(!searchBodyCache.exists(requestUrl)) {
-            synchronized (requestUrl) {
+            try {
+                locker.lock();
+                synchronized (locker) {
                     if (!searchBodyCache.exists(requestUrl)) {
                         log.trace("searchBody缓存失效, 正在更新...");
                         JsonObject jsonObject;
@@ -465,6 +506,9 @@ public final class CacheStoreCentral {
                         log.trace("搜索缓存命中.");
                     }
                 }
+            } finally {
+                locker.unlock();
+            }
         } else {
             log.trace("搜索缓存命中.");
         }
@@ -494,12 +538,12 @@ public final class CacheStoreCentral {
      * @param keys String对象
      * @return 合并后, 如果常量池存在合并后的结果, 则返回常量池中的对象, 否则存入常量池后返回.
      */
-    private static String buildSyncKey(String... keys) {
+    private Locker<String> buildSyncKey(String... keys) {
         StringBuilder sb = new StringBuilder();
         for (String string : keys) {
             sb.append(string);
         }
-        return sb.toString().intern();
+        return lockerMap.createLocker(sb.toString(), true);
     }
 
     /**

src/main/java/net/lamgc/cgj/util/Locker.java

@@ -0,0 +1,75 @@
+package net.lamgc.cgj.util;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.atomic.AtomicInteger;
+
+public final class Locker<K> {
+
+    private final static Logger log = LoggerFactory.getLogger(Locker.class);
+
+    private final LockerMap<K> fromMap;
+
+    private final K key;
+
+    private final boolean autoDestroy;
+
+    private final AtomicInteger lockCount = new AtomicInteger(0);
+
+    /**
+     * 构造一个锁对象
+     * @param map 所属LockerMap
+     * @param key 所属Key
+     */
+    Locker(LockerMap<K> map, K key, boolean autoDestroy) {
+        this.fromMap = map;
+        this.key = key;
+        this.autoDestroy = autoDestroy;
+    }
+
+    /**
+     * 上锁
+     */
+    public void lock() {
+        lockCount.incrementAndGet();
+    }
+
+    /**
+     * 解锁
+     */
+    public void unlock() {
+        int newValue = lockCount.decrementAndGet();
+        if(newValue <= 0 && autoDestroy) {
+            destroy();
+        }
+    }
+
+    /**
+     * 获取锁对象所属Key
+     */
+    public K getKey() {
+        return key;
+    }
+
+    /**
+     * 销毁锁对象
+     */
+    public void destroy() {
+        fromMap.destroyLocker(this);
+    }
+
+    @Override
+    public String toString() {
+        return "Locker@" + this.hashCode() + "{" +
+                "fromMap=" + fromMap +
+                ", key=" + key +
+                '}';
+    }
+
+    @Override
+    protected void finalize() throws Throwable {
+        super.finalize();
+        log.trace("{} 已销毁.", this.toString());
+    }
+}

src/main/java/net/lamgc/cgj/util/LockerMap.java

@@ -0,0 +1,31 @@
+package net.lamgc.cgj.util;
+
+import java.util.HashMap;
+
+public class LockerMap<K> {
+
+    private final HashMap<K, Locker<K>> lockerHashMap = new HashMap<>();
+
+    /**
+     * 创建锁
+     * @param key Key
+     * @return 如果Key所属锁存在, 则返回对应锁, 否则返回新锁
+     */
+    public Locker<K> createLocker(K key, boolean autoDestroy) {
+        if(lockerHashMap.containsKey(key)) {
+            return lockerHashMap.get(key);
+        }
+        Locker<K> newLocker = new Locker<>(this, key, autoDestroy);
+        lockerHashMap.put(key, newLocker);
+        return newLocker;
+    }
+
+    /**
+     * 销毁锁
+     * @param locker 锁对象
+     */
+    public void destroyLocker(Locker<K> locker) {
+        lockerHashMap.remove(locker.getKey());
+    }
+
+}

src/test/java/net/lamgc/cgj/util/LockerMapTest.java

@@ -0,0 +1,18 @@
+package net.lamgc.cgj.util;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+public class LockerMapTest {
+
+    @Test
+    public void createAndFinalizeTest() {
+        LockerMap<String> map = new LockerMap<>();
+        Locker<String> locker = map.createLocker("Test", true);
+        Assert.assertEquals(locker, map.createLocker("Test", true));
+        locker.lock();
+        locker.unlock();
+        System.gc();
+    }
+
+}