[Add] 调整缓存相关类的结构, 增加CacheStore接口;

This commit is contained in:
LamGC 2020-03-30 02:17:25 +08:00
parent a16238d9e7
commit 056d9740b9
3 changed files with 140 additions and 46 deletions

View File

@ -3,9 +3,7 @@ package net.lamgc.cgj;
import com.google.common.base.Strings; import com.google.common.base.Strings;
import com.google.gson.*; import com.google.gson.*;
import io.netty.handler.codec.http.HttpHeaderNames; import io.netty.handler.codec.http.HttpHeaderNames;
import net.lamgc.cgj.cache.CacheObject; import net.lamgc.cgj.cache.*;
import net.lamgc.cgj.cache.ImageCacheHandler;
import net.lamgc.cgj.cache.ImageCacheObject;
import net.lamgc.cgj.pixiv.PixivDownload; import net.lamgc.cgj.pixiv.PixivDownload;
import net.lamgc.cgj.pixiv.PixivSearchBuilder; import net.lamgc.cgj.pixiv.PixivSearchBuilder;
import net.lamgc.cgj.pixiv.PixivURL; import net.lamgc.cgj.pixiv.PixivURL;
@ -21,6 +19,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import java.io.*; import java.io.*;
import java.net.URI;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.StandardOpenOption; import java.nio.file.StandardOpenOption;
@ -38,24 +37,22 @@ public class CQProcess {
private final static File imageStoreDir = new File(System.getProperty("cgj.cqRootDir"), "data/image/cgj/"); private final static File imageStoreDir = new File(System.getProperty("cgj.cqRootDir"), "data/image/cgj/");
private final static Hashtable<String, File> imageCache = new Hashtable<>();
private final static Hashtable<Integer, JsonObject> illustInfoCache = new Hashtable<>();
private final static Hashtable<Integer, CacheObject<JsonObject>> illustPreLoadDataCache = new Hashtable<>();
private final static Hashtable<Integer, CacheObject<JsonObject>> searchBodyCache = new Hashtable<>();
private final static Hashtable<Integer, List<String>> pagesCache = new Hashtable<>();
private final static Hashtable<Integer, JsonArray> rankingCache = new Hashtable<>();
private final static Object searchCacheLock = new Object();
private final static Gson gson = new GsonBuilder() private final static Gson gson = new GsonBuilder()
.serializeNulls() .serializeNulls()
.create(); .create();
private final static Hashtable<String, File> imageCache = new Hashtable<>();
private final static CacheStore<JsonObject> illustInfoCache = new LocalHashCacheStore<>();
private final static CacheStore<JsonObject> illustPreLoadDataCache = new LocalHashCacheStore<>();
private final static CacheStore<JsonObject> searchBodyCache = new LocalHashCacheStore<>();
private final static CacheStore<List<String>> pagesCache = new LocalHashCacheStore<>();
private final static CacheStore<JsonArray> rankingCache = new LocalHashCacheStore<>();
private final static EventExecutor imageCacheExecutor = new EventExecutor(new ThreadPoolExecutor( private final static EventExecutor imageCacheExecutor = new EventExecutor(new ThreadPoolExecutor(
1, 1,
(int) Math.ceil(Runtime.getRuntime().availableProcessors() / 2F), (int) Math.ceil(Runtime.getRuntime().availableProcessors() / 2F),
@ -141,6 +138,7 @@ public class CQProcess {
return "功能未完成"; return "功能未完成";
} }
private final static Object searchCacheLock = new Object();
@Command @Command
public static String search(@Argument(name = "content") String content, public static String search(@Argument(name = "content") String content,
@Argument(name = "type", force = false) String type, @Argument(name = "type", force = false) String type,
@ -193,11 +191,9 @@ public class CQProcess {
String requestUrl = searchBuilder.buildURL(); String requestUrl = searchBuilder.buildURL();
log.info("RequestUrl: {}", requestUrl); log.info("RequestUrl: {}", requestUrl);
CacheObject<JsonObject> cacheObject = new CacheObject<>(); if(!searchBodyCache.exists(requestUrl)) {
int requestUrlSign = requestUrl.hashCode();
if(!searchBodyCache.containsKey(requestUrlSign) || (cacheObject = searchBodyCache.get(requestUrlSign)).isExpire(new Date())) {
synchronized (searchCacheLock) { synchronized (searchCacheLock) {
if (!searchBodyCache.containsKey(requestUrlSign) || (cacheObject = searchBodyCache.get(requestUrlSign)).isExpire(new Date())) { if (!searchBodyCache.exists(requestUrl)) {
log.info("searchBody缓存失效, 正在更新..."); log.info("searchBody缓存失效, 正在更新...");
JsonObject jsonObject; JsonObject jsonObject;
HttpGet httpGetRequest = pixivDownload.createHttpGetRequest(requestUrl); HttpGet httpGetRequest = pixivDownload.createHttpGetRequest(requestUrl);
@ -222,8 +218,7 @@ public class CQProcess {
} }
newExpireDate.setTime(newExpireDate.getTime() + expire); newExpireDate.setTime(newExpireDate.getTime() + expire);
cacheObject.update(jsonObject, newExpireDate); searchBodyCache.update(requestUrl, jsonObject, newExpireDate);
searchBodyCache.put(requestUrlSign, cacheObject);
log.info("searchBody缓存已更新(到期时间: {})", newExpireDate); log.info("searchBody缓存已更新(到期时间: {})", newExpireDate);
} }
} }
@ -231,7 +226,7 @@ public class CQProcess {
log.info("搜索缓存命中."); log.info("搜索缓存命中.");
} }
JsonObject resultBody = searchBodyCache.get(requestUrlSign).get().getAsJsonObject("body"); JsonObject resultBody = searchBodyCache.getCache(requestUrl).getAsJsonObject("body");
StringBuilder result = new StringBuilder("内容 " + content + " 的搜索结果:\n"); StringBuilder result = new StringBuilder("内容 " + content + " 的搜索结果:\n");
log.info("正在处理信息..."); log.info("正在处理信息...");
@ -449,9 +444,10 @@ public class CQProcess {
private final static Object illustInfoLock = new Object(); private final static Object illustInfoLock = new Object();
private static JsonObject getIllustInfo(int illustId) throws IOException { private static JsonObject getIllustInfo(int illustId) throws IOException {
if (!illustInfoCache.containsKey(illustId)) { String illustIdStr = Integer.toString(illustId);
if (!illustInfoCache.exists(illustIdStr)) {
synchronized (illustInfoLock) { synchronized (illustInfoLock) {
if (!illustInfoCache.containsKey(illustId)) { if (!illustInfoCache.exists(illustIdStr)) {
File cacheFile = new File(getImageStoreDir(), illustId + ".illustInfo.json"); File cacheFile = new File(getImageStoreDir(), illustId + ".illustInfo.json");
log.info("IllustInfoFileName: {}", cacheFile.getName()); log.info("IllustInfoFileName: {}", cacheFile.getName());
JsonObject illustInfoObj; JsonObject illustInfoObj;
@ -467,22 +463,20 @@ public class CQProcess {
} else { } else {
illustInfoObj = gson.fromJson(new FileReader(cacheFile), JsonObject.class); illustInfoObj = gson.fromJson(new FileReader(cacheFile), JsonObject.class);
} }
illustInfoCache.put(illustId, illustInfoObj); illustInfoCache.update(illustIdStr, illustInfoObj, null);
} }
} }
} }
return illustInfoCache.get(illustId); return illustInfoCache.getCache(illustIdStr);
} }
private final static Object illustPreLoadDataLock = new Object(); private final static Object illustPreLoadDataLock = new Object();
public static JsonObject getIllustPreLoadData(int illustId) throws IOException { public static JsonObject getIllustPreLoadData(int illustId) throws IOException {
CacheObject<JsonObject> cacheObject = new CacheObject<>(); String illustIdStr = Integer.toString(illustId);
Date currentDate = new Date(); if (!illustPreLoadDataCache.exists(illustIdStr)) {
if (!illustPreLoadDataCache.containsKey(illustId) || (cacheObject = illustPreLoadDataCache.get(illustId)).isExpire(currentDate)) {
synchronized (illustPreLoadDataLock) { synchronized (illustPreLoadDataLock) {
if (!illustPreLoadDataCache.containsKey(illustId) || (cacheObject = illustPreLoadDataCache.get(illustId)).isExpire(currentDate)) { if (!illustPreLoadDataCache.exists(illustIdStr)) {
File cacheFile = new File(getImageStoreDir(), illustId + ".illustPreLoadData.json"); File cacheFile = new File(getImageStoreDir(), illustId + ".illustPreLoadData.json");
log.info("因为到期而失效: {}", cacheObject.isExpire(new Date()));
log.info("缓存失效, 正在更新..."); log.info("缓存失效, 正在更新...");
log.info("illustPreLoadDataFileName: {}", cacheFile.getName()); log.info("illustPreLoadDataFileName: {}", cacheFile.getName());
JsonObject preLoadDataObj; JsonObject preLoadDataObj;
@ -512,21 +506,20 @@ public class CQProcess {
Date newExpire = new Date(); Date newExpire = new Date();
newExpire.setTime(newExpire.getTime() + expire); newExpire.setTime(newExpire.getTime() + expire);
cacheObject.update(preLoadDataObj, newExpire); illustPreLoadDataCache.update(illustIdStr, preLoadDataObj, newExpire);
illustPreLoadDataCache.put(illustId, cacheObject);
log.info("作品Id {} preLoadData缓存已更新(到期时间: {})", illustId, newExpire); log.info("作品Id {} preLoadData缓存已更新(到期时间: {})", illustId, newExpire);
} }
} }
} }
return illustPreLoadDataCache.get(illustId).get(); return illustPreLoadDataCache.getCache(illustIdStr);
} }
private final static Object illustPagesLock = new Object(); private final static Object illustPagesLock = new Object();
public static List<String> getIllustPages(int illustId, PixivDownload.PageQuality quality) throws IOException { public static List<String> getIllustPages(int illustId, PixivDownload.PageQuality quality) throws IOException {
int pagesSign = (illustId + "." + quality.name()).hashCode(); String pagesSign = illustId + "." + quality.name();
if (!pagesCache.containsKey(pagesSign)) { if (!pagesCache.exists(pagesSign)) {
synchronized (illustPagesLock) { synchronized (illustPagesLock) {
if (!pagesCache.containsKey(pagesSign)) { if (!pagesCache.exists(pagesSign)) {
File cacheFile = new File(getImageStoreDir(), illustId + "." + quality.name() + ".illustPages.json"); File cacheFile = new File(getImageStoreDir(), illustId + "." + quality.name() + ".illustPages.json");
log.info("illustPagesFileName: {}", cacheFile.getName()); log.info("illustPagesFileName: {}", cacheFile.getName());
List<String> linkList; List<String> linkList;
@ -546,12 +539,12 @@ public class CQProcess {
linkList = new ArrayList<>(jsonArray.size()); linkList = new ArrayList<>(jsonArray.size());
jsonArray.forEach(jsonElement -> linkList.add(jsonElement.getAsString())); jsonArray.forEach(jsonElement -> linkList.add(jsonElement.getAsString()));
} }
pagesCache.put(pagesSign, linkList); pagesCache.update(pagesSign, linkList, null);
} }
} }
} }
return pagesCache.get(pagesSign); return pagesCache.getCache(pagesSign);
} }
private static File getImageStoreDir() { private static File getImageStoreDir() {
@ -565,10 +558,11 @@ public class CQProcess {
private final static Object rankingLock = new Object(); private final static Object rankingLock = new Object();
private static List<JsonObject> getRankingInfoByCache(PixivURL.RankingContentType contentType, PixivURL.RankingMode mode, Date queryDate, int start, int range) throws IOException { private static List<JsonObject> getRankingInfoByCache(PixivURL.RankingContentType contentType, PixivURL.RankingMode mode, Date queryDate, int start, int range) throws IOException {
String date = new SimpleDateFormat("yyyyMMdd").format(queryDate); String date = new SimpleDateFormat("yyyyMMdd").format(queryDate);
int requestSign = ("Ranking." + contentType.name() + "." + mode.name() + "." + date).hashCode(); //int requestSign = ("Ranking." + contentType.name() + "." + mode.name() + "." + date).hashCode();
if(!rankingCache.containsKey(requestSign)) { String requestSign = "Ranking." + contentType.name() + "." + mode.name() + "." + date;
if(!rankingCache.exists(requestSign)) {
synchronized(rankingLock) { synchronized(rankingLock) {
if(!rankingCache.containsKey(requestSign)) { if(!rankingCache.exists(requestSign)) {
log.info("Ranking缓存失效, 正在更新...(RequestSign: {})", requestSign); log.info("Ranking缓存失效, 正在更新...(RequestSign: {})", requestSign);
File cacheFile = new File(getImageStoreDir(), date + "." + contentType.name() + "." + mode.modeParam + ".ranking.json"); File cacheFile = new File(getImageStoreDir(), date + "." + contentType.name() + "." + mode.modeParam + ".ranking.json");
JsonArray rankingArr; JsonArray rankingArr;
@ -589,12 +583,12 @@ public class CQProcess {
log.info("已从文件获取缓存数据."); log.info("已从文件获取缓存数据.");
} }
rankingCache.put(requestSign, rankingArr); rankingCache.update(requestSign, rankingArr, null);
} }
} }
} }
return PixivDownload.getRanking(rankingCache.get(requestSign), start, range); return PixivDownload.getRanking(rankingCache.getCache(requestSign), start, range);
} }
} }

View File

@ -0,0 +1,43 @@
package net.lamgc.cgj.cache;
import java.util.Date;
public interface CacheStore<T> {
/**
* 更新或添加缓存项
* @param key 缓存键名
* @param value 缓存值
* @param expire 过期时间, 如不过期传入null
*/
void update(String key, T value, Date expire);
/**
* 获取缓存数据
* @param key 键名
* @return 如果存在, 返回对象, 不存在或获取失败则返回null
*/
T getCache(String key);
/**
* 如果该键存在且未过期则返回true.
* @param key 要查询的键
* @return 如果存在且未过期则返回true
*/
boolean exists(String key);
/**
* 如果该键存在且未过期则返回true
* @param key 要查询的键
* @param date 检查的时间
* @return 如果存在且未过期则返回true
*/
boolean exists(String key, Date date);
/**
* 清空缓存
* @return 如果清空成功, 返回true
*/
boolean clear();
}

View File

@ -0,0 +1,57 @@
package net.lamgc.cgj.cache;
import java.util.Date;
import java.util.Hashtable;
import java.util.Objects;
public class LocalHashCacheStore<T> implements CacheStore<T> {
private final Hashtable<String, CacheObject<T>> cache = new Hashtable<>();
@Override
public void update(String key, T value, Date expire) {
if(cache.containsKey(key)) {
cache.get(key).update(value, expire);
} else {
CacheObject<T> cacheObject = new CacheObject<>(value, expire);
cache.put(key, cacheObject);
}
}
@Override
public T getCache(String key) {
if(!cache.containsKey(key)) {
return null;
}
CacheObject<T> cacheObject = cache.get(key);
if(cacheObject.isExpire(new Date())) {
cache.remove(key);
return null;
}
return cacheObject.get();
}
@Override
public boolean exists(String key) {
return exists(key, null);
}
@Override
public boolean exists(String key, Date date) {
if(!cache.containsKey(key)) {
return false;
}
CacheObject<T> cacheObject = cache.get(key);
if(cacheObject.isExpire(Objects.isNull(date) ? new Date() : date)) {
cache.remove(key);
return false;
}
return true;
}
@Override
public boolean clear() {
cache.clear();
return true;
}
}