LamGC/ContentGrabbingJi
1
0
Fork 0
mirror of https://github.com/LamGC/ContentGrabbingJi.git synced 2025-04-30 06:37:36 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

缓存优化3.0

Browse Source
This commit is contained in:
LamGC 2020-03-27 09:15:28 +08:00
parent 532d7371d8
commit 8f09dc47b4
1 changed files with 127 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

210
src/main/java/net/lamgc/cgj/CQProcess.java
View File

@ -48,6 +48,8 @@ public class CQProcess {
private final static Hashtable<String, List<String>> pagesCache = new Hashtable<>(); private final static Hashtable<String, List<String>> pagesCache = new Hashtable<>();
private final static Hashtable<String, JsonArray> rankingCache = new Hashtable<>();
private final static Object searchCacheLock = new Object(); private final static Object searchCacheLock = new Object();
private final static Gson gson = new GsonBuilder() private final static Gson gson = new GsonBuilder()
@ -113,7 +115,7 @@ public class CQProcess {
StringBuilder resultBuilder = new StringBuilder(mode.name() + " - 以下是 ").append(new SimpleDateFormat("yyyy-MM-dd").format(queryDate)).append(" 的Pixiv插画排名榜前十名：\n"); StringBuilder resultBuilder = new StringBuilder(mode.name() + " - 以下是 ").append(new SimpleDateFormat("yyyy-MM-dd").format(queryDate)).append(" 的Pixiv插画排名榜前十名：\n");
try { try {
int index = 0; int index = 0;
for (JsonObject rankInfo : pixivDownload.getRanking(PixivURL.RankingContentType.TYPE_ILLUST, mode, queryDate, 1, 10)) { for (JsonObject rankInfo : getRankingInfoByCache(PixivURL.RankingContentType.TYPE_ILLUST, mode, queryDate, 0, 10)) {
index++; index++;
int rank = rankInfo.get("rank").getAsInt(); int rank = rankInfo.get("rank").getAsInt();
int illustId = rankInfo.get("illust_id").getAsInt(); int illustId = rankInfo.get("illust_id").getAsInt();
@ -229,7 +231,7 @@ public class CQProcess {
} }
JsonObject resultBody = searchBodyCache.get(requestUrl).get().getAsJsonObject("body"); JsonObject resultBody = searchBodyCache.get(requestUrl).get().getAsJsonObject("body");
StringBuilder result = new StringBuilder("搜索结果："); StringBuilder result = new StringBuilder("内容 " + content + " 的搜索结果：\n");
log.info("正在处理信息..."); log.info("正在处理信息...");
int limit = 8; int limit = 8;
try { try {
@ -412,15 +414,19 @@ public class CQProcess {
} }
static void clearCache() { static void clearCache() {
log.warn("正在清除所有图片缓存..."); log.warn("正在清除所有缓存...");
imageCache.clear(); imageCache.clear();
illustInfoCache.clear();
illustPreLoadDataCache.clear();
pagesCache.clear();
searchBodyCache.clear();
File imageStoreDir = new File(System.getProperty("cgj.cqRootDir") + "data/image/cgj/"); File imageStoreDir = new File(System.getProperty("cgj.cqRootDir") + "data/image/cgj/");
File[] listFiles = imageStoreDir.listFiles(); File[] listFiles = imageStoreDir.listFiles();
if (listFiles == null) { if (listFiles == null) {
log.info("图片缓存目录为空或内部文件获取失败!"); log.warn("图片缓存目录为空或内部文件获取失败!");
} else { } else {
for (File file : listFiles) { for (File file : listFiles) {
log.info("图片文件 {} 删除: {}", file.getName(), file.delete()); log.debug("图片文件 {} 删除: {}", file.getName(), file.delete());
} }
} }
log.info("图片缓存目录删除: {}", imageStoreDir.delete()); log.info("图片缓存目录删除: {}", imageStoreDir.delete());
@ -441,105 +447,108 @@ public class CQProcess {
private final static Object illustInfoLock = new Object(); private final static Object illustInfoLock = new Object();
private static JsonObject getIllustInfo(int illustId) throws IOException { private static JsonObject getIllustInfo(int illustId) throws IOException {
synchronized (illustInfoLock) { if (!illustInfoCache.containsKey(illustId)) {
File cacheFile = new File(getImageStoreDir(), illustId + ".illustInfo.json"); synchronized (illustInfoLock) {
if (!illustInfoCache.containsKey(illustId)) { if (!illustInfoCache.containsKey(illustId)) {
log.info("IllustInfoFileName: {}", cacheFile.getName()); File cacheFile = new File(getImageStoreDir(), illustId + ".illustInfo.json");
JsonObject illustInfoObj; log.info("IllustInfoFileName: {}", cacheFile.getName());
if (!cacheFile.exists()) { JsonObject illustInfoObj;
try { if (!cacheFile.exists()) {
cacheFile.createNewFile(); try {
illustInfoObj = pixivDownload.getIllustInfoByIllustId(illustId); cacheFile.createNewFile();
Files.write(cacheFile.toPath(), gson.toJson(illustInfoObj).getBytes(StandardCharsets.UTF_8), StandardOpenOption.CREATE); illustInfoObj = pixivDownload.getIllustInfoByIllustId(illustId);
} catch(IOException e) { Files.write(cacheFile.toPath(), gson.toJson(illustInfoObj).getBytes(StandardCharsets.UTF_8), StandardOpenOption.CREATE);
cacheFile.delete(); } catch (IOException e) {
throw e; cacheFile.delete();
throw e;
}
} else {
illustInfoObj = gson.fromJson(new FileReader(cacheFile), JsonObject.class);
} }
} else { illustInfoCache.put(illustId, illustInfoObj);
illustInfoObj = gson.fromJson(new FileReader(cacheFile), JsonObject.class);
} }
illustInfoCache.put(illustId, illustInfoObj);
} }
return illustInfoCache.get(illustId);
} }
return illustInfoCache.get(illustId);
} }
private final static Object illustPreLoadDataLock = new Object(); private final static Object illustPreLoadDataLock = new Object();
public static JsonObject getIllustPreLoadData(int illustId) throws IOException { public static JsonObject getIllustPreLoadData(int illustId) throws IOException {
synchronized (illustPreLoadDataLock) { CacheObject<JsonObject> cacheObject = new CacheObject<>();
File cacheFile = new File(getImageStoreDir(), illustId + ".illustPreLoadData.json"); Date currentDate = new Date();
CacheObject<JsonObject> cacheObject = new CacheObject<>(); if (!illustPreLoadDataCache.containsKey(illustId) || (cacheObject = illustPreLoadDataCache.get(illustId)).isExpire(currentDate)) {
Date currentDate = new Date(); synchronized (illustPreLoadDataLock) {
if (!illustPreLoadDataCache.containsKey(illustId) || (cacheObject = illustPreLoadDataCache.get(illustId)).isExpire(currentDate)) { if (!illustPreLoadDataCache.containsKey(illustId) || (cacheObject = illustPreLoadDataCache.get(illustId)).isExpire(currentDate)) {
log.info("因为到期而失效: {}", cacheObject.isExpire(new Date())); File cacheFile = new File(getImageStoreDir(), illustId + ".illustPreLoadData.json");
log.info("因为缓存文件不存在而失效: {}", !cacheFile.exists()); log.info("因为到期而失效: {}", cacheObject.isExpire(new Date()));
log.info("缓存失效, 正在更新..."); log.info("缓存失效, 正在更新...");
log.info("illustPreLoadDataFileName: {}", cacheFile.getName()); log.info("illustPreLoadDataFileName: {}", cacheFile.getName());
JsonObject preLoadDataObj; JsonObject preLoadDataObj;
if (!cacheFile.exists()) { if (!cacheFile.exists()) {
try { try {
cacheFile.createNewFile(); cacheFile.createNewFile();
preLoadDataObj = pixivDownload.getIllustPreLoadDataById(illustId) preLoadDataObj = pixivDownload.getIllustPreLoadDataById(illustId)
.getAsJsonObject("illust") .getAsJsonObject("illust")
.getAsJsonObject(Integer.toString(illustId)); .getAsJsonObject(Integer.toString(illustId));
Files.write(cacheFile.toPath(), gson.toJson(preLoadDataObj).getBytes(StandardCharsets.UTF_8), StandardOpenOption.CREATE); Files.write(cacheFile.toPath(), gson.toJson(preLoadDataObj).getBytes(StandardCharsets.UTF_8), StandardOpenOption.CREATE);
} catch(IOException e) { } catch(IOException e) {
cacheFile.delete(); cacheFile.delete();
throw e; throw e;
}
} else {
preLoadDataObj = gson.fromJson(new FileReader(cacheFile), JsonObject.class);
} }
} else {
preLoadDataObj = gson.fromJson(new FileReader(cacheFile), JsonObject.class);
}
long expire = 7200 * 1000; long expire = 7200 * 1000;
String propValue = CQPluginMain.globalProp.getProperty("cache.illustPreLoadData.expire", "7200000"); String propValue = CQPluginMain.globalProp.getProperty("cache.illustPreLoadData.expire", "7200000");
log.info("PreLoadData有效时间设定: {}", propValue); log.info("PreLoadData有效时间设定: {}", propValue);
try { try {
expire = Long.parseLong(propValue); expire = Long.parseLong(propValue);
} catch (Exception e) { } catch (Exception e) {
log.warn("全局配置项 \"{}\" 值非法, 已使用默认值: {}", propValue, expire); log.warn("全局配置项 \"{}\" 值非法, 已使用默认值: {}", propValue, expire);
} }
Date newExpire = new Date(); Date newExpire = new Date();
newExpire.setTime(newExpire.getTime() + expire); newExpire.setTime(newExpire.getTime() + expire);
cacheObject.update(preLoadDataObj, newExpire); cacheObject.update(preLoadDataObj, newExpire);
illustPreLoadDataCache.put(illustId, cacheObject); illustPreLoadDataCache.put(illustId, cacheObject);
log.info("作品Id {} preLoadData缓存已更新(到期时间: {})", illustId, newExpire); log.info("作品Id {} preLoadData缓存已更新(到期时间: {})", illustId, newExpire);
}
} }
return illustPreLoadDataCache.get(illustId).get();
} }
return illustPreLoadDataCache.get(illustId).get();
} }
private final static Object illustPagesLock = new Object(); private final static Object illustPagesLock = new Object();
public static List<String> getIllustPages(int illustId, PixivDownload.PageQuality quality) throws IOException { public static List<String> getIllustPages(int illustId, PixivDownload.PageQuality quality) throws IOException {
synchronized (illustPagesLock) { if (!pagesCache.containsKey(illustId + "." + quality.name())) {
File cacheFile = new File(getImageStoreDir(), illustId + "." + quality.name() + ".illustPages.json"); synchronized (illustPagesLock) {
if (!pagesCache.containsKey(illustId + "." + quality.name())) { if (!pagesCache.containsKey(illustId + "." + quality.name())) {
log.info("illustPagesFileName: {}", cacheFile.getName()); File cacheFile = new File(getImageStoreDir(), illustId + "." + quality.name() + ".illustPages.json");
List<String> linkList; log.info("illustPagesFileName: {}", cacheFile.getName());
if (!cacheFile.exists()) { List<String> linkList;
try { if (!cacheFile.exists()) {
cacheFile.createNewFile(); try {
linkList = PixivDownload.getIllustAllPageDownload(pixivDownload.getHttpClient(), pixivDownload.getCookieStore(), illustId, quality); cacheFile.createNewFile();
JsonArray jsonArray = new JsonArray(linkList.size()); linkList = PixivDownload.getIllustAllPageDownload(pixivDownload.getHttpClient(), pixivDownload.getCookieStore(), illustId, quality);
linkList.forEach(jsonArray::add); JsonArray jsonArray = new JsonArray(linkList.size());
Files.write(cacheFile.toPath(), gson.toJson(jsonArray).getBytes(StandardCharsets.UTF_8), StandardOpenOption.CREATE); linkList.forEach(jsonArray::add);
} catch(IOException e) { Files.write(cacheFile.toPath(), gson.toJson(jsonArray).getBytes(StandardCharsets.UTF_8), StandardOpenOption.CREATE);
cacheFile.delete(); } catch (IOException e) {
throw e; cacheFile.delete();
throw e;
}
} else {
JsonArray jsonArray = gson.fromJson(new FileReader(cacheFile), JsonArray.class);
linkList = new ArrayList<>(jsonArray.size());
jsonArray.forEach(jsonElement -> linkList.add(jsonElement.getAsString()));
} }
} else { pagesCache.put(illustId + "." + quality.name(), linkList);
JsonArray jsonArray = gson.fromJson(new FileReader(cacheFile), JsonArray.class);
linkList = new ArrayList<>(jsonArray.size());
jsonArray.forEach(jsonElement -> linkList.add(jsonElement.getAsString()));
} }
pagesCache.put(illustId + "." + quality.name(), linkList);
} }
return pagesCache.get(illustId + "." + quality.name());
} }
return pagesCache.get(illustId + "." + quality.name());
} }
private static File getImageStoreDir() { private static File getImageStoreDir() {
@ -550,4 +559,39 @@ public class CQProcess {
return imageStoreDir; return imageStoreDir;
} }
private final static Object rankingLock = new Object();
private static List<JsonObject> getRankingInfoByCache(PixivURL.RankingContentType contentType, PixivURL.RankingMode mode, Date queryDate, int start, int range) throws IOException {
String date = new SimpleDateFormat("yyyyMMdd").format(queryDate);
String requestSign = "Ranking." + contentType.name() + "." + mode.name() + "." + date;
if(!rankingCache.containsKey(requestSign)) {
synchronized(rankingLock) {
if(!rankingCache.containsKey(requestSign)) {
log.info("Ranking缓存失效, 正在更新...(RequestSign: {})", requestSign);
File cacheFile = new File(getImageStoreDir(), date + "." + contentType.name() + "." + mode.modeParam + ".ranking.json");
JsonArray rankingArr;
if(!cacheFile.exists()) {
List<JsonObject> rankingResult = pixivDownload.getRanking(contentType, mode, queryDate, 1, 500);
rankingArr = new JsonArray(rankingResult.size());
rankingResult.forEach(rankingArr::add);
JsonObject cacheBody = new JsonObject();
cacheBody.addProperty("updateTimestamp", new Date().getTime());
cacheBody.addProperty("ContentType", contentType.name());
cacheBody.addProperty("RankingMode", mode.modeParam);
cacheBody.add("ranking", rankingArr);
Files.write(cacheFile.toPath(), gson.toJson(cacheBody).getBytes(StandardCharsets.UTF_8), StandardOpenOption.CREATE);
log.info("已从Pixiv获取数据并缓存到文件.");
} else {
JsonObject cacheBody = gson.fromJson(new FileReader(cacheFile), JsonObject.class);
rankingArr = cacheBody.getAsJsonArray("ranking");
log.info("已从文件获取缓存数据.");
}
rankingCache.put(requestSign, rankingArr);
}
}
}
return PixivDownload.getRanking(rankingCache.get(requestSign), start, range);
}
} }