LamGC/ContentGrabbingJi
1
0
Fork 0
mirror of https://github.com/LamGC/ContentGrabbingJi.git synced 2025-04-30 06:37:36 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

缓存优化3.0

Browse Source
This commit is contained in:
LamGC 2020-03-27 09:15:28 +08:00
parent 532d7371d8
commit 8f09dc47b4
1 changed files with 127 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

82
src/main/java/net/lamgc/cgj/CQProcess.java
View File

@ -48,6 +48,8 @@ public class CQProcess {
private final static Hashtable<String, List<String>> pagesCache = new Hashtable<>(); private final static Hashtable<String, List<String>> pagesCache = new Hashtable<>();
private final static Hashtable<String, JsonArray> rankingCache = new Hashtable<>();
private final static Object searchCacheLock = new Object(); private final static Object searchCacheLock = new Object();
private final static Gson gson = new GsonBuilder() private final static Gson gson = new GsonBuilder()
@ -113,7 +115,7 @@ public class CQProcess {
StringBuilder resultBuilder = new StringBuilder(mode.name() + " - 以下是 ").append(new SimpleDateFormat("yyyy-MM-dd").format(queryDate)).append(" 的Pixiv插画排名榜前十名：\n"); StringBuilder resultBuilder = new StringBuilder(mode.name() + " - 以下是 ").append(new SimpleDateFormat("yyyy-MM-dd").format(queryDate)).append(" 的Pixiv插画排名榜前十名：\n");
try { try {
int index = 0; int index = 0;
for (JsonObject rankInfo : pixivDownload.getRanking(PixivURL.RankingContentType.TYPE_ILLUST, mode, queryDate, 1, 10)) { for (JsonObject rankInfo : getRankingInfoByCache(PixivURL.RankingContentType.TYPE_ILLUST, mode, queryDate, 0, 10)) {
index++; index++;
int rank = rankInfo.get("rank").getAsInt(); int rank = rankInfo.get("rank").getAsInt();
int illustId = rankInfo.get("illust_id").getAsInt(); int illustId = rankInfo.get("illust_id").getAsInt();
@ -229,7 +231,7 @@ public class CQProcess {
} }
JsonObject resultBody = searchBodyCache.get(requestUrl).get().getAsJsonObject("body"); JsonObject resultBody = searchBodyCache.get(requestUrl).get().getAsJsonObject("body");
StringBuilder result = new StringBuilder("搜索结果："); StringBuilder result = new StringBuilder("内容 " + content + " 的搜索结果：\n");
log.info("正在处理信息..."); log.info("正在处理信息...");
int limit = 8; int limit = 8;
try { try {
@ -412,15 +414,19 @@ public class CQProcess {
} }
static void clearCache() { static void clearCache() {
log.warn("正在清除所有图片缓存..."); log.warn("正在清除所有缓存...");
imageCache.clear(); imageCache.clear();
illustInfoCache.clear();
illustPreLoadDataCache.clear();
pagesCache.clear();
searchBodyCache.clear();
File imageStoreDir = new File(System.getProperty("cgj.cqRootDir") + "data/image/cgj/"); File imageStoreDir = new File(System.getProperty("cgj.cqRootDir") + "data/image/cgj/");
File[] listFiles = imageStoreDir.listFiles(); File[] listFiles = imageStoreDir.listFiles();
if (listFiles == null) { if (listFiles == null) {
log.info("图片缓存目录为空或内部文件获取失败!"); log.warn("图片缓存目录为空或内部文件获取失败!");
} else { } else {
for (File file : listFiles) { for (File file : listFiles) {
log.info("图片文件 {} 删除: {}", file.getName(), file.delete()); log.debug("图片文件 {} 删除: {}", file.getName(), file.delete());
} }
} }
log.info("图片缓存目录删除: {}", imageStoreDir.delete()); log.info("图片缓存目录删除: {}", imageStoreDir.delete());
@ -441,9 +447,10 @@ public class CQProcess {
private final static Object illustInfoLock = new Object(); private final static Object illustInfoLock = new Object();
private static JsonObject getIllustInfo(int illustId) throws IOException { private static JsonObject getIllustInfo(int illustId) throws IOException {
synchronized (illustInfoLock) {
File cacheFile = new File(getImageStoreDir(), illustId + ".illustInfo.json");
if (!illustInfoCache.containsKey(illustId)) { if (!illustInfoCache.containsKey(illustId)) {
synchronized (illustInfoLock) {
if (!illustInfoCache.containsKey(illustId)) {
File cacheFile = new File(getImageStoreDir(), illustId + ".illustInfo.json");
log.info("IllustInfoFileName: {}", cacheFile.getName()); log.info("IllustInfoFileName: {}", cacheFile.getName());
JsonObject illustInfoObj; JsonObject illustInfoObj;
if (!cacheFile.exists()) { if (!cacheFile.exists()) {
@ -451,7 +458,7 @@ public class CQProcess {
cacheFile.createNewFile(); cacheFile.createNewFile();
illustInfoObj = pixivDownload.getIllustInfoByIllustId(illustId); illustInfoObj = pixivDownload.getIllustInfoByIllustId(illustId);
Files.write(cacheFile.toPath(), gson.toJson(illustInfoObj).getBytes(StandardCharsets.UTF_8), StandardOpenOption.CREATE); Files.write(cacheFile.toPath(), gson.toJson(illustInfoObj).getBytes(StandardCharsets.UTF_8), StandardOpenOption.CREATE);
} catch(IOException e) { } catch (IOException e) {
cacheFile.delete(); cacheFile.delete();
throw e; throw e;
} }
@ -460,20 +467,20 @@ public class CQProcess {
} }
illustInfoCache.put(illustId, illustInfoObj); illustInfoCache.put(illustId, illustInfoObj);
} }
return illustInfoCache.get(illustId);
} }
} }
return illustInfoCache.get(illustId);
}
private final static Object illustPreLoadDataLock = new Object(); private final static Object illustPreLoadDataLock = new Object();
public static JsonObject getIllustPreLoadData(int illustId) throws IOException { public static JsonObject getIllustPreLoadData(int illustId) throws IOException {
synchronized (illustPreLoadDataLock) {
File cacheFile = new File(getImageStoreDir(), illustId + ".illustPreLoadData.json");
CacheObject<JsonObject> cacheObject = new CacheObject<>(); CacheObject<JsonObject> cacheObject = new CacheObject<>();
Date currentDate = new Date(); Date currentDate = new Date();
if (!illustPreLoadDataCache.containsKey(illustId) || (cacheObject = illustPreLoadDataCache.get(illustId)).isExpire(currentDate)) { if (!illustPreLoadDataCache.containsKey(illustId) || (cacheObject = illustPreLoadDataCache.get(illustId)).isExpire(currentDate)) {
synchronized (illustPreLoadDataLock) {
if (!illustPreLoadDataCache.containsKey(illustId) || (cacheObject = illustPreLoadDataCache.get(illustId)).isExpire(currentDate)) {
File cacheFile = new File(getImageStoreDir(), illustId + ".illustPreLoadData.json");
log.info("因为到期而失效: {}", cacheObject.isExpire(new Date())); log.info("因为到期而失效: {}", cacheObject.isExpire(new Date()));
log.info("因为缓存文件不存在而失效: {}", !cacheFile.exists());
log.info("缓存失效, 正在更新..."); log.info("缓存失效, 正在更新...");
log.info("illustPreLoadDataFileName: {}", cacheFile.getName()); log.info("illustPreLoadDataFileName: {}", cacheFile.getName());
JsonObject preLoadDataObj; JsonObject preLoadDataObj;
@ -507,16 +514,17 @@ public class CQProcess {
illustPreLoadDataCache.put(illustId, cacheObject); illustPreLoadDataCache.put(illustId, cacheObject);
log.info("作品Id {} preLoadData缓存已更新(到期时间: {})", illustId, newExpire); log.info("作品Id {} preLoadData缓存已更新(到期时间: {})", illustId, newExpire);
} }
return illustPreLoadDataCache.get(illustId).get();
} }
} }
return illustPreLoadDataCache.get(illustId).get();
}
private final static Object illustPagesLock = new Object(); private final static Object illustPagesLock = new Object();
public static List<String> getIllustPages(int illustId, PixivDownload.PageQuality quality) throws IOException { public static List<String> getIllustPages(int illustId, PixivDownload.PageQuality quality) throws IOException {
synchronized (illustPagesLock) {
File cacheFile = new File(getImageStoreDir(), illustId + "." + quality.name() + ".illustPages.json");
if (!pagesCache.containsKey(illustId + "." + quality.name())) { if (!pagesCache.containsKey(illustId + "." + quality.name())) {
synchronized (illustPagesLock) {
if (!pagesCache.containsKey(illustId + "." + quality.name())) {
File cacheFile = new File(getImageStoreDir(), illustId + "." + quality.name() + ".illustPages.json");
log.info("illustPagesFileName: {}", cacheFile.getName()); log.info("illustPagesFileName: {}", cacheFile.getName());
List<String> linkList; List<String> linkList;
if (!cacheFile.exists()) { if (!cacheFile.exists()) {
@ -526,7 +534,7 @@ public class CQProcess {
JsonArray jsonArray = new JsonArray(linkList.size()); JsonArray jsonArray = new JsonArray(linkList.size());
linkList.forEach(jsonArray::add); linkList.forEach(jsonArray::add);
Files.write(cacheFile.toPath(), gson.toJson(jsonArray).getBytes(StandardCharsets.UTF_8), StandardOpenOption.CREATE); Files.write(cacheFile.toPath(), gson.toJson(jsonArray).getBytes(StandardCharsets.UTF_8), StandardOpenOption.CREATE);
} catch(IOException e) { } catch (IOException e) {
cacheFile.delete(); cacheFile.delete();
throw e; throw e;
} }
@ -537,10 +545,11 @@ public class CQProcess {
} }
pagesCache.put(illustId + "." + quality.name(), linkList); pagesCache.put(illustId + "." + quality.name(), linkList);
} }
}
}
return pagesCache.get(illustId + "." + quality.name()); return pagesCache.get(illustId + "." + quality.name());
} }
}
private static File getImageStoreDir() { private static File getImageStoreDir() {
if(!imageStoreDir.exists() && !imageStoreDir.mkdirs()) { if(!imageStoreDir.exists() && !imageStoreDir.mkdirs()) {
@ -550,4 +559,39 @@ public class CQProcess {
return imageStoreDir; return imageStoreDir;
} }
private final static Object rankingLock = new Object();
private static List<JsonObject> getRankingInfoByCache(PixivURL.RankingContentType contentType, PixivURL.RankingMode mode, Date queryDate, int start, int range) throws IOException {
String date = new SimpleDateFormat("yyyyMMdd").format(queryDate);
String requestSign = "Ranking." + contentType.name() + "." + mode.name() + "." + date;
if(!rankingCache.containsKey(requestSign)) {
synchronized(rankingLock) {
if(!rankingCache.containsKey(requestSign)) {
log.info("Ranking缓存失效, 正在更新...(RequestSign: {})", requestSign);
File cacheFile = new File(getImageStoreDir(), date + "." + contentType.name() + "." + mode.modeParam + ".ranking.json");
JsonArray rankingArr;
if(!cacheFile.exists()) {
List<JsonObject> rankingResult = pixivDownload.getRanking(contentType, mode, queryDate, 1, 500);
rankingArr = new JsonArray(rankingResult.size());
rankingResult.forEach(rankingArr::add);
JsonObject cacheBody = new JsonObject();
cacheBody.addProperty("updateTimestamp", new Date().getTime());
cacheBody.addProperty("ContentType", contentType.name());
cacheBody.addProperty("RankingMode", mode.modeParam);
cacheBody.add("ranking", rankingArr);
Files.write(cacheFile.toPath(), gson.toJson(cacheBody).getBytes(StandardCharsets.UTF_8), StandardOpenOption.CREATE);
log.info("已从Pixiv获取数据并缓存到文件.");
} else {
JsonObject cacheBody = gson.fromJson(new FileReader(cacheFile), JsonObject.class);
rankingArr = cacheBody.getAsJsonArray("ranking");
log.info("已从文件获取缓存数据.");
}
rankingCache.put(requestSign, rankingArr);
}
}
}
return PixivDownload.getRanking(rankingCache.get(requestSign), start, range);
}
} }