mirror of
https://github.com/LamGC/ContentGrabbingJi.git
synced 2025-04-30 06:37:36 +00:00
缓存优化3.0
This commit is contained in:
parent
532d7371d8
commit
8f09dc47b4
@ -48,6 +48,8 @@ public class CQProcess {
|
||||
|
||||
private final static Hashtable<String, List<String>> pagesCache = new Hashtable<>();
|
||||
|
||||
private final static Hashtable<String, JsonArray> rankingCache = new Hashtable<>();
|
||||
|
||||
private final static Object searchCacheLock = new Object();
|
||||
|
||||
private final static Gson gson = new GsonBuilder()
|
||||
@ -113,7 +115,7 @@ public class CQProcess {
|
||||
StringBuilder resultBuilder = new StringBuilder(mode.name() + " - 以下是 ").append(new SimpleDateFormat("yyyy-MM-dd").format(queryDate)).append(" 的Pixiv插画排名榜前十名：\n");
|
||||
try {
|
||||
int index = 0;
|
||||
for (JsonObject rankInfo : pixivDownload.getRanking(PixivURL.RankingContentType.TYPE_ILLUST, mode, queryDate, 1, 10)) {
|
||||
for (JsonObject rankInfo : getRankingInfoByCache(PixivURL.RankingContentType.TYPE_ILLUST, mode, queryDate, 0, 10)) {
|
||||
index++;
|
||||
int rank = rankInfo.get("rank").getAsInt();
|
||||
int illustId = rankInfo.get("illust_id").getAsInt();
|
||||
@ -229,7 +231,7 @@ public class CQProcess {
|
||||
}
|
||||
|
||||
JsonObject resultBody = searchBodyCache.get(requestUrl).get().getAsJsonObject("body");
|
||||
StringBuilder result = new StringBuilder("搜索结果：");
|
||||
StringBuilder result = new StringBuilder("内容 " + content + " 的搜索结果：\n");
|
||||
log.info("正在处理信息...");
|
||||
int limit = 8;
|
||||
try {
|
||||
@ -412,15 +414,19 @@ public class CQProcess {
|
||||
}
|
||||
|
||||
static void clearCache() {
|
||||
log.warn("正在清除所有图片缓存...");
|
||||
log.warn("正在清除所有缓存...");
|
||||
imageCache.clear();
|
||||
illustInfoCache.clear();
|
||||
illustPreLoadDataCache.clear();
|
||||
pagesCache.clear();
|
||||
searchBodyCache.clear();
|
||||
File imageStoreDir = new File(System.getProperty("cgj.cqRootDir") + "data/image/cgj/");
|
||||
File[] listFiles = imageStoreDir.listFiles();
|
||||
if (listFiles == null) {
|
||||
log.info("图片缓存目录为空或内部文件获取失败!");
|
||||
log.warn("图片缓存目录为空或内部文件获取失败!");
|
||||
} else {
|
||||
for (File file : listFiles) {
|
||||
log.info("图片文件 {} 删除: {}", file.getName(), file.delete());
|
||||
log.debug("图片文件 {} 删除: {}", file.getName(), file.delete());
|
||||
}
|
||||
}
|
||||
log.info("图片缓存目录删除: {}", imageStoreDir.delete());
|
||||
@ -441,9 +447,10 @@ public class CQProcess {
|
||||
|
||||
private final static Object illustInfoLock = new Object();
|
||||
private static JsonObject getIllustInfo(int illustId) throws IOException {
|
||||
synchronized (illustInfoLock) {
|
||||
File cacheFile = new File(getImageStoreDir(), illustId + ".illustInfo.json");
|
||||
if (!illustInfoCache.containsKey(illustId)) {
|
||||
synchronized (illustInfoLock) {
|
||||
if (!illustInfoCache.containsKey(illustId)) {
|
||||
File cacheFile = new File(getImageStoreDir(), illustId + ".illustInfo.json");
|
||||
log.info("IllustInfoFileName: {}", cacheFile.getName());
|
||||
JsonObject illustInfoObj;
|
||||
if (!cacheFile.exists()) {
|
||||
@ -451,7 +458,7 @@ public class CQProcess {
|
||||
cacheFile.createNewFile();
|
||||
illustInfoObj = pixivDownload.getIllustInfoByIllustId(illustId);
|
||||
Files.write(cacheFile.toPath(), gson.toJson(illustInfoObj).getBytes(StandardCharsets.UTF_8), StandardOpenOption.CREATE);
|
||||
} catch(IOException e) {
|
||||
} catch (IOException e) {
|
||||
cacheFile.delete();
|
||||
throw e;
|
||||
}
|
||||
@ -460,20 +467,20 @@ public class CQProcess {
|
||||
}
|
||||
illustInfoCache.put(illustId, illustInfoObj);
|
||||
}
|
||||
|
||||
return illustInfoCache.get(illustId);
|
||||
}
|
||||
}
|
||||
return illustInfoCache.get(illustId);
|
||||
}
|
||||
|
||||
private final static Object illustPreLoadDataLock = new Object();
|
||||
public static JsonObject getIllustPreLoadData(int illustId) throws IOException {
|
||||
synchronized (illustPreLoadDataLock) {
|
||||
File cacheFile = new File(getImageStoreDir(), illustId + ".illustPreLoadData.json");
|
||||
CacheObject<JsonObject> cacheObject = new CacheObject<>();
|
||||
Date currentDate = new Date();
|
||||
if (!illustPreLoadDataCache.containsKey(illustId) || (cacheObject = illustPreLoadDataCache.get(illustId)).isExpire(currentDate)) {
|
||||
synchronized (illustPreLoadDataLock) {
|
||||
if (!illustPreLoadDataCache.containsKey(illustId) || (cacheObject = illustPreLoadDataCache.get(illustId)).isExpire(currentDate)) {
|
||||
File cacheFile = new File(getImageStoreDir(), illustId + ".illustPreLoadData.json");
|
||||
log.info("因为到期而失效: {}", cacheObject.isExpire(new Date()));
|
||||
log.info("因为缓存文件不存在而失效: {}", !cacheFile.exists());
|
||||
log.info("缓存失效, 正在更新...");
|
||||
log.info("illustPreLoadDataFileName: {}", cacheFile.getName());
|
||||
JsonObject preLoadDataObj;
|
||||
@ -507,16 +514,17 @@ public class CQProcess {
|
||||
illustPreLoadDataCache.put(illustId, cacheObject);
|
||||
log.info("作品Id {} preLoadData缓存已更新(到期时间: {})", illustId, newExpire);
|
||||
}
|
||||
|
||||
return illustPreLoadDataCache.get(illustId).get();
|
||||
}
|
||||
}
|
||||
return illustPreLoadDataCache.get(illustId).get();
|
||||
}
|
||||
|
||||
private final static Object illustPagesLock = new Object();
|
||||
public static List<String> getIllustPages(int illustId, PixivDownload.PageQuality quality) throws IOException {
|
||||
synchronized (illustPagesLock) {
|
||||
File cacheFile = new File(getImageStoreDir(), illustId + "." + quality.name() + ".illustPages.json");
|
||||
if (!pagesCache.containsKey(illustId + "." + quality.name())) {
|
||||
synchronized (illustPagesLock) {
|
||||
if (!pagesCache.containsKey(illustId + "." + quality.name())) {
|
||||
File cacheFile = new File(getImageStoreDir(), illustId + "." + quality.name() + ".illustPages.json");
|
||||
log.info("illustPagesFileName: {}", cacheFile.getName());
|
||||
List<String> linkList;
|
||||
if (!cacheFile.exists()) {
|
||||
@ -526,7 +534,7 @@ public class CQProcess {
|
||||
JsonArray jsonArray = new JsonArray(linkList.size());
|
||||
linkList.forEach(jsonArray::add);
|
||||
Files.write(cacheFile.toPath(), gson.toJson(jsonArray).getBytes(StandardCharsets.UTF_8), StandardOpenOption.CREATE);
|
||||
} catch(IOException e) {
|
||||
} catch (IOException e) {
|
||||
cacheFile.delete();
|
||||
throw e;
|
||||
}
|
||||
@ -537,10 +545,11 @@ public class CQProcess {
|
||||
}
|
||||
pagesCache.put(illustId + "." + quality.name(), linkList);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return pagesCache.get(illustId + "." + quality.name());
|
||||
}
|
||||
}
|
||||
|
||||
private static File getImageStoreDir() {
|
||||
if(!imageStoreDir.exists() && !imageStoreDir.mkdirs()) {
|
||||
@ -550,4 +559,39 @@ public class CQProcess {
|
||||
return imageStoreDir;
|
||||
}
|
||||
|
||||
private final static Object rankingLock = new Object();
|
||||
private static List<JsonObject> getRankingInfoByCache(PixivURL.RankingContentType contentType, PixivURL.RankingMode mode, Date queryDate, int start, int range) throws IOException {
|
||||
String date = new SimpleDateFormat("yyyyMMdd").format(queryDate);
|
||||
String requestSign = "Ranking." + contentType.name() + "." + mode.name() + "." + date;
|
||||
if(!rankingCache.containsKey(requestSign)) {
|
||||
synchronized(rankingLock) {
|
||||
if(!rankingCache.containsKey(requestSign)) {
|
||||
log.info("Ranking缓存失效, 正在更新...(RequestSign: {})", requestSign);
|
||||
File cacheFile = new File(getImageStoreDir(), date + "." + contentType.name() + "." + mode.modeParam + ".ranking.json");
|
||||
JsonArray rankingArr;
|
||||
if(!cacheFile.exists()) {
|
||||
List<JsonObject> rankingResult = pixivDownload.getRanking(contentType, mode, queryDate, 1, 500);
|
||||
rankingArr = new JsonArray(rankingResult.size());
|
||||
rankingResult.forEach(rankingArr::add);
|
||||
JsonObject cacheBody = new JsonObject();
|
||||
cacheBody.addProperty("updateTimestamp", new Date().getTime());
|
||||
cacheBody.addProperty("ContentType", contentType.name());
|
||||
cacheBody.addProperty("RankingMode", mode.modeParam);
|
||||
cacheBody.add("ranking", rankingArr);
|
||||
Files.write(cacheFile.toPath(), gson.toJson(cacheBody).getBytes(StandardCharsets.UTF_8), StandardOpenOption.CREATE);
|
||||
log.info("已从Pixiv获取数据并缓存到文件.");
|
||||
} else {
|
||||
JsonObject cacheBody = gson.fromJson(new FileReader(cacheFile), JsonObject.class);
|
||||
rankingArr = cacheBody.getAsJsonArray("ranking");
|
||||
log.info("已从文件获取缓存数据.");
|
||||
}
|
||||
|
||||
rankingCache.put(requestSign, rankingArr);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return PixivDownload.getRanking(rankingCache.get(requestSign), start, range);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user