From bcc21149b93aab02f35ab2c134a48c455762bd80 Mon Sep 17 00:00:00 2001 From: LamGC Date: Fri, 5 Jun 2020 09:53:30 +0800 Subject: [PATCH] =?UTF-8?q?[Change=20#10]=20=E5=B0=86=E7=BC=93=E5=AD=98?= =?UTF-8?q?=E5=AD=98=E5=8F=96=E9=83=A8=E5=88=86=E4=BB=8EBotCommandProcess?= =?UTF-8?q?=E5=88=86=E7=A6=BB;=20[Change]=20=E6=95=B4=E7=90=86=E4=BB=A3?= =?UTF-8?q?=E7=A0=81;?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../net/lamgc/cgj/bot/BotCommandProcess.java | 391 +++--------------- .../cgj/bot/RandomRankingArtworksSender.java | 19 +- .../cgj/bot/cache/CacheStoreCentral.java | 345 ++++++++++++++++ .../cgj/bot/sort/PreLoadDataComparator.java | 11 +- 4 files changed, 431 insertions(+), 335 deletions(-) create mode 100644 src/main/java/net/lamgc/cgj/bot/cache/CacheStoreCentral.java diff --git a/src/main/java/net/lamgc/cgj/bot/BotCommandProcess.java b/src/main/java/net/lamgc/cgj/bot/BotCommandProcess.java index 6f6d122..3046bec 100644 --- a/src/main/java/net/lamgc/cgj/bot/BotCommandProcess.java +++ b/src/main/java/net/lamgc/cgj/bot/BotCommandProcess.java @@ -2,16 +2,18 @@ package net.lamgc.cgj.bot; import com.google.common.base.Strings; import com.google.common.base.Throwables; -import com.google.gson.*; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; import io.netty.handler.codec.http.HttpHeaderNames; import net.lamgc.cgj.bot.boot.BotGlobal; import net.lamgc.cgj.bot.cache.*; import net.lamgc.cgj.bot.event.BufferMessageEvent; import net.lamgc.cgj.bot.sort.PreLoadDataComparator; import net.lamgc.cgj.pixiv.PixivDownload; +import net.lamgc.cgj.pixiv.PixivDownload.PageQuality; import net.lamgc.cgj.pixiv.PixivSearchBuilder; import net.lamgc.cgj.pixiv.PixivURL; -import net.lamgc.cgj.pixiv.PixivDownload.PageQuality; import net.lamgc.cgj.pixiv.PixivURL.RankingContentType; import net.lamgc.cgj.pixiv.PixivURL.RankingMode; import net.lamgc.cgj.util.URLs; @@ -19,21 +21,18 @@ import net.lamgc.utils.base.runner.Argument; import net.lamgc.utils.base.runner.Command; import net.lz1998.cq.utils.CQCode; import org.apache.http.HttpResponse; -import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpHead; -import org.apache.http.util.EntityUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; -import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.text.SimpleDateFormat; import java.util.*; import java.util.concurrent.atomic.AtomicInteger; -@SuppressWarnings({"SynchronizationOnLocalVariableOrMethodParameter", "SameParameterValue"}) +@SuppressWarnings({"SameParameterValue"}) public class BotCommandProcess { private final static Logger log = LoggerFactory.getLogger(BotCommandProcess.class); @@ -44,41 +43,6 @@ public class BotCommandProcess { private final static Hashtable imageCache = new Hashtable<>(); - /** - * 作品信息缓存 - 不过期 - */ - private final static CacheStore illustInfoCache = - new JsonRedisCacheStore(BotGlobal.getGlobal().getRedisServer(), - "illustInfo", BotGlobal.getGlobal().getGson()); - - /** - * 作品信息预加载数据 - 有效期 2 小时, 本地缓存有效期1 ± 0.25 - */ - private final static CacheStore illustPreLoadDataCache = - CacheStoreUtils.hashLocalHotDataStore( - new JsonRedisCacheStore(BotGlobal.getGlobal().getRedisServer(), - "illustPreLoadData", BotGlobal.getGlobal().getGson()), - 3600000, 900000); - /** - * 搜索内容缓存, 有效期 2 小时 - */ - private final static CacheStore searchBodyCache = - new JsonRedisCacheStore(BotGlobal.getGlobal().getRedisServer(), - "searchBody", BotGlobal.getGlobal().getGson()); - - /** - * 排行榜缓存, 不过期 - */ - private final static CacheStore> rankingCache = - new JsonObjectRedisListCacheStore(BotGlobal.getGlobal().getRedisServer(), - "ranking", BotGlobal.getGlobal().getGson()); - - /** - * 作品页面下载链接缓存 - 不过期 - */ - private final static CacheStore> pagesCache = - new StringListRedisCacheStore(BotGlobal.getGlobal().getRedisServer(), "imagePages"); - /** * 作品报告存储 - 不过期 */ @@ -139,7 +103,8 @@ public class BotCommandProcess { * @return 返回作品信息 */ @Command(commandName = "info") - public static String artworkInfo(@Argument(name = "$fromGroup") long fromGroup, @Argument(name = "id") int illustId) { + public static String artworkInfo(@Argument(name = "$fromGroup") long fromGroup, + @Argument(name = "id") int illustId) { if(illustId <= 0) { return "这个作品Id是错误的！"; } @@ -149,7 +114,7 @@ public class BotCommandProcess { return "阅览禁止：该作品已被封印！！"; } - JsonObject illustPreLoadData = getIllustPreLoadData(illustId, false); + JsonObject illustPreLoadData = CacheStoreCentral.getIllustPreLoadData(illustId, false); // 在 Java 6 开始, 编译器会将用'+'进行的字符串拼接将自动转换成StringBuilder拼接 return "色图姬帮你了解了这个作品的信息！\n" + "---------------- 作品信息 ----------------" + "\n作品Id: " + illustId + @@ -213,7 +178,8 @@ public class BotCommandProcess { PixivURL.RankingMode mode; try { String rankingModeValue = contentMode.toUpperCase(); - mode = PixivURL.RankingMode.valueOf(rankingModeValue.startsWith("MODE_") ? rankingModeValue : "MODE_" + rankingModeValue); + mode = PixivURL.RankingMode.valueOf(rankingModeValue.startsWith("MODE_") ? + rankingModeValue : "MODE_" + rankingModeValue); } catch (IllegalArgumentException e) { log.warn("无效的RankingMode值: {}", contentMode); return "参数无效, 请查看帮助信息"; @@ -222,7 +188,8 @@ public class BotCommandProcess { PixivURL.RankingContentType type; try { String contentTypeValue = contentType.toUpperCase(); - type = PixivURL.RankingContentType.valueOf(contentTypeValue.startsWith("TYPE_") ? contentTypeValue : "TYPE_" + contentTypeValue); + type = PixivURL.RankingContentType.valueOf( + contentTypeValue.startsWith("TYPE_") ? contentTypeValue : "TYPE_" + contentTypeValue); } catch (IllegalArgumentException e) { log.warn("无效的RankingContentType值: {}", contentType); return "参数无效, 请查看帮助信息"; @@ -234,7 +201,8 @@ public class BotCommandProcess { return "不支持的内容类型或模式!"; } - StringBuilder resultBuilder = new StringBuilder(mode.name() + " - 以下是 ").append(new SimpleDateFormat("yyyy-MM-dd").format(queryDate)).append(" 的Pixiv插画排名榜前十名：\n"); + StringBuilder resultBuilder = new StringBuilder(mode.name() + " - 以下是 ") + .append(new SimpleDateFormat("yyyy-MM-dd").format(queryDate)).append(" 的Pixiv插画排名榜前十名：\n"); try { int index = 0; int itemLimit = 10; @@ -255,7 +223,8 @@ public class BotCommandProcess { log.warn("配置项 {} 的参数值格式有误!", imageLimitPropertyKey); } - List rankingInfoList = getRankingInfoByCache(type, mode, queryDate, 1, Math.max(0, itemLimit), false); + List rankingInfoList = CacheStoreCentral + .getRankingInfoByCache(type, mode, queryDate, 1, Math.max(0, itemLimit), false); if(rankingInfoList.isEmpty()) { return "无法查询排行榜，可能排行榜尚未更新。"; } @@ -269,16 +238,20 @@ public class BotCommandProcess { String authorName = rankInfo.get("user_name").getAsString(); String title = rankInfo.get("title").getAsString(); resultBuilder.append(rank).append(". (id: ").append(illustId).append(") ").append(title) - .append("(Author: ").append(authorName).append(",").append(authorId).append(") ").append(pagesCount).append("p.\n"); + .append("(Author: ").append(authorName).append(",").append(authorId).append(") ") + .append(pagesCount).append("p.\n"); if (index <= imageLimit) { - resultBuilder.append(getImageById(fromGroup, illustId, PixivDownload.PageQuality.REGULAR, 1)).append("\n"); + resultBuilder + .append(getImageById(fromGroup, illustId, PixivDownload.PageQuality.REGULAR, 1)) + .append("\n"); } } } catch (IOException e) { log.error("消息处理异常", e); return "排名榜获取失败！详情请查看机器人控制台。"; } - return resultBuilder.append("如查询当前时间获取到昨天时间，则今日排名榜未更新。\n如有不当作品，可使用\".cgj report -id 作品id\"向色图姬反馈。").toString(); + return resultBuilder.append("如查询当前时间获取到昨天时间，则今日排名榜未更新。\n" + + "如有不当作品，可使用\".cgj report -id 作品id\"向色图姬反馈。").toString(); } /** @@ -331,7 +304,8 @@ public class BotCommandProcess { @Argument(name = "page", force = false, defaultValue = "1") int pagesIndex ) throws IOException { log.info("正在执行搜索..."); - JsonObject resultBody = getSearchBody(content, type, area, includeKeywords, excludeKeywords, contentOption); + JsonObject resultBody = CacheStoreCentral + .getSearchBody(content, type, area, includeKeywords, excludeKeywords, contentOption); StringBuilder result = new StringBuilder("内容 " + content + " 的搜索结果：\n"); log.debug("正在处理信息..."); @@ -343,7 +317,8 @@ public class BotCommandProcess { log.warn("参数转换异常!将使用默认值(" + limit + ")", e); } for (PixivSearchBuilder.SearchArea searchArea : PixivSearchBuilder.SearchArea.values()) { - if (!resultBody.has(searchArea.jsonKey) || resultBody.getAsJsonObject(searchArea.jsonKey).getAsJsonArray("data").size() == 0) { + if (!resultBody.has(searchArea.jsonKey) || + resultBody.getAsJsonObject(searchArea.jsonKey).getAsJsonArray("data").size() == 0) { log.debug("返回数据不包含 {}", searchArea.jsonKey); continue; } @@ -367,7 +342,8 @@ public class BotCommandProcess { StringBuilder builder = new StringBuilder("["); illustObj.get("tags").getAsJsonArray().forEach(el -> builder.append(el.getAsString()).append(", ")); builder.replace(builder.length() - 2, builder.length(), "]"); - log.debug("{} ({} / {})\n\t作品id: {}, \n\t作者名(作者id): {} ({}), \n\t作品标题: {}, \n\t作品Tags: {}, \n\t页数: {}页, \n\t作品链接: {}", + log.debug("{} ({} / {})\n\t作品id: {}, \n\t作者名(作者id): {} ({}), \n\t" + + "作品标题: {}, \n\t作品Tags: {}, \n\t页数: {}页, \n\t作品链接: {}", searchArea.name(), count, illustsList.size(), @@ -389,7 +365,7 @@ public class BotCommandProcess { continue; } - JsonObject illustPreLoadData = getIllustPreLoadData(illustId, false); + JsonObject illustPreLoadData = CacheStoreCentral.getIllustPreLoadData(illustId, false); result.append(searchArea.name()).append(" (").append(count).append(" / ") .append(limit).append(")\n\t作品id: ").append(illustId) .append(", \n\t作者名: ").append(illustObj.get("userName").getAsString()) @@ -410,7 +386,10 @@ public class BotCommandProcess { break; } } - return Strings.nullToEmpty(result.toString()) + "预览图片并非原图，使用“.cgj image -id 作品id”获取原图\n如有不当作品，可使用\".cgj report -id 作品id\"向色图姬反馈。"; + return Strings.isNullOrEmpty(result.toString()) ? + "搜索完成，未找到相关作品。" : + Strings.nullToEmpty(result.toString()) + "预览图片并非原图，使用“.cgj image -id 作品id”获取原图\n" + + "如有不当作品，可使用\".cgj report -id 作品id\"向色图姬反馈。"; } /** @@ -429,8 +408,13 @@ public class BotCommandProcess { log.warn("来源群 {} 查询的作品Id {} 为R18作品, 根据配置设定, 屏蔽该作品.", fromGroup, illustId); return "该作品已被封印！"; } - List pagesList = PixivDownload.getIllustAllPageDownload(BotGlobal.getGlobal().getPixivDownload().getHttpClient(), BotGlobal.getGlobal().getPixivDownload().getCookieStore(), illustId, quality); - StringBuilder builder = new StringBuilder("作品ID ").append(illustId).append(" 共有").append(pagesList.size()).append("页：").append("\n"); + List pagesList = + PixivDownload.getIllustAllPageDownload( + BotGlobal.getGlobal().getPixivDownload().getHttpClient(), + BotGlobal.getGlobal().getPixivDownload().getCookieStore(), + illustId, quality); + StringBuilder builder = new StringBuilder("作品ID ").append(illustId) + .append(" 共有").append(pagesList.size()).append("页：").append("\n"); int index = 0; for (String link : pagesList) { builder.append("Page ").append(++index).append(": ").append(link).append("\n"); @@ -449,7 +433,8 @@ public class BotCommandProcess { * @return 返回作品在Pixiv的链接 */ @Command(commandName = "link") - public static String artworksLink(@Argument(name = "$fromGroup") long fromGroup, @Argument(name = "id") int illustId) { + public static String artworksLink(@Argument(name = "$fromGroup") long fromGroup, + @Argument(name = "id") int illustId) { try { if (isNoSafe(illustId, SettingProperties.getProperties(fromGroup), false)) { log.warn("作品Id {} 已被屏蔽.", illustId); @@ -496,7 +481,7 @@ public class BotCommandProcess { List pagesList; try { - pagesList = getIllustPages(illustId, quality, false); + pagesList = CacheStoreCentral.getIllustPages(illustId, quality, false); } catch (IOException e) { log.error("获取下载链接列表时发生异常", e); return "发生网络异常，无法获取图片！"; @@ -505,7 +490,8 @@ public class BotCommandProcess { if(log.isDebugEnabled()) { StringBuilder logBuilder = new StringBuilder("作品Id " + illustId + " 所有页面下载链接: \n"); AtomicInteger index = new AtomicInteger(); - pagesList.forEach(item -> logBuilder.append(index.incrementAndGet()).append(". ").append(item).append("\n")); + pagesList.forEach(item -> + logBuilder.append(index.incrementAndGet()).append(". ").append(item).append("\n")); log.debug(logBuilder.toString()); } @@ -529,7 +515,9 @@ public class BotCommandProcess { log.error("获取图片大小失败！", e); return "图片获取失败!"; } - String contentLengthStr = headResponse.getFirstHeader(HttpHeaderNames.CONTENT_LENGTH.toString()).getValue(); + String contentLengthStr = headResponse + .getFirstHeader(HttpHeaderNames.CONTENT_LENGTH.toString()) + .getValue(); log.debug("图片大小: {}B", contentLengthStr); if (imageFile.length() == Long.parseLong(contentLengthStr)) { imageCache.put(URLs.getResourceName(downloadLink), imageFile); @@ -539,7 +527,8 @@ public class BotCommandProcess { } try { - Throwable throwable = ImageCacheStore.executeCacheRequest(new ImageCacheObject(imageCache, illustId, downloadLink, imageFile)); + Throwable throwable = ImageCacheStore.executeCacheRequest( + new ImageCacheObject(imageCache, illustId, downloadLink, imageFile)); if(throwable != null) { throw throwable; } @@ -575,10 +564,7 @@ public class BotCommandProcess { static void clearCache() { log.warn("正在清除所有缓存..."); imageCache.clear(); - illustInfoCache.clear(); - illustPreLoadDataCache.clear(); - pagesCache.clear(); - searchBodyCache.clear(); + CacheStoreCentral.clearCache(); File imageStoreDir = new File(BotGlobal.getGlobal().getDataStoreDir(), "data/image/cgj/"); File[] listFiles = imageStoreDir.listFiles(); if (listFiles == null) { @@ -635,9 +621,22 @@ public class BotCommandProcess { * @throws IOException 获取数据时发生异常时抛出 * @throws NoSuchElementException 当作品不存在时抛出 */ - public static boolean isNoSafe(int illustId, Properties settingProp, boolean returnRaw) throws IOException, NoSuchElementException { - boolean rawValue = getIllustInfo(illustId, false).getAsJsonArray("tags").contains(new JsonPrimitive("R-18")); - return returnRaw || settingProp == null ? rawValue : rawValue && !settingProp.getProperty("image.allowR18", "false").equalsIgnoreCase("true"); + public static boolean isNoSafe(int illustId, Properties settingProp, boolean returnRaw) + throws IOException, NoSuchElementException { + // TODO(LamGC, 20200604): 看看能不能通过官方获得作品R18信息, 进而加强过滤; + JsonArray tags = CacheStoreCentral.getIllustInfo(illustId, false).getAsJsonArray("tags"); + boolean rawValue = false; + for(JsonElement tag : tags) { + boolean current = tag.getAsString().matches("R-*18") || tag.getAsString().contains("R18"); + // log.warn("Match: {}, Tag: {}", current, tag.getAsString()); + if (current) { + rawValue = true; + break; + } + } + return returnRaw || settingProp == null ? rawValue : + rawValue && !settingProp.getProperty("image.allowR18", "false") + .equalsIgnoreCase("true"); } /** @@ -656,258 +655,4 @@ public class BotCommandProcess { return imageStoreDir; } - /** - * 获取作品信息 - * @param illustId 作品Id - * @param flushCache 强制刷新缓存 - * @return 返回作品信息 - * @throws IOException 当Http请求发生异常时抛出 - * @throws NoSuchElementException 当作品未找到时抛出 - */ - private static JsonObject getIllustInfo(int illustId, boolean flushCache) throws IOException, NoSuchElementException { - String illustIdStr = buildSyncKey(Integer.toString(illustId)); - JsonObject illustInfoObj = null; - if (!illustInfoCache.exists(illustIdStr) || flushCache) { - synchronized (illustIdStr) { - if (!illustInfoCache.exists(illustIdStr) || flushCache) { - illustInfoObj = BotGlobal.getGlobal().getPixivDownload().getIllustInfoByIllustId(illustId); - illustInfoCache.update(illustIdStr, illustInfoObj, null); - } - } - } - - if(Objects.isNull(illustInfoObj)) { - illustInfoObj = illustInfoCache.getCache(illustIdStr).getAsJsonObject(); - log.debug("作品Id {} IllustInfo缓存命中.", illustId); - } - return illustInfoObj; - } - - /** - * 获取作品预加载数据. - * 可以获取作品的一些与用户相关的信息 - * @param illustId 作品Id - * @param flushCache 是否刷新缓存 - * @return 成功返回JsonObject对象 - * @throws IOException 当Http请求处理发生异常时抛出 - */ - public static JsonObject getIllustPreLoadData(int illustId, boolean flushCache) throws IOException { - String illustIdStr = buildSyncKey(Integer.toString(illustId)); - JsonObject result = null; - if (!illustPreLoadDataCache.exists(illustIdStr) || flushCache) { - synchronized (illustIdStr) { - if (!illustPreLoadDataCache.exists(illustIdStr) || flushCache) { - log.debug("IllustId {} 缓存失效, 正在更新...", illustId); - JsonObject preLoadDataObj = BotGlobal.getGlobal().getPixivDownload().getIllustPreLoadDataById(illustId) - .getAsJsonObject("illust") - .getAsJsonObject(Integer.toString(illustId)); - - long expire = 7200 * 1000; - String propValue = SettingProperties. - getProperty(SettingProperties.GLOBAL, "cache.illustPreLoadData.expire", "7200000"); - log.debug("PreLoadData有效时间设定: {}", propValue); - try { - expire = Long.parseLong(propValue); - } catch (Exception e) { - log.warn("全局配置项 \"{}\" 值非法, 已使用默认值: {}", propValue, expire); - } - - result = preLoadDataObj; - illustPreLoadDataCache.update(illustIdStr, preLoadDataObj, expire); - log.debug("作品Id {} preLoadData缓存已更新(有效时间: {})", illustId, expire); - } - } - } - - if(Objects.isNull(result)) { - result = illustPreLoadDataCache.getCache(illustIdStr).getAsJsonObject(); - log.debug("作品Id {} PreLoadData缓存命中.", illustId); - } - return result; - } - - public static List getIllustPages(int illustId, PixivDownload.PageQuality quality, boolean flushCache) throws IOException { - String pagesSign = buildSyncKey(Integer.toString(illustId), ".", quality.name()); - List result = null; - if (!pagesCache.exists(pagesSign) || flushCache) { - synchronized (pagesSign) { - if (!pagesCache.exists(pagesSign) || flushCache) { - List linkList = PixivDownload.getIllustAllPageDownload(BotGlobal.getGlobal().getPixivDownload().getHttpClient(), BotGlobal.getGlobal().getPixivDownload().getCookieStore(), illustId, quality); - result = linkList; - pagesCache.update(pagesSign, linkList, null); - } - } - } - - if(Objects.isNull(result)) { - result = pagesCache.getCache(pagesSign); - log.debug("作品Id {} Pages缓存命中.", illustId); - } - return result; - } - - private final static Random expireTimeFloatRandom = new Random(); - /** - * 获取排行榜 - * @param contentType 排行榜类型 - * @param mode 排行榜模式 - * @param queryDate 查询时间 - * @param start 开始排名, 从1开始 - * @param range 取范围 - * @param flushCache 是否强制刷新缓存 - * @return 成功返回有值List, 失败且无异常返回空 - * @throws IOException 获取异常时抛出 - */ - public static List getRankingInfoByCache(PixivURL.RankingContentType contentType, PixivURL.RankingMode mode, Date queryDate, int start, int range, boolean flushCache) throws IOException { - if(!contentType.isSupportedMode(mode)) { - log.warn("试图获取不支持的排行榜类型已拒绝.(ContentType: {}, RankingMode: {})", contentType.name(), mode.name()); - if(log.isDebugEnabled()) { - try { - Thread.dumpStack(); - } catch(Exception e) { - log.debug("本次非法请求的堆栈信息如下: \n{}", Throwables.getStackTraceAsString(e)); - } - } - return new ArrayList<>(0); - } - - String date = new SimpleDateFormat("yyyyMMdd").format(queryDate); - String requestSign = buildSyncKey(contentType.name(), ".", mode.name(), ".", date); - List result = null; - if(!rankingCache.exists(requestSign) || flushCache) { - synchronized(requestSign) { - if(!rankingCache.exists(requestSign) || flushCache) { - log.debug("Ranking缓存失效, 正在更新...(RequestSign: {})", requestSign); - List rankingResult = BotGlobal.getGlobal().getPixivDownload().getRanking(contentType, mode, queryDate, 1, 500); - long expireTime = 0; - if(rankingResult.size() == 0) { - expireTime = 5400000 + expireTimeFloatRandom.nextInt(1800000); - log.warn("数据获取失败, 将设置浮动有效时间以准备下次更新. (ExpireTime: {}ms)", expireTime); - } - result = new ArrayList<>(rankingResult).subList(start - 1, start + range - 1); - rankingCache.update(requestSign, rankingResult, expireTime); - log.debug("Ranking缓存更新完成.(RequestSign: {})", requestSign); - } - } - } - - if (Objects.isNull(result)) { - result = rankingCache.getCache(requestSign, start - 1, range); - log.debug("RequestSign [{}] 缓存命中.", requestSign); - } - log.debug("Result-Length: {}", result.size()); - return PixivDownload.getRanking(result, start - 1, range); - } - - /** - * 获取搜索结果 - * @param content 搜索内容 - * @param type 类型 - * @param area 范围 - * @param includeKeywords 包含关键词 - * @param excludeKeywords 排除关键词 - * @param contentOption 内容类型 - * @return 返回完整搜索结果 - * @throws IOException 当请求发生异常, 或接口返回异常信息时抛出. - */ - public static JsonObject getSearchBody( - String content, - String type, - String area, - String includeKeywords, - String excludeKeywords, - String contentOption) throws IOException { - PixivSearchBuilder searchBuilder = new PixivSearchBuilder(Strings.isNullOrEmpty(content) ? "" : content); - if (type != null) { - try { - searchBuilder.setSearchType(PixivSearchBuilder.SearchType.valueOf(type.toUpperCase())); - } catch (IllegalArgumentException e) { - log.warn("不支持的SearchType: {}", type); - } - } - if (area != null) { - try { - searchBuilder.setSearchArea(PixivSearchBuilder.SearchArea.valueOf(area)); - } catch (IllegalArgumentException e) { - log.warn("不支持的SearchArea: {}", area); - } - } - if (contentOption != null) { - try { - searchBuilder.setSearchContentOption(PixivSearchBuilder.SearchContentOption.valueOf(contentOption)); - } catch (IllegalArgumentException e) { - log.warn("不支持的SearchContentOption: {}", contentOption); - } - } - - if (!Strings.isNullOrEmpty(includeKeywords)) { - for (String keyword : includeKeywords.split(";")) { - searchBuilder.removeExcludeKeyword(keyword.trim()); - searchBuilder.addIncludeKeyword(keyword.trim()); - log.debug("已添加关键字: {}", keyword); - } - } - if (!Strings.isNullOrEmpty(excludeKeywords)) { - for (String keyword : excludeKeywords.split(";")) { - searchBuilder.removeIncludeKeyword(keyword.trim()); - searchBuilder.addExcludeKeyword(keyword.trim()); - log.debug("已添加排除关键字: {}", keyword); - } - } - - log.info("正在搜索作品, 条件: {}", searchBuilder.getSearchCondition()); - - String requestUrl = searchBuilder.buildURL().intern(); - log.debug("RequestUrl: {}", requestUrl); - JsonObject resultBody = null; - if(!searchBodyCache.exists(requestUrl)) { - synchronized (requestUrl) { - if (!searchBodyCache.exists(requestUrl)) { - log.debug("searchBody缓存失效, 正在更新..."); - JsonObject jsonObject; - HttpGet httpGetRequest = BotGlobal.getGlobal().getPixivDownload().createHttpGetRequest(requestUrl); - HttpResponse response = BotGlobal.getGlobal().getPixivDownload().getHttpClient().execute(httpGetRequest); - - String responseBody = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8); - log.debug("ResponseBody: {}", responseBody); - jsonObject = BotGlobal.getGlobal().getGson().fromJson(responseBody, JsonObject.class); - - if (jsonObject.get("error").getAsBoolean()) { - log.error("接口请求错误, 错误信息: {}", jsonObject.get("message").getAsString()); - throw new IOException("Interface Request Error: " + jsonObject.get("message").getAsString()); - } - - long expire = 7200 * 1000; - String propValue = SettingProperties - .getProperty(SettingProperties.GLOBAL, "cache.searchBody.expire", "7200000"); - try { - expire = Long.parseLong(propValue); - } catch (Exception e) { - log.warn("全局配置项 \"{}\" 值非法, 已使用默认值: {}", propValue, expire); - } - resultBody = jsonObject.getAsJsonObject().getAsJsonObject("body"); - searchBodyCache.update(requestUrl, jsonObject, expire); - log.debug("searchBody缓存已更新(有效时间: {})", expire); - } else { - log.debug("搜索缓存命中."); - } - } - } else { - log.debug("搜索缓存命中."); - } - - if(Objects.isNull(resultBody)) { - resultBody = searchBodyCache.getCache(requestUrl).getAsJsonObject().getAsJsonObject("body"); - } - return resultBody; - } - - - private static String buildSyncKey(String... keys) { - StringBuilder sb = new StringBuilder(); - for (String string : keys) { - sb.append(string); - } - return sb.toString().intern(); - } } diff --git a/src/main/java/net/lamgc/cgj/bot/RandomRankingArtworksSender.java b/src/main/java/net/lamgc/cgj/bot/RandomRankingArtworksSender.java index 05a3769..95e8939 100644 --- a/src/main/java/net/lamgc/cgj/bot/RandomRankingArtworksSender.java +++ b/src/main/java/net/lamgc/cgj/bot/RandomRankingArtworksSender.java @@ -1,6 +1,7 @@ package net.lamgc.cgj.bot; import com.google.gson.JsonObject; +import net.lamgc.cgj.bot.cache.CacheStoreCentral; import net.lamgc.cgj.bot.message.MessageSender; import net.lamgc.cgj.pixiv.PixivDownload; import net.lamgc.cgj.pixiv.PixivURL; @@ -61,7 +62,7 @@ public class RandomRankingArtworksSender extends AutoSender { int selectRanking = rankingStart + new Random().nextInt(rankingStop - rankingStart + 1); try { - List rankingList = BotCommandProcess.getRankingInfoByCache( + List rankingList = CacheStoreCentral.getRankingInfoByCache( contentType, mode, queryDate, @@ -76,7 +77,8 @@ public class RandomRankingArtworksSender extends AutoSender { JsonObject rankingInfo = rankingList.get(0); int illustId = rankingInfo.get("illust_id").getAsInt(); - if(BotCommandProcess.isNoSafe(illustId, SettingProperties.getProperties(SettingProperties.GLOBAL), false)) { + if(BotCommandProcess.isNoSafe(illustId, + SettingProperties.getProperties(SettingProperties.GLOBAL), false)) { log.warn("作品为r18作品, 取消本次发送."); return; } else if(BotCommandProcess.isReported(illustId)) { @@ -84,13 +86,12 @@ public class RandomRankingArtworksSender extends AutoSender { return; } - StringBuilder message = new StringBuilder(); - message.append("#美图推送 - 今日排行榜 第 ").append(rankingInfo.get("rank").getAsInt()).append(" 名\n"); - message.append("标题：").append(rankingInfo.get("title").getAsString()).append("(").append(illustId).append(")\n"); - message.append("作者：").append(rankingInfo.get("user_name").getAsString()).append("\n"); - message.append(BotCommandProcess.getImageById(0, illustId, quality, 1)); - message.append("\n如有不当作品，可使用\".cgj report -id ").append(illustId).append("\"向色图姬反馈。"); - getMessageSender().sendMessage(message.toString()); + String message = "#美图推送 - 今日排行榜 第 " + rankingInfo.get("rank").getAsInt() + " 名\n" + + "标题：" + rankingInfo.get("title").getAsString() + "(" + illustId + ")\n" + + "作者：" + rankingInfo.get("user_name").getAsString() + "\n" + + BotCommandProcess.getImageById(0, illustId, quality, 1) + + "\n如有不当作品，可使用\".cgj report -id " + illustId + "\"向色图姬反馈。"; + getMessageSender().sendMessage(message); } catch (Exception e) { log.error("发送随机作品时发生异常", e); } diff --git a/src/main/java/net/lamgc/cgj/bot/cache/CacheStoreCentral.java b/src/main/java/net/lamgc/cgj/bot/cache/CacheStoreCentral.java new file mode 100644 index 0000000..14db0f2 --- /dev/null +++ b/src/main/java/net/lamgc/cgj/bot/cache/CacheStoreCentral.java @@ -0,0 +1,345 @@ +package net.lamgc.cgj.bot.cache; + +import com.google.common.base.Strings; +import com.google.common.base.Throwables; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import net.lamgc.cgj.bot.SettingProperties; +import net.lamgc.cgj.bot.boot.BotGlobal; +import net.lamgc.cgj.pixiv.PixivDownload; +import net.lamgc.cgj.pixiv.PixivSearchBuilder; +import net.lamgc.cgj.pixiv.PixivURL; +import org.apache.http.HttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.util.EntityUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.text.SimpleDateFormat; +import java.util.*; + +@SuppressWarnings("SynchronizationOnLocalVariableOrMethodParameter") +public final class CacheStoreCentral { + + private CacheStoreCentral() {} + + private final static Logger log = LoggerFactory.getLogger(CacheStoreCentral.class); + + /** + * 作品信息缓存 - 不过期 + */ + private final static CacheStore illustInfoCache = + new JsonRedisCacheStore(BotGlobal.getGlobal().getRedisServer(), + "illustInfo", BotGlobal.getGlobal().getGson()); + + /** + * 作品信息预加载数据 - 有效期 2 小时, 本地缓存有效期1 ± 0.25 + */ + private final static CacheStore illustPreLoadDataCache = + CacheStoreUtils.hashLocalHotDataStore( + new JsonRedisCacheStore(BotGlobal.getGlobal().getRedisServer(), + "illustPreLoadData", BotGlobal.getGlobal().getGson()), + 3600000, 900000); + /** + * 搜索内容缓存, 有效期 2 小时 + */ + private final static CacheStore searchBodyCache = + new JsonRedisCacheStore(BotGlobal.getGlobal().getRedisServer(), + "searchBody", BotGlobal.getGlobal().getGson()); + + /** + * 排行榜缓存, 不过期 + */ + private final static CacheStore> rankingCache = + new JsonObjectRedisListCacheStore(BotGlobal.getGlobal().getRedisServer(), + "ranking", BotGlobal.getGlobal().getGson()); + + /** + * 作品页面下载链接缓存 - 不过期 + */ + private final static CacheStore> pagesCache = + new StringListRedisCacheStore(BotGlobal.getGlobal().getRedisServer(), "imagePages"); + + /** + * 清空所有缓存 + */ + public static void clearCache() { + illustInfoCache.clear(); + illustPreLoadDataCache.clear(); + searchBodyCache.clear(); + rankingCache.clear(); + pagesCache.clear(); + } + + /** + * 获取作品信息 + * @param illustId 作品Id + * @param flushCache 强制刷新缓存 + * @return 返回作品信息 + * @throws IOException 当Http请求发生异常时抛出 + * @throws NoSuchElementException 当作品未找到时抛出 + */ + public static JsonObject getIllustInfo(int illustId, boolean flushCache) + throws IOException, NoSuchElementException { + String illustIdStr = buildSyncKey(Integer.toString(illustId)); + JsonObject illustInfoObj = null; + if (!illustInfoCache.exists(illustIdStr) || flushCache) { + synchronized (illustIdStr) { + if (!illustInfoCache.exists(illustIdStr) || flushCache) { + illustInfoObj = BotGlobal.getGlobal().getPixivDownload().getIllustInfoByIllustId(illustId); + illustInfoCache.update(illustIdStr, illustInfoObj, null); + } + } + } + + if(Objects.isNull(illustInfoObj)) { + illustInfoObj = illustInfoCache.getCache(illustIdStr).getAsJsonObject(); + log.debug("作品Id {} IllustInfo缓存命中.", illustId); + } + return illustInfoObj; + } + + /** + * 获取作品预加载数据. + * 可以获取作品的一些与用户相关的信息 + * @param illustId 作品Id + * @param flushCache 是否刷新缓存 + * @return 成功返回JsonObject对象 + * @throws IOException 当Http请求处理发生异常时抛出 + */ + public static JsonObject getIllustPreLoadData(int illustId, boolean flushCache) throws IOException { + String illustIdStr = buildSyncKey(Integer.toString(illustId)); + JsonObject result = null; + if (!illustPreLoadDataCache.exists(illustIdStr) || flushCache) { + synchronized (illustIdStr) { + if (!illustPreLoadDataCache.exists(illustIdStr) || flushCache) { + log.debug("IllustId {} 缓存失效, 正在更新...", illustId); + JsonObject preLoadDataObj = BotGlobal.getGlobal().getPixivDownload() + .getIllustPreLoadDataById(illustId) + .getAsJsonObject("illust") + .getAsJsonObject(Integer.toString(illustId)); + + long expire = 7200 * 1000; + String propValue = SettingProperties. + getProperty(SettingProperties.GLOBAL, "cache.illustPreLoadData.expire", "7200000"); + log.debug("PreLoadData有效时间设定: {}", propValue); + try { + expire = Long.parseLong(propValue); + } catch (Exception e) { + log.warn("全局配置项 \"{}\" 值非法, 已使用默认值: {}", propValue, expire); + } + + result = preLoadDataObj; + illustPreLoadDataCache.update(illustIdStr, preLoadDataObj, expire); + log.debug("作品Id {} preLoadData缓存已更新(有效时间: {})", illustId, expire); + } + } + } + + if(Objects.isNull(result)) { + result = illustPreLoadDataCache.getCache(illustIdStr).getAsJsonObject(); + log.debug("作品Id {} PreLoadData缓存命中.", illustId); + } + return result; + } + + public static List getIllustPages(int illustId, PixivDownload.PageQuality quality, boolean flushCache) + throws IOException { + String pagesSign = buildSyncKey(Integer.toString(illustId), ".", quality.name()); + List result = null; + if (!pagesCache.exists(pagesSign) || flushCache) { + synchronized (pagesSign) { + if (!pagesCache.exists(pagesSign) || flushCache) { + List linkList = PixivDownload + .getIllustAllPageDownload(BotGlobal.getGlobal().getPixivDownload().getHttpClient(), + BotGlobal.getGlobal().getPixivDownload().getCookieStore(), illustId, quality); + result = linkList; + pagesCache.update(pagesSign, linkList, null); + } + } + } + + if(Objects.isNull(result)) { + result = pagesCache.getCache(pagesSign); + log.debug("作品Id {} Pages缓存命中.", illustId); + } + return result; + } + + private final static Random expireTimeFloatRandom = new Random(); + /** + * 获取排行榜 + * @param contentType 排行榜类型 + * @param mode 排行榜模式 + * @param queryDate 查询时间 + * @param start 开始排名, 从1开始 + * @param range 取范围 + * @param flushCache 是否强制刷新缓存 + * @return 成功返回有值List, 失败且无异常返回空 + * @throws IOException 获取异常时抛出 + */ + public static List getRankingInfoByCache(PixivURL.RankingContentType contentType, + PixivURL.RankingMode mode, + Date queryDate, int start, int range, boolean flushCache) + throws IOException { + if(!contentType.isSupportedMode(mode)) { + log.warn("试图获取不支持的排行榜类型已拒绝.(ContentType: {}, RankingMode: {})", contentType.name(), mode.name()); + if(log.isDebugEnabled()) { + try { + Thread.dumpStack(); + } catch(Exception e) { + log.debug("本次非法请求的堆栈信息如下: \n{}", Throwables.getStackTraceAsString(e)); + } + } + return new ArrayList<>(0); + } + + String date = new SimpleDateFormat("yyyyMMdd").format(queryDate); + String requestSign = buildSyncKey(contentType.name(), ".", mode.name(), ".", date); + List result = null; + if(!rankingCache.exists(requestSign) || flushCache) { + synchronized(requestSign) { + if(!rankingCache.exists(requestSign) || flushCache) { + log.debug("Ranking缓存失效, 正在更新...(RequestSign: {})", requestSign); + List rankingResult = BotGlobal.getGlobal().getPixivDownload() + .getRanking(contentType, mode, queryDate, 1, 500); + long expireTime = 0; + if(rankingResult.size() == 0) { + expireTime = 5400000 + expireTimeFloatRandom.nextInt(1800000); + log.warn("数据获取失败, 将设置浮动有效时间以准备下次更新. (ExpireTime: {}ms)", expireTime); + } + result = new ArrayList<>(rankingResult).subList(start - 1, start + range - 1); + rankingCache.update(requestSign, rankingResult, expireTime); + log.debug("Ranking缓存更新完成.(RequestSign: {})", requestSign); + } + } + } + + if (Objects.isNull(result)) { + result = rankingCache.getCache(requestSign, start - 1, range); + log.debug("RequestSign [{}] 缓存命中.", requestSign); + } + log.debug("Result-Length: {}", result.size()); + return PixivDownload.getRanking(result, start - 1, range); + } + + /** + * 获取搜索结果 + * @param content 搜索内容 + * @param type 类型 + * @param area 范围 + * @param includeKeywords 包含关键词 + * @param excludeKeywords 排除关键词 + * @param contentOption 内容类型 + * @return 返回完整搜索结果 + * @throws IOException 当请求发生异常, 或接口返回异常信息时抛出. + */ + public static JsonObject getSearchBody( + String content, + String type, + String area, + String includeKeywords, + String excludeKeywords, + String contentOption) throws IOException { + PixivSearchBuilder searchBuilder = new PixivSearchBuilder(Strings.isNullOrEmpty(content) ? "" : content); + if (type != null) { + try { + searchBuilder.setSearchType(PixivSearchBuilder.SearchType.valueOf(type.toUpperCase())); + } catch (IllegalArgumentException e) { + log.warn("不支持的SearchType: {}", type); + } + } + if (area != null) { + try { + searchBuilder.setSearchArea(PixivSearchBuilder.SearchArea.valueOf(area)); + } catch (IllegalArgumentException e) { + log.warn("不支持的SearchArea: {}", area); + } + } + if (contentOption != null) { + try { + searchBuilder.setSearchContentOption(PixivSearchBuilder.SearchContentOption.valueOf(contentOption)); + } catch (IllegalArgumentException e) { + log.warn("不支持的SearchContentOption: {}", contentOption); + } + } + + if (!Strings.isNullOrEmpty(includeKeywords)) { + for (String keyword : includeKeywords.split(";")) { + searchBuilder.removeExcludeKeyword(keyword.trim()); + searchBuilder.addIncludeKeyword(keyword.trim()); + log.debug("已添加关键字: {}", keyword); + } + } + if (!Strings.isNullOrEmpty(excludeKeywords)) { + for (String keyword : excludeKeywords.split(";")) { + searchBuilder.removeIncludeKeyword(keyword.trim()); + searchBuilder.addExcludeKeyword(keyword.trim()); + log.debug("已添加排除关键字: {}", keyword); + } + } + + log.info("正在搜索作品, 条件: {}", searchBuilder.getSearchCondition()); + + String requestUrl = searchBuilder.buildURL().intern(); + log.debug("RequestUrl: {}", requestUrl); + JsonObject resultBody = null; + if(!searchBodyCache.exists(requestUrl)) { + synchronized (requestUrl) { + if (!searchBodyCache.exists(requestUrl)) { + log.debug("searchBody缓存失效, 正在更新..."); + JsonObject jsonObject; + HttpGet httpGetRequest = BotGlobal.getGlobal().getPixivDownload(). + createHttpGetRequest(requestUrl); + HttpResponse response = BotGlobal.getGlobal().getPixivDownload(). + getHttpClient().execute(httpGetRequest); + + String responseBody = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8); + log.debug("ResponseBody: {}", responseBody); + jsonObject = BotGlobal.getGlobal().getGson().fromJson(responseBody, JsonObject.class); + + if (jsonObject.get("error").getAsBoolean()) { + log.error("接口请求错误, 错误信息: {}", jsonObject.get("message").getAsString()); + throw new IOException("Interface Request Error: " + jsonObject.get("message").getAsString()); + } + + long expire = 7200 * 1000; + String propValue = SettingProperties + .getProperty(SettingProperties.GLOBAL, "cache.searchBody.expire", "7200000"); + try { + expire = Long.parseLong(propValue); + } catch (Exception e) { + log.warn("全局配置项 \"{}\" 值非法, 已使用默认值: {}", propValue, expire); + } + resultBody = jsonObject.getAsJsonObject().getAsJsonObject("body"); + searchBodyCache.update(requestUrl, jsonObject, expire); + log.debug("searchBody缓存已更新(有效时间: {})", expire); + } else { + log.debug("搜索缓存命中."); + } + } + } else { + log.debug("搜索缓存命中."); + } + + if(Objects.isNull(resultBody)) { + resultBody = searchBodyCache.getCache(requestUrl).getAsJsonObject().getAsJsonObject("body"); + } + return resultBody; + } + + /** + * 合并String并存取到常量池, 以保证对象一致 + * @param keys String对象 + * @return 合并后, 如果常量池存在合并后的结果, 则返回常量池中的对象, 否则存入常量池后返回. + */ + private static String buildSyncKey(String... keys) { + StringBuilder sb = new StringBuilder(); + for (String string : keys) { + sb.append(string); + } + return sb.toString().intern(); + } +} diff --git a/src/main/java/net/lamgc/cgj/bot/sort/PreLoadDataComparator.java b/src/main/java/net/lamgc/cgj/bot/sort/PreLoadDataComparator.java index 05ae4f2..65673aa 100644 --- a/src/main/java/net/lamgc/cgj/bot/sort/PreLoadDataComparator.java +++ b/src/main/java/net/lamgc/cgj/bot/sort/PreLoadDataComparator.java @@ -3,6 +3,7 @@ package net.lamgc.cgj.bot.sort; import com.google.gson.JsonElement; import com.google.gson.JsonObject; import net.lamgc.cgj.bot.BotCommandProcess; +import net.lamgc.cgj.bot.cache.CacheStoreCentral; import java.io.IOException; import java.util.Comparator; @@ -30,9 +31,13 @@ public class PreLoadDataComparator implements Comparator { } } try { - JsonObject illustPreLoadData1 = BotCommandProcess.getIllustPreLoadData(o1.getAsJsonObject().get("illustId").getAsInt(), false); - JsonObject illustPreLoadData2 = BotCommandProcess.getIllustPreLoadData(o2.getAsJsonObject().get("illustId").getAsInt(), false); - return Integer.compare(illustPreLoadData2.get(attribute.attrName).getAsInt(), illustPreLoadData1.get(attribute.attrName).getAsInt()); + JsonObject illustPreLoadData1 = + CacheStoreCentral.getIllustPreLoadData(o1.getAsJsonObject().get("illustId").getAsInt(), false); + JsonObject illustPreLoadData2 = + CacheStoreCentral.getIllustPreLoadData(o2.getAsJsonObject().get("illustId").getAsInt(), false); + return Integer.compare( + illustPreLoadData2.get(attribute.attrName).getAsInt(), + illustPreLoadData1.get(attribute.attrName).getAsInt()); } catch (IOException e) { e.printStackTrace(); return 0;