From 3333f4c5c5407247c7c1e4fee62def6eccbf2d05 Mon Sep 17 00:00:00 2001 From: LamGC Date: Tue, 31 Mar 2020 11:04:08 +0800 Subject: [PATCH] =?UTF-8?q?[Add]=20=E5=A2=9E=E5=8A=A0redisAddr=E5=90=AF?= =?UTF-8?q?=E5=8A=A8=E5=8F=82=E6=95=B0=E4=BB=A5=E9=80=9A=E8=BF=87=E5=8F=82?= =?UTF-8?q?=E6=95=B0=E6=8C=87=E5=AE=9Aredis=E7=BC=93=E5=AD=98=E5=BA=93?= =?UTF-8?q?=E5=9C=B0=E5=9D=80;=20[Update]=20=E8=A1=A5=E5=85=85Javadoc;=20[?= =?UTF-8?q?Change]=20=E6=A0=B9=E6=8D=AE=E6=80=A7=E8=83=BD=E5=88=86?= =?UTF-8?q?=E6=9E=90=E7=BB=93=E6=9E=9C=E8=B0=83=E6=95=B4imageCacheExecutor?= =?UTF-8?q?=E7=BA=BF=E7=A8=8B=E6=B1=A0=E5=8F=82=E6=95=B0;=20[Change]=20?= =?UTF-8?q?=E8=B0=83=E6=95=B4CQGlobal=E5=86=85=E7=BA=BF=E7=A8=8B=E6=B1=A0?= =?UTF-8?q?=E5=8F=82=E6=95=B0;=20[Change]=20=E5=B0=86illustPages=E7=BC=93?= =?UTF-8?q?=E5=AD=98=E7=BA=B3=E5=85=A5Redis=E7=BC=93=E5=AD=98=E5=BA=93;=20?= =?UTF-8?q?[Change]=20=E6=A0=B9=E6=8D=AE=E6=80=A7=E8=83=BD=E5=88=86?= =?UTF-8?q?=E6=9E=90=E7=BB=93=E6=9E=9C=E5=B0=86SpringBoot=E7=9A=84HttpThre?= =?UTF-8?q?ads=E9=99=8D=E8=87=B31;?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/main/java/net/lamgc/cgj/CQConfig.java | 14 +++++++ src/main/java/net/lamgc/cgj/CQProcess.java | 43 +++++++++++++++++----- src/main/java/net/lamgc/cgj/Main.java | 11 +++++- src/main/resources/application.properties | 2 +- 4 files changed, 59 insertions(+), 11 deletions(-) diff --git a/src/main/java/net/lamgc/cgj/CQConfig.java b/src/main/java/net/lamgc/cgj/CQConfig.java index 161c054..801fda4 100644 --- a/src/main/java/net/lamgc/cgj/CQConfig.java +++ b/src/main/java/net/lamgc/cgj/CQConfig.java @@ -1,13 +1,27 @@ package net.lamgc.cgj; +import com.google.common.util.concurrent.ThreadFactoryBuilder; import net.lz1998.cq.CQGlobal; import net.lz1998.cq.EnableCQ; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + @EnableCQ public class CQConfig { public static void init() { CQGlobal.pluginList.add(CQPluginMain.class); + CQGlobal.executor = new ThreadPoolExecutor( + (int) Math.ceil(Runtime.getRuntime().availableProcessors() / 2F), + Runtime.getRuntime().availableProcessors(), + 25, TimeUnit.SECONDS, + new LinkedBlockingQueue<>(512), + new ThreadFactoryBuilder() + .setNameFormat("Plugin-ProcessThread-%d") + .build() + ); } } diff --git a/src/main/java/net/lamgc/cgj/CQProcess.java b/src/main/java/net/lamgc/cgj/CQProcess.java index 0bea45c..ea881dc 100644 --- a/src/main/java/net/lamgc/cgj/CQProcess.java +++ b/src/main/java/net/lamgc/cgj/CQProcess.java @@ -1,6 +1,8 @@ package net.lamgc.cgj; import com.google.common.base.Strings; +import com.google.common.reflect.TypeToken; +import com.google.common.util.concurrent.ThreadFactoryBuilder; import com.google.gson.*; import io.netty.handler.codec.http.HttpHeaderNames; import net.lamgc.cgj.cache.*; @@ -23,9 +25,7 @@ import java.net.URI; import java.nio.charset.StandardCharsets; import java.text.SimpleDateFormat; import java.util.*; -import java.util.concurrent.ArrayBlockingQueue; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; +import java.util.concurrent.*; public class CQProcess { @@ -39,7 +39,7 @@ public class CQProcess { .serializeNulls() .create(); - private final static URI redisServerUri = URI.create("redis://192.168.1.17"); + private final static URI redisServerUri = URI.create("redis://" + System.getProperty("cgj.redisAddress")); private final static Hashtable imageCache = new Hashtable<>(); @@ -49,15 +49,28 @@ public class CQProcess { private final static JsonRedisCacheStore searchBodyCache = new JsonRedisCacheStore(redisServerUri, "searchBody", gson); - private final static CacheStore> pagesCache = new LocalHashCacheStore<>(); + private final static CacheStore> pagesCache = new RedisPoolCacheStore>(redisServerUri, "imagePages") { + @Override + protected String parse(List dataObj) { + return gson.toJson(dataObj); + } + + @Override + protected List analysis(String dataStr) { + return gson.fromJson(dataStr, new TypeToken>(){}.getType()); + } + }; private final static JsonRedisCacheStore rankingCache = new JsonRedisCacheStore(redisServerUri, "ranking", gson); private final static EventExecutor imageCacheExecutor = new EventExecutor(new ThreadPoolExecutor( - 1, + Runtime.getRuntime().availableProcessors() >= 2 ? 2 : 1, (int) Math.ceil(Runtime.getRuntime().availableProcessors() / 2F), - 15L, TimeUnit.SECONDS, - new ArrayBlockingQueue<>(30), + 5L, TimeUnit.SECONDS, + new LinkedBlockingQueue<>(128), + new ThreadFactoryBuilder() + .setNameFormat("imageCacheThread-%d") + .build(), new ThreadPoolExecutor.DiscardOldestPolicy() )); @@ -164,6 +177,7 @@ public class CQProcess { @Argument(name = "contentOption", force = false) String contentOption, @Argument(name = "page", force = false, defaultValue = "1") int pagesIndex ) throws IOException { + log.info("正在执行搜索..."); PixivSearchBuilder searchBuilder = new PixivSearchBuilder(Strings.isNullOrEmpty(content) ? "" : content); if (type != null) { try { @@ -402,6 +416,7 @@ public class CQProcess { if (index == pageIndex) { try { + //TODO: 这里可以尝试改成直接提交所有所需的图片给这里，然后再获取结果 imageCacheExecutor.executorSync(new ImageCacheObject(imageCache, illustId, link, currentImageFile)); } catch (InterruptedException e) { log.warn("图片下载遭到中断!", e); @@ -522,7 +537,17 @@ public class CQProcess { return imageStoreDir; } - private static List getRankingInfoByCache(PixivURL.RankingContentType contentType, PixivURL.RankingMode mode, Date queryDate, int start, int range) throws IOException { + /** + * 获取排行榜 + * @param contentType 排行榜类型 + * @param mode 排行榜模式 + * @param queryDate 查询时间 + * @param start 开始排名, 从1开始 + * @param range 取范围 + * @return 成功返回有值List, 失败且无异常返回空 + * @throws IOException 获取异常时抛出 + */ + public static List getRankingInfoByCache(PixivURL.RankingContentType contentType, PixivURL.RankingMode mode, Date queryDate, int start, int range) throws IOException { String date = new SimpleDateFormat("yyyyMMdd").format(queryDate); //int requestSign = ("Ranking." + contentType.name() + "." + mode.name() + "." + date).hashCode(); String requestSign = buildSyncKey("Ranking.", contentType.name(), ".", mode.name(), ".", date); diff --git a/src/main/java/net/lamgc/cgj/Main.java b/src/main/java/net/lamgc/cgj/Main.java index c89b4d8..dcb9cd1 100644 --- a/src/main/java/net/lamgc/cgj/Main.java +++ b/src/main/java/net/lamgc/cgj/Main.java @@ -69,7 +69,16 @@ public class Main { log.info("cqRootDir: {}", argsProp.getValue("cqRootDir")); System.setProperty("cgj.cqRootDir", argsProp.getValue("cqRootDir")); } else { - log.info("未设置cqRootDir"); + log.info("未设置cqRootDir, 当前运行目录将作为酷Q机器人所在目录."); + System.setProperty("cgj.cqRootDir", "./"); + } + + if(argsProp.containsKey("redisAddr")) { + log.info("redisAddress: {}", argsProp.getValue("redisAddr")); + System.setProperty("cgj.redisAddress", argsProp.getValue("redisAddr")); + } else { + log.info("未设置RedisAddress, 将使用默认值连接Redis服务器(127.0.0.1:6379)"); + System.setProperty("cgj.redisAddress", "127.0.0.1"); } File cookieStoreFile = new File("cookies.store"); diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index d2a8109..aa21d2c 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -1,2 +1,2 @@ server.port=8081 -server.tomcat.max-threads=10 \ No newline at end of file +server.tomcat.max-threads=1 \ No newline at end of file