From fba6d3532c503866a4897ba81789dc2c1c3104d1 Mon Sep 17 00:00:00 2001 From: LamGC Date: Thu, 30 Apr 2020 02:23:31 +0800 Subject: [PATCH] =?UTF-8?q?[Delete]=20=E5=88=A0=E9=99=A4=E4=B8=8D=E5=86=8D?= =?UTF-8?q?=E4=BD=BF=E7=94=A8=E7=9A=84=E7=B1=BB=E5=92=8C=E6=96=B9=E6=B3=95?= =?UTF-8?q?;=20[Update]=20=E4=BC=98=E5=8C=96Javadoc=E5=B9=B6=E8=AE=BE?= =?UTF-8?q?=E7=BD=AE=E7=BC=96=E8=AF=91=E6=B3=A8=E8=A7=A3,=20=E4=BC=98?= =?UTF-8?q?=E5=8C=96=E4=BB=A3=E7=A0=81;=20[Change]=20=E8=B0=83=E6=95=B4?= =?UTF-8?q?=E9=83=A8=E5=88=86=E6=97=A5=E5=BF=97=E7=9A=84=E8=BE=93=E5=87=BA?= =?UTF-8?q?=E7=BA=A7=E5=88=AB;=20[Change]=20RedisPoolCacheStore=20clear?= =?UTF-8?q?=E6=96=B9=E6=B3=95=E5=B0=86=E6=A0=B9=E6=8D=AE=E8=BF=94=E5=9B=9E?= =?UTF-8?q?=E4=BF=A1=E6=81=AF=E7=A1=AE=E5=AE=9A=E6=98=AF=E5=90=A6=E6=B8=85?= =?UTF-8?q?=E7=A9=BA=E6=88=90=E5=8A=9F;=20[Change]=20Dockerfile.sample=20?= =?UTF-8?q?=E8=B0=83=E6=95=B4=E9=95=9C=E5=83=8F=E6=9E=84=E5=BB=BA=E6=AD=A5?= =?UTF-8?q?=E9=AA=A4;?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Dockerfile.sample | 3 +- src/main/java/net/lamgc/cgj/Main.java | 148 +--------- .../net/lamgc/cgj/bot/BotCommandProcess.java | 15 +- .../cgj/bot/RandomRankingArtworksSender.java | 2 +- .../cgj/bot/cache/ImageCacheHandler.java | 6 +- .../cgj/bot/cache/RedisPoolCacheStore.java | 13 +- .../cgj/bot/framework/coolq/CQPluginMain.java | 3 - .../mirai/message/MiraiMessageSender.java | 2 +- .../java/net/lamgc/cgj/pixiv/IllustInfo.java | 69 ----- src/main/java/net/lamgc/cgj/pixiv/Pixiv.java | 267 ------------------ .../lamgc/cgj/pixiv/PixivSearchBuilder.java | 7 + .../net/lamgc/cgj/pixiv/PixivSession.java | 171 ----------- .../java/net/lamgc/cgj/pixiv/PixivTag.java | 16 -- .../java/net/lamgc/cgj/pixiv/PixivURL.java | 1 + .../lamgc/cgj/pixiv/PixivUgoiraBuilder.java | 3 +- .../java/net/lamgc/cgj/util/CookieUtil.java | 31 -- .../lamgc/cgj/util/PagesQualityParser.java | 2 +- .../lamgc/cgj/pixiv/PixivDownloadTest.java | 13 - 18 files changed, 29 insertions(+), 743 deletions(-) delete mode 100644 src/main/java/net/lamgc/cgj/pixiv/IllustInfo.java delete mode 100644 src/main/java/net/lamgc/cgj/pixiv/Pixiv.java delete mode 100644 src/main/java/net/lamgc/cgj/pixiv/PixivSession.java delete mode 100644 src/main/java/net/lamgc/cgj/pixiv/PixivTag.java delete mode 100644 src/main/java/net/lamgc/cgj/util/CookieUtil.java diff --git a/Dockerfile.sample b/Dockerfile.sample index 4d468ab..aa2fb14 100644 --- a/Dockerfile.sample +++ b/Dockerfile.sample @@ -4,4 +4,5 @@ ENV jarFileName=ContentGrabbingJi-exec.jar COPY ${jarFileName} /program/CGJ.jar RUN mkdir /program/data -CMD cd /program/data && java -jar /program/CGJ.jar botMode \ No newline at end of file +WORKDIR /program/data +CMD java -jar /program/CGJ.jar botMode \ No newline at end of file diff --git a/src/main/java/net/lamgc/cgj/Main.java b/src/main/java/net/lamgc/cgj/Main.java index d89c513..efc7963 100644 --- a/src/main/java/net/lamgc/cgj/Main.java +++ b/src/main/java/net/lamgc/cgj/Main.java @@ -9,8 +9,9 @@ import com.google.gson.JsonElement; import com.google.gson.JsonObject; import net.lamgc.cgj.bot.framework.coolq.CQConfig; import net.lamgc.cgj.bot.framework.mirai.MiraiMain; -import net.lamgc.cgj.pixiv.*; -import net.lamgc.cgj.proxy.PixivAccessProxyServer; +import net.lamgc.cgj.pixiv.PixivDownload; +import net.lamgc.cgj.pixiv.PixivSearchBuilder; +import net.lamgc.cgj.pixiv.PixivURL; import net.lamgc.plps.PixivLoginProxyServer; import net.lamgc.utils.base.ArgumentsProperties; import net.lamgc.utils.base.runner.Argument; @@ -20,9 +21,7 @@ import org.apache.commons.io.IOUtils; import org.apache.http.HttpHost; import org.apache.http.HttpResponse; import org.apache.http.client.CookieStore; -import org.apache.http.client.HttpClient; import org.apache.http.client.methods.HttpGet; -import org.apache.http.message.BasicHeader; import org.apache.http.util.EntityUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -355,86 +354,16 @@ public class Main { builder, PixivURL.getPixivRefererLink(illustId) ); - - /*log.info("正在下载..."); - List list = PixivDownload.getIllustAllPageDownload( - HttpClientBuilder.create() - .setProxy(proxy) - .build(), - illustId, PixivDownload.PageQuality.ORIGINAL);*/ } } - /* - - if(searchBuilder.getSearchArea().equals(PixivSearchBuilder.SearchArea.TOP)) { - - } else { - JsonArray illustsArray = resultBody - .getAsJsonObject(searchBuilder.getSearchArea().jsonKey).getAsJsonArray("data"); - log.info("已找到与 {} 相关插图信息：", content); - int count = 1; - for (JsonElement jsonElement : illustsArray) { - JsonObject illustObj = jsonElement.getAsJsonObject(); - //TODO: 防止数据内混入无效内容, 需要检查对象是否有illustId - if(!illustObj.has("illustId")) { - continue; - } - int illustId = illustObj.get("illustId").getAsInt(); - StringBuilder builder = new StringBuilder("["); - illustObj.get("tags").getAsJsonArray().forEach(el -> builder.append(el.getAsString()).append(", ")); - builder.replace(builder.length() - 2, builder.length(), "]"); - log.info("({} / {})\n\t作品id: {}, \n\t作者名(作者id): {} ({}), \n\t作品标题: {}, \n\t作品Tags: {}, \n\t作品链接: {}", - count++, - illustsArray.size(), - illustId, - illustObj.get("userName").getAsString(), - illustObj.get("userId").getAsInt(), - illustObj.get("illustTitle").getAsString(), - builder, - PixivURL.getPixivRefererLink(illustId) - ); - } - } - */ } @Command(defaultCommand = true) public static void testRun() { - /*loadCookieStoreFromFile(); - - if(cookieStore == null){ - startPixivLoginProxyServer(); - }*/ - - //accessPixivToFile(); - - //startPixivAccessProxyServer(); - - //saveCookieStoreToFile(); log.info("这里啥都没有哟w"); } - private static void loadCookieStoreFromFile() throws IOException { - log.info("正在加载CookieStore..."); - File storeFile = new File("./cookies.store"); - if(!storeFile.exists()){ - log.info("未找到CookieStore, 跳过加载."); - return; - } - ObjectInputStream stream = new ObjectInputStream(new FileInputStream(storeFile)); - Object result; - try { - result = stream.readObject(); - } catch (ClassNotFoundException e) { - log.error("加载出错", e); - return; - } - cookieStore = (CookieStore) result; - cookieStore.getCookies().forEach(cookie -> log.debug(cookie.getName() + ": " + cookie.getValue() + ", isExpired: " + cookie.isExpired(new Date()))); - log.info("CookieStore加载完成."); - } - private static void saveCookieStoreToFile() throws IOException { log.info("正在保存CookieStore..."); File outputFile = new File("./cookies.store"); @@ -484,75 +413,4 @@ public class Main { } } - private static void accessPixivToFile() throws IOException { - File cookieStoreFile = new File("./cookie.txt"); - if (!cookieStoreFile.exists() && !cookieStoreFile.createNewFile()) { - log.info("Cookie文件存储失败"); - } - /*log.info("正在写出Cookie, Cookie count: " + cookieStore.getCookies().size()); - FileWriter cookieWriter = new FileWriter(cookieStoreFile); - cookieStore.getCookies().forEach(cookie -> { - try { - StringBuilder sb = new StringBuilder().append(cookie.getName()).append(" = ").append(cookie.getValue()); - log.info("正在导出Cookie: " + sb.toString()); - cookieWriter.append(sb.toString()).append("\n").flush(); - } catch (IOException e) { - e.printStackTrace(); - } - }); - log.info("Cookie写出完成");*/ - - log.info("尝试通过捕获的Cookie访问Pixiv..."); - HttpClient httpClient = new PixivSession(new HttpHost("127.0.0.1", 1080), cookieStore).getHttpClient(); - HttpGet request = new HttpGet("https://www.pixiv.net"); - request.addHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:73.0) Gecko/20100101 Firefox/73.0"); - request.addHeader(new BasicHeader("accept-encoding", "gzip, deflate, br")); - request.addHeader(new BasicHeader("accept-language", "zh-CN,zh;q=0.9")); - StringBuilder cookieBuilder = new StringBuilder(); - cookieStore.getCookies().forEach(cookie -> { - if(cookie.isExpired(new Date())){ - return; - } - cookieBuilder.append(cookie.getName()).append("=").append(cookie.getValue()).append("; "); - }); - request.addHeader("cookie", cookieBuilder.toString()); - - HttpResponse response = httpClient.execute(request); - log.info("正在写入文件..."); - File outFile = new File("./pixiv.html"); - if (outFile.createNewFile() && !outFile.exists()) { - log.info("文件创建失败!"); - }else { - new FileWriter(outFile).append(EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8)).flush(); - } - - Pixiv pixiv = new Pixiv(httpClient); - pixiv.getRecommend().forEach(illustMap -> { - StringBuilder builder = new StringBuilder(); - illustMap.forEach((key, value) -> builder.append(key).append(": ").append(value).append("\n")); - try { - builder.append("download Link: ").append(Arrays.toString(pixiv.getAllDownloadLink(Integer.parseInt(illustMap.get(Pixiv.ATTR_ILLUST_ID))))); - } catch (IOException e) { - log.error("获取下载链接时出错!", e); - } - log.info(builder.append("\n").toString()); - }); - - } - - private static void startPixivAccessProxyServer(){ - log.info("正在启动访问代理服务器, 将浏览器相关缓存清空后, 使用浏览器进行访问以尝试Cookie正确性."); - final PixivAccessProxyServer accessProxyServer = new PixivAccessProxyServer(cookieStore, new ProxyConfig(ProxyType.SOCKS5, "127.0.0.1", 1080)); - Thread accessProxyServerThread = new Thread(() -> { - log.info("正在启动PAPS..."); - accessProxyServer.start(1007); - log.info("PAPS已关闭."); - }); - accessProxyServerThread.setName("AccessProxyServerThread"); - accessProxyServerThread.start(); - new Scanner(System.in).nextLine(); - log.info("关闭PAPS服务器..."); - accessProxyServer.close(); - } - } diff --git a/src/main/java/net/lamgc/cgj/bot/BotCommandProcess.java b/src/main/java/net/lamgc/cgj/bot/BotCommandProcess.java index 7bd80f2..96d012a 100644 --- a/src/main/java/net/lamgc/cgj/bot/BotCommandProcess.java +++ b/src/main/java/net/lamgc/cgj/bot/BotCommandProcess.java @@ -684,7 +684,7 @@ public class BotCommandProcess { if (!illustPreLoadDataCache.exists(illustIdStr) || flushCache) { synchronized (illustIdStr) { if (!illustPreLoadDataCache.exists(illustIdStr) || flushCache) { - log.info("IllustId {} 缓存失效, 正在更新...", illustId); + log.debug("IllustId {} 缓存失效, 正在更新...", illustId); JsonObject preLoadDataObj = pixivDownload.getIllustPreLoadDataById(illustId) .getAsJsonObject("illust") .getAsJsonObject(Integer.toString(illustId)); @@ -700,7 +700,7 @@ public class BotCommandProcess { result = preLoadDataObj; illustPreLoadDataCache.update(illustIdStr, preLoadDataObj, expire); - log.info("作品Id {} preLoadData缓存已更新(有效时间: {})", illustId, expire); + log.debug("作品Id {} preLoadData缓存已更新(有效时间: {})", illustId, expire); } } } @@ -772,15 +772,16 @@ public class BotCommandProcess { if(!rankingCache.exists(requestSign) || flushCache) { synchronized(requestSign) { if(!rankingCache.exists(requestSign) || flushCache) { - log.info("Ranking缓存失效, 正在更新...(RequestSign: {})", requestSign); + log.debug("Ranking缓存失效, 正在更新...(RequestSign: {})", requestSign); List rankingResult = pixivDownload.getRanking(contentType, mode, queryDate, 1, 500); + long expireTime = 0; if(rankingResult.size() == 0) { - log.info("数据获取失败, 将设置浮动有效时间以准备下次更新."); + expireTime = 5400000 + expireTimeFloatRandom.nextInt(1800000); + log.warn("数据获取失败, 将设置浮动有效时间以准备下次更新. (ExpireTime: {}ms)", expireTime); } result = new ArrayList<>(rankingResult).subList(start - 1, start + range - 1); - rankingCache.update(requestSign, rankingResult, - rankingResult.size() == 0 ? 5400000 + expireTimeFloatRandom.nextInt(1800000) : 0); - log.info("Ranking缓存更新完成.(RequestSign: {})", requestSign); + rankingCache.update(requestSign, rankingResult, expireTime); + log.debug("Ranking缓存更新完成.(RequestSign: {})", requestSign); } } } diff --git a/src/main/java/net/lamgc/cgj/bot/RandomRankingArtworksSender.java b/src/main/java/net/lamgc/cgj/bot/RandomRankingArtworksSender.java index d0479db..4335f0a 100644 --- a/src/main/java/net/lamgc/cgj/bot/RandomRankingArtworksSender.java +++ b/src/main/java/net/lamgc/cgj/bot/RandomRankingArtworksSender.java @@ -69,7 +69,7 @@ public class RandomRankingArtworksSender extends AutoSender { selectRanking, 1, false); - log.info("RankingResult.size: {}", rankingList.size()); + log.debug("RankingResult.size: {}", rankingList.size()); if(rankingList.size() != 1) { log.error("排行榜选取失败!(获取到了多个结果)"); return; diff --git a/src/main/java/net/lamgc/cgj/bot/cache/ImageCacheHandler.java b/src/main/java/net/lamgc/cgj/bot/cache/ImageCacheHandler.java index 3bc61bc..9113386 100644 --- a/src/main/java/net/lamgc/cgj/bot/cache/ImageCacheHandler.java +++ b/src/main/java/net/lamgc/cgj/bot/cache/ImageCacheHandler.java @@ -30,7 +30,7 @@ public class ImageCacheHandler implements EventHandler { @SuppressWarnings("unused") public void getImageToCache(ImageCacheObject event) { if(cacheQueue.contains(event)) { - log.info("图片 {} 已存在相同缓存任务, 跳过.", event.getStoreFile().getName()); + log.debug("图片 {} 已存在相同缓存任务, 跳过.", event.getStoreFile().getName()); return; } else { cacheQueue.add(event); @@ -39,7 +39,7 @@ public class ImageCacheHandler implements EventHandler { try { log.info("图片 {} Event正在进行...({})", event.getStoreFile().getName(), Integer.toHexString(event.hashCode())); File storeFile = event.getStoreFile(); - log.info("正在缓存图片 {} (Path: {})", storeFile.getName(), storeFile.getAbsolutePath()); + log.debug("正在缓存图片 {} (Path: {})", storeFile.getName(), storeFile.getAbsolutePath()); try { if(!storeFile.exists() && !storeFile.createNewFile()) { log.error("无法创建文件(Path: {})", storeFile.getAbsolutePath()); @@ -64,7 +64,7 @@ public class ImageCacheHandler implements EventHandler { return; } - log.info("正在下载...(Content-Length: {}KB)", response.getEntity().getContentLength() / 1024); + log.debug("正在下载...(Content-Length: {}KB)", response.getEntity().getContentLength() / 1024); try(FileOutputStream fos = new FileOutputStream(storeFile)) { IOUtils.copy(response.getEntity().getContent(), fos); } catch (IOException e) { diff --git a/src/main/java/net/lamgc/cgj/bot/cache/RedisPoolCacheStore.java b/src/main/java/net/lamgc/cgj/bot/cache/RedisPoolCacheStore.java index 0645aaf..8f79d87 100644 --- a/src/main/java/net/lamgc/cgj/bot/cache/RedisPoolCacheStore.java +++ b/src/main/java/net/lamgc/cgj/bot/cache/RedisPoolCacheStore.java @@ -82,9 +82,7 @@ public abstract class RedisPoolCacheStore implements CacheStore { @Override public boolean clear() { try (Jedis jedis = jedisPool.getResource()) { - String result = jedis.flushDB(); - log.info("flushDB返回结果: {}", result); - return true; + return jedis.flushDB().equalsIgnoreCase("ok"); } } @@ -158,13 +156,4 @@ public abstract class RedisPoolCacheStore implements CacheStore { return false; } - /** - * 替换原本的分隔符(.)为(:).
- * 即将启用 - * @param key 要处理的键名 - * @return 处理后的键名 - */ - public static String replaceKey(String key) { - return key.replaceAll("\\.", ":"); - } } diff --git a/src/main/java/net/lamgc/cgj/bot/framework/coolq/CQPluginMain.java b/src/main/java/net/lamgc/cgj/bot/framework/coolq/CQPluginMain.java index 74a00a5..34c34d1 100644 --- a/src/main/java/net/lamgc/cgj/bot/framework/coolq/CQPluginMain.java +++ b/src/main/java/net/lamgc/cgj/bot/framework/coolq/CQPluginMain.java @@ -24,19 +24,16 @@ public class CQPluginMain extends CQPlugin implements EventHandler { @Override public int onPrivateMessage(CoolQ cq, CQPrivateMessageEvent event) { - //log.info("私聊消息到达: 发送者[{}], 消息内容: {}", event.getSender().getUserId(), event.getMessage()); return processMessage(cq, event); } @Override public int onGroupMessage(CoolQ cq, CQGroupMessageEvent event) { - //log.info("群消息到达: 群[{}], 发送者[{}], 消息内容: {}", event.getGroupId(), event.getSender().getUserId(), event.getMessage()); return processMessage(cq, event); } @Override public int onDiscussMessage(CoolQ cq, CQDiscussMessageEvent event) { - //log.info("讨论组消息到达: 群[{}], 发送者[{}], 消息内容: {}", event.getDiscussId(), event.getSender().getUserId(), event.getMessage()); return processMessage(cq, event); } diff --git a/src/main/java/net/lamgc/cgj/bot/framework/mirai/message/MiraiMessageSender.java b/src/main/java/net/lamgc/cgj/bot/framework/mirai/message/MiraiMessageSender.java index cdf2073..4f50bd2 100644 --- a/src/main/java/net/lamgc/cgj/bot/framework/mirai/message/MiraiMessageSender.java +++ b/src/main/java/net/lamgc/cgj/bot/framework/mirai/message/MiraiMessageSender.java @@ -156,7 +156,7 @@ public class MiraiMessageSender implements MessageSender { } } imageIdCache.update(imageName, image.getImageId(), expireTime); - log.info("imageName [{}] 缓存更新完成.(有效时间: {})", imageName, expireTime); + log.debug("imageName [{}] 缓存更新完成.(有效时间: {})", imageName, expireTime); } else { log.debug("ImageName: [{}] 缓存命中.", imageName); } diff --git a/src/main/java/net/lamgc/cgj/pixiv/IllustInfo.java b/src/main/java/net/lamgc/cgj/pixiv/IllustInfo.java deleted file mode 100644 index 07c28e4..0000000 --- a/src/main/java/net/lamgc/cgj/pixiv/IllustInfo.java +++ /dev/null @@ -1,69 +0,0 @@ -package net.lamgc.cgj.pixiv; - -/** - * 插图(集)的信息 - */ -public class IllustInfo{ - - /** - * 插图(集)ID - */ - public final int illustID; - - /** - * 第几页 - */ - public final int page; - - /** - * 插图标题 - */ - public final String title; - - /** - * 插图说明 - */ - public final String description; - - /** - * 插图标签 - */ - public final String[] tags; - - /** - * 插图图片长度 - */ - public final int width; - - /** - * 插图图片高度 - */ - public final int height; - - /** - * 作者名 - */ - public final String authorName; - - /** - * 作者用户ID - */ - public final int authorUserID; - - public IllustInfo(int illustID, String title, String description, String[] tags, int width, int height, String authorName, int authorUserID){ - this(illustID, 0, title, description, tags, width, height, authorName, authorUserID); - } - - public IllustInfo(int illustID, int p, String title, String description, String[] tags, int width, int height, String authorName, int authorUserID){ - this.illustID = illustID; - this.page = p; - this.title = title; - this.description = description; - this.tags = tags; - this.width = width; - this.height = height; - this.authorName = authorName; - this.authorUserID = authorUserID; - } - -} diff --git a/src/main/java/net/lamgc/cgj/pixiv/Pixiv.java b/src/main/java/net/lamgc/cgj/pixiv/Pixiv.java deleted file mode 100644 index 15835db..0000000 --- a/src/main/java/net/lamgc/cgj/pixiv/Pixiv.java +++ /dev/null @@ -1,267 +0,0 @@ -package net.lamgc.cgj.pixiv; - -import com.google.gson.JsonArray; -import com.google.gson.JsonObject; -import com.google.gson.JsonParser; -import org.apache.http.HttpHost; -import org.apache.http.HttpResponse; -import org.apache.http.client.HttpClient; -import org.apache.http.client.config.RequestConfig; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.conn.ConnectionPoolTimeoutException; -import org.apache.http.impl.client.HttpClientBuilder; -import org.apache.http.util.EntityUtils; -import org.jsoup.Jsoup; -import org.jsoup.nodes.Document; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.nio.charset.StandardCharsets; -import java.util.*; - -public class Pixiv { - - /** - * illust Link - */ - public final static String ATTR_LINK = "link"; - - /** - * illust Id - */ - public final static String ATTR_ILLUST_ID = "illustId"; - - /** - * illust Title - */ - public final static String ATTR_TITLE = "title"; - - /** - * illust Author Name - */ - public final static String ATTR_AUTHOR_NAME = "authorName"; - - /** - * illust Author UserID - */ - public final static String ATTR_AUTHOR_ID = "authorId"; - - private final HttpClient httpClient; - - public Pixiv(HttpClient client){ - this.httpClient = client; - } - - /** - * 使用帐号密码登录Pixiv - * @param PixivID Pixiv账户登录名 - * @param Password Pixiv帐号密码 - * @throws IOException 当登录连接出现异常时抛出 - * @deprecated {@link PixivSession#Login(String, String)} 已经废除, 故本方法不可用 - */ - public Pixiv(String PixivID, String Password) throws IOException { - this(PixivID, Password, null); - } - - /** - * 使用帐号密码登录Pixiv - * @param PixivID Pixiv账户登录名 - * @param Password Pixiv帐号密码 - * @param proxy 代理设置 - * @throws IOException 当登录连接出现异常时抛出 - * @deprecated {@link PixivSession#Login(String, String)} 已经废除, 故本方法不可用 - */ - public Pixiv(String PixivID, String Password, HttpHost proxy) throws IOException { - PixivSession pixivSession = new PixivSession(proxy, null); - if(pixivSession.Login(PixivID, Password)){ - System.out.println("P站登录成功!"); - }else{ - System.out.println("P站登录失败!错误信息: " + pixivSession.getErrMsg()); - throw new RuntimeException(pixivSession.getErrMsg()); - } - //httpClient = pixivSession.getHttpClient(); - httpClient = HttpClientBuilder.create() - .setDefaultCookieStore(pixivSession.getCookieStore()) - .build(); - } - - /** - * 获取首页推荐列表 - * @return 首页推荐列表, 一个Map对应一个推荐项, 使用ATTR_开头常量访问即可 - * @throws IOException - */ - public List> getRecommend() throws IOException { - HttpGet getRequest = new HttpGet(PixivURL.PIXIV_INDEX_URL); - HttpResponse response = httpClient.execute(getRequest); - String pageAsXML = EntityUtils.toString(response.getEntity(),"utf-8"); - - //获取推荐图列表(li) - //System.out.println(pageAsXML); - - Document document = Jsoup.parse(pageAsXML); - - List links = document.select("._image-items.gtm-illust-recommend-zone>li>.gtm-illust-recommend-thumbnail-link").eachAttr("href"); - List illustId = document.select("._image-items.gtm-illust-recommend-zone>li>.gtm-illust-recommend-thumbnail-link").eachAttr("data-gtm-recommend-illust-id"); - List title = document.select("._image-items.gtm-illust-recommend-zone>li>.gtm-illust-recommend-title>h1").eachAttr("title"); - List authorName = document.select("._image-items.gtm-illust-recommend-zone>li>.gtm-illust-recommend-user-name").eachText(); - List authorId = document.select("._image-items.gtm-illust-recommend-zone>li>.gtm-illust-recommend-user-name").eachAttr("data-user_id"); - - List> recommendList = new ArrayList<>(); - for(int i = 0; i < links.size(); i++){ - //System.out.println(links.get(i)); - Map info = new HashMap<>(); - info.put(ATTR_LINK, PixivURL.PIXIV_INDEX_URL + links.get(i)); - info.put(ATTR_ILLUST_ID, illustId.get(i)); - info.put(ATTR_TITLE, title.get(i)); - info.put(ATTR_AUTHOR_NAME, authorName.get(i)); - info.put(ATTR_AUTHOR_ID, authorId.get(i)); - recommendList.add(info); - } - return recommendList; - } - - public String[] getAllDownloadLink(int illustID) throws IOException { - HttpGet illustPage = new HttpGet(PixivURL.PIXIV_ILLUST_API_URL.replaceAll("\\{illustId}", String.valueOf(illustID))); - HttpResponse response = httpClient.execute(illustPage); - String pageAsXML = EntityUtils.toString(response.getEntity(),"utf-8"); - //System.out.println(pageAsXML); - JsonObject resultObj = (JsonObject) new JsonParser().parse(pageAsXML); - if(!resultObj.get("error").getAsBoolean()){ - JsonArray bodyArray = resultObj.get("body").getAsJsonArray(); - int length = bodyArray.size(); - String[] result = new String[length]; - for(int i = 0; i < length; i++){ - JsonObject childObj = bodyArray.get(i).getAsJsonObject(); - result[i] = childObj.get("urls").getAsJsonObject().get("original").getAsString(); - } - return result; - }else{ - return null; - } - } - - - - /** - * 下载P站图片 - * @param illustID 插图ID - * @return 成功返回图片输入流,失败或为多图则返回null - */ - public InputStream[] downloadIllustImage(int illustID) throws IOException { - String[] links = getAllDownloadLink(illustID); - List inputStreamList = new ArrayList<>(); - int count = 1; - boolean retry = false; - for(int i = 0; i < links.length; i++){ - try { - long sleepTime = (new Random().nextInt(4) + 2) * 1000; - System.out.println("nextTime: " + (float)(sleepTime / 1000)); - Thread.sleep(sleepTime); - } catch (InterruptedException ignored) {} - String link = links[i]; - System.out.print("page:" + count++ + "/" + links.length + " ..."); - HttpGet imgGet = new HttpGet(link); - //关键!如果不加上Referer的话,会返回403 - imgGet.setHeader("Referer", PixivURL.PIXIV_ILLUST_MEDIUM_URL.replaceAll("\\{illustId}", String.valueOf(illustID))); - RequestConfig config = RequestConfig.custom() - .setConnectTimeout(20 * 1000) - .setConnectionRequestTimeout(20 * 1000) - .setSocketTimeout(30 * 1000) - .build(); - imgGet.setConfig(config); - HttpResponse response; - try { - response = httpClient.execute(imgGet); - }catch(ConnectionPoolTimeoutException e){ - if(retry){ - retry = false; - System.out.println("获取失败,跳过..."); - continue; - } - System.out.println("连接超时,重新获取..."); - retry = true; - i--; - continue; - } - retry = false; - - ByteArrayOutputStream cacheOutputStream = new ByteArrayOutputStream((int)response.getEntity().getContentLength()); - InputStream content = response.getEntity().getContent(); - int readLength; - byte[] cache = new byte[4096]; - while((readLength = content.read(cache)) != -1){ - cacheOutputStream.write(cache, 0, readLength); - } - byte[] data = cacheOutputStream.toByteArray(); - //System.out.println("读到数据: " + data.length); - inputStreamList.add(new ByteArrayInputStream(data)); - System.out.println("done!（length: " + response.getEntity().getContentLength() + ")"); - - } - return inputStreamList.toArray(new InputStream[0]); - } - - /** - * 下载P站图片 - * @param illustID 插图ID - * @return 成功返回图片输入流,失败或为多图则返回null - */ - public InputStream downloadIllustImages(int illustID){ - throw new UnsupportedOperationException(); - } - - /** - * 通过解析插图详情页获取 - * - 插图标题 - * - 插图作者(及其UserId) - * - 插图上传时间 - * - 插图标签(原始标签) - * ... - * @return 成功返回IllustInfo对象,失败返回null - */ - public IllustInfo[] getIllustInfo(int[] illustIDs) throws IOException { - //获取Api - HttpGet apiRequest = new HttpGet(PixivURL.getPixivIllustInfoAPI(illustIDs)); - HttpResponse response = httpClient.execute(apiRequest); - String resultText = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8); - System.out.println(resultText); - JsonObject resultObj = ((JsonObject) new JsonParser().parse(resultText)); - if(resultObj.get("error").getAsBoolean()){ - System.err.println("获取失败!"); - return null; - } - List illustInfoList = new ArrayList<>(); - JsonArray illustArray = resultObj.get("body").getAsJsonObject().get("illusts").getAsJsonArray(); - illustArray.forEach(jsonElement -> { - JsonObject illustInfoObj = jsonElement.getAsJsonObject(); - JsonArray tagsArray = illustInfoObj.get("tags").getAsJsonArray(); - String[] tags = new String[tagsArray.size()]; - for(int i = 0; i < tags.length; i++){ - tags[i] = tagsArray.get(i).getAsString(); - } - //TODO: 通过不需要作者id就能获取图片信息的api无法获取图片尺寸 - IllustInfo illustInfo = new IllustInfo( - illustInfoObj.get("workId").getAsInt(), - illustInfoObj.get("title").getAsString(), - null, - tags, - -1, - -1, - illustInfoObj.get("userName").getAsString(), - illustInfoObj.get("userId").getAsInt() - ); - }); - return null; - } - - /** - * 获取指定用户的所有插画 - */ - public void getUserAllIllustTest() { - - } - - -} diff --git a/src/main/java/net/lamgc/cgj/pixiv/PixivSearchBuilder.java b/src/main/java/net/lamgc/cgj/pixiv/PixivSearchBuilder.java index ef4e73c..88959bb 100644 --- a/src/main/java/net/lamgc/cgj/pixiv/PixivSearchBuilder.java +++ b/src/main/java/net/lamgc/cgj/pixiv/PixivSearchBuilder.java @@ -9,6 +9,13 @@ import java.util.Date; import java.util.HashSet; import java.util.Objects; +/** + * Pixiv搜索URL构造器 + *

该构造器通过分析Pixiv搜索链接可用的参数而开发, 对搜索链接的构造有高度自定义能力.

+ * @author LamGC + * @see PixivURL#PIXIV_SEARCH_CONTENT_URL + */ +@SuppressWarnings("ALL") public class PixivSearchBuilder { private final String content; diff --git a/src/main/java/net/lamgc/cgj/pixiv/PixivSession.java b/src/main/java/net/lamgc/cgj/pixiv/PixivSession.java deleted file mode 100644 index f89505a..0000000 --- a/src/main/java/net/lamgc/cgj/pixiv/PixivSession.java +++ /dev/null @@ -1,171 +0,0 @@ -package net.lamgc.cgj.pixiv; - -import com.google.gson.JsonObject; -import com.google.gson.JsonParser; -import org.apache.http.Header; -import org.apache.http.HttpHost; -import org.apache.http.HttpResponse; -import org.apache.http.NameValuePair; -import org.apache.http.client.CookieStore; -import org.apache.http.client.HttpClient; -import org.apache.http.client.entity.UrlEncodedFormEntity; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.impl.client.BasicCookieStore; -import org.apache.http.impl.client.HttpClients; -import org.apache.http.message.BasicHeader; -import org.apache.http.message.BasicNameValuePair; -import org.apache.http.util.EntityUtils; -import org.jsoup.Jsoup; -import org.jsoup.nodes.Document; - -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.List; - -public class PixivSession { - - public static final String USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36"; - - /** - * 全登陆过程的关键, - * 保存会话用的cookieStore! - */ - private CookieStore cookieStore = new BasicCookieStore(); - - /** - * 可以直接使用的HttpClient对象 - */ - private HttpClient httpClient; - - /** - * 最后一次登录的错误信息 - */ - private String errMsg; - - public PixivSession(){ - this(null); - } - - public PixivSession(CookieStore cookieStore){ - this(null, cookieStore); - } - - /** - * 创建一个Pixiv登录会话 - */ - public PixivSession(HttpHost proxy, CookieStore cookieStore) { - if(cookieStore != null){ - this.cookieStore = cookieStore; - } - List
defaultHeader = new ArrayList<>(); - defaultHeader.add(new BasicHeader("accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9")); - defaultHeader.add(new BasicHeader("user-agent", PixivSession.USER_AGENT)); - defaultHeader.add(new BasicHeader("accept-encoding", "gzip, deflate, br")); - defaultHeader.add(new BasicHeader("accept-language", "zh-CN,zh;q=0.9")); - - /*defaultHeader.add(new BasicHeader("sec-fetch-mode", "navigate")); - defaultHeader.add(new BasicHeader("sec-fetch-site", "same-origin")); - defaultHeader.add(new BasicHeader("upgrade-insecure-requests", "1"));*/ - //创建一个Http访问器 - httpClient = HttpClients.custom() - .setDefaultCookieStore(cookieStore) - .setDefaultHeaders(defaultHeader) - .setProxy(proxy) - .build(); - } - - /** - * 程序自行通过帐号密码登录Pixiv. - * @param PixivID Pixiv帐号 - * @param Password Pixiv密码 - * @return 登录成功返回true - * @throws IOException 当登录抛出异常时返回 - * @deprecated Pixiv已经新增Google人机验证, 程序已无法自行登录Pixiv - */ - public boolean Login(String PixivID, String Password) throws IOException { - // 获取登录接口所需的PostKey - String post_key = getPostKey(); - HttpPost postRequest = new HttpPost(PixivURL.PIXIV_LOGIN_URL); //https://accounts.pixiv.net/api/login?lang=zh - List params = new ArrayList<>(); - params.add(new BasicNameValuePair("pixiv_id", PixivID)); - params.add(new BasicNameValuePair("password", Password)); - params.add(new BasicNameValuePair("post_key", post_key)); - //Form编码表单,作为Post的数据 - postRequest.setEntity(new UrlEncodedFormEntity(params, StandardCharsets.UTF_8)); - //启动访问 - HttpResponse response = httpClient.execute(postRequest); - //获取接口返回数据 - String httpXML = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8); - System.out.println(httpXML); - JsonObject responseJson = (JsonObject) new JsonParser().parse(httpXML); - if(!responseJson.get("error").getAsBoolean() && !responseJson.get("body").getAsJsonObject().has("validation_errors")){ - errMsg = null; - return true; - }else{ - errMsg = responseJson.get("body").getAsJsonObject().get("validation_errors").toString(); - //System.err.println("登录失败！MSG: " + errMsg); - return false; - } - } - - /** - * 登录前准备, 获取PostKey - * @return Post_Key - */ - private String getPostKey() throws IOException { - //创建请求,获取PostKey - HttpGet getRequest = new HttpGet(PixivURL.PIXIV_LOGIN_PAGE_URL); - //设置请求 - //getRequest.setConfig(config); - getRequest.setHeader("User-Agent", USER_AGENT); - //启动访问 - HttpResponse response = httpClient.execute(getRequest); - //获取网页内容 - String pageAsXML = EntityUtils.toString(response.getEntity(),"utf-8"); - //创建Http解析器 - Document document = Jsoup.parse(pageAsXML); - //获取init-config内容 - String init_config = document.getElementById("init-config").val(); - //System.out.println(init_config); - //创建Json解析器解析init-config - JsonObject initConfigObj = (JsonObject) new JsonParser().parse(init_config); - //检查是否有postKey - if(!initConfigObj.has("pixivAccount.postKey")){ - throw new RuntimeException("postKey获取失败!可能是Pixiv修改了登录过程!"); - } - //获取postKey - return initConfigObj.get("pixivAccount.postKey").getAsString(); - } - - /** - * 获取CookieStore - * @return CookieStore - */ - public CookieStore getCookieStore(){ - return cookieStore; - } - - /** - * 获取可直接使用的HttpClient对象 - * @return 已配置好的HttpClient对象 - */ - public HttpClient getHttpClient(){ - return this.httpClient; - } - - public boolean hasError(){ - return errMsg == null; - } - - /** - * 获取错误信息 - * @return 返回登录错误信息 - * @deprecated {@link #Login(String, String)}已经废除, 故本接口废除 - */ - public String getErrMsg(){ - return errMsg; - } - -} diff --git a/src/main/java/net/lamgc/cgj/pixiv/PixivTag.java b/src/main/java/net/lamgc/cgj/pixiv/PixivTag.java deleted file mode 100644 index 25cfaa2..0000000 --- a/src/main/java/net/lamgc/cgj/pixiv/PixivTag.java +++ /dev/null @@ -1,16 +0,0 @@ -package net.lamgc.cgj.pixiv; - -public class PixivTag { - - public static PixivTag create(String tagName) { - - - return null; - } - - private PixivTag() { - - } - - -} diff --git a/src/main/java/net/lamgc/cgj/pixiv/PixivURL.java b/src/main/java/net/lamgc/cgj/pixiv/PixivURL.java index b0bd4fb..e6e7e31 100644 --- a/src/main/java/net/lamgc/cgj/pixiv/PixivURL.java +++ b/src/main/java/net/lamgc/cgj/pixiv/PixivURL.java @@ -9,6 +9,7 @@ import java.util.GregorianCalendar; /** * 目前已整理的一些Pixiv接口列表 */ +@SuppressWarnings("unused") public class PixivURL { diff --git a/src/main/java/net/lamgc/cgj/pixiv/PixivUgoiraBuilder.java b/src/main/java/net/lamgc/cgj/pixiv/PixivUgoiraBuilder.java index 0a76cd5..b5d024e 100644 --- a/src/main/java/net/lamgc/cgj/pixiv/PixivUgoiraBuilder.java +++ b/src/main/java/net/lamgc/cgj/pixiv/PixivUgoiraBuilder.java @@ -26,8 +26,7 @@ import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; /** - * Pixiv动图构建器. - * 可便捷的接收并处理动图. + * Pixiv动图构建器 */ public final class PixivUgoiraBuilder { diff --git a/src/main/java/net/lamgc/cgj/util/CookieUtil.java b/src/main/java/net/lamgc/cgj/util/CookieUtil.java deleted file mode 100644 index ae06dfb..0000000 --- a/src/main/java/net/lamgc/cgj/util/CookieUtil.java +++ /dev/null @@ -1,31 +0,0 @@ -package net.lamgc.cgj.util; - -import org.apache.http.client.CookieStore; -import org.apache.http.impl.client.BasicCookieStore; -import org.apache.http.impl.cookie.BasicClientCookie; - -import java.util.Date; - -public class CookieUtil { - - /** - * 将{@link java.net.CookieStore}转换到{@link CookieStore} - * @param netCookieStore java.net.CookieStore - * @return org.apache.http.client.CookieStore - */ - public static CookieStore NAParse(java.net.CookieStore netCookieStore){ - CookieStore apacheCookieStore = new BasicCookieStore(); - netCookieStore.getCookies().forEach(netCookie -> { - BasicClientCookie aCookie = new BasicClientCookie(netCookie.getName(), netCookie.getValue()); - aCookie.setComment(netCookie.getComment()); - aCookie.setDomain(netCookie.getDomain()); - aCookie.setExpiryDate(new Date(netCookie.getMaxAge())); - aCookie.setPath(netCookie.getPath()); - aCookie.setSecure(netCookie.getSecure()); - aCookie.setVersion(netCookie.getVersion()); - apacheCookieStore.addCookie(aCookie); - }); - return apacheCookieStore; - } - -} diff --git a/src/main/java/net/lamgc/cgj/util/PagesQualityParser.java b/src/main/java/net/lamgc/cgj/util/PagesQualityParser.java index 52b4bc9..c25b41c 100644 --- a/src/main/java/net/lamgc/cgj/util/PagesQualityParser.java +++ b/src/main/java/net/lamgc/cgj/util/PagesQualityParser.java @@ -7,7 +7,7 @@ import net.lamgc.utils.base.runner.StringParameterParser; public class PagesQualityParser implements StringParameterParser { @Override - public PixivDownload.PageQuality parse(String strValue) throws Exception { + public PixivDownload.PageQuality parse(String strValue) { return PixivDownload.PageQuality.valueOf(strValue.toUpperCase()); } diff --git a/src/test/java/net/lamgc/cgj/pixiv/PixivDownloadTest.java b/src/test/java/net/lamgc/cgj/pixiv/PixivDownloadTest.java index 9745fed..6a19830 100644 --- a/src/test/java/net/lamgc/cgj/pixiv/PixivDownloadTest.java +++ b/src/test/java/net/lamgc/cgj/pixiv/PixivDownloadTest.java @@ -181,19 +181,6 @@ public class PixivDownloadTest { log.info("正在调用方法..."); try { pixivDownload.getRankingAsInputStream(null, null, queryDate, 5, 50, PixivDownload.PageQuality.ORIGINAL, (rank, link, rankInfo, inputStream) -> { - /*try { - - ZipEntry entry = new ZipEntry("Rank" + rank + "-" + link.substring(link.lastIndexOf("/") + 1)); - entry.setComment(rankInfo.toString()); - log.info("正在写入: " + entry.getName()); - zos.putNextEntry(entry); - IOUtils.copy(inputStream, zos); - zos.flush(); - log.info("已成功写入 {}", entry.getName()); - inputStream.close(); - } catch (IOException e) { - log.error("写入文件项时发生异常", e); - }*/ log.info("空操作"); }); } finally {