From 532d7371d86ebf346a9c79cdf3eddfee39d77d3b Mon Sep 17 00:00:00 2001 From: LamGC Date: Thu, 26 Mar 2020 14:39:59 +0800 Subject: [PATCH] initial commit --- .gitignore | 6 + pom.xml | 150 +++++ search.txt | 24 + .../java/net/lamgc/cgj/CQBotAdminProcess.java | 58 ++ src/main/java/net/lamgc/cgj/CQConfig.java | 13 + src/main/java/net/lamgc/cgj/CQPluginMain.java | 147 +++++ src/main/java/net/lamgc/cgj/CQProcess.java | 553 ++++++++++++++++++ src/main/java/net/lamgc/cgj/Main.java | 518 ++++++++++++++++ .../net/lamgc/cgj/bot/AutoArtworksSender.java | 57 ++ .../java/net/lamgc/cgj/cache/CacheObject.java | 40 ++ .../lamgc/cgj/cache/ImageCacheHandler.java | 79 +++ .../net/lamgc/cgj/cache/ImageCacheObject.java | 57 ++ src/main/java/net/lamgc/cgj/db/RankingDB.java | 28 + .../java/net/lamgc/cgj/pixiv/IllustInfo.java | 69 +++ src/main/java/net/lamgc/cgj/pixiv/Pixiv.java | 267 +++++++++ .../net/lamgc/cgj/pixiv/PixivDownload.java | 533 +++++++++++++++++ .../lamgc/cgj/pixiv/PixivSearchBuilder.java | 340 +++++++++++ .../net/lamgc/cgj/pixiv/PixivSession.java | 171 ++++++ .../java/net/lamgc/cgj/pixiv/PixivTag.java | 16 + .../java/net/lamgc/cgj/pixiv/PixivURL.java | 290 +++++++++ .../cgj/proxy/PixivAccessProxyServer.java | 149 +++++ .../cgj/proxy/PixivLoginProxyServer.java | 157 +++++ .../cgj/proxy/PixivLoginProxyServer_Old.java | 185 ++++++ .../java/net/lamgc/cgj/util/CookieUtil.java | 31 + .../java/net/lamgc/cgj/util/DateParser.java | 25 + .../lamgc/cgj/util/PagesQualityParser.java | 18 + src/main/resources/application.properties | 2 + src/main/resources/log4j.properties | 15 + src/main/resources/simple.properties | 9 + .../lamgc/cgj/pixiv/PixivDownloadTest.java | 202 +++++++ .../cgj/pixiv/PixivSearchBuilderTest.java | 15 + 31 files changed, 4224 insertions(+) create mode 100644 .gitignore create mode 100644 pom.xml create mode 100644 search.txt create mode 100644 src/main/java/net/lamgc/cgj/CQBotAdminProcess.java create mode 100644 src/main/java/net/lamgc/cgj/CQConfig.java create mode 100644 src/main/java/net/lamgc/cgj/CQPluginMain.java create mode 100644 src/main/java/net/lamgc/cgj/CQProcess.java create mode 100644 src/main/java/net/lamgc/cgj/Main.java create mode 100644 src/main/java/net/lamgc/cgj/bot/AutoArtworksSender.java create mode 100644 src/main/java/net/lamgc/cgj/cache/CacheObject.java create mode 100644 src/main/java/net/lamgc/cgj/cache/ImageCacheHandler.java create mode 100644 src/main/java/net/lamgc/cgj/cache/ImageCacheObject.java create mode 100644 src/main/java/net/lamgc/cgj/db/RankingDB.java create mode 100644 src/main/java/net/lamgc/cgj/pixiv/IllustInfo.java create mode 100644 src/main/java/net/lamgc/cgj/pixiv/Pixiv.java create mode 100644 src/main/java/net/lamgc/cgj/pixiv/PixivDownload.java create mode 100644 src/main/java/net/lamgc/cgj/pixiv/PixivSearchBuilder.java create mode 100644 src/main/java/net/lamgc/cgj/pixiv/PixivSession.java create mode 100644 src/main/java/net/lamgc/cgj/pixiv/PixivTag.java create mode 100644 src/main/java/net/lamgc/cgj/pixiv/PixivURL.java create mode 100644 src/main/java/net/lamgc/cgj/proxy/PixivAccessProxyServer.java create mode 100644 src/main/java/net/lamgc/cgj/proxy/PixivLoginProxyServer.java create mode 100644 src/main/java/net/lamgc/cgj/proxy/PixivLoginProxyServer_Old.java create mode 100644 src/main/java/net/lamgc/cgj/util/CookieUtil.java create mode 100644 src/main/java/net/lamgc/cgj/util/DateParser.java create mode 100644 src/main/java/net/lamgc/cgj/util/PagesQualityParser.java create mode 100644 src/main/resources/application.properties create mode 100644 src/main/resources/log4j.properties create mode 100644 src/main/resources/simple.properties create mode 100644 src/test/java/net/lamgc/cgj/pixiv/PixivDownloadTest.java create mode 100644 src/test/java/net/lamgc/cgj/pixiv/PixivSearchBuilderTest.java diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..bf2955d --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +/pluginData/ +/logs/ +/.idea/ +/CGJ_2.iml +/cookies.store +/target/ diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..ccd3c22 --- /dev/null +++ b/pom.xml @@ -0,0 +1,150 @@ + + + 4.0.0 + + org.example + CGJ_2 + 1.0-SNAPSHOT + + + UTF-8 + UTF-8 + UTF-8 + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + 8 + 8 + + + + + org.springframework.boot + spring-boot-maven-plugin + + net.lamgc.cgj.Main + exec + + + + + repackage + + + + + + + + + + com.github.monkeywie + proxyee + 1.0.4 + + + org.slf4j + slf4j-api + 1.7.30 + + + org.slf4j + slf4j-log4j12 + 1.7.30 + + + net.lamgc + java-utils + 1.1.0_5-SNAPSHOT + + + + junit + junit + 4.12 + test + + + org.apache.httpcomponents + httpclient + 4.5.12 + + + org.jsoup + jsoup + 1.11.3 + + + com.google.code.gson + gson + 2.8.5 + + + net.sourceforge.htmlunit + htmlunit + 2.29 + + + com.github.rholder + guava-retrying + 2.0.0 + + + mysql + mysql-connector-java + 8.0.19 + + + net.lz1998 + spring-cq + 4.14.0.6 + + + ch.qos.logback + logback-classic + + + ch.qos.logback + logback-core + + + + + + \ No newline at end of file diff --git a/search.txt b/search.txt new file mode 100644 index 0000000..f6d74fb --- /dev/null +++ b/search.txt @@ -0,0 +1,24 @@ +搜索标签信息：https://www.pixiv.net/ajax/search/tags/标签名 + +搜索接口： +https://www.pixiv.net/ajax/search/{Type}/搜索内容 +Type = illustrations(插画) / top(顶部?) / manga(漫画) / novels(小说) + +word=搜索内容 [参数可能不是必须的] +s_mode=s_tag(标签-部分一致) / s_tag_full(标签-完全一致) / s_tc(标题、说明文字) +type= all(插画、漫画、动图_动态插图) / illust_and_ugoira(插画、动图) / illust(插画) / manga(漫画) / ugoira(动图) +p=页数 [超出页数的情况下将获取不到数据(即"body.illust.data"是空数组)] +order=date(按旧排序) / date_d(按新排序) / Unknown(按热门度排序, 需要会员) +mode= all(全部) / safe(全年龄) / r18(咳咳) + +可选参数： +wlt=最小宽度像素 +wgt=最高宽度像素 +hlt=最小高度像素 +hgt=最高高度像素 +ratio=0.5(横图) / -0.5(纵图) / 0(正方形) [可能不能改变参数, 三个值是固定的] +tool=使用工具, 不是很重要晚些再加 +scd=开始时间(yyyy-MM-dd) +ecd=结束时间(yyyy-MM-dd) + +最小收藏数 = 收藏数限定参数为会员功能, 无法获取 \ No newline at end of file diff --git a/src/main/java/net/lamgc/cgj/CQBotAdminProcess.java b/src/main/java/net/lamgc/cgj/CQBotAdminProcess.java new file mode 100644 index 0000000..6de7f43 --- /dev/null +++ b/src/main/java/net/lamgc/cgj/CQBotAdminProcess.java @@ -0,0 +1,58 @@ +package net.lamgc.cgj; + +import com.google.common.base.Strings; +import net.lamgc.utils.base.runner.Argument; +import net.lamgc.utils.base.runner.Command; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; + +public class CQBotAdminProcess { + + private final static Logger log = LoggerFactory.getLogger("CQBotAdminProcess"); + + @Command + public String clearCache() { + CQProcess.clearCache(); + return "操作已完成."; + } + + @Command + public String setGlobalProperty(@Argument(name = "key") String key, @Argument(name = "value") String value, @Argument(name = "save", force = false) boolean saveNow) { + String lastValue = CQPluginMain.globalProp.getProperty(key); + CQPluginMain.globalProp.setProperty(key, Strings.nullToEmpty(value)); + if(saveNow) { + saveGlobalProperties(); + } + return "全局配置项 " + key + " 现已设置为: " + value + " (设置前的值: " + lastValue + ")"; + } + + @Command + public String getGlobalProperty(@Argument(name = "key") String key) { + return "全局配置项 " + key + " 当前值: " + CQPluginMain.globalProp.getProperty(key, "(Empty)"); + } + + @Command + public String saveGlobalProperties() { + log.info("正在保存全局配置文件..."); + File globalPropFile = new File("./global.properties"); + try { + if(!globalPropFile.exists()) { + if(!globalPropFile.createNewFile()) { + log.error("全局配置项文件保存失败！({})", "文件创建失败"); + return "全局配置项文件保存失败！"; + } + } + CQPluginMain.globalProp.store(new FileOutputStream(globalPropFile), ""); + log.info("全局配置文件保存成功！"); + return "保存全局配置文件 - 操作已完成."; + } catch (IOException e) { + log.error("全局配置项文件保存失败！", e); + return "全局配置项文件保存失败！"; + } + } + +} diff --git a/src/main/java/net/lamgc/cgj/CQConfig.java b/src/main/java/net/lamgc/cgj/CQConfig.java new file mode 100644 index 0000000..161c054 --- /dev/null +++ b/src/main/java/net/lamgc/cgj/CQConfig.java @@ -0,0 +1,13 @@ +package net.lamgc.cgj; + +import net.lz1998.cq.CQGlobal; +import net.lz1998.cq.EnableCQ; + +@EnableCQ +public class CQConfig { + + public static void init() { + CQGlobal.pluginList.add(CQPluginMain.class); + } + +} diff --git a/src/main/java/net/lamgc/cgj/CQPluginMain.java b/src/main/java/net/lamgc/cgj/CQPluginMain.java new file mode 100644 index 0000000..eb8e66b --- /dev/null +++ b/src/main/java/net/lamgc/cgj/CQPluginMain.java @@ -0,0 +1,147 @@ +package net.lamgc.cgj; + +import com.google.common.base.Strings; +import net.lamgc.cgj.util.DateParser; +import net.lamgc.cgj.util.PagesQualityParser; +import net.lamgc.utils.base.runner.ArgumentsRunner; +import net.lamgc.utils.base.runner.ArgumentsRunnerConfig; +import net.lamgc.utils.base.runner.exception.DeveloperRunnerException; +import net.lamgc.utils.base.runner.exception.NoSuchCommandException; +import net.lamgc.utils.base.runner.exception.ParameterNoFoundException; +import net.lz1998.cq.event.message.CQDiscussMessageEvent; +import net.lz1998.cq.event.message.CQGroupMessageEvent; +import net.lz1998.cq.event.message.CQMessageEvent; +import net.lz1998.cq.event.message.CQPrivateMessageEvent; +import net.lz1998.cq.robot.CQPlugin; +import net.lz1998.cq.robot.CoolQ; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Objects; +import java.util.Properties; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +@Component +public class CQPluginMain extends CQPlugin { + + private final static String COMMAND_PREFIX = ".cgj"; + private final Logger log = LoggerFactory.getLogger("CQPluginMain@" + Integer.toHexString(this.hashCode())); + private final ArgumentsRunnerConfig runnerConfig = new ArgumentsRunnerConfig(); + public final static Properties globalProp = new Properties(); + + public CQPluginMain() { + runnerConfig.setUseDefaultValueInsteadOfException(true); + runnerConfig.setCommandIgnoreCase(true); + runnerConfig.addStringParameterParser(new DateParser(new SimpleDateFormat("yyyy-MM-dd"))); + runnerConfig.addStringParameterParser(new PagesQualityParser()); + + File globalPropFile = new File("./global.properties"); + if(globalPropFile.exists() && globalPropFile.isFile()) { + log.info("正在加载全局配置文件..."); + try { + globalProp.load(new FileInputStream(globalPropFile)); + log.info("全局配置文件加载完成."); + } catch (IOException e) { + log.error("加载全局配置文件时发生异常", e); + } + } else { + log.info("未找到全局配置文件，跳过加载."); + } + } + + @Override + public int onPrivateMessage(CoolQ cq, CQPrivateMessageEvent event) { + //log.info("私聊消息到达: 发送者[{}], 消息内容: {}", event.getSender().getUserId(), event.getMessage()); + return processMessage(cq, event); + } + + @Override + public int onGroupMessage(CoolQ cq, CQGroupMessageEvent event) { + //log.info("群消息到达: 群[{}], 发送者[{}], 消息内容: {}", event.getGroupId(), event.getSender().getUserId(), event.getMessage()); + return processMessage(cq, event); + } + + @Override + public int onDiscussMessage(CoolQ cq, CQDiscussMessageEvent event) { + //log.info("讨论组消息到达: 群[{}], 发送者[{}], 消息内容: {}", event.getDiscussId(), event.getSender().getUserId(), event.getMessage()); + return processMessage(cq, event); + } + + public int processMessage(CoolQ cq, CQMessageEvent event) { + String msg = event.getMessage(); + if(!msg.startsWith(COMMAND_PREFIX)) { + return MESSAGE_IGNORE; + } + + Pattern pattern = Pattern.compile("/\\s*(\".+?\"|[^:\\s])+((\\s*:\\s*(\".+?\"|[^\\s])+)|)|(\".+?\"|[^\"\\s])+"); + Matcher matcher = pattern.matcher(Strings.nullToEmpty(msg)); + ArrayList argsList = new ArrayList<>(); + while (matcher.find()) { + argsList.add(matcher.group()); + } + String[] args = new String[argsList.size()]; + argsList.toArray(args); + + log.info("正在处理命令..."); + long time = System.currentTimeMillis(); + Object result; + try { + if(msg.toLowerCase().startsWith(COMMAND_PREFIX + "admin")) { + if(!String.valueOf(event.getUserId()).equals(globalProp.getProperty("admin.adminId"))) { + sendMessage(cq, event, "你没有执行该命令的权限！", false); + return MESSAGE_BLOCK; + } else { + result = new ArgumentsRunner(CQBotAdminProcess.class, runnerConfig) + .run(new CQBotAdminProcess(), args.length <= 1 ? new String[0] : Arrays.copyOfRange(args, 1, args.length)); + } + } else { + result = new ArgumentsRunner(CQProcess.class, runnerConfig).run(args.length <= 1 ? new String[0] : Arrays.copyOfRange(args, 1, args.length)); + } + } catch(NoSuchCommandException e) { + result = "没有这个命令！请使用“.cgj”查看帮助说明！"; + } catch(ParameterNoFoundException e) { + result = "命令缺少参数: " + e.getParameterName(); + } catch(DeveloperRunnerException e) { + log.error("执行命令时发生异常", e); + result = "命令执行时发生错误，无法完成！"; + } + log.info("命令处理完成(耗时: {}ms)", System.currentTimeMillis() - time); + if(Objects.requireNonNull(result) instanceof String) { + sendMessage(cq, event, (String) result, false); + } + return MESSAGE_BLOCK; + } + + private final static Logger msgLog = LoggerFactory.getLogger("SendMsg"); + + /** + * 发送消息 + * @param cq CoolQ对象 + * @param event 消息事件对象 + * @param message 消息内容 + * @param auto_escape 消息内容是否作为纯文本发送（即不解析 CQ 码），只在 message 字段是字符串时有效. + */ + public static void sendMessage(CoolQ cq, CQMessageEvent event, String message, boolean auto_escape) { + msgLog.debug("发送消息：{}", message); + if(event instanceof CQPrivateMessageEvent) { + CQPrivateMessageEvent _event = (CQPrivateMessageEvent) event; + cq.sendPrivateMsg(_event.getSender().getUserId(), message, auto_escape); + } else if(event instanceof CQGroupMessageEvent) { + CQGroupMessageEvent _event = (CQGroupMessageEvent) event; + cq.sendGroupMsg(_event.getGroupId(), message, auto_escape); + } else if(event instanceof CQDiscussMessageEvent) { + CQDiscussMessageEvent _event = (CQDiscussMessageEvent) event; + cq.sendGroupMsg(_event.getDiscussId(), message, auto_escape).getData().getMessageId(); + } + } + + +} diff --git a/src/main/java/net/lamgc/cgj/CQProcess.java b/src/main/java/net/lamgc/cgj/CQProcess.java new file mode 100644 index 0000000..81887d8 --- /dev/null +++ b/src/main/java/net/lamgc/cgj/CQProcess.java @@ -0,0 +1,553 @@ +package net.lamgc.cgj; + +import com.google.common.base.Strings; +import com.google.gson.*; +import io.netty.handler.codec.http.HttpHeaderNames; +import net.lamgc.cgj.cache.CacheObject; +import net.lamgc.cgj.cache.ImageCacheHandler; +import net.lamgc.cgj.cache.ImageCacheObject; +import net.lamgc.cgj.pixiv.PixivDownload; +import net.lamgc.cgj.pixiv.PixivSearchBuilder; +import net.lamgc.cgj.pixiv.PixivURL; +import net.lamgc.utils.base.runner.Argument; +import net.lamgc.utils.base.runner.Command; +import net.lamgc.utils.event.EventExecutor; +import net.lz1998.cq.utils.CQCode; +import org.apache.http.HttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpHead; +import org.apache.http.util.EntityUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.*; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.StandardOpenOption; +import java.text.SimpleDateFormat; +import java.util.*; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +public class CQProcess { + + private final static PixivDownload pixivDownload = new PixivDownload(Main.cookieStore, Main.proxy); + + private final static Logger log = LoggerFactory.getLogger("CQProcess"); + + private final static File imageStoreDir = new File(System.getProperty("cgj.cqRootDir") + "data/image/cgj/"); + + private final static Hashtable imageCache = new Hashtable<>(); + + private final static Hashtable illustInfoCache = new Hashtable<>(); + + private final static Hashtable> illustPreLoadDataCache = new Hashtable<>(); + + private final static Hashtable> searchBodyCache = new Hashtable<>(); + + private final static Hashtable> pagesCache = new Hashtable<>(); + + private final static Object searchCacheLock = new Object(); + + private final static Gson gson = new GsonBuilder() + .serializeNulls() + .create(); + + private final static EventExecutor imageCacheExecutor = new EventExecutor(new ThreadPoolExecutor( + 1, + (int) Math.ceil(Runtime.getRuntime().availableProcessors() / 2F), + 15L, TimeUnit.SECONDS, + new ArrayBlockingQueue<>(30), + new ThreadPoolExecutor.DiscardOldestPolicy() + )); + + static { + try { + imageCacheExecutor.addHandler(new ImageCacheHandler()); + } catch (IllegalAccessException e) { + e.printStackTrace(); + } + } + + @Command(defaultCommand = true) + public static String help() { + StringBuilder helpStrBuilder = new StringBuilder(); + helpStrBuilder.append("CGJ Bot使用指南").append("\n"); + helpStrBuilder.append("目前可用的命令:").append("\n"); + helpStrBuilder.append("\t").append("ranking - 获取今天或指定日期排行榜的前10名作品").append("\n"); + helpStrBuilder.append("\t\t").append("date - 指定查询日期(年年年年-月月-日日)").append("\n"); + helpStrBuilder.append("\t").append("search - 搜索指定关键词并显示前10个作品").append("\n"); + helpStrBuilder.append("\t\t").append("content - 搜索内容").append("\n"); + helpStrBuilder.append("\t").append("artworks - 获取作品的Pixiv页面").append("\n"); + helpStrBuilder.append("\t\t").append("id - 作品id").append("\n"); + return helpStrBuilder.toString(); + } + + @Command + public static String ranking( + @Argument(force = false, name = "date") Date queryTime, + @Argument(force = false, name = "contentMode", defaultValue = "DAILY") String contentMode + ) { + Date queryDate = queryTime; + if (queryDate == null) { + queryDate = new Date(); + GregorianCalendar gregorianCalendar = new GregorianCalendar(); + log.info("CurrentDate: {}", queryDate); + gregorianCalendar.setTime(queryDate); + if (gregorianCalendar.get(Calendar.HOUR_OF_DAY) < 12) { + gregorianCalendar.add(Calendar.DATE, -2); + } else { + gregorianCalendar.add(Calendar.DATE, -1); + } + queryDate = gregorianCalendar.getTime(); + } + + PixivURL.RankingMode mode = PixivURL.RankingMode.MODE_DAILY; + try { + mode = PixivURL.RankingMode.valueOf("MODE_" + contentMode.toUpperCase()); + } catch (IllegalArgumentException e) { + log.warn("无效的RankingMode值: {}", contentMode); + } + + StringBuilder resultBuilder = new StringBuilder(mode.name() + " - 以下是 ").append(new SimpleDateFormat("yyyy-MM-dd").format(queryDate)).append(" 的Pixiv插画排名榜前十名：\n"); + try { + int index = 0; + for (JsonObject rankInfo : pixivDownload.getRanking(PixivURL.RankingContentType.TYPE_ILLUST, mode, queryDate, 1, 10)) { + index++; + int rank = rankInfo.get("rank").getAsInt(); + int illustId = rankInfo.get("illust_id").getAsInt(); + int authorId = rankInfo.get("user_id").getAsInt(); + String authorName = rankInfo.get("user_name").getAsString(); + String title = rankInfo.get("title").getAsString(); + resultBuilder.append(rank).append(". (id: ").append(illustId).append(") ").append(title) + .append("(Author: ").append(authorName).append(",").append(authorId).append(")\n"); + if (index < 4) { + resultBuilder.append(getImageById(illustId, PixivDownload.PageQuality.REGULAR, 1)).append("\n"); + } + } + } catch (IOException e) { + log.error("消息处理异常", e); + return "排名榜获取失败！详情请查看机器人控制台。"; + } + return resultBuilder.append("如查询当前时间获取到昨天时间，则今日排名榜未更新。").toString(); + } + + @Command(commandName = "userArt") + public static String userArtworks() { + + return "功能未完成"; + } + + @Command + public static String search(@Argument(name = "content") String content, + @Argument(name = "type", force = false) String type, + @Argument(name = "area", force = false) String area, + @Argument(name = "in", force = false) String includeKeywords, + @Argument(name = "ex", force = false) String excludeKeywords, + @Argument(name = "contentOption", force = false) String contentOption, + @Argument(name = "page", force = false, defaultValue = "1") int pagesIndex + ) throws IOException { + PixivSearchBuilder searchBuilder = new PixivSearchBuilder(Strings.isNullOrEmpty(content) ? "" : content); + if (type != null) { + try { + searchBuilder.setSearchType(PixivSearchBuilder.SearchType.valueOf(type.toUpperCase())); + } catch (IllegalArgumentException e) { + log.warn("不支持的SearchType: {}", type); + } + } + if (area != null) { + try { + searchBuilder.setSearchArea(PixivSearchBuilder.SearchArea.valueOf(area)); + } catch (IllegalArgumentException e) { + log.warn("不支持的SearchArea: {}", area); + } + } + if (contentOption != null) { + try { + searchBuilder.setSearchContentOption(PixivSearchBuilder.SearchContentOption.valueOf(contentOption)); + } catch (IllegalArgumentException e) { + log.warn("不支持的SearchContentOption: {}", contentOption); + } + } + + if (!Strings.isNullOrEmpty(includeKeywords)) { + for (String keyword : includeKeywords.split(";")) { + searchBuilder.removeExcludeKeyword(keyword); + searchBuilder.addIncludeKeyword(keyword); + log.info("已添加关键字: {}", keyword); + } + } + if (!Strings.isNullOrEmpty(excludeKeywords)) { + for (String keyword : excludeKeywords.split(";")) { + searchBuilder.removeIncludeKeyword(keyword); + searchBuilder.addExcludeKeyword(keyword); + log.info("已添加排除关键字: {}", keyword); + } + } + + log.info("正在搜索作品, 条件: {}", searchBuilder.getSearchCondition()); + + String requestUrl = searchBuilder.buildURL(); + log.info("RequestUrl: {}", requestUrl); + + CacheObject cacheObject = new CacheObject<>(); + if(!searchBodyCache.containsKey(requestUrl) || (cacheObject = searchBodyCache.get(requestUrl)).isExpire(new Date())) { + synchronized (searchCacheLock) { + if (!searchBodyCache.containsKey(requestUrl) || (cacheObject = searchBodyCache.get(requestUrl)).isExpire(new Date())) { + log.info("searchBody缓存失效, 正在更新..."); + JsonObject jsonObject; + HttpGet httpGetRequest = pixivDownload.createHttpGetRequest(requestUrl); + HttpResponse response = pixivDownload.getHttpClient().execute(httpGetRequest); + + String responseBody = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8); + log.info("ResponseBody: {}", responseBody); + jsonObject = gson.fromJson(responseBody, JsonObject.class); + + if (jsonObject.get("error").getAsBoolean()) { + log.error("接口请求错误, 错误信息: {}", jsonObject.get("message").getAsString()); + return "处理命令时发生错误！"; + } + + Date newExpireDate = new Date(); + long expire = 7200 * 1000; + String propValue = CQPluginMain.globalProp.getProperty("cache.searchBody.expire", "7200000"); + try { + expire = Long.parseLong(propValue); + } catch (Exception e) { + log.warn("全局配置项 \"{}\" 值非法, 已使用默认值: {}", propValue, expire); + } + + newExpireDate.setTime(newExpireDate.getTime() + expire); + cacheObject.update(jsonObject, newExpireDate); + searchBodyCache.put(requestUrl, cacheObject); + log.info("searchBody缓存已更新(到期时间: {})", newExpireDate); + } + } + } else { + log.info("搜索缓存命中."); + } + + JsonObject resultBody = searchBodyCache.get(requestUrl).get().getAsJsonObject("body"); + StringBuilder result = new StringBuilder("搜索结果："); + log.info("正在处理信息..."); + int limit = 8; + try { + limit = Integer.parseInt(CQPluginMain.globalProp.getProperty("search.ItemCountLimit", "8")); + } catch (Exception e) { + log.warn("参数转换异常!将使用默认值(" + limit + ")", e); + } + for (PixivSearchBuilder.SearchArea searchArea : PixivSearchBuilder.SearchArea.values()) { + if (!resultBody.has(searchArea.jsonKey) || resultBody.getAsJsonObject(searchArea.jsonKey).getAsJsonArray("data").size() == 0) { + log.debug("返回数据不包含 {}", searchArea.jsonKey); + continue; + } + JsonArray illustsArray = resultBody + .getAsJsonObject(searchArea.jsonKey).getAsJsonArray("data"); + ArrayList illustsList = new ArrayList<>(); + illustsArray.forEach(illustsList::add); + illustsList.sort((o1, o2) -> { + try { + int illustLikeCount1 = getIllustPreLoadData(o1.getAsJsonObject().get("illustId").getAsInt()).get("likeCount").getAsInt(); + int illustLikeCount2 = getIllustPreLoadData(o2.getAsJsonObject().get("illustId").getAsInt()).get("likeCount").getAsInt(); + return Integer.compare(illustLikeCount2, illustLikeCount1); + } catch (IOException e) { + e.printStackTrace(); + return 0; + } + }); + + log.info("已找到与 {} 相关插图信息({})：", content, searchArea.name().toLowerCase()); + int count = 1; + for (JsonElement jsonElement : illustsList) { + if (count > limit) { + break; + } + JsonObject illustObj = jsonElement.getAsJsonObject(); + if (!illustObj.has("illustId")) { + continue; + } + int illustId = illustObj.get("illustId").getAsInt(); + StringBuilder builder = new StringBuilder("["); + illustObj.get("tags").getAsJsonArray().forEach(el -> builder.append(el.getAsString()).append(", ")); + builder.replace(builder.length() - 2, builder.length(), "]"); + log.debug("{} ({} / {})\n\t作品id: {}, \n\t作者名(作者id): {} ({}), \n\t作品标题: {}, \n\t作品Tags: {}, \n\t作品链接: {}", + searchArea.name(), + count, + illustsList.size(), + illustId, + illustObj.get("userName").getAsString(), + illustObj.get("userId").getAsInt(), + illustObj.get("illustTitle").getAsString(), + builder, + PixivURL.getPixivRefererLink(illustId) + ); + + String imageMsg = getImageById(illustId, PixivDownload.PageQuality.REGULAR, 1); + if (isNoSafe(illustId, CQPluginMain.globalProp, true)) { + log.warn("作品Id {} 为R-18作品, 跳过.", illustId); + count--; + continue; + } + + result.append(searchArea.name()).append(" (").append(count).append(" / ").append(illustsList.size()).append(")\n\t作品id: ").append(illustId) + .append(", \n\t作者名: ").append(illustObj.get("userName").getAsString()) + .append("\n\t作品标题: ").append(illustObj.get("illustTitle").getAsString()).append("\n").append(imageMsg).append("\n"); + count++; + } + if (count > limit) { + break; + } + } + return Strings.nullToEmpty(result.toString()) + "预览图片并非原图，使用“.cgj image -id 作品id”获取原图"; + } + + @Command(commandName = "pages") + public static String getPagesList(@Argument(name = "id") int illustId, @Argument(name = "quality", force = false) PixivDownload.PageQuality quality) { + try { + List pagesList = PixivDownload.getIllustAllPageDownload(pixivDownload.getHttpClient(), pixivDownload.getCookieStore(), illustId, quality); + StringBuilder builder = new StringBuilder("作品ID ").append(illustId).append(" 共有").append(pagesList.size()).append("页：").append("\n"); + int index = 0; + for (String link : pagesList) { + builder.append("Page ").append(++index).append(": ").append(link).append("\n"); + } + return builder.toString(); + } catch (IOException e) { + log.error("获取作品所有页面下载链接失败！", e); + return "发生错误，无法完成命令"; + } + } + + @Command(commandName = "artworks") + public static String artworksLink(@Argument(name = "id") int illustId) { + try { + if (isNoSafe(illustId, CQPluginMain.globalProp, false)) { + log.warn("作品Id {} 已被屏蔽.", illustId); + return "由于相关设置，该作品已被屏蔽！"; + } + } catch (IOException e) { + log.error("获取作品信息失败!", e); + return "作品信息无法获取！"; + } + return PixivURL.getPixivRefererLink(illustId); + } + + @Command(commandName = "image") + public static String getImageById(@Argument(name = "id") int illustId, + @Argument(name = "quality", force = false) PixivDownload.PageQuality quality, + @Argument(name = "page", force = false, defaultValue = "1") int pageIndex) { + log.info("IllustId: {}, Quality: {}, PageIndex: {}", illustId, quality.name(), pageIndex); + List pagesList; + try { + pagesList = getIllustPages(illustId, quality); + } catch (IOException e) { + log.error("获取下载链接列表时发生异常", e); + return "发生网络异常，无法获取图片！"; + } + if (pagesList.size() < pageIndex || pageIndex <= 0) { + log.warn("指定的页数超出了总页数({} / {})", pageIndex, pagesList.size()); + return "指定的页数超出了范围(总共 " + pagesList.size() + " 页)"; + } + + try { + if (isNoSafe(illustId, CQPluginMain.globalProp, false)) { + log.warn("作品 {} 存在R-18内容且设置\"image.allowR18\"为false，将屏蔽该作品不发送.", illustId); + pageIndex = -1; + } + } catch (IOException e) { + log.warn("作品信息无法获取!", e); + return "发生网络异常，无法获取图片！"; + } + + int index = 0; + String targetLink = null; + File targetFile; + File currentImageFile; + for (String link : pagesList) { + index++; + if (index == pageIndex) { + targetLink = link; + } + currentImageFile = new File(getImageStoreDir(), link.substring(link.lastIndexOf("/") + 1)); + if (!imageCache.containsKey(link)) { + HttpHead headRequest = new HttpHead(link); + headRequest.addHeader("Referer", PixivURL.getPixivRefererLink(illustId)); + HttpResponse headResponse; + try { + headResponse = pixivDownload.getHttpClient().execute(headRequest); + } catch (IOException e) { + log.error("获取图片大小失败！", e); + return "图片获取失败!"; + } + String contentLengthStr = headResponse.getFirstHeader(HttpHeaderNames.CONTENT_LENGTH.toString()).getValue(); + if (currentImageFile.exists() && currentImageFile.length() == Long.parseLong(contentLengthStr)) { + imageCache.put(link, currentImageFile); + log.info("作品Id {} 第 {} 页缓存已补充.", illustId, index); + continue; + } + + if (index == pageIndex) { + try { + imageCacheExecutor.executorSync(new ImageCacheObject(imageCache, illustId, link, currentImageFile)); + } catch (InterruptedException e) { + log.warn("图片下载遭到中断!", e); + } + } else { + imageCacheExecutor.executor( + new ImageCacheObject(imageCache, illustId, link, currentImageFile)); + } + } + } + + if (pageIndex == -1) { + return "（根据设置，该作品已被屏蔽！）"; + } + + if (targetLink == null) { + return "未找到对应的图片！"; + } + + targetFile = imageCache.get(targetLink); + return targetFile == null ? "未找到对应的图片！" : CQCode.image(getImageStoreDir().getName() + "/" + targetFile.getName()); + } + + static void clearCache() { + log.warn("正在清除所有图片缓存..."); + imageCache.clear(); + File imageStoreDir = new File(System.getProperty("cgj.cqRootDir") + "data/image/cgj/"); + File[] listFiles = imageStoreDir.listFiles(); + if (listFiles == null) { + log.info("图片缓存目录为空或内部文件获取失败!"); + } else { + for (File file : listFiles) { + log.info("图片文件 {} 删除: {}", file.getName(), file.delete()); + } + } + log.info("图片缓存目录删除: {}", imageStoreDir.delete()); + log.warn("缓存删除完成."); + } + + /* + 下一目标： + 添加定时发图 + 定时发图支持设置关注标签 + 标签....标签支持搜索吧 + */ + + private static boolean isNoSafe(int illustId, Properties settingProp, boolean returnRaw) throws IOException { + boolean rawValue = getIllustInfo(illustId).getAsJsonArray("tags").contains(new JsonPrimitive("R-18")); + return returnRaw || settingProp == null ? rawValue : rawValue && !settingProp.getProperty("image.allowR18", "false").equalsIgnoreCase("true"); + } + + private final static Object illustInfoLock = new Object(); + private static JsonObject getIllustInfo(int illustId) throws IOException { + synchronized (illustInfoLock) { + File cacheFile = new File(getImageStoreDir(), illustId + ".illustInfo.json"); + if (!illustInfoCache.containsKey(illustId)) { + log.info("IllustInfoFileName: {}", cacheFile.getName()); + JsonObject illustInfoObj; + if (!cacheFile.exists()) { + try { + cacheFile.createNewFile(); + illustInfoObj = pixivDownload.getIllustInfoByIllustId(illustId); + Files.write(cacheFile.toPath(), gson.toJson(illustInfoObj).getBytes(StandardCharsets.UTF_8), StandardOpenOption.CREATE); + } catch(IOException e) { + cacheFile.delete(); + throw e; + } + } else { + illustInfoObj = gson.fromJson(new FileReader(cacheFile), JsonObject.class); + } + illustInfoCache.put(illustId, illustInfoObj); + } + + return illustInfoCache.get(illustId); + } + } + + private final static Object illustPreLoadDataLock = new Object(); + public static JsonObject getIllustPreLoadData(int illustId) throws IOException { + synchronized (illustPreLoadDataLock) { + File cacheFile = new File(getImageStoreDir(), illustId + ".illustPreLoadData.json"); + CacheObject cacheObject = new CacheObject<>(); + Date currentDate = new Date(); + if (!illustPreLoadDataCache.containsKey(illustId) || (cacheObject = illustPreLoadDataCache.get(illustId)).isExpire(currentDate)) { + log.info("因为到期而失效: {}", cacheObject.isExpire(new Date())); + log.info("因为缓存文件不存在而失效: {}", !cacheFile.exists()); + log.info("缓存失效, 正在更新..."); + log.info("illustPreLoadDataFileName: {}", cacheFile.getName()); + JsonObject preLoadDataObj; + if (!cacheFile.exists()) { + try { + cacheFile.createNewFile(); + preLoadDataObj = pixivDownload.getIllustPreLoadDataById(illustId) + .getAsJsonObject("illust") + .getAsJsonObject(Integer.toString(illustId)); + Files.write(cacheFile.toPath(), gson.toJson(preLoadDataObj).getBytes(StandardCharsets.UTF_8), StandardOpenOption.CREATE); + } catch(IOException e) { + cacheFile.delete(); + throw e; + } + } else { + preLoadDataObj = gson.fromJson(new FileReader(cacheFile), JsonObject.class); + } + + long expire = 7200 * 1000; + String propValue = CQPluginMain.globalProp.getProperty("cache.illustPreLoadData.expire", "7200000"); + log.info("PreLoadData有效时间设定: {}", propValue); + try { + expire = Long.parseLong(propValue); + } catch (Exception e) { + log.warn("全局配置项 \"{}\" 值非法, 已使用默认值: {}", propValue, expire); + } + + Date newExpire = new Date(); + newExpire.setTime(newExpire.getTime() + expire); + cacheObject.update(preLoadDataObj, newExpire); + illustPreLoadDataCache.put(illustId, cacheObject); + log.info("作品Id {} preLoadData缓存已更新(到期时间: {})", illustId, newExpire); + } + + return illustPreLoadDataCache.get(illustId).get(); + } + } + + private final static Object illustPagesLock = new Object(); + public static List getIllustPages(int illustId, PixivDownload.PageQuality quality) throws IOException { + synchronized (illustPagesLock) { + File cacheFile = new File(getImageStoreDir(), illustId + "." + quality.name() + ".illustPages.json"); + if (!pagesCache.containsKey(illustId + "." + quality.name())) { + log.info("illustPagesFileName: {}", cacheFile.getName()); + List linkList; + if (!cacheFile.exists()) { + try { + cacheFile.createNewFile(); + linkList = PixivDownload.getIllustAllPageDownload(pixivDownload.getHttpClient(), pixivDownload.getCookieStore(), illustId, quality); + JsonArray jsonArray = new JsonArray(linkList.size()); + linkList.forEach(jsonArray::add); + Files.write(cacheFile.toPath(), gson.toJson(jsonArray).getBytes(StandardCharsets.UTF_8), StandardOpenOption.CREATE); + } catch(IOException e) { + cacheFile.delete(); + throw e; + } + } else { + JsonArray jsonArray = gson.fromJson(new FileReader(cacheFile), JsonArray.class); + linkList = new ArrayList<>(jsonArray.size()); + jsonArray.forEach(jsonElement -> linkList.add(jsonElement.getAsString())); + } + pagesCache.put(illustId + "." + quality.name(), linkList); + } + + return pagesCache.get(illustId + "." + quality.name()); + } + } + + private static File getImageStoreDir() { + if(!imageStoreDir.exists() && !imageStoreDir.mkdirs()) { + log.warn("酷Q图片缓存目录失效！(Path: {} )", imageStoreDir.getAbsolutePath()); + throw new RuntimeException(new IOException("文件夹创建失败!")); + } + return imageStoreDir; + } + +} diff --git a/src/main/java/net/lamgc/cgj/Main.java b/src/main/java/net/lamgc/cgj/Main.java new file mode 100644 index 0000000..c89b4d8 --- /dev/null +++ b/src/main/java/net/lamgc/cgj/Main.java @@ -0,0 +1,518 @@ +package net.lamgc.cgj; + +import com.github.monkeywie.proxyee.proxy.ProxyConfig; +import com.github.monkeywie.proxyee.proxy.ProxyType; +import com.google.common.base.Strings; +import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import net.lamgc.cgj.pixiv.*; +import net.lamgc.cgj.proxy.PixivAccessProxyServer; +import net.lamgc.cgj.proxy.PixivLoginProxyServer; +import net.lamgc.utils.base.ArgumentsProperties; +import net.lamgc.utils.base.runner.Argument; +import net.lamgc.utils.base.runner.ArgumentsRunner; +import net.lamgc.utils.base.runner.Command; +import org.apache.commons.io.IOUtils; +import org.apache.http.HttpHost; +import org.apache.http.HttpResponse; +import org.apache.http.client.CookieStore; +import org.apache.http.client.HttpClient; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.message.BasicHeader; +import org.apache.http.util.EntityUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; + +import java.io.*; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +@SpringBootApplication +public class Main { + + private final static Logger log = LoggerFactory.getLogger("Main"); + + private final static File storeDir = new File("store/"); + + public static CookieStore cookieStore; + + public static HttpHost proxy; + + static { + if(!storeDir.exists() && !storeDir.mkdirs()) { + log.error("创建文件夹失败!"); + } + } + + public static void main(String[] args) throws IOException, ClassNotFoundException { + ArgumentsProperties argsProp = new ArgumentsProperties(args); + if(argsProp.containsKey("proxy")) { + URL proxyUrl = new URL(argsProp.getValue("proxy")); + proxy = new HttpHost(proxyUrl.getHost(), proxyUrl.getPort()); + log.info("已启用Http协议代理：{}", proxy.toHostString()); + } else { + proxy = null; + } + + if(argsProp.containsKey("cqRootDir")) { + log.info("cqRootDir: {}", argsProp.getValue("cqRootDir")); + System.setProperty("cgj.cqRootDir", argsProp.getValue("cqRootDir")); + } else { + log.info("未设置cqRootDir"); + } + + File cookieStoreFile = new File("cookies.store"); + if(!cookieStoreFile.exists()) { + log.error("未找到cookies.store文件, 请确保文件存在于运行目录下!"); + System.exit(1); + return; + } + ObjectInputStream ois = new ObjectInputStream(new FileInputStream(cookieStoreFile)); + cookieStore = (CookieStore) ois.readObject(); + ois.close(); + log.info("已载入CookieStore"); + + log.debug(Arrays.toString(args)); + + ArgumentsRunner.run(Main.class, args); + } + + @Command + public static void pluginMode(@Argument(name = "args", force = false) String argsStr) { + if(!System.getProperty("cgj.cqRootDir").endsWith("\\") && !System.getProperty("cgj.cqRootDir").endsWith("/")) { + System.setProperty("cgj.cqRootDir", System.getProperty("cgj.cqRootDir") + "/"); + } + log.info("酷Q机器人根目录: {}", System.getProperty("cgj.cqRootDir")); + CQConfig.init(); + Pattern pattern = Pattern.compile("/\\s*(\".+?\"|[^:\\s])+((\\s*:\\s*(\".+?\"|[^\\s])+)|)|(\".+?\"|[^\"\\s])+"); + Matcher matcher = pattern.matcher(Strings.nullToEmpty(argsStr)); + ArrayList argsList = new ArrayList<>(); + while (matcher.find()) { + argsList.add(matcher.group()); + } + String[] args = new String[argsList.size()]; + argsList.toArray(args); + SpringApplication.run(Main.class, args); + } + + @Command + public static void collectionDownload() throws IOException { + PixivDownload pixivDownload = new PixivDownload(Objects.requireNonNull(cookieStore), proxy); + File outputFile = new File(storeDir, "collection.zip"); + if(!outputFile.exists() && !outputFile.createNewFile()) { + log.error("文件创建失败: " + outputFile.getAbsolutePath()); + } + ZipOutputStream zos = new ZipOutputStream(new FileOutputStream(outputFile)); + zos.setLevel(9); + log.info("正在调用方法..."); + pixivDownload.getCollectionAsInputStream(PixivDownload.PageQuality.ORIGINAL, (link, inputStream) -> { + try { + ZipEntry entry = new ZipEntry(link.substring(link.lastIndexOf("/") + 1)); + log.info("正在写入: " + entry.getName()); + zos.putNextEntry(entry); + IOUtils.copy(inputStream, zos); + zos.flush(); + } catch (IOException e) { + log.error("写入文件项时发生异常", e); + } + }); + log.info("调用完成."); + zos.close(); + } + + @Command + public static void getRecommends() throws IOException { + PixivDownload pixivDownload = new PixivDownload(Objects.requireNonNull(cookieStore), proxy); + String date = new SimpleDateFormat("yyyyMMdd").format(new Date()); + int id = 1; + File outputFile = new File(storeDir, "recommends-" + date + "-" + id + ".zip"); + while(outputFile.exists()) { + id++; + outputFile = new File(storeDir, "recommends-" + date + "-" + id + ".zip"); + } + + if(!outputFile.createNewFile()) { + log.error("文件创建失败: " + outputFile.getAbsolutePath()); + } + ZipOutputStream zos = new ZipOutputStream(new FileOutputStream(outputFile)); + zos.setLevel(9); + log.info("正在调用方法..."); + pixivDownload.getRecommendAsInputStream(PixivDownload.PageQuality.ORIGINAL, (link, inputStream) -> { + try { + ZipEntry entry = new ZipEntry(link.substring(link.lastIndexOf("/") + 1)); + log.info("正在写入: " + entry.getName()); + zos.putNextEntry(entry); + IOUtils.copy(inputStream, zos); + zos.flush(); + log.info("已成功写入 {}", entry.getName()); + } catch (IOException e) { + log.error("写入文件项时发生异常", e); + } + }); + log.info("调用完成."); + zos.close(); + } + + @Command + public static void getRankingIllust(@Argument(name = "range", force = false, defaultValue = "100") int range, + @Argument(name = "mode", force = false) String mode, + @Argument(name = "content", force = false) String content, + @Argument(name = "queryTime", force = false) String queryTime) throws IOException, ParseException { + PixivDownload pixivDownload = new PixivDownload(cookieStore, proxy); + SimpleDateFormat format = new SimpleDateFormat("yyyyMMdd"); + Date queryDate; + String date; + if (queryTime == null) { + GregorianCalendar gregorianCalendar = new GregorianCalendar(); + gregorianCalendar.setTime(new Date()); + gregorianCalendar.add(Calendar.DATE, -1); + queryDate = gregorianCalendar.getTime(); + } else { + queryDate = format.parse(queryTime); + } + + date = format.format(queryDate); + + log.info("查询时间: {}", date); + PixivURL.RankingMode rankingMode = PixivURL.RankingMode.MODE_DAILY; + PixivURL.RankingContentType contentType = null; + if(mode != null) { + try { + rankingMode = PixivURL.RankingMode.valueOf(mode); + } catch (IllegalArgumentException e) { + log.warn("不支持的RankingMode: {}", mode); + } + } + if(content != null) { + try { + contentType = PixivURL.RankingContentType.valueOf(content); + } catch (IllegalArgumentException e) { + log.warn("不支持的RankingContentType: {}", content); + } + } + + int id = 1; + File outputFile = new File(storeDir, "ranking" + rankingMode.modeParam + "-" + date + "-" + id + ".zip"); + while(outputFile.exists()) { + id++; + outputFile = new File(storeDir, "ranking" + rankingMode.modeParam + "-" + date + "-" + id + ".zip"); + } + + if(!outputFile.createNewFile()) { + log.error("文件创建失败: " + outputFile.getAbsolutePath()); + return; + } + ZipOutputStream zos = new ZipOutputStream(new FileOutputStream(outputFile)); + zos.setLevel(9); + + log.info("正在调用方法..."); + try { + pixivDownload.getRankingAsInputStream(contentType, rankingMode, queryDate, range, PixivDownload.PageQuality.ORIGINAL, (rank, link, rankInfo, inputStream) -> { + try { + ZipEntry entry = new ZipEntry("Rank" + rank + "-" + link.substring(link.lastIndexOf("/") + 1)); + entry.setComment(rankInfo.toString()); + log.info("正在写入: " + entry.getName()); + zos.putNextEntry(entry); + IOUtils.copy(inputStream, zos); + zos.flush(); + log.info("已成功写入 {}", entry.getName()); + } catch (IOException e) { + log.error("写入文件项时发生异常", e); + } + }); + } finally { + zos.finish(); + zos.flush(); + zos.close(); + } + log.info("调用完成."); + } + + @Command + public static void search( + @Argument(name = "content") String content, + @Argument(name = "type", force = false) String type, + @Argument(name = "area", force = false) String area, + @Argument(name = "includeKeywords", force = false) String includeKeywords, + @Argument(name = "excludeKeywords", force = false) String excludeKeywords, + @Argument(name = "contentOption", force = false) String contentOption + ) throws IOException { + PixivSearchBuilder searchBuilder = new PixivSearchBuilder(Strings.isNullOrEmpty(content) ? "" : content); + if (type != null) { + try { + searchBuilder.setSearchType(PixivSearchBuilder.SearchType.valueOf(type.toUpperCase())); + } catch (IllegalArgumentException e) { + log.warn("不支持的SearchType: {}", type); + } + } + if(area != null) { + try { + searchBuilder.setSearchArea(PixivSearchBuilder.SearchArea.valueOf(area)); + } catch (IllegalArgumentException e) { + log.warn("不支持的SearchArea: {}", area); + } + } + if(contentOption != null) { + try { + searchBuilder.setSearchContentOption(PixivSearchBuilder.SearchContentOption.valueOf(contentOption)); + } catch (IllegalArgumentException e) { + log.warn("不支持的SearchContentOption: {}", contentOption); + } + } + + if(!Strings.isNullOrEmpty(includeKeywords)) { + for (String keyword : includeKeywords.split(";")) { + searchBuilder.removeExcludeKeyword(keyword); + searchBuilder.addIncludeKeyword(keyword); + log.info("已添加关键字: {}", keyword); + } + } + if(!Strings.isNullOrEmpty(excludeKeywords)) { + for (String keyword : excludeKeywords.split(";")) { + searchBuilder.removeIncludeKeyword(keyword); + searchBuilder.addExcludeKeyword(keyword); + log.info("已添加排除关键字: {}", keyword); + } + } + + log.info("正在搜索作品, 条件: {}", searchBuilder.getSearchCondition()); + + String requestUrl = searchBuilder.buildURL(); + log.info("RequestUrl: {}", requestUrl); + PixivDownload pixivDownload = new PixivDownload(cookieStore, proxy); + HttpGet httpGetRequest = pixivDownload.createHttpGetRequest(requestUrl); + HttpResponse response = pixivDownload.getHttpClient().execute(httpGetRequest); + + String responseBody = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8); + log.info("ResponseBody: {}", responseBody); + JsonObject jsonObject = new Gson().fromJson(responseBody, JsonObject.class); + if(jsonObject.get("error").getAsBoolean()) { + log.error("接口请求错误, 错误信息: {}", jsonObject.get("message").getAsString()); + return; + } + + JsonObject resultBody = jsonObject.getAsJsonObject("body"); + + for(PixivSearchBuilder.SearchArea searchArea : PixivSearchBuilder.SearchArea.values()) { + if(!resultBody.has(searchArea.jsonKey) || resultBody.getAsJsonObject(searchArea.jsonKey).getAsJsonArray("data").size() == 0) { + //log.info("返回数据不包含 {}", searchArea.jsonKey); + continue; + } + JsonArray illustsArray = resultBody + .getAsJsonObject(searchArea.jsonKey).getAsJsonArray("data"); + log.info("已找到与 {} 相关插图信息({})：", content, searchArea.name().toLowerCase()); + int count = 1; + for (JsonElement jsonElement : illustsArray) { + JsonObject illustObj = jsonElement.getAsJsonObject(); + if(!illustObj.has("illustId")) { + continue; + } + int illustId = illustObj.get("illustId").getAsInt(); + StringBuilder builder = new StringBuilder("["); + illustObj.get("tags").getAsJsonArray().forEach(el -> builder.append(el.getAsString()).append(", ")); + builder.replace(builder.length() - 2, builder.length(), "]"); + log.info("{} ({} / {})\n\t作品id: {}, \n\t作者名(作者id): {} ({}), \n\t作品标题: {}, \n\t作品Tags: {}, \n\t作品链接: {}", + searchArea.name(), + count++, + illustsArray.size(), + illustId, + illustObj.get("userName").getAsString(), + illustObj.get("userId").getAsInt(), + illustObj.get("illustTitle").getAsString(), + builder, + PixivURL.getPixivRefererLink(illustId) + ); + + /*log.info("正在下载..."); + List list = PixivDownload.getIllustAllPageDownload( + HttpClientBuilder.create() + .setProxy(proxy) + .build(), + illustId, PixivDownload.PageQuality.ORIGINAL);*/ + } + } + /* + + if(searchBuilder.getSearchArea().equals(PixivSearchBuilder.SearchArea.TOP)) { + + } else { + JsonArray illustsArray = resultBody + .getAsJsonObject(searchBuilder.getSearchArea().jsonKey).getAsJsonArray("data"); + log.info("已找到与 {} 相关插图信息：", content); + int count = 1; + for (JsonElement jsonElement : illustsArray) { + JsonObject illustObj = jsonElement.getAsJsonObject(); + //TODO: 防止数据内混入无效内容, 需要检查对象是否有illustId + if(!illustObj.has("illustId")) { + continue; + } + int illustId = illustObj.get("illustId").getAsInt(); + StringBuilder builder = new StringBuilder("["); + illustObj.get("tags").getAsJsonArray().forEach(el -> builder.append(el.getAsString()).append(", ")); + builder.replace(builder.length() - 2, builder.length(), "]"); + log.info("({} / {})\n\t作品id: {}, \n\t作者名(作者id): {} ({}), \n\t作品标题: {}, \n\t作品Tags: {}, \n\t作品链接: {}", + count++, + illustsArray.size(), + illustId, + illustObj.get("userName").getAsString(), + illustObj.get("userId").getAsInt(), + illustObj.get("illustTitle").getAsString(), + builder, + PixivURL.getPixivRefererLink(illustId) + ); + } + } + */ + } + + + @Command(defaultCommand = true) + public static void testRun() throws IOException { + /*loadCookieStoreFromFile(); + + if(cookieStore == null){ + startPixivLoginProxyServer(); + }*/ + + //accessPixivToFile(); + + //startPixivAccessProxyServer(); + + //saveCookieStoreToFile(); + log.info("这里啥都没有哟w"); + } + + private static void loadCookieStoreFromFile() throws IOException { + log.info("正在加载CookieStore..."); + File storeFile = new File("./cookies.store"); + if(!storeFile.exists()){ + log.info("未找到CookieStore, 跳过加载."); + return; + } + ObjectInputStream stream = new ObjectInputStream(new FileInputStream(storeFile)); + Object result; + try { + result = stream.readObject(); + } catch (ClassNotFoundException e) { + log.error("加载出错", e); + return; + } + cookieStore = (CookieStore) result; + cookieStore.getCookies().forEach(cookie -> log.debug(cookie.getName() + ": " + cookie.getValue() + ", isExpired: " + cookie.isExpired(new Date()))); + log.info("CookieStore加载完成."); + } + + private static void saveCookieStoreToFile() throws IOException { + log.info("正在保存CookieStore..."); + File outputFile = new File("./cookies.store"); + if(!outputFile.exists() && !outputFile.delete() && !outputFile.createNewFile()){ + log.error("保存CookieStore失败."); + return; + } + ObjectOutputStream stream = new ObjectOutputStream(new FileOutputStream(outputFile)); + stream.writeObject(cookieStore); + stream.flush(); + stream.close(); + log.info("CookieStore保存成功."); + } + + private static void startPixivLoginProxyServer(){ + ProxyConfig proxyConfig = new ProxyConfig(ProxyType.SOCKS5, "127.0.0.1", 1080); + PixivLoginProxyServer proxyServer = new PixivLoginProxyServer(proxyConfig); + Thread proxyServerStartThread = new Thread(() -> { + log.info("启动代理服务器..."); + proxyServer.start(1006); + log.info("代理服务器已关闭."); + }); + proxyServerStartThread.setName("LoginProxyServerThread"); + proxyServerStartThread.start(); + //System.console().readLine(); + new Scanner(System.in).nextLine(); + log.info("关闭PLPS服务器..."); + proxyServer.close(); + cookieStore = proxyServer.getCookieStore(); + } + + private static void accessPixivToFile() throws IOException { + File cookieStoreFile = new File("./cookie.txt"); + if (!cookieStoreFile.exists() && !cookieStoreFile.createNewFile()) { + log.info("Cookie文件存储失败"); + } + /*log.info("正在写出Cookie, Cookie count: " + cookieStore.getCookies().size()); + FileWriter cookieWriter = new FileWriter(cookieStoreFile); + cookieStore.getCookies().forEach(cookie -> { + try { + StringBuilder sb = new StringBuilder().append(cookie.getName()).append(" = ").append(cookie.getValue()); + log.info("正在导出Cookie: " + sb.toString()); + cookieWriter.append(sb.toString()).append("\n").flush(); + } catch (IOException e) { + e.printStackTrace(); + } + }); + log.info("Cookie写出完成");*/ + + log.info("尝试通过捕获的Cookie访问Pixiv..."); + HttpClient httpClient = new PixivSession(new HttpHost("127.0.0.1", 1080), cookieStore).getHttpClient(); + HttpGet request = new HttpGet("https://www.pixiv.net"); + request.addHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:73.0) Gecko/20100101 Firefox/73.0"); + request.addHeader(new BasicHeader("accept-encoding", "gzip, deflate, br")); + request.addHeader(new BasicHeader("accept-language", "zh-CN,zh;q=0.9")); + StringBuilder cookieBuilder = new StringBuilder(); + cookieStore.getCookies().forEach(cookie -> { + if(cookie.isExpired(new Date())){ + return; + } + cookieBuilder.append(cookie.getName()).append("=").append(cookie.getValue()).append("; "); + }); + request.addHeader("cookie", cookieBuilder.toString()); + + HttpResponse response = httpClient.execute(request); + log.info("正在写入文件..."); + File outFile = new File("./pixiv.html"); + if (outFile.createNewFile() && !outFile.exists()) { + log.info("文件创建失败!"); + }else { + new FileWriter(outFile).append(EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8)).flush(); + } + + Pixiv pixiv = new Pixiv(httpClient); + pixiv.getRecommend().forEach(illustMap -> { + StringBuilder builder = new StringBuilder(); + illustMap.forEach((key, value) -> builder.append(key).append(": ").append(value).append("\n")); + try { + builder.append("download Link: ").append(Arrays.toString(pixiv.getAllDownloadLink(Integer.parseInt(illustMap.get(Pixiv.ATTR_ILLUST_ID))))); + } catch (IOException e) { + log.error("获取下载链接时出错!", e); + } + log.info(builder.append("\n").toString()); + }); + + } + + private static void startPixivAccessProxyServer(){ + log.info("正在启动访问代理服务器, 将浏览器相关缓存清空后, 使用浏览器进行访问以尝试Cookie正确性."); + final PixivAccessProxyServer accessProxyServer = new PixivAccessProxyServer(cookieStore, new ProxyConfig(ProxyType.SOCKS5, "127.0.0.1", 1080)); + Thread accessProxyServerThread = new Thread(() -> { + log.info("正在启动PAPS..."); + accessProxyServer.start(1007); + log.info("PAPS已关闭."); + }); + accessProxyServerThread.setName("AccessProxyServerThread"); + accessProxyServerThread.start(); + new Scanner(System.in).nextLine(); + log.info("关闭PAPS服务器..."); + accessProxyServer.close(); + } + +} diff --git a/src/main/java/net/lamgc/cgj/bot/AutoArtworksSender.java b/src/main/java/net/lamgc/cgj/bot/AutoArtworksSender.java new file mode 100644 index 0000000..aff1d4e --- /dev/null +++ b/src/main/java/net/lamgc/cgj/bot/AutoArtworksSender.java @@ -0,0 +1,57 @@ +package net.lamgc.cgj.bot; + +import net.lz1998.cq.robot.CoolQ; +import org.apache.http.client.methods.HttpGet; + +import java.util.Random; +import java.util.Timer; +import java.util.TimerTask; + +public class AutoArtworksSender { + + private final CoolQ CQ; + private final ReceiveType receiveType; + private final long targetReceiveId; + private Timer timer = new Timer(); + private TimerTask task = new TimerTask() { + @Override + public void run() { + HttpGet request = new HttpGet(); + + // https://api.imjad.cn/pixiv/v2/?type=tags + } + }; + + public AutoArtworksSender(CoolQ cq, ReceiveType receiveType, long receiveId) { + this.CQ = cq; + this.receiveType = receiveType; + this.targetReceiveId = receiveId; + } + + public void reset(long time) { + if(time <= 0) { + timer.schedule(task, new Random().nextInt(10 * 60 * 60 * 1000) + 7200000L); //2H ~ 12H + } else { + timer.schedule(task, time); + } + } + + public void sendMessage(String message, boolean auto_escape) { + switch (receiveType) { + case GROUP: + CQ.sendGroupMsg(targetReceiveId, message, auto_escape); + break; + case Discuss: + CQ.sendDiscussMsg(targetReceiveId, message, auto_escape); + break; + case PRIVATE: + CQ.sendPrivateMsg(targetReceiveId, message, auto_escape); + break; + } + } + + public enum ReceiveType { + PRIVATE, GROUP, Discuss + } + +} diff --git a/src/main/java/net/lamgc/cgj/cache/CacheObject.java b/src/main/java/net/lamgc/cgj/cache/CacheObject.java new file mode 100644 index 0000000..dc7b49a --- /dev/null +++ b/src/main/java/net/lamgc/cgj/cache/CacheObject.java @@ -0,0 +1,40 @@ +package net.lamgc.cgj.cache; + +import java.util.Date; +import java.util.concurrent.atomic.AtomicReference; + +public class CacheObject { + + private AtomicReference value; + private AtomicReference expire; + + public CacheObject() { + this(null, null); + } + + public CacheObject(T value, Date expire) { + this.value = new AtomicReference<>(value); + this.expire = new AtomicReference<>(expire); + } + + public synchronized void update(T value, Date newExpire) { + if(new Date().after(newExpire)) { + throw new IllegalArgumentException("Due earlier than current time"); + } + this.expire.set(newExpire); + this.value.set(value); + } + + public synchronized T get() { + return value.get(); + } + + public Date getExpireDate() { + return expire.get(); + } + + public boolean isExpire(Date time) { + Date expireDate = getExpireDate(); + return expireDate != null && expireDate.before(time); + } +} diff --git a/src/main/java/net/lamgc/cgj/cache/ImageCacheHandler.java b/src/main/java/net/lamgc/cgj/cache/ImageCacheHandler.java new file mode 100644 index 0000000..a19b005 --- /dev/null +++ b/src/main/java/net/lamgc/cgj/cache/ImageCacheHandler.java @@ -0,0 +1,79 @@ +package net.lamgc.cgj.cache; + +import net.lamgc.cgj.Main; +import net.lamgc.cgj.pixiv.PixivURL; +import net.lamgc.utils.event.EventHandler; +import org.apache.commons.io.IOUtils; +import org.apache.http.HttpResponse; +import org.apache.http.client.HttpClient; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.HttpClientBuilder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; + +public class ImageCacheHandler implements EventHandler { + + private final static Logger log = LoggerFactory.getLogger("ImageCacheHandler"); + + private final static HttpClient httpClient = HttpClientBuilder.create().setProxy(Main.proxy).build(); + + private final static Set cacheQueue = Collections.synchronizedSet(new HashSet<>()); + + public void getImageToCache(ImageCacheObject event) { + if(cacheQueue.contains(event)) { + log.info("图片 {} 已存在相同缓存任务, 跳过.", event.getStoreFile().getName()); + return; + } else { + cacheQueue.add(event); + } + + try { + log.info("图片 {} Event正在进行...({})", event.getStoreFile().getName(), Integer.toHexString(event.hashCode())); + File storeFile = event.getStoreFile(); + log.info("正在缓存图片 {} (Path: {})", storeFile.getName(), storeFile.getAbsolutePath()); + try { + if(!storeFile.exists() && !storeFile.createNewFile()) { + log.error("无法创建文件(Path: {})", storeFile.getAbsolutePath()); + return; + } + } catch (IOException e) { + log.error("无法创建文件(Path: {})", storeFile.getAbsolutePath()); + e.printStackTrace(); + } + + HttpGet request = new HttpGet(event.getDownloadLink()); + request.addHeader("Referer", PixivURL.getPixivRefererLink(event.getIllustId())); + HttpResponse response; + try { + response = httpClient.execute(request); + } catch (IOException e) { + log.error("Http请求时发生异常", e); + return; + } + if(response.getStatusLine().getStatusCode() != 200) { + log.warn("Http请求异常：{}", response.getStatusLine()); + return; + } + + log.info("正在下载...(Content-Length: {}KB)", response.getEntity().getContentLength() / 1024); + try(FileOutputStream fos = new FileOutputStream(storeFile)) { + IOUtils.copy(response.getEntity().getContent(), fos); + } catch (IOException e) { + log.error("下载图片时发生异常", e); + return; + } + event.getImageCache().put(event.getDownloadLink(), storeFile); + } finally { + log.info("图片 {} Event结束({})", event.getStoreFile().getName(), Integer.toHexString(event.hashCode())); + cacheQueue.remove(event); + } + } + +} diff --git a/src/main/java/net/lamgc/cgj/cache/ImageCacheObject.java b/src/main/java/net/lamgc/cgj/cache/ImageCacheObject.java new file mode 100644 index 0000000..75187f4 --- /dev/null +++ b/src/main/java/net/lamgc/cgj/cache/ImageCacheObject.java @@ -0,0 +1,57 @@ +package net.lamgc.cgj.cache; + +import net.lamgc.utils.event.EventObject; + +import java.io.File; +import java.util.Map; +import java.util.Objects; + +public class ImageCacheObject implements EventObject { + + private final Map imageCache; + + private final int illustId; + + private final String downloadLink; + + private final File storeFile; + + public ImageCacheObject(Map imageCache, int illustId, String downloadLink, File storeFile) { + this.imageCache = imageCache; + this.illustId = illustId; + this.downloadLink = downloadLink; + this.storeFile = storeFile; + } + + public Map getImageCache() { + return imageCache; + } + + public String getDownloadLink() { + return downloadLink; + } + + public File getStoreFile() { + return storeFile; + } + + public int getIllustId() { + return illustId; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ImageCacheObject that = (ImageCacheObject) o; + return illustId == that.illustId && + Objects.equals(imageCache, that.imageCache) && + Objects.equals(downloadLink, that.downloadLink) && + Objects.equals(storeFile, that.storeFile); + } + + @Override + public int hashCode() { + return Objects.hash(imageCache, illustId, downloadLink, storeFile); + } +} diff --git a/src/main/java/net/lamgc/cgj/db/RankingDB.java b/src/main/java/net/lamgc/cgj/db/RankingDB.java new file mode 100644 index 0000000..30d56c0 --- /dev/null +++ b/src/main/java/net/lamgc/cgj/db/RankingDB.java @@ -0,0 +1,28 @@ +package net.lamgc.cgj.db; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; + +public class RankingDB { + + private final Connection dbConnection; + + public RankingDB(String dbUrl, String username, String password) throws SQLException { + try { + Class.forName("com.mysql.cj.jdbc.Driver"); + } catch (ClassNotFoundException e) { + e.printStackTrace(); + } + dbConnection = DriverManager.getConnection("jdbc:mysql://" + dbUrl + "/pixivRanking?useSSL=false&serverTimezone=UTC",username,password); + checkAndFix(dbConnection); + } + + private static void checkAndFix(Connection connection) throws SQLException { + Statement statement = connection.createStatement(); + statement.execute(""); + } + + +} diff --git a/src/main/java/net/lamgc/cgj/pixiv/IllustInfo.java b/src/main/java/net/lamgc/cgj/pixiv/IllustInfo.java new file mode 100644 index 0000000..07c28e4 --- /dev/null +++ b/src/main/java/net/lamgc/cgj/pixiv/IllustInfo.java @@ -0,0 +1,69 @@ +package net.lamgc.cgj.pixiv; + +/** + * 插图(集)的信息 + */ +public class IllustInfo{ + + /** + * 插图(集)ID + */ + public final int illustID; + + /** + * 第几页 + */ + public final int page; + + /** + * 插图标题 + */ + public final String title; + + /** + * 插图说明 + */ + public final String description; + + /** + * 插图标签 + */ + public final String[] tags; + + /** + * 插图图片长度 + */ + public final int width; + + /** + * 插图图片高度 + */ + public final int height; + + /** + * 作者名 + */ + public final String authorName; + + /** + * 作者用户ID + */ + public final int authorUserID; + + public IllustInfo(int illustID, String title, String description, String[] tags, int width, int height, String authorName, int authorUserID){ + this(illustID, 0, title, description, tags, width, height, authorName, authorUserID); + } + + public IllustInfo(int illustID, int p, String title, String description, String[] tags, int width, int height, String authorName, int authorUserID){ + this.illustID = illustID; + this.page = p; + this.title = title; + this.description = description; + this.tags = tags; + this.width = width; + this.height = height; + this.authorName = authorName; + this.authorUserID = authorUserID; + } + +} diff --git a/src/main/java/net/lamgc/cgj/pixiv/Pixiv.java b/src/main/java/net/lamgc/cgj/pixiv/Pixiv.java new file mode 100644 index 0000000..15835db --- /dev/null +++ b/src/main/java/net/lamgc/cgj/pixiv/Pixiv.java @@ -0,0 +1,267 @@ +package net.lamgc.cgj.pixiv; + +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import org.apache.http.HttpHost; +import org.apache.http.HttpResponse; +import org.apache.http.client.HttpClient; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.conn.ConnectionPoolTimeoutException; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.util.EntityUtils; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.*; + +public class Pixiv { + + /** + * illust Link + */ + public final static String ATTR_LINK = "link"; + + /** + * illust Id + */ + public final static String ATTR_ILLUST_ID = "illustId"; + + /** + * illust Title + */ + public final static String ATTR_TITLE = "title"; + + /** + * illust Author Name + */ + public final static String ATTR_AUTHOR_NAME = "authorName"; + + /** + * illust Author UserID + */ + public final static String ATTR_AUTHOR_ID = "authorId"; + + private final HttpClient httpClient; + + public Pixiv(HttpClient client){ + this.httpClient = client; + } + + /** + * 使用帐号密码登录Pixiv + * @param PixivID Pixiv账户登录名 + * @param Password Pixiv帐号密码 + * @throws IOException 当登录连接出现异常时抛出 + * @deprecated {@link PixivSession#Login(String, String)} 已经废除, 故本方法不可用 + */ + public Pixiv(String PixivID, String Password) throws IOException { + this(PixivID, Password, null); + } + + /** + * 使用帐号密码登录Pixiv + * @param PixivID Pixiv账户登录名 + * @param Password Pixiv帐号密码 + * @param proxy 代理设置 + * @throws IOException 当登录连接出现异常时抛出 + * @deprecated {@link PixivSession#Login(String, String)} 已经废除, 故本方法不可用 + */ + public Pixiv(String PixivID, String Password, HttpHost proxy) throws IOException { + PixivSession pixivSession = new PixivSession(proxy, null); + if(pixivSession.Login(PixivID, Password)){ + System.out.println("P站登录成功!"); + }else{ + System.out.println("P站登录失败!错误信息: " + pixivSession.getErrMsg()); + throw new RuntimeException(pixivSession.getErrMsg()); + } + //httpClient = pixivSession.getHttpClient(); + httpClient = HttpClientBuilder.create() + .setDefaultCookieStore(pixivSession.getCookieStore()) + .build(); + } + + /** + * 获取首页推荐列表 + * @return 首页推荐列表, 一个Map对应一个推荐项, 使用ATTR_开头常量访问即可 + * @throws IOException + */ + public List> getRecommend() throws IOException { + HttpGet getRequest = new HttpGet(PixivURL.PIXIV_INDEX_URL); + HttpResponse response = httpClient.execute(getRequest); + String pageAsXML = EntityUtils.toString(response.getEntity(),"utf-8"); + + //获取推荐图列表(li) + //System.out.println(pageAsXML); + + Document document = Jsoup.parse(pageAsXML); + + List links = document.select("._image-items.gtm-illust-recommend-zone>li>.gtm-illust-recommend-thumbnail-link").eachAttr("href"); + List illustId = document.select("._image-items.gtm-illust-recommend-zone>li>.gtm-illust-recommend-thumbnail-link").eachAttr("data-gtm-recommend-illust-id"); + List title = document.select("._image-items.gtm-illust-recommend-zone>li>.gtm-illust-recommend-title>h1").eachAttr("title"); + List authorName = document.select("._image-items.gtm-illust-recommend-zone>li>.gtm-illust-recommend-user-name").eachText(); + List authorId = document.select("._image-items.gtm-illust-recommend-zone>li>.gtm-illust-recommend-user-name").eachAttr("data-user_id"); + + List> recommendList = new ArrayList<>(); + for(int i = 0; i < links.size(); i++){ + //System.out.println(links.get(i)); + Map info = new HashMap<>(); + info.put(ATTR_LINK, PixivURL.PIXIV_INDEX_URL + links.get(i)); + info.put(ATTR_ILLUST_ID, illustId.get(i)); + info.put(ATTR_TITLE, title.get(i)); + info.put(ATTR_AUTHOR_NAME, authorName.get(i)); + info.put(ATTR_AUTHOR_ID, authorId.get(i)); + recommendList.add(info); + } + return recommendList; + } + + public String[] getAllDownloadLink(int illustID) throws IOException { + HttpGet illustPage = new HttpGet(PixivURL.PIXIV_ILLUST_API_URL.replaceAll("\\{illustId}", String.valueOf(illustID))); + HttpResponse response = httpClient.execute(illustPage); + String pageAsXML = EntityUtils.toString(response.getEntity(),"utf-8"); + //System.out.println(pageAsXML); + JsonObject resultObj = (JsonObject) new JsonParser().parse(pageAsXML); + if(!resultObj.get("error").getAsBoolean()){ + JsonArray bodyArray = resultObj.get("body").getAsJsonArray(); + int length = bodyArray.size(); + String[] result = new String[length]; + for(int i = 0; i < length; i++){ + JsonObject childObj = bodyArray.get(i).getAsJsonObject(); + result[i] = childObj.get("urls").getAsJsonObject().get("original").getAsString(); + } + return result; + }else{ + return null; + } + } + + + + /** + * 下载P站图片 + * @param illustID 插图ID + * @return 成功返回图片输入流,失败或为多图则返回null + */ + public InputStream[] downloadIllustImage(int illustID) throws IOException { + String[] links = getAllDownloadLink(illustID); + List inputStreamList = new ArrayList<>(); + int count = 1; + boolean retry = false; + for(int i = 0; i < links.length; i++){ + try { + long sleepTime = (new Random().nextInt(4) + 2) * 1000; + System.out.println("nextTime: " + (float)(sleepTime / 1000)); + Thread.sleep(sleepTime); + } catch (InterruptedException ignored) {} + String link = links[i]; + System.out.print("page:" + count++ + "/" + links.length + " ..."); + HttpGet imgGet = new HttpGet(link); + //关键!如果不加上Referer的话,会返回403 + imgGet.setHeader("Referer", PixivURL.PIXIV_ILLUST_MEDIUM_URL.replaceAll("\\{illustId}", String.valueOf(illustID))); + RequestConfig config = RequestConfig.custom() + .setConnectTimeout(20 * 1000) + .setConnectionRequestTimeout(20 * 1000) + .setSocketTimeout(30 * 1000) + .build(); + imgGet.setConfig(config); + HttpResponse response; + try { + response = httpClient.execute(imgGet); + }catch(ConnectionPoolTimeoutException e){ + if(retry){ + retry = false; + System.out.println("获取失败,跳过..."); + continue; + } + System.out.println("连接超时,重新获取..."); + retry = true; + i--; + continue; + } + retry = false; + + ByteArrayOutputStream cacheOutputStream = new ByteArrayOutputStream((int)response.getEntity().getContentLength()); + InputStream content = response.getEntity().getContent(); + int readLength; + byte[] cache = new byte[4096]; + while((readLength = content.read(cache)) != -1){ + cacheOutputStream.write(cache, 0, readLength); + } + byte[] data = cacheOutputStream.toByteArray(); + //System.out.println("读到数据: " + data.length); + inputStreamList.add(new ByteArrayInputStream(data)); + System.out.println("done!（length: " + response.getEntity().getContentLength() + ")"); + + } + return inputStreamList.toArray(new InputStream[0]); + } + + /** + * 下载P站图片 + * @param illustID 插图ID + * @return 成功返回图片输入流,失败或为多图则返回null + */ + public InputStream downloadIllustImages(int illustID){ + throw new UnsupportedOperationException(); + } + + /** + * 通过解析插图详情页获取 + * - 插图标题 + * - 插图作者(及其UserId) + * - 插图上传时间 + * - 插图标签(原始标签) + * ... + * @return 成功返回IllustInfo对象,失败返回null + */ + public IllustInfo[] getIllustInfo(int[] illustIDs) throws IOException { + //获取Api + HttpGet apiRequest = new HttpGet(PixivURL.getPixivIllustInfoAPI(illustIDs)); + HttpResponse response = httpClient.execute(apiRequest); + String resultText = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8); + System.out.println(resultText); + JsonObject resultObj = ((JsonObject) new JsonParser().parse(resultText)); + if(resultObj.get("error").getAsBoolean()){ + System.err.println("获取失败!"); + return null; + } + List illustInfoList = new ArrayList<>(); + JsonArray illustArray = resultObj.get("body").getAsJsonObject().get("illusts").getAsJsonArray(); + illustArray.forEach(jsonElement -> { + JsonObject illustInfoObj = jsonElement.getAsJsonObject(); + JsonArray tagsArray = illustInfoObj.get("tags").getAsJsonArray(); + String[] tags = new String[tagsArray.size()]; + for(int i = 0; i < tags.length; i++){ + tags[i] = tagsArray.get(i).getAsString(); + } + //TODO: 通过不需要作者id就能获取图片信息的api无法获取图片尺寸 + IllustInfo illustInfo = new IllustInfo( + illustInfoObj.get("workId").getAsInt(), + illustInfoObj.get("title").getAsString(), + null, + tags, + -1, + -1, + illustInfoObj.get("userName").getAsString(), + illustInfoObj.get("userId").getAsInt() + ); + }); + return null; + } + + /** + * 获取指定用户的所有插画 + */ + public void getUserAllIllustTest() { + + } + + +} diff --git a/src/main/java/net/lamgc/cgj/pixiv/PixivDownload.java b/src/main/java/net/lamgc/cgj/pixiv/PixivDownload.java new file mode 100644 index 0000000..7b4e504 --- /dev/null +++ b/src/main/java/net/lamgc/cgj/pixiv/PixivDownload.java @@ -0,0 +1,533 @@ +package net.lamgc.cgj.pixiv; + +import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import io.netty.handler.codec.http.HttpHeaderNames; +import org.apache.http.Header; +import org.apache.http.HttpHost; +import org.apache.http.HttpRequest; +import org.apache.http.HttpResponse; +import org.apache.http.client.CookieStore; +import org.apache.http.client.HttpClient; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.message.BasicHeader; +import org.apache.http.util.EntityUtils; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.select.Elements; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.*; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.BiConsumer; + +public class PixivDownload { + + private final static Logger log = LoggerFactory.getLogger("PixivDownload"); + + private final HttpClient httpClient; + + private final CookieStore cookieStore; + + /** + * 构造一个PixivDownload对象 + * @param cookieStore 存在已登录Pixiv的CookieStore对象 + */ + public PixivDownload(CookieStore cookieStore) { + this(cookieStore, null); + } + + /** + * 构造一个PixivDownload对象 + * @param cookieStore 存在已登录Pixiv的CookieStore对象 + * @param proxy 访问代理 + */ + public PixivDownload(CookieStore cookieStore, HttpHost proxy) { + this.cookieStore = cookieStore; + HttpClientBuilder builder = HttpClientBuilder.create(); + builder.setDefaultCookieStore(cookieStore); + // UA: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.122 Safari/537.36 + ArrayList
defaultHeaders = new ArrayList<>(2); + defaultHeaders.add(new BasicHeader("User-Agent", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.122 Safari/537.36")); + builder.setDefaultHeaders(defaultHeaders); + builder.setProxy(proxy); + httpClient = builder.build(); + } + + public HttpClient getHttpClient() { + return httpClient; + } + + public CookieStore getCookieStore() { + return cookieStore; + } + + /** + * 获取帐号所有的收藏插画，并以输入流形式提供 + * @param fn 回调函数，函数传进的InputStream无需手动关闭 + * @throws IOException 当获取时发生异常则直接抛出 + */ + public void getCollectionAsInputStream(PageQuality quality, BiConsumer fn) throws IOException { + int pageIndex = 0; + HttpGet request; + Document document; + ArrayList linkList = new ArrayList<>(); + do { + request = new HttpGet(PixivURL.PIXIV_USER_COLLECTION_PAGE.replace("{pageIndex}", Integer.toString(++pageIndex))); + setCookieInRequest(request, cookieStore); + log.info("Request Link: " + request.getURI().toString()); + HttpResponse response = httpClient.execute(request); + // 解析网页内容，获得所有的收藏信息 + document = Jsoup.parse(EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8)); + Elements items = document.select(".display_editable_works .image-item a.work"); + List hrefList = items.eachAttr("href"); + log.info("第 {} 页获取到的图片项数量: {}", pageIndex, hrefList.size()); + if(hrefList.size() == 0) { + break; + } + + Gson gson = new Gson(); + for (String href : hrefList) { + HttpGet linkApiRequest = createHttpGetRequest(PixivURL.PIXIV_ILLUST_API_URL.replace("{illustId}", href.substring(href.lastIndexOf("/") + 1))); + log.debug(linkApiRequest.getURI().toString()); + HttpResponse httpResponse = httpClient.execute(linkApiRequest); + JsonObject linkResult = gson.fromJson(EntityUtils.toString(httpResponse.getEntity()), JsonObject.class); + if(linkResult.get("error").getAsBoolean()) { + log.error("接口返回错误信息: {}", linkResult.get("message").getAsString()); + continue; + } + + JsonArray linkArray = linkResult.get("body").getAsJsonArray(); + for (int i = 0; i < linkArray.size(); i++) { + JsonObject linkObject = linkArray.get(i).getAsJsonObject().get("urls").getAsJsonObject(); + linkList.add(linkObject.get((quality == null ? PageQuality.ORIGINAL : quality).toString().toLowerCase()).getAsString()); + } + } + } while(!document.select(".pager-container>.next").isEmpty()); + log.info("获取完成."); + AtomicInteger count = new AtomicInteger(1); + linkList.forEach(link -> { + log.info("Next Link [{}]: {}", count.getAndIncrement(), link); + InputStream imageInputStream = null; + int tryCount = 0; + do { + try { + imageInputStream = getImageAsInputStream(httpClient, link); + } catch (IOException e) { + log.error("获取图片数据时发生异常", e); + if(++tryCount < 5) { + log.warn("即将重试[{} / 5]", tryCount); + } + } + } while(imageInputStream == null); + + try(InputStream imageInput = new BufferedInputStream(imageInputStream, 256 * 1024)) { + log.debug("调用回调方法..."); + fn.accept(link, imageInput); + log.debug("回调方法调用完成."); + } catch (IOException e) { + log.error("图片获取失败", e); + } + }); + } + + /** + * 获取推荐插图 + * @param quality 图片质量 + * @param fn 回调函数 + * @throws IOException 当获取时发生异常则直接抛出 + */ + public void getRecommendAsInputStream(PageQuality quality, BiConsumer fn) throws IOException { + HttpResponse response = httpClient.execute(createHttpGetRequest(PixivURL.PIXIV_INDEX_URL)); + Document document = Jsoup.parse(EntityUtils.toString(response.getEntity())); + + HttpClient imageClient = HttpClientBuilder.create().build(); + Elements elements = document.select(".gtm-illust-recommend-zone>.image-item>.gtm-illust-recommend-thumbnail-link"); + for(int illustIndex = 0; illustIndex < elements.size(); illustIndex++){ + String href = elements.get(illustIndex).attr("href"); + int illustId = Integer.parseInt(href.substring(href.lastIndexOf("/") + 1)); + log.info("({}/{}) Href: {}, IllustID: {}", illustIndex + 1, elements.size(), href, illustId); + List pageLinkList = getIllustAllPageDownload(httpClient, this.cookieStore, illustId, quality); + for (int linkIndex = 0; linkIndex < pageLinkList.size(); linkIndex++) { + String link = pageLinkList.get(linkIndex); + String fileName = link.substring(link.lastIndexOf("/") + 1); + log.info("({}/{})正在处理 {}", linkIndex, pageLinkList.size(), fileName); + InputStream imageInputStream = null; + int tryCount = 0; + do { + try { + imageInputStream = getImageAsInputStream(imageClient, link); + } catch (IOException e) { + log.error("获取图片数据时发生异常", e); + if(++tryCount < 5) { + log.warn("即将重试[{} / 5]", tryCount); + } + } + } while(imageInputStream == null); + + try(InputStream pageInputStream = new BufferedInputStream(imageInputStream, 256 * 1024)) { + fn.accept(fileName, pageInputStream); + } + log.info("Done!"); + + } + log.info("IllustId {} 处理完成.", illustId); + } + } + + /** + * 获取排行榜 + * @param contentType 内容类型 + * @param mode 查询模式 + * @param time 查询时间 + * @param range 从第一名开始的范围 + * @param quality 图片质量 + * @param fn 回调函数 + * @throws IOException 当请求发生异常时抛出 + */ + public void getRankingAsInputStream(PixivURL.RankingContentType contentType, PixivURL.RankingMode mode, + Date time, int range, PageQuality quality, RankingDownloadFunction fn) throws IOException { + getRankingAsInputStream(contentType, mode, time, 1, range, quality, fn); + } + + /** + * 获取排行榜 + * @param contentType 内容类型 + * @param mode 查询模式 + * @param time 查询时间 + * @param rankStart 开始排行位(包括) + * @param range 范围 + * @param quality 图片质量 + * @param fn 回调函数 + * @throws IOException 当请求发生异常时抛出 + */ + public void getRankingAsInputStream(PixivURL.RankingContentType contentType, PixivURL.RankingMode mode, + Date time, int rankStart, int range, PageQuality quality, RankingDownloadFunction fn) throws IOException { + getRanking(contentType, mode, time, rankStart, range).forEach(rankInfo -> { + int rank = rankInfo.get("rank").getAsInt(); + int illustId = rankInfo.get("illust_id").getAsInt(); + int authorId = rankInfo.get("user_id").getAsInt(); + String authorName = rankInfo.get("user_name").getAsString(); + String title = rankInfo.get("title").getAsString(); + log.info("当前到第 {}/{} 名(总共 {} 名), IllustID: {}, Author: ({}) {}, Title: {}", rank, rankStart + range - 1, range, illustId, authorId, authorName, title); + log.info("正在获取PagesLink..."); + List linkList; + try { + linkList = getIllustAllPageDownload(httpClient, this.cookieStore, illustId, quality); + } catch (IOException e) { + if(e.getMessage().equals("該当作品は削除されたか、存在しない作品IDです。")) { + log.warn("作品 {} 不存在.", illustId); + } else { + e.printStackTrace(); + } + return; + } + log.info("PagesLink 获取完成, 总数: {}", linkList.size()); + for (int pageIndex = 0; pageIndex < linkList.size(); pageIndex++) { + String downloadLink = linkList.get(pageIndex); + log.info("当前Page: {}/{}", pageIndex + 1, linkList.size()); + try(InputStream imageInputStream = new BufferedInputStream(getImageAsInputStream(HttpClientBuilder.create().build(), downloadLink), 256 * 1024)) { + fn.download(rank, downloadLink, rankInfo.deepCopy(), imageInputStream); + } catch(IOException e) { + log.error("下载插画时发生异常", e); + return; + } + log.info("完成."); + } + }); + } + + public List getRanking(PixivURL.RankingContentType contentType, PixivURL.RankingMode mode, + Date time, int rankStart, int range) throws IOException { + if(rankStart <= 0) { + throw new IllegalArgumentException("rankStart cannot be less than or equal to zero"); + } + if(range <= 0) { + throw new IllegalArgumentException("range cannot be less than or equal to zero"); + } + + int startPage = (int) Math.ceil(rankStart / 50F); + int requestFrequency = (int) Math.ceil((rankStart + (range - 1)) / 50F); + int surplusQuantity = range; + boolean firstRequest = true; + Gson gson = new Gson(); + ArrayList results = new ArrayList<>(); + for (int requestCount = startPage; requestCount <= requestFrequency && requestCount <= 10; requestCount++) { + int rangeStart = (requestCount - 1) * 50 + 1; + log.info("正在请求第 {} 到 {} 位排名榜 (第{}次请求, 共 {} 次)", rangeStart, rangeStart + 49, requestCount - startPage + 1, requestFrequency - startPage); + HttpGet request = createHttpGetRequest(PixivURL.getRankingLink(contentType, mode, time, requestCount, true)); + log.info("Request URL: {}", request.getURI()); + HttpResponse response = httpClient.execute(request); + String content = EntityUtils.toString(response.getEntity()); + log.debug("Content: " + content); + JsonObject contentObject = gson.fromJson(content, JsonObject.class); + if(contentObject.has("error")) { + log.warn("接口报错, 返回信息: {}", contentObject.get("error").getAsString()); + break; + } + JsonArray rankingArray = contentObject.getAsJsonArray("contents"); + log.info("正在解析数据..."); + + //需要添加一个总量, 否则会完整跑完一次. + //检查是否为最后一次请求，和剩余量有多少 + int firstRequestStartIndex = (rankStart % 50) - 1; + for (int rankIndex = firstRequest ? firstRequestStartIndex : 0; rankIndex < rankingArray.size() && surplusQuantity > 0; rankIndex++, surplusQuantity--) { + JsonElement jsonElement = rankingArray.get(rankIndex); + JsonObject rankInfo = jsonElement.getAsJsonObject(); + int rank = rankInfo.get("rank").getAsInt(); + int illustId = rankInfo.get("illust_id").getAsInt(); + int authorId = rankInfo.get("user_id").getAsInt(); + String authorName = rankInfo.get("user_name").getAsString(); + String title = rankInfo.get("title").getAsString(); + log.info("当前到第 {}/{} 名(总共 {} 名), IllustID: {}, Author: ({}) {}, Title: {}", rank, rankStart + range - 1, range, illustId, authorId, authorName, title); + results.add(rankInfo); + } + firstRequest = false; + log.info("第 {} 到 {} 位排名榜完成. (第{}次请求)", rangeStart, rangeStart + 49, requestCount); + } + + if(requestFrequency > 10) { + log.warn("请求的排名榜范围超出所支持的范围, 已终止请求."); + } + + return results; + } + + /** + * 获取作品的预加载数据 + * @param illustId 作品id + * @return 如果请求成功返回JsonObject, 失败返回null + * @throws IOException 当请求响应非200或请求发生异常时抛出. + */ + public JsonObject getIllustPreLoadDataById(int illustId) throws IOException { + HttpGet request = createHttpGetRequest(PixivURL.getPixivRefererLink(illustId)); + HttpResponse response = httpClient.execute(request); + + if(response.getStatusLine().getStatusCode() != 200) { + throw new IOException("Http响应码非200: " + response.getStatusLine()); + } + + Document document = Jsoup.parse(EntityUtils.toString(response.getEntity())); + Elements selectElements = document.select("#meta-preload-data"); + if(selectElements.size() == 0) { + return null; + } + + return new Gson().fromJson(selectElements.attr("content"), JsonObject.class); + } + + + @FunctionalInterface + public interface RankingDownloadFunction { + /** + * 接收图片InputStream + * @param rank 当前作品排名 + * @param link 作品下载链接 + * @param inputStream 作品下载输入流, InputStream会自动关闭 + */ + void download(int rank, String link, JsonObject rankInfo, InputStream inputStream); + } + + public HttpGet createHttpGetRequest(String url) { + HttpGet request = new HttpGet(url); + setCookieInRequest(request, cookieStore); + return request; + } + + + /** + * 取Illust所有页的原图下载链接 + * @param httpClient 用于发起请求的HttpClient对象 + * @param illustId 插画ID + * @param quality 页质量, 见{@link PageQuality} + * @return 返回该illust所有Page的下载链接 + * @throws IOException 当HttpClient在请求时发生异常, 或接口报错时抛出, 注意{@link IOException#getMessage()} + */ + public static List getIllustAllPageDownload(HttpClient httpClient, CookieStore cookieStore, int illustId, PageQuality quality) throws IOException { + HttpGet linkApiRequest = new HttpGet(PixivURL.PIXIV_ILLUST_API_URL.replace("{illustId}", Integer.toString(illustId))); + setCookieInRequest(linkApiRequest, cookieStore); + HttpResponse response = httpClient.execute(linkApiRequest); + JsonObject resultObject = new Gson().fromJson(EntityUtils.toString(response.getEntity()), JsonObject.class); + + if(resultObject.get("error").getAsBoolean()) { + String message = resultObject.get("message").getAsString(); + log.info("请求错误, 错误信息: {}", message); + throw new IOException(message); + } + + JsonArray linkArray = resultObject.getAsJsonArray("body"); + + ArrayList resultList = new ArrayList<>(); + String qualityType = quality == null ? "original" : quality.toString().toLowerCase(); + log.info("已选择插画类型: {}", qualityType); + linkArray.forEach(el -> { + JsonObject urlObj = el.getAsJsonObject().getAsJsonObject("urls"); + resultList.add(urlObj.get(qualityType).getAsString()); + }); + + return resultList; + } + + /** + * 插图质量 + */ + public enum PageQuality{ + /** + * 原图画质 + */ + ORIGINAL, + /** + * 常规画质 + */ + REGULAR, + /** + * 小图画质 + */ + SMALL, + /** + * 迷你画质 + */ + THUMB_MINI + } + + + + /** + * 获取帐号所有的收藏插画，并以输入流形式提供 + * @return 获取所有链接的InputStream, 请注意关闭InputStream + * @throws IOException 当获取时发生异常则直接抛出 + */ + public Set> getCollectionAsInputStream(PageQuality quality) throws IOException { + HashSet> illustInputStreamSet = new HashSet<>(); + getCollectionAsInputStream(quality, (link, inputStream) -> illustInputStreamSet.add(new AbstractMap.SimpleEntry<>(link, inputStream))); + return illustInputStreamSet; + } + + + /** + * 获取Pixiv图片 + * @param httpClient HttpClient对象 + * @param link Pixiv图片链接 + * @return 返回图片InputStream，注意关闭InputStream + * @throws IOException 获取失败时抛出 + * @throws IllegalArgumentException 当链接无法处理时抛出 + */ + public static InputStream getImageAsInputStream(HttpClient httpClient, String link) throws IOException { + HttpGet request = new HttpGet(link); + int startIndex = link.lastIndexOf("/"); + int endIndex = link.lastIndexOf("_"); + if(startIndex == -1 || endIndex == -1) { + throw new IllegalArgumentException("无法从链接获取illustID: " + link); + } + + String referer = PixivURL.getPixivRefererLink(link.substring(startIndex + 1, endIndex)); + request.addHeader(HttpHeaderNames.REFERER.toString(), referer); + + HttpResponse response = httpClient.execute(request); + log.debug("response: {}", response); + log.info("Content Length: {}KB", Float.parseFloat(response.getFirstHeader(HttpHeaderNames.CONTENT_LENGTH.toString()).getValue()) / 1024F); + log.info("{}", response.getFirstHeader(HttpHeaderNames.CONTENT_TYPE.toString())); + return response.getEntity().getContent(); + } + + /** + * 登出当前会话.
+ * 登出成功后, 该Cookies作废. + * @return 返回是否成功登出 + * @throws IOException 当登出请求异常时抛出 + */ + public boolean logOut() throws IOException { + HttpGet request = new HttpGet(PixivURL.PIXIV_LOGOUT_URL); + request.setConfig(RequestConfig.custom().setRedirectsEnabled(false).build()); + setCookieInRequest(request, cookieStore); + HttpResponse response = httpClient.execute(request); + if (response.getStatusLine().getStatusCode() == 302) { + cookieStore.clear(); + return true; + } else { + return false; + } + + } + + /** + * 获取作品信息 + * @param illustId 作品ID + * @return 成功获取返回JsonObject, 失败返回null,
+ * Json示例:
+ * 
+     *          {
+     *         "illustId": "79584670",
+     *         "illustTitle": "このヤンキーはウブすぎる",
+     *         "id": "79584670",
+     *         "title": "このヤンキーはウブすぎる",
+     *         "illustType": 1,
+     *         "xRestrict": 0,
+     *         "restrict": 0,
+     *         "sl": 2,
+     *         "url": "https://i.pximg.net/c/360x360_70/img-master/img/2020/02/19/00/38/23/79584670_p0_square1200.jpg",
+     *         "description": "",
+     *         "tags": [
+     *           "漫画",
+     *           "オリジナル",
+     *           "創作",
+     *           "創作男女",
+     *           "コロさん、ポリさん此方です!",
+     *           "恋の予感",
+     *           "あまずっぺー",
+     *           "交換日記",
+     *           "続編希望!!",
+     *           "オリジナル10000users入り"
+     *         ],
+     *         "userId": "4778293",
+     *         "userName": "隈浪さえ",
+     *         "width": 3288,
+     *         "height": 4564,
+     *         "pageCount": 4,
+     *         "isBookmarkable": true,
+     *         "bookmarkData": null,
+     *         "alt": "#オリジナル このヤンキーはウブすぎる - 隈浪さえ的漫画",
+     *         "isAdContainer": false,
+     *         "profileImageUrl": "https://i.pximg.net/user-profile/img/2019/12/04/18/56/19/16639046_fea29ce38ea89b0cb2313b40b3a72f9a_50.jpg",
+     *         "type": "illust"
+     *       }
+     *
+ * @throws IOException 当请求发生异常, 或接口返回错误信息时抛出. + */ + public JsonObject getIllustInfoByIllustId(int illustId) throws IOException { + HttpGet request = createHttpGetRequest(PixivURL.getPixivIllustInfoAPI(new int[] {illustId}));; + HttpResponse response = httpClient.execute(request); + String responseStr = EntityUtils.toString(response.getEntity()); + log.debug("Response Content: {}", responseStr); + JsonObject responseObj = new Gson().fromJson(responseStr, JsonObject.class); + + if(responseObj.get("error").getAsBoolean()) { + throw new IOException(responseObj.get("message").getAsString()); + } + + JsonArray illustsArray = responseObj.getAsJsonObject("body").getAsJsonArray("illusts"); + if(illustsArray.size() == 1) { + return illustsArray.get(0).getAsJsonObject(); + } else { + return null; + } + } + + + public static void setCookieInRequest(HttpRequest request, CookieStore cookieStore) { + StringBuilder builder = new StringBuilder(); + cookieStore.getCookies().forEach(cookie -> builder.append(cookie.getName()).append("=").append(cookie.getValue()).append("; ")); + request.setHeader(HttpHeaderNames.COOKIE.toString(), builder.toString()); + } + +} diff --git a/src/main/java/net/lamgc/cgj/pixiv/PixivSearchBuilder.java b/src/main/java/net/lamgc/cgj/pixiv/PixivSearchBuilder.java new file mode 100644 index 0000000..466ebc5 --- /dev/null +++ b/src/main/java/net/lamgc/cgj/pixiv/PixivSearchBuilder.java @@ -0,0 +1,340 @@ +package net.lamgc.cgj.pixiv; + +import com.google.common.base.Strings; + +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.HashSet; +import java.util.Objects; + +public class PixivSearchBuilder { + + private final String content; + + private SearchArea searchArea = SearchArea.ARTWORKS; + private SearchMode searchMode = SearchMode.TAG_FULL; + private SearchType searchType = SearchType.ILLUST_AND_UGOIRA; + private SearchOrder searchOrder = SearchOrder.DATE_D; + private SearchContentOption searchContentOption = SearchContentOption.ALL; + + private HashSet includeKeywords = new HashSet<>(0); + private HashSet excludeKeywords = new HashSet<>(0); + + private int page = 1; + + private int wgt = 0; + private int hgt = 0; + + private int wlt = 0; + private int hlt = 0; + + private RatioOption ratioOption = null; + + private Date startDate = null; + private Date endDate = null; + + public PixivSearchBuilder(String searchContent) { + this.content = Objects.requireNonNull(searchContent); + } + + public String buildURL() { + StringBuilder builder; + try { + builder = new StringBuilder(PixivURL.PIXIV_SEARCH_CONTENT_URL.replaceAll("\\{area}", searchArea.name().toLowerCase()) + .replaceAll("\\{content}", + URLEncoder.encode(getSearchCondition(), "UTF-8").replaceAll("\\+", "%20") + ) + ); + } catch (UnsupportedEncodingException e) { + throw new RuntimeException(e); + } + if(searchArea.equals(SearchArea.TOP)) { + return builder.toString(); + } + + builder.append("&s_mode=").append(searchMode.name().toLowerCase()); + builder.append("&type=").append(searchType.name().toLowerCase()); + builder.append("&p=").append(page); + builder.append("&order=").append(searchOrder.name().toLowerCase()); + builder.append("&mode=").append(searchContentOption.name().toLowerCase()); + + //可选参数 + if(wgt > 0 && hgt > 0) { + builder.append("&wgt=").append(wgt); + builder.append("&hgt").append(hgt); + } + + //可选参数 + if(wlt > 0 && hlt > 0) { + builder.append("&wlt=").append(wlt); + builder.append("&hlt").append(hlt); + } + + if (ratioOption != null) { + builder.append("&ratio=").append(ratioOption.value); + } + + + SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd"); + if (startDate != null) { + builder.append("&scd=").append(format.format(startDate)); + } + if (endDate != null) { + builder.append("&ecd=").append(format.format(endDate)); + } + + return builder.toString(); + } + + public PixivSearchBuilder setSearchArea(SearchArea searchArea) { + this.searchArea = searchArea; + return this; + } + + /** + * 获取搜索区域 + * @return 返回搜索区域对象 + */ + public SearchArea getSearchArea() { + return searchArea; + } + + /** + * 获取搜索条件. + * @return 搜索条件内容 + */ + public String getSearchCondition() { + StringBuilder searchContent = new StringBuilder(Strings.nullToEmpty(this.content)); + if(searchArea.equals(SearchArea.TOP)) { + return searchContent.toString(); + } + + excludeKeywords.forEach(keyword -> searchContent.append(" -").append(keyword)); + if(!includeKeywords.isEmpty()) { + if(!Strings.isNullOrEmpty(searchContent.toString())) { + searchContent.append(" ("); + } + includeKeywords.forEach(keyword -> searchContent.append(keyword).append(" OR ")); + int deleteStart = searchContent.lastIndexOf(" OR "); + if(searchContent.length() >= 4 && deleteStart != -1) { + searchContent.delete(deleteStart, searchContent.length()); + } + if(!Strings.isNullOrEmpty(searchContent.toString())) { + searchContent.append(")"); + } + } + + return searchContent.toString(); + } + + public PixivSearchBuilder setSearchMode(SearchMode searchMode) { + this.searchMode = Objects.requireNonNull(searchMode); + return this; + } + + public PixivSearchBuilder setSearchType(SearchType searchType) { + this.searchType = Objects.requireNonNull(searchType); + return this; + } + + public PixivSearchBuilder setSearchOrder(SearchOrder searchOrder) { + this.searchOrder = Objects.requireNonNull(searchOrder); + return this; + } + + public PixivSearchBuilder setSearchContentOption(SearchContentOption searchContentOption) { + this.searchContentOption = Objects.requireNonNull(searchContentOption); + return this; + } + + public PixivSearchBuilder setRatioOption(RatioOption ratioOption) { + this.ratioOption = Objects.requireNonNull(ratioOption); + return this; + } + + public PixivSearchBuilder setDateRange(Date startDate, Date endDate) { + this.startDate = startDate; + this.endDate = endDate; + return this; + } + + public PixivSearchBuilder setMaxSize(int width, int height) { + this.wgt = width; + this.hgt = height; + return this; + } + + public PixivSearchBuilder setMinSize(int width, int height) { + this.wlt = width; + this.hlt = height; + return this; + } + + public PixivSearchBuilder setPage(int pageIndex) { + if (pageIndex <= 0) { + throw new IllegalArgumentException("Invalid pageIndex: " + pageIndex); + } + this.page = pageIndex; + return this; + } + + public PixivSearchBuilder addExcludeKeyword(String keyword) { + excludeKeywords.add(keyword); + return this; + } + + public PixivSearchBuilder removeExcludeKeyword(String keyword) { + excludeKeywords.remove(keyword); + return this; + } + + public PixivSearchBuilder addIncludeKeyword(String keyword) { + includeKeywords.add(keyword); + return this; + } + + public PixivSearchBuilder removeIncludeKeyword(String keyword) { + includeKeywords.remove(keyword); + return this; + } + + /** + * 搜索区域 + */ + public enum SearchArea { + /** + * 所有(可能是 插画 + 漫画) + */ + ARTWORKS("illustManga"), + /** + * 顶部(所有内容) + * 同时包含了: + * {@link #ILLUSTRATIONS} + * {@link #MANGA} + * {@link #NOVELS} + * 选择此项后, 将直接显示所有与content相关内容, 而忽略所有附加搜索条件. + * 因为无法指定pageIndex, 数据只有24项 + */ + TOP(null), + /** + * 插画 + */ + ILLUSTRATIONS("illust"), + /** + * 漫画 + */ + MANGA("manga"), + /** + * 小说 + */ + NOVELS("novel"); + + public final String jsonKey; + + SearchArea(String jsonKey) { + this.jsonKey = jsonKey; + } + + + } + + /** + * 搜索模式 + */ + public enum SearchMode { + /** + * 按标签搜索, 部分一致 + */ + TAG, + /** + * 按标签搜索, 完全一致 + */ + TAG_FULL, + /** + * 按标题和说明文字搜索 + */ + TC + } + + /** + * 搜索内容类型 + */ + public enum SearchType { + /** + * 全部内容(插画、漫画、动图) + */ + ALL, + /** + * 插画和动图(不包括漫画) + */ + ILLUST_AND_UGOIRA, + /** + * 插图 + */ + ILLUST, + /** + * 漫画 + */ + MANGA, + /** + * 动图 + */ + UGOIRA + } + + public enum SearchOrder { + /** + * 按旧排序 + */ + DATE, + /** + * 按新排序 + */ + DATE_D + } + + /** + * 搜索内容选项 + */ + public enum SearchContentOption { + /** + * 所有内容 + */ + ALL, + /** + * 全年龄 + */ + SAFE, + /** + * R18 + */ + R18 + } + + public enum RatioOption { + /** + * 横向 + */ + TRANSVERSE(0.5F), + /** + * 纵向 + */ + PORTRAIT(-0.5F), + /** + * 正方形 + */ + SQUARE(0F) + ; + + public final float value; + + RatioOption(float ratio) { + this.value = ratio; + } + + } + + +} diff --git a/src/main/java/net/lamgc/cgj/pixiv/PixivSession.java b/src/main/java/net/lamgc/cgj/pixiv/PixivSession.java new file mode 100644 index 0000000..f89505a --- /dev/null +++ b/src/main/java/net/lamgc/cgj/pixiv/PixivSession.java @@ -0,0 +1,171 @@ +package net.lamgc.cgj.pixiv; + +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import org.apache.http.Header; +import org.apache.http.HttpHost; +import org.apache.http.HttpResponse; +import org.apache.http.NameValuePair; +import org.apache.http.client.CookieStore; +import org.apache.http.client.HttpClient; +import org.apache.http.client.entity.UrlEncodedFormEntity; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.impl.client.BasicCookieStore; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.message.BasicHeader; +import org.apache.http.message.BasicNameValuePair; +import org.apache.http.util.EntityUtils; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; + +public class PixivSession { + + public static final String USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36"; + + /** + * 全登陆过程的关键, + * 保存会话用的cookieStore! + */ + private CookieStore cookieStore = new BasicCookieStore(); + + /** + * 可以直接使用的HttpClient对象 + */ + private HttpClient httpClient; + + /** + * 最后一次登录的错误信息 + */ + private String errMsg; + + public PixivSession(){ + this(null); + } + + public PixivSession(CookieStore cookieStore){ + this(null, cookieStore); + } + + /** + * 创建一个Pixiv登录会话 + */ + public PixivSession(HttpHost proxy, CookieStore cookieStore) { + if(cookieStore != null){ + this.cookieStore = cookieStore; + } + List
defaultHeader = new ArrayList<>(); + defaultHeader.add(new BasicHeader("accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9")); + defaultHeader.add(new BasicHeader("user-agent", PixivSession.USER_AGENT)); + defaultHeader.add(new BasicHeader("accept-encoding", "gzip, deflate, br")); + defaultHeader.add(new BasicHeader("accept-language", "zh-CN,zh;q=0.9")); + + /*defaultHeader.add(new BasicHeader("sec-fetch-mode", "navigate")); + defaultHeader.add(new BasicHeader("sec-fetch-site", "same-origin")); + defaultHeader.add(new BasicHeader("upgrade-insecure-requests", "1"));*/ + //创建一个Http访问器 + httpClient = HttpClients.custom() + .setDefaultCookieStore(cookieStore) + .setDefaultHeaders(defaultHeader) + .setProxy(proxy) + .build(); + } + + /** + * 程序自行通过帐号密码登录Pixiv. + * @param PixivID Pixiv帐号 + * @param Password Pixiv密码 + * @return 登录成功返回true + * @throws IOException 当登录抛出异常时返回 + * @deprecated Pixiv已经新增Google人机验证, 程序已无法自行登录Pixiv + */ + public boolean Login(String PixivID, String Password) throws IOException { + // 获取登录接口所需的PostKey + String post_key = getPostKey(); + HttpPost postRequest = new HttpPost(PixivURL.PIXIV_LOGIN_URL); //https://accounts.pixiv.net/api/login?lang=zh + List params = new ArrayList<>(); + params.add(new BasicNameValuePair("pixiv_id", PixivID)); + params.add(new BasicNameValuePair("password", Password)); + params.add(new BasicNameValuePair("post_key", post_key)); + //Form编码表单,作为Post的数据 + postRequest.setEntity(new UrlEncodedFormEntity(params, StandardCharsets.UTF_8)); + //启动访问 + HttpResponse response = httpClient.execute(postRequest); + //获取接口返回数据 + String httpXML = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8); + System.out.println(httpXML); + JsonObject responseJson = (JsonObject) new JsonParser().parse(httpXML); + if(!responseJson.get("error").getAsBoolean() && !responseJson.get("body").getAsJsonObject().has("validation_errors")){ + errMsg = null; + return true; + }else{ + errMsg = responseJson.get("body").getAsJsonObject().get("validation_errors").toString(); + //System.err.println("登录失败！MSG: " + errMsg); + return false; + } + } + + /** + * 登录前准备, 获取PostKey + * @return Post_Key + */ + private String getPostKey() throws IOException { + //创建请求,获取PostKey + HttpGet getRequest = new HttpGet(PixivURL.PIXIV_LOGIN_PAGE_URL); + //设置请求 + //getRequest.setConfig(config); + getRequest.setHeader("User-Agent", USER_AGENT); + //启动访问 + HttpResponse response = httpClient.execute(getRequest); + //获取网页内容 + String pageAsXML = EntityUtils.toString(response.getEntity(),"utf-8"); + //创建Http解析器 + Document document = Jsoup.parse(pageAsXML); + //获取init-config内容 + String init_config = document.getElementById("init-config").val(); + //System.out.println(init_config); + //创建Json解析器解析init-config + JsonObject initConfigObj = (JsonObject) new JsonParser().parse(init_config); + //检查是否有postKey + if(!initConfigObj.has("pixivAccount.postKey")){ + throw new RuntimeException("postKey获取失败!可能是Pixiv修改了登录过程!"); + } + //获取postKey + return initConfigObj.get("pixivAccount.postKey").getAsString(); + } + + /** + * 获取CookieStore + * @return CookieStore + */ + public CookieStore getCookieStore(){ + return cookieStore; + } + + /** + * 获取可直接使用的HttpClient对象 + * @return 已配置好的HttpClient对象 + */ + public HttpClient getHttpClient(){ + return this.httpClient; + } + + public boolean hasError(){ + return errMsg == null; + } + + /** + * 获取错误信息 + * @return 返回登录错误信息 + * @deprecated {@link #Login(String, String)}已经废除, 故本接口废除 + */ + public String getErrMsg(){ + return errMsg; + } + +} diff --git a/src/main/java/net/lamgc/cgj/pixiv/PixivTag.java b/src/main/java/net/lamgc/cgj/pixiv/PixivTag.java new file mode 100644 index 0000000..25cfaa2 --- /dev/null +++ b/src/main/java/net/lamgc/cgj/pixiv/PixivTag.java @@ -0,0 +1,16 @@ +package net.lamgc.cgj.pixiv; + +public class PixivTag { + + public static PixivTag create(String tagName) { + + + return null; + } + + private PixivTag() { + + } + + +} diff --git a/src/main/java/net/lamgc/cgj/pixiv/PixivURL.java b/src/main/java/net/lamgc/cgj/pixiv/PixivURL.java new file mode 100644 index 0000000..3fab25a --- /dev/null +++ b/src/main/java/net/lamgc/cgj/pixiv/PixivURL.java @@ -0,0 +1,290 @@ +package net.lamgc.cgj.pixiv; + +import java.text.SimpleDateFormat; +import java.util.Calendar; +import java.util.Date; +import java.util.GregorianCalendar; + +/** + * 目前已整理的一些Pixiv接口列表 + */ +public class PixivURL { + + + public static final String PIXIV_INDEX_URL = "https://www.pixiv.net"; + + /** + * P站预登陆url + */ + public static final String PIXIV_LOGIN_PAGE_URL = "https://accounts.pixiv.net/login?lang=zh&source=pc&view_type=page&ref=wwwtop_accounts_index"; + + /** + * P站登录请求url + */ + public static final String PIXIV_LOGIN_URL = "https://accounts.pixiv.net/api/login?lang=zh"; + + /** + * P站搜索请求url + */ + private static final String PIXIV_SEARCH_URL = "https://www.pixiv.net/search.php"; + + /** + * P站搜索用户url + * 需要替换的参数: + * {nick} - 用户昵称、部分名称 + */ + public static final String PIXIV_SEARCH_USER_URL = PIXIV_SEARCH_URL + "?s_mode=s_usr&nick={nick}"; + + /** + * P站搜索插画url + * 需要替换的参数: + * {word} - 插画相关文本 + */ + public static final String PIXIV_SEARCH_TAG_URL = PIXIV_SEARCH_URL + "?s_mode=s_tag&word={word}"; + + /** + * P站插图下载链接获取url + * 需要替换的文本: + * {illustId} - 插画ID + */ + public static final String PIXIV_ILLUST_API_URL = "https://www.pixiv.net/ajax/illust/{illustId}/pages"; + + /* + * P站用户插图列表获取API + * 需要替换的文本: + * {userId} - 用户ID + */ + //TODO: 所需数据在 body属性内的 illusts(属性名,属性值不重要), manga(多图) pickup(精选) + //{"error":false,"message":"","body":{"illusts":{"74369837":null,"70990542":null,"70608653":null,"69755191":null,"69729450":null,"69729416":null,"69503608":null,"69288766":null,"69083882":null,"69051458":null,"68484200":null,"68216927":null,"68216866":null,"68192333":null,"67915106":null,"67914932":null,"67854803":null,"67854745":null,"67854670":null,"67787211":null,"67772199":null,"67770637":null,"67754861":null,"67754804":null,"67754726":null,"67740486":null,"67740480":null,"67740450":null,"67740434":null,"67726337":null,"67499196":null,"67499163":null,"67499145":null,"67499111":null,"67499085":null,"67499038":null,"67498987":null,"67473178":null,"66271465":null,"63682753":null,"63682697":null,"59385148":null,"59383265":null,"59383240":null,"59383227":null,"59383173":null},"manga":[],"novels":[],"mangaSeries":[],"novelSeries":[],"pickup":[],"bookmarkCount":{"public":{"illust":1,"novel":0},"private":{"illust":0,"novel":0}}}} + //public static final String PIXIV_USER_ILLUST_LIST_URL = "https://www.pixiv.net/ajax/user/{userId}/profile/all"; + + /** + * 能够同时获取插图信息的用户插图列表获取API + */ + public static final String PIXIV_USER_ILLUSTINFO_LIST_URL = "https://www.pixiv.net/ajax/user/{userId}/profile/top"; + + /** + * P站单图详情页url + * 需要替换的文本: + * {illustId} - 插画ID + */ + public static final String PIXIV_ILLUST_MEDIUM_URL = "https://www.pixiv.net/member_illust.php?mode=medium&illust_id={illustId}"; + + /** + * P站多图详情页url + * 需要替换的文本: + * {illustId} - 插画ID + */ + public static final String PIXIV_ILLUST_MANGA_URL = "https://www.pixiv.net/member_illust.php?mode=manga&illust_id={illustId}"; + + /** + * P站用户页面url + * 需要替换的文本: + * {userId} - 用户ID + */ + public static final String PIXIV_USER_URL = "https://www.pixiv.net/member.php?id={userId}"; + + /** + * P站插图信息获取API + * 这个API能获取插图基本信息，但不能获取大小 + * 请使用{@link #getPixivIllustInfoAPI(int[])}获取URL + */ + private static final String PIXIV_GET_ILLUST_INFO_URL = "https://www.pixiv.net/ajax/illust/recommend/illusts?"; + + /** + * P站获取用户所有插图ID的Api + * 这个API只能获取该用户的插图ID，不能获取图片信息(图片信息要另外获取) + * 需要替换的文本: + * {userId} - 用户ID + */ + public static final String PIXIV_GET_USER_ALL_ILLUST_ID_URL = "https://www.pixiv.net/ajax/user/{userId}/profile/all"; + + /** + * P站标签搜索URL + * 可以将Tag的大概内容搜索成P站精确的Tag，以搜索其他接口返回的Tags数组; + * 需要替换的文本: + * {content} - 大致tag内容 + */ + public static final String PIXIV_TAG_SEARCH_URL = "https://www.pixiv.net/ajax/search/tags/{content}"; + + /** + * 请求时带上需要退出的Cookies + * 无论成功与否都会返回302重定向到{@linkplain #PIXIV_LOGIN_PAGE_URL 登录页面} + */ + public static final String PIXIV_LOGOUT_URL = "https://www.pixiv.net/logout.php"; + + /** + * 构造P站获取插图信息的Api Url + * @param illustIds 要查询的插图ID数组 + * @return 对应查询的API Url + */ + public static String getPixivIllustInfoAPI(int[] illustIds){ + StringBuilder strBuilder = new StringBuilder().append(PIXIV_GET_ILLUST_INFO_URL); + for(int illustId : illustIds){ + strBuilder.append("illust_ids[]=").append(illustId).append("&"); + } + return strBuilder.toString(); + } + + /** + * 获取用于下载图片时防盗链所需Referer的链接 + * @param illustId 欲下载图片所属illustId + * @return 返回Referer链接, 也可以作为作品链接使用 + */ + public static String getPixivRefererLink(int illustId){ + return "https://www.pixiv.net/artworks/" + illustId; + } + + /** + * 获取用于下载图片时防盗链所需Referer的链接 + * @param illustId 欲下载图片所属illustId + * @return 返回Referer链接, 也可以作为作品链接使用 + */ + public static String getPixivRefererLink(String illustId){ + return "https://www.pixiv.net/artworks/" + illustId; + } + + /** + * 排行榜接口, 需加入"&format=json" + */ + private final static String PIXIV_RANKING_LINK = "https://www.pixiv.net/ranking.php?"; + + /** + * 查询用户收藏.
+ * 该URL返回HTML页面，需要进行解析.
+ * 需要替换的文本:
+ * {pageIndex} - 页数, 超出了则结果为空
+ */ + public final static String PIXIV_USER_COLLECTION_PAGE = "https://www.pixiv.net/bookmark.php?rest=show&p={pageIndex}"; + + /** + * 获取排名榜 + * @param mode 查询类型, 详细信息看{@link RankingMode}, 如本参数为null, 则为每天 + * @param contentType 排名榜类型, 如为null则为综合 + * @param time 欲查询的时间, 最新只能查询昨天, 根据mode不同: + * 每天 - 查询指定日期的排名榜 + * 每周 - 查询指定时间结束(含)到七天前一段时间内的排名榜 + * 每月 - 查询指定日期结束(含)到28天时间内的排名榜 + * 新人 - 与每周相同 + * 受男性欢迎 - 与每天相同 + * 受女性欢迎 - 与每天相同 + * 默认值为昨天 + * @param pageIndex 页数，一页50位，总共10页 + * @return 返回构建好的链接 + */ + public static String getRankingLink(RankingContentType contentType, RankingMode mode, Date time, int pageIndex, boolean json){ + StringBuilder linkBuilder = new StringBuilder(PIXIV_RANKING_LINK); + linkBuilder.append("mode=").append(mode == null ? RankingMode.MODE_DAILY.modeParam : mode.modeParam); + if(contentType != null){ + linkBuilder.append("&content=").append(contentType.typeName); + } + SimpleDateFormat format = new SimpleDateFormat("yyyyMMdd"); + Date queryDate; + if(time == null){ + GregorianCalendar gregorianCalendar = new GregorianCalendar(); + gregorianCalendar.setTime(new Date()); + gregorianCalendar.add(Calendar.DATE, -1); + queryDate = gregorianCalendar.getTime(); + } else { + queryDate = time; + } + linkBuilder.append("&date=").append(format.format(queryDate)); + if(pageIndex > 0 && pageIndex <= 10) { + linkBuilder.append("&p=").append(pageIndex); + } + if(json) { + linkBuilder.append("&format=").append("json"); + } + return linkBuilder.toString(); + } + + /** + * 排名榜模式 + */ + public enum RankingMode{ + /** + * 每天 + */ + MODE_DAILY("daily"), + /** + * 每周 + */ + MODE_WEEKLY("weekly"), + /** + * 每月 + */ + MODE_MONTHLY("monthly"), + /** + * 新人 + */ + MODE_ROOKIE("rookie"), + /** + * 受男性喜欢 + */ + MODE_MALE("male"), + /** + * 受女性喜欢 + */ + MODE_FEMALE("female"), + + /** + * 每天 - R18 + */ + MODE_DAILY_R18("daily_r18"), + /** + * 每周 - R18 + */ + MODE_WEEKLY_R18("weekly_r18"), + /** + * 受男性喜欢 - R18 + */ + MODE_MALE_R18("male_r18"), + /** + * 受女性喜欢 - R18 + */ + MODE_FEMALE_R18("female_r18"), + ; + public String modeParam; + + RankingMode(String modeParamName){ + this.modeParam = modeParamName; + } + } + + /** + * Pixiv搜索接口.
+ * 要使用该链接请使用{@link PixivSearchBuilder}构造链接.
+ * 需要替换的参数:
+ * content - 搜索内容 + */ + final static String PIXIV_SEARCH_CONTENT_URL = "https://www.pixiv.net/ajax/search/{area}/{content}?word={content}"; + + /** + * 排名榜类型 + */ + public enum RankingContentType{ + /** + * 插画 + * 支持的时间类型: 每天, 每周, 每月, 新人 + */ + TYPE_ILLUST("illust"), + /** + * 动图 + * 支持的时间类型:每天, 每周 + */ + TYPE_UGOIRA("ugoira"), + /** + * 漫画 + * 支持的时间类型: 每天, 每周, 每月, 新人 + */ + TYPE_MANGA("manga") + ; + + String typeName; + + RankingContentType(String typeName){ + this.typeName = typeName; + } + } + +} diff --git a/src/main/java/net/lamgc/cgj/proxy/PixivAccessProxyServer.java b/src/main/java/net/lamgc/cgj/proxy/PixivAccessProxyServer.java new file mode 100644 index 0000000..f589f9b --- /dev/null +++ b/src/main/java/net/lamgc/cgj/proxy/PixivAccessProxyServer.java @@ -0,0 +1,149 @@ +package net.lamgc.cgj.proxy; + +import com.github.monkeywie.proxyee.intercept.HttpProxyIntercept; +import com.github.monkeywie.proxyee.intercept.HttpProxyInterceptInitializer; +import com.github.monkeywie.proxyee.intercept.HttpProxyInterceptPipeline; +import com.github.monkeywie.proxyee.intercept.common.CertDownIntercept; +import com.github.monkeywie.proxyee.proxy.ProxyConfig; +import com.github.monkeywie.proxyee.server.HttpProxyServer; +import com.github.monkeywie.proxyee.server.HttpProxyServerConfig; +import io.netty.channel.Channel; +import io.netty.handler.codec.http.HttpHeaderNames; +import io.netty.handler.codec.http.HttpHeaders; +import io.netty.handler.codec.http.HttpRequest; +import io.netty.handler.codec.http.HttpResponse; +import org.apache.http.client.CookieStore; +import org.apache.http.impl.cookie.BasicClientCookie; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.net.HttpCookie; +import java.util.Date; +import java.util.List; + +/** + * 登录成功后提供CookieStore, 然后由程序自动登录Pixiv + * @author LamGC + */ +public class PixivAccessProxyServer { + + private final Logger log = LoggerFactory.getLogger(this.getClass().getSimpleName()); + + private final HttpProxyServer proxyServer; + + private final CookieStore cookieStore; + + public PixivAccessProxyServer(CookieStore cookieStore){ + this(cookieStore, null); + } + + public PixivAccessProxyServer(CookieStore cookieStore, ProxyConfig proxyConfig){ + HttpProxyServerConfig config = new HttpProxyServerConfig(); + this.cookieStore = cookieStore; + config.setHandleSsl(true); + this.proxyServer = new HttpProxyServer(); + this.proxyServer + .serverConfig(config) + .proxyConfig(proxyConfig) + .proxyInterceptInitializer(new HttpProxyInterceptInitializer(){ + @Override + public void init(HttpProxyInterceptPipeline pipeline) { + pipeline.addLast(new CertDownIntercept()); + pipeline.addLast(new HttpProxyIntercept(){ + + private boolean match(HttpRequest request){ + String host = request.headers().get(HttpHeaderNames.HOST); + return host.equalsIgnoreCase("pixiv.net") || host.contains(".pixiv.net"); + } + + @Override + public void beforeRequest(Channel clientChannel, HttpRequest httpRequest, HttpProxyInterceptPipeline pipeline) throws Exception { + log.info("URL: " + httpRequest.headers().get(HttpHeaderNames.HOST) + httpRequest.uri()); + if(!match(httpRequest)){ + super.beforeRequest(clientChannel, httpRequest, pipeline); + return; + } + log.info("正在注入Cookies..."); + HttpHeaders requestHeaders = httpRequest.headers(); + if(requestHeaders.contains(HttpHeaderNames.COOKIE)){ + log.info("原请求存在自带Cookies, 正在清除Cookies..."); + log.debug("原Cookies: {}", requestHeaders.getAsString(HttpHeaderNames.COOKIE)); + requestHeaders.remove(HttpHeaderNames.COOKIE); + } + StringBuilder cookieBuilder = new StringBuilder(); + cookieStore.getCookies().forEach(cookie -> { + if(cookie.isExpired(new Date())){ + return; + } + cookieBuilder.append(cookie.getName()).append("=").append(cookie.getValue()).append("; "); + }); + log.info("Cookies构造完成, 结果: " + cookieBuilder.toString()); + requestHeaders.add(HttpHeaderNames.COOKIE, cookieBuilder.toString()); + log.info("Cookies注入完成."); + + super.beforeRequest(clientChannel, httpRequest, pipeline); + } + + @Override + public void afterResponse(Channel clientChannel, Channel proxyChannel, HttpResponse httpResponse, HttpProxyInterceptPipeline pipeline) throws Exception { + if(!match(pipeline.getHttpRequest())){ + super.afterResponse(clientChannel, proxyChannel, httpResponse, pipeline); + return; + } + log.info("正在更新Response Cookie...(Header Name: " + HttpHeaderNames.SET_COOKIE + ")"); + List responseCookies = httpResponse.headers().getAll(HttpHeaderNames.SET_COOKIE); + responseCookies.forEach(value -> { + /*if(check(value)){ + log.info("黑名单Cookie, 已忽略: " + value); + return; + }*/ + log.info("Response Cookie: " + value); + BasicClientCookie cookie = parseRawCookie(value); + cookieStore.addCookie(null); + }); + httpResponse.headers().remove(HttpHeaderNames.SET_COOKIE); + super.afterResponse(clientChannel, proxyChannel, httpResponse, pipeline); + } + + protected BasicClientCookie parseRawCookie(String rawCookie) { + List cookies = HttpCookie.parse(rawCookie); + if (cookies.size() < 1) + return null; + HttpCookie httpCookie = cookies.get(0); + BasicClientCookie cookie = new BasicClientCookie(httpCookie.getName(), httpCookie.getValue()); + if (httpCookie.getMaxAge() >= 0) { + Date expiryDate = new Date(System.currentTimeMillis() + httpCookie.getMaxAge() * 1000); + cookie.setExpiryDate(expiryDate); + } + if (httpCookie.getDomain() != null) + cookie.setDomain(httpCookie.getDomain()); + if (httpCookie.getPath() != null) + cookie.setPath(httpCookie.getPath()); + if (httpCookie.getComment() != null) + cookie.setComment(httpCookie.getComment()); + cookie.setSecure(httpCookie.getSecure()); + return cookie; + } + }); + } + }); + } + + public void start(int port){ + this.proxyServer.start(port); + } + + public void close(){ + this.proxyServer.close(); + } + + /** + * 导出CookieStore. + * 注意!该方法导出的CookieStore不适用于ApacheHttpClient, 如需使用则需要进行转换. + * @return CookieStore对象 + */ + public CookieStore getCookieStore(){ + return this.cookieStore; + } + +} diff --git a/src/main/java/net/lamgc/cgj/proxy/PixivLoginProxyServer.java b/src/main/java/net/lamgc/cgj/proxy/PixivLoginProxyServer.java new file mode 100644 index 0000000..ed5b7de --- /dev/null +++ b/src/main/java/net/lamgc/cgj/proxy/PixivLoginProxyServer.java @@ -0,0 +1,157 @@ +package net.lamgc.cgj.proxy; + +import com.github.monkeywie.proxyee.intercept.HttpProxyInterceptInitializer; +import com.github.monkeywie.proxyee.intercept.HttpProxyInterceptPipeline; +import com.github.monkeywie.proxyee.intercept.common.CertDownIntercept; +import com.github.monkeywie.proxyee.intercept.common.FullResponseIntercept; +import com.github.monkeywie.proxyee.proxy.ProxyConfig; +import com.github.monkeywie.proxyee.server.HttpProxyServer; +import com.github.monkeywie.proxyee.server.HttpProxyServerConfig; +import com.google.gson.Gson; +import com.google.gson.JsonObject; +import io.netty.handler.codec.http.FullHttpResponse; +import io.netty.handler.codec.http.HttpHeaderNames; +import io.netty.handler.codec.http.HttpRequest; +import io.netty.handler.codec.http.HttpResponse; +import org.apache.commons.text.StringEscapeUtils; +import org.apache.http.client.CookieStore; +import org.apache.http.impl.client.BasicCookieStore; +import org.apache.http.impl.cookie.BasicClientCookie; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.net.HttpCookie; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.Date; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * 由用户介入, 让用户手动登录Pixiv的方式, 再通过代理服务器捕获Cookie来绕过Google人机验证 + * @author LamGC + */ +public class PixivLoginProxyServer { + + private final Logger log = LoggerFactory.getLogger(this.getClass().getSimpleName()); + + private boolean login = false; + + private final HttpProxyServer proxyServer; + + private final CookieStore cookieStore = new BasicCookieStore(); + + public PixivLoginProxyServer(){ + this(null); + } + + public PixivLoginProxyServer(ProxyConfig proxyConfig){ + HttpProxyServerConfig config = new HttpProxyServerConfig(); + config.setHandleSsl(true); + this.proxyServer = new HttpProxyServer(); + this.proxyServer + .serverConfig(config) + .proxyConfig(proxyConfig) + .proxyInterceptInitializer(new HttpProxyInterceptInitializer(){ + @Override + public void init(HttpProxyInterceptPipeline pipeline) { + pipeline.addLast(new CertDownIntercept()); + pipeline.addLast(new FullResponseIntercept() { + @Override + public boolean match(HttpRequest httpRequest, HttpResponse httpResponse, HttpProxyInterceptPipeline httpProxyInterceptPipeline) { + String host = httpRequest.headers().get(HttpHeaderNames.HOST); + return host.equalsIgnoreCase("pixiv.net") || host.contains(".pixiv.net"); + } + + @Override + public void handelResponse(HttpRequest httpRequest, FullHttpResponse fullHttpResponse, HttpProxyInterceptPipeline httpProxyInterceptPipeline) { + String url = httpRequest.headers().get(HttpHeaderNames.HOST) + httpRequest.uri(); + log.info("拦截到Pixiv请求, URL: " + url); + + log.info("正在导出Response Cookie...(Header Name: " + HttpHeaderNames.SET_COOKIE + ")"); + List responseCookies = fullHttpResponse.headers().getAll(HttpHeaderNames.SET_COOKIE); + AtomicInteger responseCookieCount = new AtomicInteger(); + responseCookies.forEach(value -> { + log.debug("Response Cookie: " + value); + cookieStore.addCookie(parseRawCookie(value)); + responseCookieCount.incrementAndGet(); + }); + log.info("Cookie导出完成(已导出 " + responseCookieCount.get() + " 条Cookie)"); + + //登录检查 + // 如果用户在登录界面登录成功后反复刷新，会出现登录返回不对但已经成功登录的情况， + // 故此处在登录完成后不再判断是否成功登录 + if(!isLogin() && url.contains("accounts.pixiv.net/api/login")){ + log.info("正在检查登录结果..."); + //拷贝一份以防止对原响应造成影响 + FullHttpResponse copyResponse = fullHttpResponse.copy(); + ByteBuffer buffer = ByteBuffer.allocate(copyResponse.content().capacity()); + String contentStr; + copyResponse.content().readBytes(buffer); + contentStr = new String(buffer.array(), StandardCharsets.UTF_8); + log.debug("Login Result: " + contentStr); + + JsonObject resultObject = new Gson().fromJson(contentStr, JsonObject.class); + //只要error:false, body存在(应该是会存在的)且success字段存在, 即为登录成功 + login = !resultObject.get("error").getAsBoolean() && + resultObject.has("body") && + resultObject.get("body").getAsJsonObject().has("success"); + log.info("登录状态确认: " + (login ? "登录成功" : "登录失败")); + + fullHttpResponse.content().clear().writeBytes( + ("{\"error\":false,\"message\":\"\",\"body\":{\"validation_errors\":{\"etc\":\"" + + StringEscapeUtils.escapeJava("Pixiv登录代理器已确认登录") + "\"}}}") + .getBytes(StandardCharsets.UTF_8)); + } + } + + protected BasicClientCookie parseRawCookie(String rawCookie) { + List cookies = HttpCookie.parse(rawCookie); + if (cookies.size() < 1) + return null; + HttpCookie httpCookie = cookies.get(0); + BasicClientCookie cookie = new BasicClientCookie(httpCookie.getName(), httpCookie.getValue()); + if (httpCookie.getMaxAge() >= 0) { + Date expiryDate = new Date(System.currentTimeMillis() + httpCookie.getMaxAge() * 1000); + cookie.setExpiryDate(expiryDate); + } + if (httpCookie.getDomain() != null) + cookie.setDomain(httpCookie.getDomain()); + if (httpCookie.getPath() != null) + cookie.setPath(httpCookie.getPath()); + if (httpCookie.getComment() != null) + cookie.setComment(httpCookie.getComment()); + cookie.setSecure(httpCookie.getSecure()); + return cookie; + } + }); + } + }); + } + + public void start(int port){ + this.proxyServer.start(port); + } + + public void close(){ + this.proxyServer.close(); + } + + /** + * 是否已登录Pixiv + * @return 如已登录返回true + */ + public boolean isLogin(){ + return login; + } + + /** + * 导出CookieStore. + * 注意!该方法导出的CookieStore不适用于ApacheHttpClient, 如需使用则需要进行转换. + * @return CookieStore对象 + */ + public CookieStore getCookieStore(){ + return this.cookieStore; + } + +} diff --git a/src/main/java/net/lamgc/cgj/proxy/PixivLoginProxyServer_Old.java b/src/main/java/net/lamgc/cgj/proxy/PixivLoginProxyServer_Old.java new file mode 100644 index 0000000..965df1e --- /dev/null +++ b/src/main/java/net/lamgc/cgj/proxy/PixivLoginProxyServer_Old.java @@ -0,0 +1,185 @@ +package net.lamgc.cgj.proxy; + +import com.github.monkeywie.proxyee.intercept.HttpProxyInterceptInitializer; +import com.github.monkeywie.proxyee.intercept.HttpProxyInterceptPipeline; +import com.github.monkeywie.proxyee.intercept.common.CertDownIntercept; +import com.github.monkeywie.proxyee.intercept.common.FullResponseIntercept; +import com.github.monkeywie.proxyee.proxy.ProxyConfig; +import com.github.monkeywie.proxyee.server.HttpProxyServer; +import com.github.monkeywie.proxyee.server.HttpProxyServerConfig; +import com.google.gson.Gson; +import com.google.gson.JsonObject; +import io.netty.handler.codec.http.*; +import org.apache.http.client.CookieStore; +import org.apache.http.impl.client.BasicCookieStore; +import org.apache.http.impl.cookie.BasicClientCookie; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.net.HttpCookie; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.*; + +/** + * 由用户介入, 让用户手动登录Pixiv的方式, 再通过代理服务器捕获Cookie来绕过Google人机验证 + * @author LamGC + */ +public class PixivLoginProxyServer_Old { + + private final Logger log = LoggerFactory.getLogger(this.getClass().getSimpleName()); + + private boolean login = false; + + private final HttpProxyServer proxyServer; + + //private final CookieManager cookieManager = new CookieManager(); + + private final CookieStore cookieStore = new BasicCookieStore(); + + public PixivLoginProxyServer_Old(){ + this(null); + } + + public PixivLoginProxyServer_Old(ProxyConfig proxyConfig){ + HttpProxyServerConfig config = new HttpProxyServerConfig(); + config.setHandleSsl(true); + this.proxyServer = new HttpProxyServer(); + this.proxyServer + .serverConfig(config) + .proxyConfig(proxyConfig) + .proxyInterceptInitializer(new HttpProxyInterceptInitializer(){ + @Override + public void init(HttpProxyInterceptPipeline pipeline) { + pipeline.addLast(new CertDownIntercept()); + pipeline.addLast(new FullResponseIntercept() { + @Override + public boolean match(HttpRequest httpRequest, HttpResponse httpResponse, HttpProxyInterceptPipeline httpProxyInterceptPipeline) { + //log.info("Match: " + httpRequest.headers().get(HttpHeaderNames.HOST) + httpRequest.uri()); + //return HttpUtil.checkUrl(httpRequest, ".*pixiv\\.net"); + String host = httpRequest.headers().get(HttpHeaderNames.HOST); + return host.equalsIgnoreCase("pixiv.net") || host.contains(".pixiv.net"); + } + + @Override + public void handelResponse(HttpRequest httpRequest, FullHttpResponse fullHttpResponse, HttpProxyInterceptPipeline httpProxyInterceptPipeline) { + log.info("拦截到Pixiv请求, 正在导出Response Cookie..."); + String url = httpRequest.headers().get(HttpHeaderNames.HOST) + httpRequest.uri(); + log.info("URL: " + url); + /*URI requestURI = URI.create(httpRequest.headers().get(HttpHeaderNames.HOST) + httpRequest.uri()); + log.info("正在导出Request Cookie...(Header Name: " + HttpHeaderNames.COOKIE + ")"); + List requestCookie = httpRequest.headers().getAll(HttpHeaderNames.COOKIE); + //requestCookie.forEach(value -> log.info("Request Cookie: " + value)); + requestCookie.forEach(value -> { + String[] items = value.split(";"); + for(String item : items){ + log.info("Raw Request Cookie: " + item); + String[] keyValueSet = item.split("="); + if(keyValueSet.length == 2){ + //TODO: 会出现重复的情况 + log.info("Request Cookie: " + keyValueSet[0].trim() + "=" + keyValueSet[1].trim()); + cookieStore.addCookie(new BasicClientCookie(keyValueSet[0].trim(), keyValueSet[1].trim())); + } + } + //cookieStore.addCookie(parseRawCookie(value)); + });*/ + log.info("正在导出Response Cookie...(Header Name: " + HttpHeaderNames.SET_COOKIE + ")"); + List responseCookies = fullHttpResponse.headers().getAll(HttpHeaderNames.SET_COOKIE); + responseCookies.forEach(value -> { + /*if(check(value)){ + log.info("黑名单Cookie, 已忽略: " + value); + return; + }*/ + log.info("Response Cookie: " + value); + cookieStore.addCookie(parseRawCookie(value)); + }); + + log.info("Cookie导出完成"); + + if(url.contains("accounts.pixiv.net/api/login")){ + log.info("正在检查登录结果..."); + FullHttpResponse copyResponse = fullHttpResponse.copy(); + //ByteArrayOutputStream contentOS = new ByteArrayOutputStream(copyResponse.content().capacity()); + ByteBuffer buffer = ByteBuffer.allocate(copyResponse.content().capacity()); + String contentStr; + copyResponse.content().readBytes(buffer); + contentStr = new String(buffer.array(), StandardCharsets.UTF_8); + //log.info("Login Result: " + contentStr); + JsonObject resultObject = new Gson().fromJson(contentStr, JsonObject.class); + login = !resultObject.get("error").getAsBoolean() && + resultObject.has("body") && + resultObject.get("body").getAsJsonObject().has("success"); + if(login) { + log.info("登录状态确认: 登录成功"); + } else { + log.info("登录状态确认: 登录失败"); + } + } + } + + protected BasicClientCookie parseRawCookie(String rawCookie) { + List cookies = HttpCookie.parse(rawCookie); + if (cookies.size() < 1) + return null; + HttpCookie httpCookie = cookies.get(0); + BasicClientCookie cookie = new BasicClientCookie(httpCookie.getName(), httpCookie.getValue()); + if (httpCookie.getMaxAge() >= 0) { + Date expiryDate = new Date(System.currentTimeMillis() + httpCookie.getMaxAge() * 1000); + cookie.setExpiryDate(expiryDate); + } + if (httpCookie.getDomain() != null) + cookie.setDomain(httpCookie.getDomain()); + if (httpCookie.getPath() != null) + cookie.setPath(httpCookie.getPath()); + if (httpCookie.getComment() != null) + cookie.setComment(httpCookie.getComment()); + cookie.setSecure(httpCookie.getSecure()); + return cookie; + } + + private boolean check(String cookieValue){ + for(String blackItem : new String[]{ + "a_type", + "b_type", + "c_type", + "is_sensei_service_user", + "module_orders_mypage", + }){ + if(cookieValue.startsWith(blackItem)){ + return true; + } + } + return false; + } + + }); + } + }); + } + + public void start(int port){ + this.proxyServer.start(port); + } + + public void close(){ + this.proxyServer.close(); + } + + /** + * 是否已登录Pixiv + * @return 如已登录返回true + */ + public boolean isLogin(){ + return login; + } + + /** + * 导出CookieStore. + * 注意!该方法导出的CookieStore不适用于ApacheHttpClient, 如需使用则需要进行转换. + * @return CookieStore对象 + */ + public CookieStore getCookieStore(){ + return this.cookieStore; + } + +} diff --git a/src/main/java/net/lamgc/cgj/util/CookieUtil.java b/src/main/java/net/lamgc/cgj/util/CookieUtil.java new file mode 100644 index 0000000..ae06dfb --- /dev/null +++ b/src/main/java/net/lamgc/cgj/util/CookieUtil.java @@ -0,0 +1,31 @@ +package net.lamgc.cgj.util; + +import org.apache.http.client.CookieStore; +import org.apache.http.impl.client.BasicCookieStore; +import org.apache.http.impl.cookie.BasicClientCookie; + +import java.util.Date; + +public class CookieUtil { + + /** + * 将{@link java.net.CookieStore}转换到{@link CookieStore} + * @param netCookieStore java.net.CookieStore + * @return org.apache.http.client.CookieStore + */ + public static CookieStore NAParse(java.net.CookieStore netCookieStore){ + CookieStore apacheCookieStore = new BasicCookieStore(); + netCookieStore.getCookies().forEach(netCookie -> { + BasicClientCookie aCookie = new BasicClientCookie(netCookie.getName(), netCookie.getValue()); + aCookie.setComment(netCookie.getComment()); + aCookie.setDomain(netCookie.getDomain()); + aCookie.setExpiryDate(new Date(netCookie.getMaxAge())); + aCookie.setPath(netCookie.getPath()); + aCookie.setSecure(netCookie.getSecure()); + aCookie.setVersion(netCookie.getVersion()); + apacheCookieStore.addCookie(aCookie); + }); + return apacheCookieStore; + } + +} diff --git a/src/main/java/net/lamgc/cgj/util/DateParser.java b/src/main/java/net/lamgc/cgj/util/DateParser.java new file mode 100644 index 0000000..bd4269e --- /dev/null +++ b/src/main/java/net/lamgc/cgj/util/DateParser.java @@ -0,0 +1,25 @@ +package net.lamgc.cgj.util; + +import net.lamgc.utils.base.runner.StringParameterParser; + +import java.text.DateFormat; +import java.util.Date; + +public class DateParser implements StringParameterParser { + + private final DateFormat dateFormat; + + public DateParser(DateFormat dateFormat) { + this.dateFormat = dateFormat; + } + + @Override + public Date parse(String strValue) throws Exception { + return dateFormat.parse(strValue); + } + + @Override + public Date defaultValue() { + return null; + } +} diff --git a/src/main/java/net/lamgc/cgj/util/PagesQualityParser.java b/src/main/java/net/lamgc/cgj/util/PagesQualityParser.java new file mode 100644 index 0000000..52b4bc9 --- /dev/null +++ b/src/main/java/net/lamgc/cgj/util/PagesQualityParser.java @@ -0,0 +1,18 @@ +package net.lamgc.cgj.util; + +import net.lamgc.cgj.pixiv.PixivDownload; +import net.lamgc.cgj.pixiv.PixivURL; +import net.lamgc.utils.base.runner.StringParameterParser; + +public class PagesQualityParser implements StringParameterParser { + + @Override + public PixivDownload.PageQuality parse(String strValue) throws Exception { + return PixivDownload.PageQuality.valueOf(strValue.toUpperCase()); + } + + @Override + public PixivDownload.PageQuality defaultValue() { + return PixivDownload.PageQuality.ORIGINAL; + } +} diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties new file mode 100644 index 0000000..d2a8109 --- /dev/null +++ b/src/main/resources/application.properties @@ -0,0 +1,2 @@ +server.port=8081 +server.tomcat.max-threads=10 \ No newline at end of file diff --git a/src/main/resources/log4j.properties b/src/main/resources/log4j.properties new file mode 100644 index 0000000..16165dc --- /dev/null +++ b/src/main/resources/log4j.properties @@ -0,0 +1,15 @@ +log4j.rootLogger = info,console,logfile + +#\u8BBE\u7F6Edebug\u65E5\u5FD7\u4FE1\u606F +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.Target=System.out +log4j.appender.console.layout=org.apache.log4j.PatternLayout +#[\u5E74.\u6708.\u65E5 \u65F6:\u5206:\u79D2,\u6BEB\u79D2 \u4F18\u5148\u7EA7] [\u7C7B\u5168\u540D.\u65B9\u6CD5\u540D:\u884C\u53F7][\u7EBF\u7A0B\u540D] \u65E5\u5FD7\u5185\u5BB9 +log4j.appender.console.layout.ConversionPattern=[%d{HH:mm:ss,SSS} %5p][%c.%M():%-3L][%t]: %m%n + +log4j.appender.logfile=org.apache.log4j.DailyRollingFileAppender +log4j.appender.logfile.File=logs/running.log. +log4j.appender.logfile.DatePattern=yyyy-MM-dd +log4j.appender.logfile.layout=org.apache.log4j.PatternLayout +#[\u65F6:\u5206:\u79D2,\u6BEB\u79D2 \u4F18\u5148\u7EA7] [\u7C7B\u5168\u540D.\u65B9\u6CD5\u540D:\u884C\u53F7][\u7EBF\u7A0B\u540D] \u65E5\u5FD7\u5185\u5BB9 +log4j.appender.logfile.layout.ConversionPattern=[%d{HH:mm:ss,SSS} %5p][%c.%M():%-3L][%t]: %m%n \ No newline at end of file diff --git a/src/main/resources/simple.properties b/src/main/resources/simple.properties new file mode 100644 index 0000000..4d96c1c --- /dev/null +++ b/src/main/resources/simple.properties @@ -0,0 +1,9 @@ +#用于访问Pixiv的代理服务器 +reptile.proxy.type=socks5/socks4/http +reptile.proxy.host=127.0.0.1 +reptile.proxy.port=1080 +reptile.proxy.username= +reptile.proxy.password= +#登录用代理, 需要让浏览器使用该代理, 访问Pixiv并登录 +login.proxy.host=127.0.0.1 +login.proxy.port=1080 diff --git a/src/test/java/net/lamgc/cgj/pixiv/PixivDownloadTest.java b/src/test/java/net/lamgc/cgj/pixiv/PixivDownloadTest.java new file mode 100644 index 0000000..97ce6a8 --- /dev/null +++ b/src/test/java/net/lamgc/cgj/pixiv/PixivDownloadTest.java @@ -0,0 +1,202 @@ +package net.lamgc.cgj.pixiv; + +import org.apache.commons.io.IOUtils; +import org.apache.http.HttpHost; +import org.apache.http.client.CookieStore; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Ignore; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.*; +import java.text.SimpleDateFormat; +import java.util.Calendar; +import java.util.Date; +import java.util.GregorianCalendar; +import java.util.Objects; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +@Ignore +public class PixivDownloadTest { + + private static CookieStore cookieStore; + + private final static Logger log = LoggerFactory.getLogger("PixivDownloadTest"); + + private static HttpHost proxy = new HttpHost("127.0.0.1", 1001); + + @BeforeClass + public static void before() throws IOException, ClassNotFoundException { + ObjectInputStream ois = new ObjectInputStream(new FileInputStream(new File("cookies.store"))); + cookieStore = (CookieStore) ois.readObject(); + ois.close(); + log.info("已载入CookieStore"); + } + + @Test + public void logOutTest() throws IOException { + PixivDownload pixivDownload = new PixivDownload(Objects.requireNonNull(cookieStore), proxy); + Assert.assertTrue(pixivDownload.logOut()); + } + + @Test + public void collectionDownloadTest() throws IOException { + PixivDownload pixivDownload = new PixivDownload(Objects.requireNonNull(cookieStore), proxy); + File outputFile = new File("collection.zip"); + if(!outputFile.exists() && !outputFile.createNewFile()) { + Assert.fail("文件创建失败: " + outputFile.getAbsolutePath()); + } + ZipOutputStream zos = new ZipOutputStream(new FileOutputStream(outputFile)); + zos.setLevel(9); + log.info("正在调用方法..."); + pixivDownload.getCollectionAsInputStream(PixivDownload.PageQuality.ORIGINAL, (link, inputStream) -> { + try { + ZipEntry entry = new ZipEntry(link.substring(link.lastIndexOf("/") + 1)); + log.info("正在写入: " + entry.getName()); + zos.putNextEntry(entry); + IOUtils.copy(inputStream, zos); + zos.flush(); + } catch (IOException e) { + log.error("写入文件项时发生异常", e); + } + }); + log.info("调用完成."); + zos.close(); + } + + @Test + public void getRecommendsTest() throws IOException { + PixivDownload pixivDownload = new PixivDownload(Objects.requireNonNull(cookieStore), proxy); + String date = new SimpleDateFormat("yyyyMMdd").format(new Date()); + int id = 1; + File outputFile = new File("recommends-" + date + "-" + id + ".zip"); + while(outputFile.exists()) { + id++; + outputFile = new File("recommends-" + date + "-" + id + ".zip"); + } + + if(!outputFile.createNewFile()) { + Assert.fail("文件创建失败: " + outputFile.getAbsolutePath()); + } + ZipOutputStream zos = new ZipOutputStream(new FileOutputStream(outputFile)); + zos.setLevel(9); + log.info("正在调用方法..."); + pixivDownload.getRecommendAsInputStream(PixivDownload.PageQuality.ORIGINAL, (link, inputStream) -> { + try { + ZipEntry entry = new ZipEntry(link.substring(link.lastIndexOf("/") + 1)); + log.info("正在写入: " + entry.getName()); + zos.putNextEntry(entry); + IOUtils.copy(inputStream, zos); + zos.flush(); + log.info("已成功写入 {}", entry.getName()); + } catch (IOException e) { + log.error("写入文件项时发生异常", e); + } + }); + log.info("调用完成."); + zos.close(); + } + + @Test + public void getRankingIllustByRangeTest() throws IOException { + PixivDownload pixivDownload = new PixivDownload(cookieStore, proxy); + + GregorianCalendar gregorianCalendar = new GregorianCalendar(); + gregorianCalendar.setTime(new Date()); + gregorianCalendar.add(Calendar.DATE, -1); + Date queryDate = gregorianCalendar.getTime(); + String date = new SimpleDateFormat("yyyyMMdd").format(queryDate); + log.info("查询时间: {}", date); + + + int id = 1; + File outputFile = new File("ranking-" + date + "-" + id + ".zip"); + while(outputFile.exists()) { + id++; + outputFile = new File("ranking-" + date + "-" + id + ".zip"); + } + + if(!outputFile.createNewFile()) { + Assert.fail("文件创建失败: " + outputFile.getAbsolutePath()); + } + ZipOutputStream zos = new ZipOutputStream(new FileOutputStream(outputFile)); + zos.setLevel(9); + log.info("正在调用方法..."); + try { + pixivDownload.getRankingAsInputStream(null, PixivURL.RankingMode.MODE_DAILY_R18, queryDate, 15, PixivDownload.PageQuality.ORIGINAL, (rank, link, rankInfo, inputStream) -> { + try { + ZipEntry entry = new ZipEntry("Rank" + rank + "-" + link.substring(link.lastIndexOf("/") + 1)); + entry.setComment(rankInfo.toString()); + log.info("正在写入: " + entry.getName()); + zos.putNextEntry(entry); + IOUtils.copy(inputStream, zos); + zos.flush(); + log.info("已成功写入 {}", entry.getName()); + } catch (IOException e) { + log.error("写入文件项时发生异常", e); + } + }); + } finally { + zos.finish(); + zos.flush(); + zos.close(); + } + log.info("调用完成."); + } + + @Test + public void getRankingIllustTest() throws IOException { + PixivDownload pixivDownload = new PixivDownload(cookieStore, proxy); + + GregorianCalendar gregorianCalendar = new GregorianCalendar(); + gregorianCalendar.setTime(new Date()); + gregorianCalendar.add(Calendar.DATE, -1); + Date queryDate = gregorianCalendar.getTime(); + String date = new SimpleDateFormat("yyyyMMdd").format(queryDate); + log.info("查询时间: {}", date); + + int id = 1; + File outputFile = new File("ranking-" + date + "-" + id + ".zip"); + while(outputFile.exists()) { + id++; + outputFile = new File("ranking-" + date + "-" + id + ".zip"); + } + + if(!outputFile.createNewFile()) { + Assert.fail("文件创建失败: " + outputFile.getAbsolutePath()); + } + ZipOutputStream zos = new ZipOutputStream(new FileOutputStream(outputFile)); + zos.setLevel(9); + + log.info("正在调用方法..."); + try { + pixivDownload.getRankingAsInputStream(null, null, queryDate, 65, 55, PixivDownload.PageQuality.ORIGINAL, (rank, link, rankInfo, inputStream) -> { + try { + ZipEntry entry = new ZipEntry("Rank" + rank + "-" + link.substring(link.lastIndexOf("/") + 1)); + entry.setComment(rankInfo.toString()); + log.info("正在写入: " + entry.getName()); + zos.putNextEntry(entry); + IOUtils.copy(inputStream, zos); + zos.flush(); + log.info("已成功写入 {}", entry.getName()); + } catch (IOException e) { + log.error("写入文件项时发生异常", e); + } + }); + } finally { + zos.finish(); + zos.flush(); + zos.close(); + } + log.info("调用完成."); + } + + @Test + public void getIllustPreLoadDataByIdTest() throws IOException { + log.info(new PixivDownload(cookieStore, proxy).getIllustPreLoadDataById(64076261).toString()); + } + +} diff --git a/src/test/java/net/lamgc/cgj/pixiv/PixivSearchBuilderTest.java b/src/test/java/net/lamgc/cgj/pixiv/PixivSearchBuilderTest.java new file mode 100644 index 0000000..2066cbc --- /dev/null +++ b/src/test/java/net/lamgc/cgj/pixiv/PixivSearchBuilderTest.java @@ -0,0 +1,15 @@ +package net.lamgc.cgj.pixiv; + +import org.junit.Test; + +public class PixivSearchBuilderTest { + + @Test + public void buildTest() { + PixivSearchBuilder builder = new PixivSearchBuilder("hololive"); + //builder.addExcludeKeyword("fubuki").addExcludeKeyword("minato"); + builder.addIncludeKeyword("35").addIncludeKeyword("okayu").addIncludeKeyword("百鬼あやめ"); + System.out.println(builder.buildURL()); + } + +}