mirror of
https://github.com/LamGC/ContentGrabbingJi.git
synced 2025-04-30 06:37:36 +00:00
initial commit
This commit is contained in:
commit
532d7371d8
6
.gitignore
vendored
Normal file
6
.gitignore
vendored
Normal file
@ -0,0 +1,6 @@
|
|||||||
|
/pluginData/
|
||||||
|
/logs/
|
||||||
|
/.idea/
|
||||||
|
/CGJ_2.iml
|
||||||
|
/cookies.store
|
||||||
|
/target/
|
150
pom.xml
Normal file
150
pom.xml
Normal file
@ -0,0 +1,150 @@
|
|||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||||
|
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||||
|
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||||
|
<modelVersion>4.0.0</modelVersion>
|
||||||
|
|
||||||
|
<groupId>org.example</groupId>
|
||||||
|
<artifactId>CGJ_2</artifactId>
|
||||||
|
<version>1.0-SNAPSHOT</version>
|
||||||
|
|
||||||
|
<properties>
|
||||||
|
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||||
|
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
|
||||||
|
<maven.compiler.encoding>UTF-8</maven.compiler.encoding>
|
||||||
|
</properties>
|
||||||
|
|
||||||
|
<build>
|
||||||
|
<plugins>
|
||||||
|
<plugin>
|
||||||
|
<groupId>org.apache.maven.plugins</groupId>
|
||||||
|
<artifactId>maven-compiler-plugin</artifactId>
|
||||||
|
<configuration>
|
||||||
|
<source>8</source>
|
||||||
|
<target>8</target>
|
||||||
|
</configuration>
|
||||||
|
</plugin>
|
||||||
|
<!--<plugin>
|
||||||
|
<groupId>org.apache.maven.plugins</groupId>
|
||||||
|
<artifactId>maven-shade-plugin</artifactId>
|
||||||
|
<version>3.2.2</version>
|
||||||
|
<executions>
|
||||||
|
<execution>
|
||||||
|
<phase>package</phase>
|
||||||
|
<goals>
|
||||||
|
<goal>shade</goal>
|
||||||
|
</goals>
|
||||||
|
<configuration>
|
||||||
|
<transformers>
|
||||||
|
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
|
||||||
|
<mainClass>net.lamgc.cgj.Main</mainClass>
|
||||||
|
</transformer>
|
||||||
|
</transformers>
|
||||||
|
<filters>
|
||||||
|
<filter>
|
||||||
|
<artifact>*:*</artifact>
|
||||||
|
<excludes>
|
||||||
|
<exclude>META-INF/*.SF</exclude>
|
||||||
|
<exclude>META-INF/*.DSA</exclude>
|
||||||
|
<exclude>META-INF/*.RSA</exclude>
|
||||||
|
</excludes>
|
||||||
|
</filter>
|
||||||
|
</filters>
|
||||||
|
</configuration>
|
||||||
|
</execution>
|
||||||
|
</executions>
|
||||||
|
</plugin>-->
|
||||||
|
<plugin>
|
||||||
|
<groupId>org.springframework.boot</groupId>
|
||||||
|
<artifactId>spring-boot-maven-plugin</artifactId>
|
||||||
|
<configuration>
|
||||||
|
<mainClass>net.lamgc.cgj.Main</mainClass>
|
||||||
|
<classifier>exec</classifier>
|
||||||
|
</configuration>
|
||||||
|
<executions>
|
||||||
|
<execution>
|
||||||
|
<goals>
|
||||||
|
<goal>repackage</goal>
|
||||||
|
</goals>
|
||||||
|
</execution>
|
||||||
|
</executions>
|
||||||
|
</plugin>
|
||||||
|
</plugins>
|
||||||
|
</build>
|
||||||
|
|
||||||
|
<dependencies>
|
||||||
|
<dependency>
|
||||||
|
<groupId>com.github.monkeywie</groupId>
|
||||||
|
<artifactId>proxyee</artifactId>
|
||||||
|
<version>1.0.4</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.slf4j</groupId>
|
||||||
|
<artifactId>slf4j-api</artifactId>
|
||||||
|
<version>1.7.30</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.slf4j</groupId>
|
||||||
|
<artifactId>slf4j-log4j12</artifactId>
|
||||||
|
<version>1.7.30</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>net.lamgc</groupId>
|
||||||
|
<artifactId>java-utils</artifactId>
|
||||||
|
<version>1.1.0_5-SNAPSHOT</version>
|
||||||
|
</dependency>
|
||||||
|
|
||||||
|
<dependency>
|
||||||
|
<groupId>junit</groupId>
|
||||||
|
<artifactId>junit</artifactId>
|
||||||
|
<version>4.12</version>
|
||||||
|
<scope>test</scope>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.httpcomponents</groupId>
|
||||||
|
<artifactId>httpclient</artifactId>
|
||||||
|
<version>4.5.12</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.jsoup</groupId>
|
||||||
|
<artifactId>jsoup</artifactId>
|
||||||
|
<version>1.11.3</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>com.google.code.gson</groupId>
|
||||||
|
<artifactId>gson</artifactId>
|
||||||
|
<version>2.8.5</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>net.sourceforge.htmlunit</groupId>
|
||||||
|
<artifactId>htmlunit</artifactId>
|
||||||
|
<version>2.29</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>com.github.rholder</groupId>
|
||||||
|
<artifactId>guava-retrying</artifactId>
|
||||||
|
<version>2.0.0</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>mysql</groupId>
|
||||||
|
<artifactId>mysql-connector-java</artifactId>
|
||||||
|
<version>8.0.19</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>net.lz1998</groupId>
|
||||||
|
<artifactId>spring-cq</artifactId>
|
||||||
|
<version>4.14.0.6</version>
|
||||||
|
<exclusions>
|
||||||
|
<exclusion>
|
||||||
|
<groupId>ch.qos.logback</groupId>
|
||||||
|
<artifactId>logback-classic</artifactId>
|
||||||
|
</exclusion>
|
||||||
|
<exclusion>
|
||||||
|
<groupId>ch.qos.logback</groupId>
|
||||||
|
<artifactId>logback-core</artifactId>
|
||||||
|
</exclusion>
|
||||||
|
</exclusions>
|
||||||
|
</dependency>
|
||||||
|
</dependencies>
|
||||||
|
|
||||||
|
</project>
|
24
search.txt
Normal file
24
search.txt
Normal file
@ -0,0 +1,24 @@
|
|||||||
|
搜索标签信息:https://www.pixiv.net/ajax/search/tags/标签名
|
||||||
|
|
||||||
|
搜索接口:
|
||||||
|
https://www.pixiv.net/ajax/search/{Type}/搜索内容
|
||||||
|
Type = illustrations(插画) / top(顶部?) / manga(漫画) / novels(小说)
|
||||||
|
|
||||||
|
word=搜索内容 [参数可能不是必须的]
|
||||||
|
s_mode=s_tag(标签-部分一致) / s_tag_full(标签-完全一致) / s_tc(标题、说明文字)
|
||||||
|
type= all(插画、漫画、动图_动态插图) / illust_and_ugoira(插画、动图) / illust(插画) / manga(漫画) / ugoira(动图)
|
||||||
|
p=页数 [超出页数的情况下将获取不到数据(即"body.illust.data"是空数组)]
|
||||||
|
order=date(按旧排序) / date_d(按新排序) / Unknown(按热门度排序, 需要会员)
|
||||||
|
mode= all(全部) / safe(全年龄) / r18(咳咳)
|
||||||
|
|
||||||
|
可选参数:
|
||||||
|
wlt=最小宽度像素
|
||||||
|
wgt=最高宽度像素
|
||||||
|
hlt=最小高度像素
|
||||||
|
hgt=最高高度像素
|
||||||
|
ratio=0.5(横图) / -0.5(纵图) / 0(正方形) [可能不能改变参数, 三个值是固定的]
|
||||||
|
tool=使用工具, 不是很重要晚些再加
|
||||||
|
scd=开始时间(yyyy-MM-dd)
|
||||||
|
ecd=结束时间(yyyy-MM-dd)
|
||||||
|
|
||||||
|
最小收藏数 = 收藏数限定参数为会员功能, 无法获取
|
58
src/main/java/net/lamgc/cgj/CQBotAdminProcess.java
Normal file
58
src/main/java/net/lamgc/cgj/CQBotAdminProcess.java
Normal file
@ -0,0 +1,58 @@
|
|||||||
|
package net.lamgc.cgj;
|
||||||
|
|
||||||
|
import com.google.common.base.Strings;
|
||||||
|
import net.lamgc.utils.base.runner.Argument;
|
||||||
|
import net.lamgc.utils.base.runner.Command;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
|
import java.io.FileOutputStream;
|
||||||
|
import java.io.IOException;
|
||||||
|
|
||||||
|
public class CQBotAdminProcess {
|
||||||
|
|
||||||
|
private final static Logger log = LoggerFactory.getLogger("CQBotAdminProcess");
|
||||||
|
|
||||||
|
@Command
|
||||||
|
public String clearCache() {
|
||||||
|
CQProcess.clearCache();
|
||||||
|
return "操作已完成.";
|
||||||
|
}
|
||||||
|
|
||||||
|
@Command
|
||||||
|
public String setGlobalProperty(@Argument(name = "key") String key, @Argument(name = "value") String value, @Argument(name = "save", force = false) boolean saveNow) {
|
||||||
|
String lastValue = CQPluginMain.globalProp.getProperty(key);
|
||||||
|
CQPluginMain.globalProp.setProperty(key, Strings.nullToEmpty(value));
|
||||||
|
if(saveNow) {
|
||||||
|
saveGlobalProperties();
|
||||||
|
}
|
||||||
|
return "全局配置项 " + key + " 现已设置为: " + value + " (设置前的值: " + lastValue + ")";
|
||||||
|
}
|
||||||
|
|
||||||
|
@Command
|
||||||
|
public String getGlobalProperty(@Argument(name = "key") String key) {
|
||||||
|
return "全局配置项 " + key + " 当前值: " + CQPluginMain.globalProp.getProperty(key, "(Empty)");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Command
|
||||||
|
public String saveGlobalProperties() {
|
||||||
|
log.info("正在保存全局配置文件...");
|
||||||
|
File globalPropFile = new File("./global.properties");
|
||||||
|
try {
|
||||||
|
if(!globalPropFile.exists()) {
|
||||||
|
if(!globalPropFile.createNewFile()) {
|
||||||
|
log.error("全局配置项文件保存失败!({})", "文件创建失败");
|
||||||
|
return "全局配置项文件保存失败!";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
CQPluginMain.globalProp.store(new FileOutputStream(globalPropFile), "");
|
||||||
|
log.info("全局配置文件保存成功!");
|
||||||
|
return "保存全局配置文件 - 操作已完成.";
|
||||||
|
} catch (IOException e) {
|
||||||
|
log.error("全局配置项文件保存失败!", e);
|
||||||
|
return "全局配置项文件保存失败!";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
13
src/main/java/net/lamgc/cgj/CQConfig.java
Normal file
13
src/main/java/net/lamgc/cgj/CQConfig.java
Normal file
@ -0,0 +1,13 @@
|
|||||||
|
package net.lamgc.cgj;
|
||||||
|
|
||||||
|
import net.lz1998.cq.CQGlobal;
|
||||||
|
import net.lz1998.cq.EnableCQ;
|
||||||
|
|
||||||
|
@EnableCQ
|
||||||
|
public class CQConfig {
|
||||||
|
|
||||||
|
public static void init() {
|
||||||
|
CQGlobal.pluginList.add(CQPluginMain.class);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
147
src/main/java/net/lamgc/cgj/CQPluginMain.java
Normal file
147
src/main/java/net/lamgc/cgj/CQPluginMain.java
Normal file
@ -0,0 +1,147 @@
|
|||||||
|
package net.lamgc.cgj;
|
||||||
|
|
||||||
|
import com.google.common.base.Strings;
|
||||||
|
import net.lamgc.cgj.util.DateParser;
|
||||||
|
import net.lamgc.cgj.util.PagesQualityParser;
|
||||||
|
import net.lamgc.utils.base.runner.ArgumentsRunner;
|
||||||
|
import net.lamgc.utils.base.runner.ArgumentsRunnerConfig;
|
||||||
|
import net.lamgc.utils.base.runner.exception.DeveloperRunnerException;
|
||||||
|
import net.lamgc.utils.base.runner.exception.NoSuchCommandException;
|
||||||
|
import net.lamgc.utils.base.runner.exception.ParameterNoFoundException;
|
||||||
|
import net.lz1998.cq.event.message.CQDiscussMessageEvent;
|
||||||
|
import net.lz1998.cq.event.message.CQGroupMessageEvent;
|
||||||
|
import net.lz1998.cq.event.message.CQMessageEvent;
|
||||||
|
import net.lz1998.cq.event.message.CQPrivateMessageEvent;
|
||||||
|
import net.lz1998.cq.robot.CQPlugin;
|
||||||
|
import net.lz1998.cq.robot.CoolQ;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
import org.springframework.stereotype.Component;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
|
import java.io.FileInputStream;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.text.SimpleDateFormat;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.Arrays;
|
||||||
|
import java.util.Objects;
|
||||||
|
import java.util.Properties;
|
||||||
|
import java.util.regex.Matcher;
|
||||||
|
import java.util.regex.Pattern;
|
||||||
|
|
||||||
|
@Component
|
||||||
|
public class CQPluginMain extends CQPlugin {
|
||||||
|
|
||||||
|
private final static String COMMAND_PREFIX = ".cgj";
|
||||||
|
private final Logger log = LoggerFactory.getLogger("CQPluginMain@" + Integer.toHexString(this.hashCode()));
|
||||||
|
private final ArgumentsRunnerConfig runnerConfig = new ArgumentsRunnerConfig();
|
||||||
|
public final static Properties globalProp = new Properties();
|
||||||
|
|
||||||
|
public CQPluginMain() {
|
||||||
|
runnerConfig.setUseDefaultValueInsteadOfException(true);
|
||||||
|
runnerConfig.setCommandIgnoreCase(true);
|
||||||
|
runnerConfig.addStringParameterParser(new DateParser(new SimpleDateFormat("yyyy-MM-dd")));
|
||||||
|
runnerConfig.addStringParameterParser(new PagesQualityParser());
|
||||||
|
|
||||||
|
File globalPropFile = new File("./global.properties");
|
||||||
|
if(globalPropFile.exists() && globalPropFile.isFile()) {
|
||||||
|
log.info("正在加载全局配置文件...");
|
||||||
|
try {
|
||||||
|
globalProp.load(new FileInputStream(globalPropFile));
|
||||||
|
log.info("全局配置文件加载完成.");
|
||||||
|
} catch (IOException e) {
|
||||||
|
log.error("加载全局配置文件时发生异常", e);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
log.info("未找到全局配置文件,跳过加载.");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public int onPrivateMessage(CoolQ cq, CQPrivateMessageEvent event) {
|
||||||
|
//log.info("私聊消息到达: 发送者[{}], 消息内容: {}", event.getSender().getUserId(), event.getMessage());
|
||||||
|
return processMessage(cq, event);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public int onGroupMessage(CoolQ cq, CQGroupMessageEvent event) {
|
||||||
|
//log.info("群消息到达: 群[{}], 发送者[{}], 消息内容: {}", event.getGroupId(), event.getSender().getUserId(), event.getMessage());
|
||||||
|
return processMessage(cq, event);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public int onDiscussMessage(CoolQ cq, CQDiscussMessageEvent event) {
|
||||||
|
//log.info("讨论组消息到达: 群[{}], 发送者[{}], 消息内容: {}", event.getDiscussId(), event.getSender().getUserId(), event.getMessage());
|
||||||
|
return processMessage(cq, event);
|
||||||
|
}
|
||||||
|
|
||||||
|
public int processMessage(CoolQ cq, CQMessageEvent event) {
|
||||||
|
String msg = event.getMessage();
|
||||||
|
if(!msg.startsWith(COMMAND_PREFIX)) {
|
||||||
|
return MESSAGE_IGNORE;
|
||||||
|
}
|
||||||
|
|
||||||
|
Pattern pattern = Pattern.compile("/\\s*(\".+?\"|[^:\\s])+((\\s*:\\s*(\".+?\"|[^\\s])+)|)|(\".+?\"|[^\"\\s])+");
|
||||||
|
Matcher matcher = pattern.matcher(Strings.nullToEmpty(msg));
|
||||||
|
ArrayList<String> argsList = new ArrayList<>();
|
||||||
|
while (matcher.find()) {
|
||||||
|
argsList.add(matcher.group());
|
||||||
|
}
|
||||||
|
String[] args = new String[argsList.size()];
|
||||||
|
argsList.toArray(args);
|
||||||
|
|
||||||
|
log.info("正在处理命令...");
|
||||||
|
long time = System.currentTimeMillis();
|
||||||
|
Object result;
|
||||||
|
try {
|
||||||
|
if(msg.toLowerCase().startsWith(COMMAND_PREFIX + "admin")) {
|
||||||
|
if(!String.valueOf(event.getUserId()).equals(globalProp.getProperty("admin.adminId"))) {
|
||||||
|
sendMessage(cq, event, "你没有执行该命令的权限!", false);
|
||||||
|
return MESSAGE_BLOCK;
|
||||||
|
} else {
|
||||||
|
result = new ArgumentsRunner(CQBotAdminProcess.class, runnerConfig)
|
||||||
|
.run(new CQBotAdminProcess(), args.length <= 1 ? new String[0] : Arrays.copyOfRange(args, 1, args.length));
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
result = new ArgumentsRunner(CQProcess.class, runnerConfig).run(args.length <= 1 ? new String[0] : Arrays.copyOfRange(args, 1, args.length));
|
||||||
|
}
|
||||||
|
} catch(NoSuchCommandException e) {
|
||||||
|
result = "没有这个命令!请使用“.cgj”查看帮助说明!";
|
||||||
|
} catch(ParameterNoFoundException e) {
|
||||||
|
result = "命令缺少参数: " + e.getParameterName();
|
||||||
|
} catch(DeveloperRunnerException e) {
|
||||||
|
log.error("执行命令时发生异常", e);
|
||||||
|
result = "命令执行时发生错误,无法完成!";
|
||||||
|
}
|
||||||
|
log.info("命令处理完成(耗时: {}ms)", System.currentTimeMillis() - time);
|
||||||
|
if(Objects.requireNonNull(result) instanceof String) {
|
||||||
|
sendMessage(cq, event, (String) result, false);
|
||||||
|
}
|
||||||
|
return MESSAGE_BLOCK;
|
||||||
|
}
|
||||||
|
|
||||||
|
private final static Logger msgLog = LoggerFactory.getLogger("SendMsg");
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 发送消息
|
||||||
|
* @param cq CoolQ对象
|
||||||
|
* @param event 消息事件对象
|
||||||
|
* @param message 消息内容
|
||||||
|
* @param auto_escape 消息内容是否作为纯文本发送(即不解析 CQ 码),只在 message 字段是字符串时有效.
|
||||||
|
*/
|
||||||
|
public static void sendMessage(CoolQ cq, CQMessageEvent event, String message, boolean auto_escape) {
|
||||||
|
msgLog.debug("发送消息:{}", message);
|
||||||
|
if(event instanceof CQPrivateMessageEvent) {
|
||||||
|
CQPrivateMessageEvent _event = (CQPrivateMessageEvent) event;
|
||||||
|
cq.sendPrivateMsg(_event.getSender().getUserId(), message, auto_escape);
|
||||||
|
} else if(event instanceof CQGroupMessageEvent) {
|
||||||
|
CQGroupMessageEvent _event = (CQGroupMessageEvent) event;
|
||||||
|
cq.sendGroupMsg(_event.getGroupId(), message, auto_escape);
|
||||||
|
} else if(event instanceof CQDiscussMessageEvent) {
|
||||||
|
CQDiscussMessageEvent _event = (CQDiscussMessageEvent) event;
|
||||||
|
cq.sendGroupMsg(_event.getDiscussId(), message, auto_escape).getData().getMessageId();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
}
|
553
src/main/java/net/lamgc/cgj/CQProcess.java
Normal file
553
src/main/java/net/lamgc/cgj/CQProcess.java
Normal file
@ -0,0 +1,553 @@
|
|||||||
|
package net.lamgc.cgj;
|
||||||
|
|
||||||
|
import com.google.common.base.Strings;
|
||||||
|
import com.google.gson.*;
|
||||||
|
import io.netty.handler.codec.http.HttpHeaderNames;
|
||||||
|
import net.lamgc.cgj.cache.CacheObject;
|
||||||
|
import net.lamgc.cgj.cache.ImageCacheHandler;
|
||||||
|
import net.lamgc.cgj.cache.ImageCacheObject;
|
||||||
|
import net.lamgc.cgj.pixiv.PixivDownload;
|
||||||
|
import net.lamgc.cgj.pixiv.PixivSearchBuilder;
|
||||||
|
import net.lamgc.cgj.pixiv.PixivURL;
|
||||||
|
import net.lamgc.utils.base.runner.Argument;
|
||||||
|
import net.lamgc.utils.base.runner.Command;
|
||||||
|
import net.lamgc.utils.event.EventExecutor;
|
||||||
|
import net.lz1998.cq.utils.CQCode;
|
||||||
|
import org.apache.http.HttpResponse;
|
||||||
|
import org.apache.http.client.methods.HttpGet;
|
||||||
|
import org.apache.http.client.methods.HttpHead;
|
||||||
|
import org.apache.http.util.EntityUtils;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
import java.io.*;
|
||||||
|
import java.nio.charset.StandardCharsets;
|
||||||
|
import java.nio.file.Files;
|
||||||
|
import java.nio.file.StandardOpenOption;
|
||||||
|
import java.text.SimpleDateFormat;
|
||||||
|
import java.util.*;
|
||||||
|
import java.util.concurrent.ArrayBlockingQueue;
|
||||||
|
import java.util.concurrent.ThreadPoolExecutor;
|
||||||
|
import java.util.concurrent.TimeUnit;
|
||||||
|
|
||||||
|
public class CQProcess {
|
||||||
|
|
||||||
|
private final static PixivDownload pixivDownload = new PixivDownload(Main.cookieStore, Main.proxy);
|
||||||
|
|
||||||
|
private final static Logger log = LoggerFactory.getLogger("CQProcess");
|
||||||
|
|
||||||
|
private final static File imageStoreDir = new File(System.getProperty("cgj.cqRootDir") + "data/image/cgj/");
|
||||||
|
|
||||||
|
private final static Hashtable<String, File> imageCache = new Hashtable<>();
|
||||||
|
|
||||||
|
private final static Hashtable<Integer, JsonObject> illustInfoCache = new Hashtable<>();
|
||||||
|
|
||||||
|
private final static Hashtable<Integer, CacheObject<JsonObject>> illustPreLoadDataCache = new Hashtable<>();
|
||||||
|
|
||||||
|
private final static Hashtable<String, CacheObject<JsonObject>> searchBodyCache = new Hashtable<>();
|
||||||
|
|
||||||
|
private final static Hashtable<String, List<String>> pagesCache = new Hashtable<>();
|
||||||
|
|
||||||
|
private final static Object searchCacheLock = new Object();
|
||||||
|
|
||||||
|
private final static Gson gson = new GsonBuilder()
|
||||||
|
.serializeNulls()
|
||||||
|
.create();
|
||||||
|
|
||||||
|
private final static EventExecutor imageCacheExecutor = new EventExecutor(new ThreadPoolExecutor(
|
||||||
|
1,
|
||||||
|
(int) Math.ceil(Runtime.getRuntime().availableProcessors() / 2F),
|
||||||
|
15L, TimeUnit.SECONDS,
|
||||||
|
new ArrayBlockingQueue<>(30),
|
||||||
|
new ThreadPoolExecutor.DiscardOldestPolicy()
|
||||||
|
));
|
||||||
|
|
||||||
|
static {
|
||||||
|
try {
|
||||||
|
imageCacheExecutor.addHandler(new ImageCacheHandler());
|
||||||
|
} catch (IllegalAccessException e) {
|
||||||
|
e.printStackTrace();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Command(defaultCommand = true)
|
||||||
|
public static String help() {
|
||||||
|
StringBuilder helpStrBuilder = new StringBuilder();
|
||||||
|
helpStrBuilder.append("CGJ Bot使用指南").append("\n");
|
||||||
|
helpStrBuilder.append("目前可用的命令:").append("\n");
|
||||||
|
helpStrBuilder.append("\t").append("ranking - 获取今天或指定日期排行榜的前10名作品").append("\n");
|
||||||
|
helpStrBuilder.append("\t\t").append("date - 指定查询日期(年年年年-月月-日日)").append("\n");
|
||||||
|
helpStrBuilder.append("\t").append("search - 搜索指定关键词并显示前10个作品").append("\n");
|
||||||
|
helpStrBuilder.append("\t\t").append("content - 搜索内容").append("\n");
|
||||||
|
helpStrBuilder.append("\t").append("artworks - 获取作品的Pixiv页面").append("\n");
|
||||||
|
helpStrBuilder.append("\t\t").append("id - 作品id").append("\n");
|
||||||
|
return helpStrBuilder.toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Command
|
||||||
|
public static String ranking(
|
||||||
|
@Argument(force = false, name = "date") Date queryTime,
|
||||||
|
@Argument(force = false, name = "contentMode", defaultValue = "DAILY") String contentMode
|
||||||
|
) {
|
||||||
|
Date queryDate = queryTime;
|
||||||
|
if (queryDate == null) {
|
||||||
|
queryDate = new Date();
|
||||||
|
GregorianCalendar gregorianCalendar = new GregorianCalendar();
|
||||||
|
log.info("CurrentDate: {}", queryDate);
|
||||||
|
gregorianCalendar.setTime(queryDate);
|
||||||
|
if (gregorianCalendar.get(Calendar.HOUR_OF_DAY) < 12) {
|
||||||
|
gregorianCalendar.add(Calendar.DATE, -2);
|
||||||
|
} else {
|
||||||
|
gregorianCalendar.add(Calendar.DATE, -1);
|
||||||
|
}
|
||||||
|
queryDate = gregorianCalendar.getTime();
|
||||||
|
}
|
||||||
|
|
||||||
|
PixivURL.RankingMode mode = PixivURL.RankingMode.MODE_DAILY;
|
||||||
|
try {
|
||||||
|
mode = PixivURL.RankingMode.valueOf("MODE_" + contentMode.toUpperCase());
|
||||||
|
} catch (IllegalArgumentException e) {
|
||||||
|
log.warn("无效的RankingMode值: {}", contentMode);
|
||||||
|
}
|
||||||
|
|
||||||
|
StringBuilder resultBuilder = new StringBuilder(mode.name() + " - 以下是 ").append(new SimpleDateFormat("yyyy-MM-dd").format(queryDate)).append(" 的Pixiv插画排名榜前十名:\n");
|
||||||
|
try {
|
||||||
|
int index = 0;
|
||||||
|
for (JsonObject rankInfo : pixivDownload.getRanking(PixivURL.RankingContentType.TYPE_ILLUST, mode, queryDate, 1, 10)) {
|
||||||
|
index++;
|
||||||
|
int rank = rankInfo.get("rank").getAsInt();
|
||||||
|
int illustId = rankInfo.get("illust_id").getAsInt();
|
||||||
|
int authorId = rankInfo.get("user_id").getAsInt();
|
||||||
|
String authorName = rankInfo.get("user_name").getAsString();
|
||||||
|
String title = rankInfo.get("title").getAsString();
|
||||||
|
resultBuilder.append(rank).append(". (id: ").append(illustId).append(") ").append(title)
|
||||||
|
.append("(Author: ").append(authorName).append(",").append(authorId).append(")\n");
|
||||||
|
if (index < 4) {
|
||||||
|
resultBuilder.append(getImageById(illustId, PixivDownload.PageQuality.REGULAR, 1)).append("\n");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (IOException e) {
|
||||||
|
log.error("消息处理异常", e);
|
||||||
|
return "排名榜获取失败!详情请查看机器人控制台。";
|
||||||
|
}
|
||||||
|
return resultBuilder.append("如查询当前时间获取到昨天时间,则今日排名榜未更新。").toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Command(commandName = "userArt")
|
||||||
|
public static String userArtworks() {
|
||||||
|
|
||||||
|
return "功能未完成";
|
||||||
|
}
|
||||||
|
|
||||||
|
@Command
|
||||||
|
public static String search(@Argument(name = "content") String content,
|
||||||
|
@Argument(name = "type", force = false) String type,
|
||||||
|
@Argument(name = "area", force = false) String area,
|
||||||
|
@Argument(name = "in", force = false) String includeKeywords,
|
||||||
|
@Argument(name = "ex", force = false) String excludeKeywords,
|
||||||
|
@Argument(name = "contentOption", force = false) String contentOption,
|
||||||
|
@Argument(name = "page", force = false, defaultValue = "1") int pagesIndex
|
||||||
|
) throws IOException {
|
||||||
|
PixivSearchBuilder searchBuilder = new PixivSearchBuilder(Strings.isNullOrEmpty(content) ? "" : content);
|
||||||
|
if (type != null) {
|
||||||
|
try {
|
||||||
|
searchBuilder.setSearchType(PixivSearchBuilder.SearchType.valueOf(type.toUpperCase()));
|
||||||
|
} catch (IllegalArgumentException e) {
|
||||||
|
log.warn("不支持的SearchType: {}", type);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (area != null) {
|
||||||
|
try {
|
||||||
|
searchBuilder.setSearchArea(PixivSearchBuilder.SearchArea.valueOf(area));
|
||||||
|
} catch (IllegalArgumentException e) {
|
||||||
|
log.warn("不支持的SearchArea: {}", area);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (contentOption != null) {
|
||||||
|
try {
|
||||||
|
searchBuilder.setSearchContentOption(PixivSearchBuilder.SearchContentOption.valueOf(contentOption));
|
||||||
|
} catch (IllegalArgumentException e) {
|
||||||
|
log.warn("不支持的SearchContentOption: {}", contentOption);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!Strings.isNullOrEmpty(includeKeywords)) {
|
||||||
|
for (String keyword : includeKeywords.split(";")) {
|
||||||
|
searchBuilder.removeExcludeKeyword(keyword);
|
||||||
|
searchBuilder.addIncludeKeyword(keyword);
|
||||||
|
log.info("已添加关键字: {}", keyword);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (!Strings.isNullOrEmpty(excludeKeywords)) {
|
||||||
|
for (String keyword : excludeKeywords.split(";")) {
|
||||||
|
searchBuilder.removeIncludeKeyword(keyword);
|
||||||
|
searchBuilder.addExcludeKeyword(keyword);
|
||||||
|
log.info("已添加排除关键字: {}", keyword);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
log.info("正在搜索作品, 条件: {}", searchBuilder.getSearchCondition());
|
||||||
|
|
||||||
|
String requestUrl = searchBuilder.buildURL();
|
||||||
|
log.info("RequestUrl: {}", requestUrl);
|
||||||
|
|
||||||
|
CacheObject<JsonObject> cacheObject = new CacheObject<>();
|
||||||
|
if(!searchBodyCache.containsKey(requestUrl) || (cacheObject = searchBodyCache.get(requestUrl)).isExpire(new Date())) {
|
||||||
|
synchronized (searchCacheLock) {
|
||||||
|
if (!searchBodyCache.containsKey(requestUrl) || (cacheObject = searchBodyCache.get(requestUrl)).isExpire(new Date())) {
|
||||||
|
log.info("searchBody缓存失效, 正在更新...");
|
||||||
|
JsonObject jsonObject;
|
||||||
|
HttpGet httpGetRequest = pixivDownload.createHttpGetRequest(requestUrl);
|
||||||
|
HttpResponse response = pixivDownload.getHttpClient().execute(httpGetRequest);
|
||||||
|
|
||||||
|
String responseBody = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8);
|
||||||
|
log.info("ResponseBody: {}", responseBody);
|
||||||
|
jsonObject = gson.fromJson(responseBody, JsonObject.class);
|
||||||
|
|
||||||
|
if (jsonObject.get("error").getAsBoolean()) {
|
||||||
|
log.error("接口请求错误, 错误信息: {}", jsonObject.get("message").getAsString());
|
||||||
|
return "处理命令时发生错误!";
|
||||||
|
}
|
||||||
|
|
||||||
|
Date newExpireDate = new Date();
|
||||||
|
long expire = 7200 * 1000;
|
||||||
|
String propValue = CQPluginMain.globalProp.getProperty("cache.searchBody.expire", "7200000");
|
||||||
|
try {
|
||||||
|
expire = Long.parseLong(propValue);
|
||||||
|
} catch (Exception e) {
|
||||||
|
log.warn("全局配置项 \"{}\" 值非法, 已使用默认值: {}", propValue, expire);
|
||||||
|
}
|
||||||
|
|
||||||
|
newExpireDate.setTime(newExpireDate.getTime() + expire);
|
||||||
|
cacheObject.update(jsonObject, newExpireDate);
|
||||||
|
searchBodyCache.put(requestUrl, cacheObject);
|
||||||
|
log.info("searchBody缓存已更新(到期时间: {})", newExpireDate);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
log.info("搜索缓存命中.");
|
||||||
|
}
|
||||||
|
|
||||||
|
JsonObject resultBody = searchBodyCache.get(requestUrl).get().getAsJsonObject("body");
|
||||||
|
StringBuilder result = new StringBuilder("搜索结果:");
|
||||||
|
log.info("正在处理信息...");
|
||||||
|
int limit = 8;
|
||||||
|
try {
|
||||||
|
limit = Integer.parseInt(CQPluginMain.globalProp.getProperty("search.ItemCountLimit", "8"));
|
||||||
|
} catch (Exception e) {
|
||||||
|
log.warn("参数转换异常!将使用默认值(" + limit + ")", e);
|
||||||
|
}
|
||||||
|
for (PixivSearchBuilder.SearchArea searchArea : PixivSearchBuilder.SearchArea.values()) {
|
||||||
|
if (!resultBody.has(searchArea.jsonKey) || resultBody.getAsJsonObject(searchArea.jsonKey).getAsJsonArray("data").size() == 0) {
|
||||||
|
log.debug("返回数据不包含 {}", searchArea.jsonKey);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
JsonArray illustsArray = resultBody
|
||||||
|
.getAsJsonObject(searchArea.jsonKey).getAsJsonArray("data");
|
||||||
|
ArrayList<JsonElement> illustsList = new ArrayList<>();
|
||||||
|
illustsArray.forEach(illustsList::add);
|
||||||
|
illustsList.sort((o1, o2) -> {
|
||||||
|
try {
|
||||||
|
int illustLikeCount1 = getIllustPreLoadData(o1.getAsJsonObject().get("illustId").getAsInt()).get("likeCount").getAsInt();
|
||||||
|
int illustLikeCount2 = getIllustPreLoadData(o2.getAsJsonObject().get("illustId").getAsInt()).get("likeCount").getAsInt();
|
||||||
|
return Integer.compare(illustLikeCount2, illustLikeCount1);
|
||||||
|
} catch (IOException e) {
|
||||||
|
e.printStackTrace();
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
log.info("已找到与 {} 相关插图信息({}):", content, searchArea.name().toLowerCase());
|
||||||
|
int count = 1;
|
||||||
|
for (JsonElement jsonElement : illustsList) {
|
||||||
|
if (count > limit) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
JsonObject illustObj = jsonElement.getAsJsonObject();
|
||||||
|
if (!illustObj.has("illustId")) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
int illustId = illustObj.get("illustId").getAsInt();
|
||||||
|
StringBuilder builder = new StringBuilder("[");
|
||||||
|
illustObj.get("tags").getAsJsonArray().forEach(el -> builder.append(el.getAsString()).append(", "));
|
||||||
|
builder.replace(builder.length() - 2, builder.length(), "]");
|
||||||
|
log.debug("{} ({} / {})\n\t作品id: {}, \n\t作者名(作者id): {} ({}), \n\t作品标题: {}, \n\t作品Tags: {}, \n\t作品链接: {}",
|
||||||
|
searchArea.name(),
|
||||||
|
count,
|
||||||
|
illustsList.size(),
|
||||||
|
illustId,
|
||||||
|
illustObj.get("userName").getAsString(),
|
||||||
|
illustObj.get("userId").getAsInt(),
|
||||||
|
illustObj.get("illustTitle").getAsString(),
|
||||||
|
builder,
|
||||||
|
PixivURL.getPixivRefererLink(illustId)
|
||||||
|
);
|
||||||
|
|
||||||
|
String imageMsg = getImageById(illustId, PixivDownload.PageQuality.REGULAR, 1);
|
||||||
|
if (isNoSafe(illustId, CQPluginMain.globalProp, true)) {
|
||||||
|
log.warn("作品Id {} 为R-18作品, 跳过.", illustId);
|
||||||
|
count--;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
result.append(searchArea.name()).append(" (").append(count).append(" / ").append(illustsList.size()).append(")\n\t作品id: ").append(illustId)
|
||||||
|
.append(", \n\t作者名: ").append(illustObj.get("userName").getAsString())
|
||||||
|
.append("\n\t作品标题: ").append(illustObj.get("illustTitle").getAsString()).append("\n").append(imageMsg).append("\n");
|
||||||
|
count++;
|
||||||
|
}
|
||||||
|
if (count > limit) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return Strings.nullToEmpty(result.toString()) + "预览图片并非原图,使用“.cgj image -id 作品id”获取原图";
|
||||||
|
}
|
||||||
|
|
||||||
|
@Command(commandName = "pages")
|
||||||
|
public static String getPagesList(@Argument(name = "id") int illustId, @Argument(name = "quality", force = false) PixivDownload.PageQuality quality) {
|
||||||
|
try {
|
||||||
|
List<String> pagesList = PixivDownload.getIllustAllPageDownload(pixivDownload.getHttpClient(), pixivDownload.getCookieStore(), illustId, quality);
|
||||||
|
StringBuilder builder = new StringBuilder("作品ID ").append(illustId).append(" 共有").append(pagesList.size()).append("页:").append("\n");
|
||||||
|
int index = 0;
|
||||||
|
for (String link : pagesList) {
|
||||||
|
builder.append("Page ").append(++index).append(": ").append(link).append("\n");
|
||||||
|
}
|
||||||
|
return builder.toString();
|
||||||
|
} catch (IOException e) {
|
||||||
|
log.error("获取作品所有页面下载链接失败!", e);
|
||||||
|
return "发生错误,无法完成命令";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Command(commandName = "artworks")
|
||||||
|
public static String artworksLink(@Argument(name = "id") int illustId) {
|
||||||
|
try {
|
||||||
|
if (isNoSafe(illustId, CQPluginMain.globalProp, false)) {
|
||||||
|
log.warn("作品Id {} 已被屏蔽.", illustId);
|
||||||
|
return "由于相关设置,该作品已被屏蔽!";
|
||||||
|
}
|
||||||
|
} catch (IOException e) {
|
||||||
|
log.error("获取作品信息失败!", e);
|
||||||
|
return "作品信息无法获取!";
|
||||||
|
}
|
||||||
|
return PixivURL.getPixivRefererLink(illustId);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Command(commandName = "image")
|
||||||
|
public static String getImageById(@Argument(name = "id") int illustId,
|
||||||
|
@Argument(name = "quality", force = false) PixivDownload.PageQuality quality,
|
||||||
|
@Argument(name = "page", force = false, defaultValue = "1") int pageIndex) {
|
||||||
|
log.info("IllustId: {}, Quality: {}, PageIndex: {}", illustId, quality.name(), pageIndex);
|
||||||
|
List<String> pagesList;
|
||||||
|
try {
|
||||||
|
pagesList = getIllustPages(illustId, quality);
|
||||||
|
} catch (IOException e) {
|
||||||
|
log.error("获取下载链接列表时发生异常", e);
|
||||||
|
return "发生网络异常,无法获取图片!";
|
||||||
|
}
|
||||||
|
if (pagesList.size() < pageIndex || pageIndex <= 0) {
|
||||||
|
log.warn("指定的页数超出了总页数({} / {})", pageIndex, pagesList.size());
|
||||||
|
return "指定的页数超出了范围(总共 " + pagesList.size() + " 页)";
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
if (isNoSafe(illustId, CQPluginMain.globalProp, false)) {
|
||||||
|
log.warn("作品 {} 存在R-18内容且设置\"image.allowR18\"为false,将屏蔽该作品不发送.", illustId);
|
||||||
|
pageIndex = -1;
|
||||||
|
}
|
||||||
|
} catch (IOException e) {
|
||||||
|
log.warn("作品信息无法获取!", e);
|
||||||
|
return "发生网络异常,无法获取图片!";
|
||||||
|
}
|
||||||
|
|
||||||
|
int index = 0;
|
||||||
|
String targetLink = null;
|
||||||
|
File targetFile;
|
||||||
|
File currentImageFile;
|
||||||
|
for (String link : pagesList) {
|
||||||
|
index++;
|
||||||
|
if (index == pageIndex) {
|
||||||
|
targetLink = link;
|
||||||
|
}
|
||||||
|
currentImageFile = new File(getImageStoreDir(), link.substring(link.lastIndexOf("/") + 1));
|
||||||
|
if (!imageCache.containsKey(link)) {
|
||||||
|
HttpHead headRequest = new HttpHead(link);
|
||||||
|
headRequest.addHeader("Referer", PixivURL.getPixivRefererLink(illustId));
|
||||||
|
HttpResponse headResponse;
|
||||||
|
try {
|
||||||
|
headResponse = pixivDownload.getHttpClient().execute(headRequest);
|
||||||
|
} catch (IOException e) {
|
||||||
|
log.error("获取图片大小失败!", e);
|
||||||
|
return "图片获取失败!";
|
||||||
|
}
|
||||||
|
String contentLengthStr = headResponse.getFirstHeader(HttpHeaderNames.CONTENT_LENGTH.toString()).getValue();
|
||||||
|
if (currentImageFile.exists() && currentImageFile.length() == Long.parseLong(contentLengthStr)) {
|
||||||
|
imageCache.put(link, currentImageFile);
|
||||||
|
log.info("作品Id {} 第 {} 页缓存已补充.", illustId, index);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (index == pageIndex) {
|
||||||
|
try {
|
||||||
|
imageCacheExecutor.executorSync(new ImageCacheObject(imageCache, illustId, link, currentImageFile));
|
||||||
|
} catch (InterruptedException e) {
|
||||||
|
log.warn("图片下载遭到中断!", e);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
imageCacheExecutor.executor(
|
||||||
|
new ImageCacheObject(imageCache, illustId, link, currentImageFile));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (pageIndex == -1) {
|
||||||
|
return "(根据设置,该作品已被屏蔽!)";
|
||||||
|
}
|
||||||
|
|
||||||
|
if (targetLink == null) {
|
||||||
|
return "未找到对应的图片!";
|
||||||
|
}
|
||||||
|
|
||||||
|
targetFile = imageCache.get(targetLink);
|
||||||
|
return targetFile == null ? "未找到对应的图片!" : CQCode.image(getImageStoreDir().getName() + "/" + targetFile.getName());
|
||||||
|
}
|
||||||
|
|
||||||
|
static void clearCache() {
|
||||||
|
log.warn("正在清除所有图片缓存...");
|
||||||
|
imageCache.clear();
|
||||||
|
File imageStoreDir = new File(System.getProperty("cgj.cqRootDir") + "data/image/cgj/");
|
||||||
|
File[] listFiles = imageStoreDir.listFiles();
|
||||||
|
if (listFiles == null) {
|
||||||
|
log.info("图片缓存目录为空或内部文件获取失败!");
|
||||||
|
} else {
|
||||||
|
for (File file : listFiles) {
|
||||||
|
log.info("图片文件 {} 删除: {}", file.getName(), file.delete());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
log.info("图片缓存目录删除: {}", imageStoreDir.delete());
|
||||||
|
log.warn("缓存删除完成.");
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
下一目标:
|
||||||
|
添加定时发图
|
||||||
|
定时发图支持设置关注标签
|
||||||
|
标签....标签支持搜索吧
|
||||||
|
*/
|
||||||
|
|
||||||
|
private static boolean isNoSafe(int illustId, Properties settingProp, boolean returnRaw) throws IOException {
|
||||||
|
boolean rawValue = getIllustInfo(illustId).getAsJsonArray("tags").contains(new JsonPrimitive("R-18"));
|
||||||
|
return returnRaw || settingProp == null ? rawValue : rawValue && !settingProp.getProperty("image.allowR18", "false").equalsIgnoreCase("true");
|
||||||
|
}
|
||||||
|
|
||||||
|
private final static Object illustInfoLock = new Object();
|
||||||
|
private static JsonObject getIllustInfo(int illustId) throws IOException {
|
||||||
|
synchronized (illustInfoLock) {
|
||||||
|
File cacheFile = new File(getImageStoreDir(), illustId + ".illustInfo.json");
|
||||||
|
if (!illustInfoCache.containsKey(illustId)) {
|
||||||
|
log.info("IllustInfoFileName: {}", cacheFile.getName());
|
||||||
|
JsonObject illustInfoObj;
|
||||||
|
if (!cacheFile.exists()) {
|
||||||
|
try {
|
||||||
|
cacheFile.createNewFile();
|
||||||
|
illustInfoObj = pixivDownload.getIllustInfoByIllustId(illustId);
|
||||||
|
Files.write(cacheFile.toPath(), gson.toJson(illustInfoObj).getBytes(StandardCharsets.UTF_8), StandardOpenOption.CREATE);
|
||||||
|
} catch(IOException e) {
|
||||||
|
cacheFile.delete();
|
||||||
|
throw e;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
illustInfoObj = gson.fromJson(new FileReader(cacheFile), JsonObject.class);
|
||||||
|
}
|
||||||
|
illustInfoCache.put(illustId, illustInfoObj);
|
||||||
|
}
|
||||||
|
|
||||||
|
return illustInfoCache.get(illustId);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private final static Object illustPreLoadDataLock = new Object();
|
||||||
|
public static JsonObject getIllustPreLoadData(int illustId) throws IOException {
|
||||||
|
synchronized (illustPreLoadDataLock) {
|
||||||
|
File cacheFile = new File(getImageStoreDir(), illustId + ".illustPreLoadData.json");
|
||||||
|
CacheObject<JsonObject> cacheObject = new CacheObject<>();
|
||||||
|
Date currentDate = new Date();
|
||||||
|
if (!illustPreLoadDataCache.containsKey(illustId) || (cacheObject = illustPreLoadDataCache.get(illustId)).isExpire(currentDate)) {
|
||||||
|
log.info("因为到期而失效: {}", cacheObject.isExpire(new Date()));
|
||||||
|
log.info("因为缓存文件不存在而失效: {}", !cacheFile.exists());
|
||||||
|
log.info("缓存失效, 正在更新...");
|
||||||
|
log.info("illustPreLoadDataFileName: {}", cacheFile.getName());
|
||||||
|
JsonObject preLoadDataObj;
|
||||||
|
if (!cacheFile.exists()) {
|
||||||
|
try {
|
||||||
|
cacheFile.createNewFile();
|
||||||
|
preLoadDataObj = pixivDownload.getIllustPreLoadDataById(illustId)
|
||||||
|
.getAsJsonObject("illust")
|
||||||
|
.getAsJsonObject(Integer.toString(illustId));
|
||||||
|
Files.write(cacheFile.toPath(), gson.toJson(preLoadDataObj).getBytes(StandardCharsets.UTF_8), StandardOpenOption.CREATE);
|
||||||
|
} catch(IOException e) {
|
||||||
|
cacheFile.delete();
|
||||||
|
throw e;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
preLoadDataObj = gson.fromJson(new FileReader(cacheFile), JsonObject.class);
|
||||||
|
}
|
||||||
|
|
||||||
|
long expire = 7200 * 1000;
|
||||||
|
String propValue = CQPluginMain.globalProp.getProperty("cache.illustPreLoadData.expire", "7200000");
|
||||||
|
log.info("PreLoadData有效时间设定: {}", propValue);
|
||||||
|
try {
|
||||||
|
expire = Long.parseLong(propValue);
|
||||||
|
} catch (Exception e) {
|
||||||
|
log.warn("全局配置项 \"{}\" 值非法, 已使用默认值: {}", propValue, expire);
|
||||||
|
}
|
||||||
|
|
||||||
|
Date newExpire = new Date();
|
||||||
|
newExpire.setTime(newExpire.getTime() + expire);
|
||||||
|
cacheObject.update(preLoadDataObj, newExpire);
|
||||||
|
illustPreLoadDataCache.put(illustId, cacheObject);
|
||||||
|
log.info("作品Id {} preLoadData缓存已更新(到期时间: {})", illustId, newExpire);
|
||||||
|
}
|
||||||
|
|
||||||
|
return illustPreLoadDataCache.get(illustId).get();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private final static Object illustPagesLock = new Object();
|
||||||
|
public static List<String> getIllustPages(int illustId, PixivDownload.PageQuality quality) throws IOException {
|
||||||
|
synchronized (illustPagesLock) {
|
||||||
|
File cacheFile = new File(getImageStoreDir(), illustId + "." + quality.name() + ".illustPages.json");
|
||||||
|
if (!pagesCache.containsKey(illustId + "." + quality.name())) {
|
||||||
|
log.info("illustPagesFileName: {}", cacheFile.getName());
|
||||||
|
List<String> linkList;
|
||||||
|
if (!cacheFile.exists()) {
|
||||||
|
try {
|
||||||
|
cacheFile.createNewFile();
|
||||||
|
linkList = PixivDownload.getIllustAllPageDownload(pixivDownload.getHttpClient(), pixivDownload.getCookieStore(), illustId, quality);
|
||||||
|
JsonArray jsonArray = new JsonArray(linkList.size());
|
||||||
|
linkList.forEach(jsonArray::add);
|
||||||
|
Files.write(cacheFile.toPath(), gson.toJson(jsonArray).getBytes(StandardCharsets.UTF_8), StandardOpenOption.CREATE);
|
||||||
|
} catch(IOException e) {
|
||||||
|
cacheFile.delete();
|
||||||
|
throw e;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
JsonArray jsonArray = gson.fromJson(new FileReader(cacheFile), JsonArray.class);
|
||||||
|
linkList = new ArrayList<>(jsonArray.size());
|
||||||
|
jsonArray.forEach(jsonElement -> linkList.add(jsonElement.getAsString()));
|
||||||
|
}
|
||||||
|
pagesCache.put(illustId + "." + quality.name(), linkList);
|
||||||
|
}
|
||||||
|
|
||||||
|
return pagesCache.get(illustId + "." + quality.name());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static File getImageStoreDir() {
|
||||||
|
if(!imageStoreDir.exists() && !imageStoreDir.mkdirs()) {
|
||||||
|
log.warn("酷Q图片缓存目录失效!(Path: {} )", imageStoreDir.getAbsolutePath());
|
||||||
|
throw new RuntimeException(new IOException("文件夹创建失败!"));
|
||||||
|
}
|
||||||
|
return imageStoreDir;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
518
src/main/java/net/lamgc/cgj/Main.java
Normal file
518
src/main/java/net/lamgc/cgj/Main.java
Normal file
@ -0,0 +1,518 @@
|
|||||||
|
package net.lamgc.cgj;
|
||||||
|
|
||||||
|
import com.github.monkeywie.proxyee.proxy.ProxyConfig;
|
||||||
|
import com.github.monkeywie.proxyee.proxy.ProxyType;
|
||||||
|
import com.google.common.base.Strings;
|
||||||
|
import com.google.gson.Gson;
|
||||||
|
import com.google.gson.JsonArray;
|
||||||
|
import com.google.gson.JsonElement;
|
||||||
|
import com.google.gson.JsonObject;
|
||||||
|
import net.lamgc.cgj.pixiv.*;
|
||||||
|
import net.lamgc.cgj.proxy.PixivAccessProxyServer;
|
||||||
|
import net.lamgc.cgj.proxy.PixivLoginProxyServer;
|
||||||
|
import net.lamgc.utils.base.ArgumentsProperties;
|
||||||
|
import net.lamgc.utils.base.runner.Argument;
|
||||||
|
import net.lamgc.utils.base.runner.ArgumentsRunner;
|
||||||
|
import net.lamgc.utils.base.runner.Command;
|
||||||
|
import org.apache.commons.io.IOUtils;
|
||||||
|
import org.apache.http.HttpHost;
|
||||||
|
import org.apache.http.HttpResponse;
|
||||||
|
import org.apache.http.client.CookieStore;
|
||||||
|
import org.apache.http.client.HttpClient;
|
||||||
|
import org.apache.http.client.methods.HttpGet;
|
||||||
|
import org.apache.http.message.BasicHeader;
|
||||||
|
import org.apache.http.util.EntityUtils;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
import org.springframework.boot.SpringApplication;
|
||||||
|
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||||
|
|
||||||
|
import java.io.*;
|
||||||
|
import java.net.URL;
|
||||||
|
import java.nio.charset.StandardCharsets;
|
||||||
|
import java.text.ParseException;
|
||||||
|
import java.text.SimpleDateFormat;
|
||||||
|
import java.util.*;
|
||||||
|
import java.util.regex.Matcher;
|
||||||
|
import java.util.regex.Pattern;
|
||||||
|
import java.util.zip.ZipEntry;
|
||||||
|
import java.util.zip.ZipOutputStream;
|
||||||
|
|
||||||
|
@SpringBootApplication
|
||||||
|
public class Main {
|
||||||
|
|
||||||
|
private final static Logger log = LoggerFactory.getLogger("Main");
|
||||||
|
|
||||||
|
private final static File storeDir = new File("store/");
|
||||||
|
|
||||||
|
public static CookieStore cookieStore;
|
||||||
|
|
||||||
|
public static HttpHost proxy;
|
||||||
|
|
||||||
|
static {
|
||||||
|
if(!storeDir.exists() && !storeDir.mkdirs()) {
|
||||||
|
log.error("创建文件夹失败!");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static void main(String[] args) throws IOException, ClassNotFoundException {
|
||||||
|
ArgumentsProperties argsProp = new ArgumentsProperties(args);
|
||||||
|
if(argsProp.containsKey("proxy")) {
|
||||||
|
URL proxyUrl = new URL(argsProp.getValue("proxy"));
|
||||||
|
proxy = new HttpHost(proxyUrl.getHost(), proxyUrl.getPort());
|
||||||
|
log.info("已启用Http协议代理:{}", proxy.toHostString());
|
||||||
|
} else {
|
||||||
|
proxy = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
if(argsProp.containsKey("cqRootDir")) {
|
||||||
|
log.info("cqRootDir: {}", argsProp.getValue("cqRootDir"));
|
||||||
|
System.setProperty("cgj.cqRootDir", argsProp.getValue("cqRootDir"));
|
||||||
|
} else {
|
||||||
|
log.info("未设置cqRootDir");
|
||||||
|
}
|
||||||
|
|
||||||
|
File cookieStoreFile = new File("cookies.store");
|
||||||
|
if(!cookieStoreFile.exists()) {
|
||||||
|
log.error("未找到cookies.store文件, 请确保文件存在于运行目录下!");
|
||||||
|
System.exit(1);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
ObjectInputStream ois = new ObjectInputStream(new FileInputStream(cookieStoreFile));
|
||||||
|
cookieStore = (CookieStore) ois.readObject();
|
||||||
|
ois.close();
|
||||||
|
log.info("已载入CookieStore");
|
||||||
|
|
||||||
|
log.debug(Arrays.toString(args));
|
||||||
|
|
||||||
|
ArgumentsRunner.run(Main.class, args);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Command
|
||||||
|
public static void pluginMode(@Argument(name = "args", force = false) String argsStr) {
|
||||||
|
if(!System.getProperty("cgj.cqRootDir").endsWith("\\") && !System.getProperty("cgj.cqRootDir").endsWith("/")) {
|
||||||
|
System.setProperty("cgj.cqRootDir", System.getProperty("cgj.cqRootDir") + "/");
|
||||||
|
}
|
||||||
|
log.info("酷Q机器人根目录: {}", System.getProperty("cgj.cqRootDir"));
|
||||||
|
CQConfig.init();
|
||||||
|
Pattern pattern = Pattern.compile("/\\s*(\".+?\"|[^:\\s])+((\\s*:\\s*(\".+?\"|[^\\s])+)|)|(\".+?\"|[^\"\\s])+");
|
||||||
|
Matcher matcher = pattern.matcher(Strings.nullToEmpty(argsStr));
|
||||||
|
ArrayList<String> argsList = new ArrayList<>();
|
||||||
|
while (matcher.find()) {
|
||||||
|
argsList.add(matcher.group());
|
||||||
|
}
|
||||||
|
String[] args = new String[argsList.size()];
|
||||||
|
argsList.toArray(args);
|
||||||
|
SpringApplication.run(Main.class, args);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Command
|
||||||
|
public static void collectionDownload() throws IOException {
|
||||||
|
PixivDownload pixivDownload = new PixivDownload(Objects.requireNonNull(cookieStore), proxy);
|
||||||
|
File outputFile = new File(storeDir, "collection.zip");
|
||||||
|
if(!outputFile.exists() && !outputFile.createNewFile()) {
|
||||||
|
log.error("文件创建失败: " + outputFile.getAbsolutePath());
|
||||||
|
}
|
||||||
|
ZipOutputStream zos = new ZipOutputStream(new FileOutputStream(outputFile));
|
||||||
|
zos.setLevel(9);
|
||||||
|
log.info("正在调用方法...");
|
||||||
|
pixivDownload.getCollectionAsInputStream(PixivDownload.PageQuality.ORIGINAL, (link, inputStream) -> {
|
||||||
|
try {
|
||||||
|
ZipEntry entry = new ZipEntry(link.substring(link.lastIndexOf("/") + 1));
|
||||||
|
log.info("正在写入: " + entry.getName());
|
||||||
|
zos.putNextEntry(entry);
|
||||||
|
IOUtils.copy(inputStream, zos);
|
||||||
|
zos.flush();
|
||||||
|
} catch (IOException e) {
|
||||||
|
log.error("写入文件项时发生异常", e);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
log.info("调用完成.");
|
||||||
|
zos.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Command
|
||||||
|
public static void getRecommends() throws IOException {
|
||||||
|
PixivDownload pixivDownload = new PixivDownload(Objects.requireNonNull(cookieStore), proxy);
|
||||||
|
String date = new SimpleDateFormat("yyyyMMdd").format(new Date());
|
||||||
|
int id = 1;
|
||||||
|
File outputFile = new File(storeDir, "recommends-" + date + "-" + id + ".zip");
|
||||||
|
while(outputFile.exists()) {
|
||||||
|
id++;
|
||||||
|
outputFile = new File(storeDir, "recommends-" + date + "-" + id + ".zip");
|
||||||
|
}
|
||||||
|
|
||||||
|
if(!outputFile.createNewFile()) {
|
||||||
|
log.error("文件创建失败: " + outputFile.getAbsolutePath());
|
||||||
|
}
|
||||||
|
ZipOutputStream zos = new ZipOutputStream(new FileOutputStream(outputFile));
|
||||||
|
zos.setLevel(9);
|
||||||
|
log.info("正在调用方法...");
|
||||||
|
pixivDownload.getRecommendAsInputStream(PixivDownload.PageQuality.ORIGINAL, (link, inputStream) -> {
|
||||||
|
try {
|
||||||
|
ZipEntry entry = new ZipEntry(link.substring(link.lastIndexOf("/") + 1));
|
||||||
|
log.info("正在写入: " + entry.getName());
|
||||||
|
zos.putNextEntry(entry);
|
||||||
|
IOUtils.copy(inputStream, zos);
|
||||||
|
zos.flush();
|
||||||
|
log.info("已成功写入 {}", entry.getName());
|
||||||
|
} catch (IOException e) {
|
||||||
|
log.error("写入文件项时发生异常", e);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
log.info("调用完成.");
|
||||||
|
zos.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Command
|
||||||
|
public static void getRankingIllust(@Argument(name = "range", force = false, defaultValue = "100") int range,
|
||||||
|
@Argument(name = "mode", force = false) String mode,
|
||||||
|
@Argument(name = "content", force = false) String content,
|
||||||
|
@Argument(name = "queryTime", force = false) String queryTime) throws IOException, ParseException {
|
||||||
|
PixivDownload pixivDownload = new PixivDownload(cookieStore, proxy);
|
||||||
|
SimpleDateFormat format = new SimpleDateFormat("yyyyMMdd");
|
||||||
|
Date queryDate;
|
||||||
|
String date;
|
||||||
|
if (queryTime == null) {
|
||||||
|
GregorianCalendar gregorianCalendar = new GregorianCalendar();
|
||||||
|
gregorianCalendar.setTime(new Date());
|
||||||
|
gregorianCalendar.add(Calendar.DATE, -1);
|
||||||
|
queryDate = gregorianCalendar.getTime();
|
||||||
|
} else {
|
||||||
|
queryDate = format.parse(queryTime);
|
||||||
|
}
|
||||||
|
|
||||||
|
date = format.format(queryDate);
|
||||||
|
|
||||||
|
log.info("查询时间: {}", date);
|
||||||
|
PixivURL.RankingMode rankingMode = PixivURL.RankingMode.MODE_DAILY;
|
||||||
|
PixivURL.RankingContentType contentType = null;
|
||||||
|
if(mode != null) {
|
||||||
|
try {
|
||||||
|
rankingMode = PixivURL.RankingMode.valueOf(mode);
|
||||||
|
} catch (IllegalArgumentException e) {
|
||||||
|
log.warn("不支持的RankingMode: {}", mode);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if(content != null) {
|
||||||
|
try {
|
||||||
|
contentType = PixivURL.RankingContentType.valueOf(content);
|
||||||
|
} catch (IllegalArgumentException e) {
|
||||||
|
log.warn("不支持的RankingContentType: {}", content);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
int id = 1;
|
||||||
|
File outputFile = new File(storeDir, "ranking" + rankingMode.modeParam + "-" + date + "-" + id + ".zip");
|
||||||
|
while(outputFile.exists()) {
|
||||||
|
id++;
|
||||||
|
outputFile = new File(storeDir, "ranking" + rankingMode.modeParam + "-" + date + "-" + id + ".zip");
|
||||||
|
}
|
||||||
|
|
||||||
|
if(!outputFile.createNewFile()) {
|
||||||
|
log.error("文件创建失败: " + outputFile.getAbsolutePath());
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
ZipOutputStream zos = new ZipOutputStream(new FileOutputStream(outputFile));
|
||||||
|
zos.setLevel(9);
|
||||||
|
|
||||||
|
log.info("正在调用方法...");
|
||||||
|
try {
|
||||||
|
pixivDownload.getRankingAsInputStream(contentType, rankingMode, queryDate, range, PixivDownload.PageQuality.ORIGINAL, (rank, link, rankInfo, inputStream) -> {
|
||||||
|
try {
|
||||||
|
ZipEntry entry = new ZipEntry("Rank" + rank + "-" + link.substring(link.lastIndexOf("/") + 1));
|
||||||
|
entry.setComment(rankInfo.toString());
|
||||||
|
log.info("正在写入: " + entry.getName());
|
||||||
|
zos.putNextEntry(entry);
|
||||||
|
IOUtils.copy(inputStream, zos);
|
||||||
|
zos.flush();
|
||||||
|
log.info("已成功写入 {}", entry.getName());
|
||||||
|
} catch (IOException e) {
|
||||||
|
log.error("写入文件项时发生异常", e);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
} finally {
|
||||||
|
zos.finish();
|
||||||
|
zos.flush();
|
||||||
|
zos.close();
|
||||||
|
}
|
||||||
|
log.info("调用完成.");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Command
|
||||||
|
public static void search(
|
||||||
|
@Argument(name = "content") String content,
|
||||||
|
@Argument(name = "type", force = false) String type,
|
||||||
|
@Argument(name = "area", force = false) String area,
|
||||||
|
@Argument(name = "includeKeywords", force = false) String includeKeywords,
|
||||||
|
@Argument(name = "excludeKeywords", force = false) String excludeKeywords,
|
||||||
|
@Argument(name = "contentOption", force = false) String contentOption
|
||||||
|
) throws IOException {
|
||||||
|
PixivSearchBuilder searchBuilder = new PixivSearchBuilder(Strings.isNullOrEmpty(content) ? "" : content);
|
||||||
|
if (type != null) {
|
||||||
|
try {
|
||||||
|
searchBuilder.setSearchType(PixivSearchBuilder.SearchType.valueOf(type.toUpperCase()));
|
||||||
|
} catch (IllegalArgumentException e) {
|
||||||
|
log.warn("不支持的SearchType: {}", type);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if(area != null) {
|
||||||
|
try {
|
||||||
|
searchBuilder.setSearchArea(PixivSearchBuilder.SearchArea.valueOf(area));
|
||||||
|
} catch (IllegalArgumentException e) {
|
||||||
|
log.warn("不支持的SearchArea: {}", area);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if(contentOption != null) {
|
||||||
|
try {
|
||||||
|
searchBuilder.setSearchContentOption(PixivSearchBuilder.SearchContentOption.valueOf(contentOption));
|
||||||
|
} catch (IllegalArgumentException e) {
|
||||||
|
log.warn("不支持的SearchContentOption: {}", contentOption);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if(!Strings.isNullOrEmpty(includeKeywords)) {
|
||||||
|
for (String keyword : includeKeywords.split(";")) {
|
||||||
|
searchBuilder.removeExcludeKeyword(keyword);
|
||||||
|
searchBuilder.addIncludeKeyword(keyword);
|
||||||
|
log.info("已添加关键字: {}", keyword);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if(!Strings.isNullOrEmpty(excludeKeywords)) {
|
||||||
|
for (String keyword : excludeKeywords.split(";")) {
|
||||||
|
searchBuilder.removeIncludeKeyword(keyword);
|
||||||
|
searchBuilder.addExcludeKeyword(keyword);
|
||||||
|
log.info("已添加排除关键字: {}", keyword);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
log.info("正在搜索作品, 条件: {}", searchBuilder.getSearchCondition());
|
||||||
|
|
||||||
|
String requestUrl = searchBuilder.buildURL();
|
||||||
|
log.info("RequestUrl: {}", requestUrl);
|
||||||
|
PixivDownload pixivDownload = new PixivDownload(cookieStore, proxy);
|
||||||
|
HttpGet httpGetRequest = pixivDownload.createHttpGetRequest(requestUrl);
|
||||||
|
HttpResponse response = pixivDownload.getHttpClient().execute(httpGetRequest);
|
||||||
|
|
||||||
|
String responseBody = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8);
|
||||||
|
log.info("ResponseBody: {}", responseBody);
|
||||||
|
JsonObject jsonObject = new Gson().fromJson(responseBody, JsonObject.class);
|
||||||
|
if(jsonObject.get("error").getAsBoolean()) {
|
||||||
|
log.error("接口请求错误, 错误信息: {}", jsonObject.get("message").getAsString());
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
JsonObject resultBody = jsonObject.getAsJsonObject("body");
|
||||||
|
|
||||||
|
for(PixivSearchBuilder.SearchArea searchArea : PixivSearchBuilder.SearchArea.values()) {
|
||||||
|
if(!resultBody.has(searchArea.jsonKey) || resultBody.getAsJsonObject(searchArea.jsonKey).getAsJsonArray("data").size() == 0) {
|
||||||
|
//log.info("返回数据不包含 {}", searchArea.jsonKey);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
JsonArray illustsArray = resultBody
|
||||||
|
.getAsJsonObject(searchArea.jsonKey).getAsJsonArray("data");
|
||||||
|
log.info("已找到与 {} 相关插图信息({}):", content, searchArea.name().toLowerCase());
|
||||||
|
int count = 1;
|
||||||
|
for (JsonElement jsonElement : illustsArray) {
|
||||||
|
JsonObject illustObj = jsonElement.getAsJsonObject();
|
||||||
|
if(!illustObj.has("illustId")) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
int illustId = illustObj.get("illustId").getAsInt();
|
||||||
|
StringBuilder builder = new StringBuilder("[");
|
||||||
|
illustObj.get("tags").getAsJsonArray().forEach(el -> builder.append(el.getAsString()).append(", "));
|
||||||
|
builder.replace(builder.length() - 2, builder.length(), "]");
|
||||||
|
log.info("{} ({} / {})\n\t作品id: {}, \n\t作者名(作者id): {} ({}), \n\t作品标题: {}, \n\t作品Tags: {}, \n\t作品链接: {}",
|
||||||
|
searchArea.name(),
|
||||||
|
count++,
|
||||||
|
illustsArray.size(),
|
||||||
|
illustId,
|
||||||
|
illustObj.get("userName").getAsString(),
|
||||||
|
illustObj.get("userId").getAsInt(),
|
||||||
|
illustObj.get("illustTitle").getAsString(),
|
||||||
|
builder,
|
||||||
|
PixivURL.getPixivRefererLink(illustId)
|
||||||
|
);
|
||||||
|
|
||||||
|
/*log.info("正在下载...");
|
||||||
|
List<String> list = PixivDownload.getIllustAllPageDownload(
|
||||||
|
HttpClientBuilder.create()
|
||||||
|
.setProxy(proxy)
|
||||||
|
.build(),
|
||||||
|
illustId, PixivDownload.PageQuality.ORIGINAL);*/
|
||||||
|
}
|
||||||
|
}
|
||||||
|
/*
|
||||||
|
|
||||||
|
if(searchBuilder.getSearchArea().equals(PixivSearchBuilder.SearchArea.TOP)) {
|
||||||
|
|
||||||
|
} else {
|
||||||
|
JsonArray illustsArray = resultBody
|
||||||
|
.getAsJsonObject(searchBuilder.getSearchArea().jsonKey).getAsJsonArray("data");
|
||||||
|
log.info("已找到与 {} 相关插图信息:", content);
|
||||||
|
int count = 1;
|
||||||
|
for (JsonElement jsonElement : illustsArray) {
|
||||||
|
JsonObject illustObj = jsonElement.getAsJsonObject();
|
||||||
|
//TODO: 防止数据内混入无效内容, 需要检查对象是否有illustId
|
||||||
|
if(!illustObj.has("illustId")) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
int illustId = illustObj.get("illustId").getAsInt();
|
||||||
|
StringBuilder builder = new StringBuilder("[");
|
||||||
|
illustObj.get("tags").getAsJsonArray().forEach(el -> builder.append(el.getAsString()).append(", "));
|
||||||
|
builder.replace(builder.length() - 2, builder.length(), "]");
|
||||||
|
log.info("({} / {})\n\t作品id: {}, \n\t作者名(作者id): {} ({}), \n\t作品标题: {}, \n\t作品Tags: {}, \n\t作品链接: {}",
|
||||||
|
count++,
|
||||||
|
illustsArray.size(),
|
||||||
|
illustId,
|
||||||
|
illustObj.get("userName").getAsString(),
|
||||||
|
illustObj.get("userId").getAsInt(),
|
||||||
|
illustObj.get("illustTitle").getAsString(),
|
||||||
|
builder,
|
||||||
|
PixivURL.getPixivRefererLink(illustId)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
*/
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@Command(defaultCommand = true)
|
||||||
|
public static void testRun() throws IOException {
|
||||||
|
/*loadCookieStoreFromFile();
|
||||||
|
|
||||||
|
if(cookieStore == null){
|
||||||
|
startPixivLoginProxyServer();
|
||||||
|
}*/
|
||||||
|
|
||||||
|
//accessPixivToFile();
|
||||||
|
|
||||||
|
//startPixivAccessProxyServer();
|
||||||
|
|
||||||
|
//saveCookieStoreToFile();
|
||||||
|
log.info("这里啥都没有哟w");
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void loadCookieStoreFromFile() throws IOException {
|
||||||
|
log.info("正在加载CookieStore...");
|
||||||
|
File storeFile = new File("./cookies.store");
|
||||||
|
if(!storeFile.exists()){
|
||||||
|
log.info("未找到CookieStore, 跳过加载.");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
ObjectInputStream stream = new ObjectInputStream(new FileInputStream(storeFile));
|
||||||
|
Object result;
|
||||||
|
try {
|
||||||
|
result = stream.readObject();
|
||||||
|
} catch (ClassNotFoundException e) {
|
||||||
|
log.error("加载出错", e);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
cookieStore = (CookieStore) result;
|
||||||
|
cookieStore.getCookies().forEach(cookie -> log.debug(cookie.getName() + ": " + cookie.getValue() + ", isExpired: " + cookie.isExpired(new Date())));
|
||||||
|
log.info("CookieStore加载完成.");
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void saveCookieStoreToFile() throws IOException {
|
||||||
|
log.info("正在保存CookieStore...");
|
||||||
|
File outputFile = new File("./cookies.store");
|
||||||
|
if(!outputFile.exists() && !outputFile.delete() && !outputFile.createNewFile()){
|
||||||
|
log.error("保存CookieStore失败.");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
ObjectOutputStream stream = new ObjectOutputStream(new FileOutputStream(outputFile));
|
||||||
|
stream.writeObject(cookieStore);
|
||||||
|
stream.flush();
|
||||||
|
stream.close();
|
||||||
|
log.info("CookieStore保存成功.");
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void startPixivLoginProxyServer(){
|
||||||
|
ProxyConfig proxyConfig = new ProxyConfig(ProxyType.SOCKS5, "127.0.0.1", 1080);
|
||||||
|
PixivLoginProxyServer proxyServer = new PixivLoginProxyServer(proxyConfig);
|
||||||
|
Thread proxyServerStartThread = new Thread(() -> {
|
||||||
|
log.info("启动代理服务器...");
|
||||||
|
proxyServer.start(1006);
|
||||||
|
log.info("代理服务器已关闭.");
|
||||||
|
});
|
||||||
|
proxyServerStartThread.setName("LoginProxyServerThread");
|
||||||
|
proxyServerStartThread.start();
|
||||||
|
//System.console().readLine();
|
||||||
|
new Scanner(System.in).nextLine();
|
||||||
|
log.info("关闭PLPS服务器...");
|
||||||
|
proxyServer.close();
|
||||||
|
cookieStore = proxyServer.getCookieStore();
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void accessPixivToFile() throws IOException {
|
||||||
|
File cookieStoreFile = new File("./cookie.txt");
|
||||||
|
if (!cookieStoreFile.exists() && !cookieStoreFile.createNewFile()) {
|
||||||
|
log.info("Cookie文件存储失败");
|
||||||
|
}
|
||||||
|
/*log.info("正在写出Cookie, Cookie count: " + cookieStore.getCookies().size());
|
||||||
|
FileWriter cookieWriter = new FileWriter(cookieStoreFile);
|
||||||
|
cookieStore.getCookies().forEach(cookie -> {
|
||||||
|
try {
|
||||||
|
StringBuilder sb = new StringBuilder().append(cookie.getName()).append(" = ").append(cookie.getValue());
|
||||||
|
log.info("正在导出Cookie: " + sb.toString());
|
||||||
|
cookieWriter.append(sb.toString()).append("\n").flush();
|
||||||
|
} catch (IOException e) {
|
||||||
|
e.printStackTrace();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
log.info("Cookie写出完成");*/
|
||||||
|
|
||||||
|
log.info("尝试通过捕获的Cookie访问Pixiv...");
|
||||||
|
HttpClient httpClient = new PixivSession(new HttpHost("127.0.0.1", 1080), cookieStore).getHttpClient();
|
||||||
|
HttpGet request = new HttpGet("https://www.pixiv.net");
|
||||||
|
request.addHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:73.0) Gecko/20100101 Firefox/73.0");
|
||||||
|
request.addHeader(new BasicHeader("accept-encoding", "gzip, deflate, br"));
|
||||||
|
request.addHeader(new BasicHeader("accept-language", "zh-CN,zh;q=0.9"));
|
||||||
|
StringBuilder cookieBuilder = new StringBuilder();
|
||||||
|
cookieStore.getCookies().forEach(cookie -> {
|
||||||
|
if(cookie.isExpired(new Date())){
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
cookieBuilder.append(cookie.getName()).append("=").append(cookie.getValue()).append("; ");
|
||||||
|
});
|
||||||
|
request.addHeader("cookie", cookieBuilder.toString());
|
||||||
|
|
||||||
|
HttpResponse response = httpClient.execute(request);
|
||||||
|
log.info("正在写入文件...");
|
||||||
|
File outFile = new File("./pixiv.html");
|
||||||
|
if (outFile.createNewFile() && !outFile.exists()) {
|
||||||
|
log.info("文件创建失败!");
|
||||||
|
}else {
|
||||||
|
new FileWriter(outFile).append(EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8)).flush();
|
||||||
|
}
|
||||||
|
|
||||||
|
Pixiv pixiv = new Pixiv(httpClient);
|
||||||
|
pixiv.getRecommend().forEach(illustMap -> {
|
||||||
|
StringBuilder builder = new StringBuilder();
|
||||||
|
illustMap.forEach((key, value) -> builder.append(key).append(": ").append(value).append("\n"));
|
||||||
|
try {
|
||||||
|
builder.append("download Link: ").append(Arrays.toString(pixiv.getAllDownloadLink(Integer.parseInt(illustMap.get(Pixiv.ATTR_ILLUST_ID)))));
|
||||||
|
} catch (IOException e) {
|
||||||
|
log.error("获取下载链接时出错!", e);
|
||||||
|
}
|
||||||
|
log.info(builder.append("\n").toString());
|
||||||
|
});
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void startPixivAccessProxyServer(){
|
||||||
|
log.info("正在启动访问代理服务器, 将浏览器相关缓存清空后, 使用浏览器进行访问以尝试Cookie正确性.");
|
||||||
|
final PixivAccessProxyServer accessProxyServer = new PixivAccessProxyServer(cookieStore, new ProxyConfig(ProxyType.SOCKS5, "127.0.0.1", 1080));
|
||||||
|
Thread accessProxyServerThread = new Thread(() -> {
|
||||||
|
log.info("正在启动PAPS...");
|
||||||
|
accessProxyServer.start(1007);
|
||||||
|
log.info("PAPS已关闭.");
|
||||||
|
});
|
||||||
|
accessProxyServerThread.setName("AccessProxyServerThread");
|
||||||
|
accessProxyServerThread.start();
|
||||||
|
new Scanner(System.in).nextLine();
|
||||||
|
log.info("关闭PAPS服务器...");
|
||||||
|
accessProxyServer.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
57
src/main/java/net/lamgc/cgj/bot/AutoArtworksSender.java
Normal file
57
src/main/java/net/lamgc/cgj/bot/AutoArtworksSender.java
Normal file
@ -0,0 +1,57 @@
|
|||||||
|
package net.lamgc.cgj.bot;
|
||||||
|
|
||||||
|
import net.lz1998.cq.robot.CoolQ;
|
||||||
|
import org.apache.http.client.methods.HttpGet;
|
||||||
|
|
||||||
|
import java.util.Random;
|
||||||
|
import java.util.Timer;
|
||||||
|
import java.util.TimerTask;
|
||||||
|
|
||||||
|
public class AutoArtworksSender {
|
||||||
|
|
||||||
|
private final CoolQ CQ;
|
||||||
|
private final ReceiveType receiveType;
|
||||||
|
private final long targetReceiveId;
|
||||||
|
private Timer timer = new Timer();
|
||||||
|
private TimerTask task = new TimerTask() {
|
||||||
|
@Override
|
||||||
|
public void run() {
|
||||||
|
HttpGet request = new HttpGet();
|
||||||
|
|
||||||
|
// https://api.imjad.cn/pixiv/v2/?type=tags
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
public AutoArtworksSender(CoolQ cq, ReceiveType receiveType, long receiveId) {
|
||||||
|
this.CQ = cq;
|
||||||
|
this.receiveType = receiveType;
|
||||||
|
this.targetReceiveId = receiveId;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void reset(long time) {
|
||||||
|
if(time <= 0) {
|
||||||
|
timer.schedule(task, new Random().nextInt(10 * 60 * 60 * 1000) + 7200000L); //2H ~ 12H
|
||||||
|
} else {
|
||||||
|
timer.schedule(task, time);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void sendMessage(String message, boolean auto_escape) {
|
||||||
|
switch (receiveType) {
|
||||||
|
case GROUP:
|
||||||
|
CQ.sendGroupMsg(targetReceiveId, message, auto_escape);
|
||||||
|
break;
|
||||||
|
case Discuss:
|
||||||
|
CQ.sendDiscussMsg(targetReceiveId, message, auto_escape);
|
||||||
|
break;
|
||||||
|
case PRIVATE:
|
||||||
|
CQ.sendPrivateMsg(targetReceiveId, message, auto_escape);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public enum ReceiveType {
|
||||||
|
PRIVATE, GROUP, Discuss
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
40
src/main/java/net/lamgc/cgj/cache/CacheObject.java
vendored
Normal file
40
src/main/java/net/lamgc/cgj/cache/CacheObject.java
vendored
Normal file
@ -0,0 +1,40 @@
|
|||||||
|
package net.lamgc.cgj.cache;
|
||||||
|
|
||||||
|
import java.util.Date;
|
||||||
|
import java.util.concurrent.atomic.AtomicReference;
|
||||||
|
|
||||||
|
public class CacheObject<T> {
|
||||||
|
|
||||||
|
private AtomicReference<T> value;
|
||||||
|
private AtomicReference<Date> expire;
|
||||||
|
|
||||||
|
public CacheObject() {
|
||||||
|
this(null, null);
|
||||||
|
}
|
||||||
|
|
||||||
|
public CacheObject(T value, Date expire) {
|
||||||
|
this.value = new AtomicReference<>(value);
|
||||||
|
this.expire = new AtomicReference<>(expire);
|
||||||
|
}
|
||||||
|
|
||||||
|
public synchronized void update(T value, Date newExpire) {
|
||||||
|
if(new Date().after(newExpire)) {
|
||||||
|
throw new IllegalArgumentException("Due earlier than current time");
|
||||||
|
}
|
||||||
|
this.expire.set(newExpire);
|
||||||
|
this.value.set(value);
|
||||||
|
}
|
||||||
|
|
||||||
|
public synchronized T get() {
|
||||||
|
return value.get();
|
||||||
|
}
|
||||||
|
|
||||||
|
public Date getExpireDate() {
|
||||||
|
return expire.get();
|
||||||
|
}
|
||||||
|
|
||||||
|
public boolean isExpire(Date time) {
|
||||||
|
Date expireDate = getExpireDate();
|
||||||
|
return expireDate != null && expireDate.before(time);
|
||||||
|
}
|
||||||
|
}
|
79
src/main/java/net/lamgc/cgj/cache/ImageCacheHandler.java
vendored
Normal file
79
src/main/java/net/lamgc/cgj/cache/ImageCacheHandler.java
vendored
Normal file
@ -0,0 +1,79 @@
|
|||||||
|
package net.lamgc.cgj.cache;
|
||||||
|
|
||||||
|
import net.lamgc.cgj.Main;
|
||||||
|
import net.lamgc.cgj.pixiv.PixivURL;
|
||||||
|
import net.lamgc.utils.event.EventHandler;
|
||||||
|
import org.apache.commons.io.IOUtils;
|
||||||
|
import org.apache.http.HttpResponse;
|
||||||
|
import org.apache.http.client.HttpClient;
|
||||||
|
import org.apache.http.client.methods.HttpGet;
|
||||||
|
import org.apache.http.impl.client.HttpClientBuilder;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
|
import java.io.FileOutputStream;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.util.Collections;
|
||||||
|
import java.util.HashSet;
|
||||||
|
import java.util.Set;
|
||||||
|
|
||||||
|
public class ImageCacheHandler implements EventHandler {
|
||||||
|
|
||||||
|
private final static Logger log = LoggerFactory.getLogger("ImageCacheHandler");
|
||||||
|
|
||||||
|
private final static HttpClient httpClient = HttpClientBuilder.create().setProxy(Main.proxy).build();
|
||||||
|
|
||||||
|
private final static Set<ImageCacheObject> cacheQueue = Collections.synchronizedSet(new HashSet<>());
|
||||||
|
|
||||||
|
public void getImageToCache(ImageCacheObject event) {
|
||||||
|
if(cacheQueue.contains(event)) {
|
||||||
|
log.info("图片 {} 已存在相同缓存任务, 跳过.", event.getStoreFile().getName());
|
||||||
|
return;
|
||||||
|
} else {
|
||||||
|
cacheQueue.add(event);
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
log.info("图片 {} Event正在进行...({})", event.getStoreFile().getName(), Integer.toHexString(event.hashCode()));
|
||||||
|
File storeFile = event.getStoreFile();
|
||||||
|
log.info("正在缓存图片 {} (Path: {})", storeFile.getName(), storeFile.getAbsolutePath());
|
||||||
|
try {
|
||||||
|
if(!storeFile.exists() && !storeFile.createNewFile()) {
|
||||||
|
log.error("无法创建文件(Path: {})", storeFile.getAbsolutePath());
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
} catch (IOException e) {
|
||||||
|
log.error("无法创建文件(Path: {})", storeFile.getAbsolutePath());
|
||||||
|
e.printStackTrace();
|
||||||
|
}
|
||||||
|
|
||||||
|
HttpGet request = new HttpGet(event.getDownloadLink());
|
||||||
|
request.addHeader("Referer", PixivURL.getPixivRefererLink(event.getIllustId()));
|
||||||
|
HttpResponse response;
|
||||||
|
try {
|
||||||
|
response = httpClient.execute(request);
|
||||||
|
} catch (IOException e) {
|
||||||
|
log.error("Http请求时发生异常", e);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if(response.getStatusLine().getStatusCode() != 200) {
|
||||||
|
log.warn("Http请求异常:{}", response.getStatusLine());
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
log.info("正在下载...(Content-Length: {}KB)", response.getEntity().getContentLength() / 1024);
|
||||||
|
try(FileOutputStream fos = new FileOutputStream(storeFile)) {
|
||||||
|
IOUtils.copy(response.getEntity().getContent(), fos);
|
||||||
|
} catch (IOException e) {
|
||||||
|
log.error("下载图片时发生异常", e);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
event.getImageCache().put(event.getDownloadLink(), storeFile);
|
||||||
|
} finally {
|
||||||
|
log.info("图片 {} Event结束({})", event.getStoreFile().getName(), Integer.toHexString(event.hashCode()));
|
||||||
|
cacheQueue.remove(event);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
57
src/main/java/net/lamgc/cgj/cache/ImageCacheObject.java
vendored
Normal file
57
src/main/java/net/lamgc/cgj/cache/ImageCacheObject.java
vendored
Normal file
@ -0,0 +1,57 @@
|
|||||||
|
package net.lamgc.cgj.cache;
|
||||||
|
|
||||||
|
import net.lamgc.utils.event.EventObject;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.Objects;
|
||||||
|
|
||||||
|
public class ImageCacheObject implements EventObject {
|
||||||
|
|
||||||
|
private final Map<String, File> imageCache;
|
||||||
|
|
||||||
|
private final int illustId;
|
||||||
|
|
||||||
|
private final String downloadLink;
|
||||||
|
|
||||||
|
private final File storeFile;
|
||||||
|
|
||||||
|
public ImageCacheObject(Map<String, File> imageCache, int illustId, String downloadLink, File storeFile) {
|
||||||
|
this.imageCache = imageCache;
|
||||||
|
this.illustId = illustId;
|
||||||
|
this.downloadLink = downloadLink;
|
||||||
|
this.storeFile = storeFile;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Map<String, File> getImageCache() {
|
||||||
|
return imageCache;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getDownloadLink() {
|
||||||
|
return downloadLink;
|
||||||
|
}
|
||||||
|
|
||||||
|
public File getStoreFile() {
|
||||||
|
return storeFile;
|
||||||
|
}
|
||||||
|
|
||||||
|
public int getIllustId() {
|
||||||
|
return illustId;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean equals(Object o) {
|
||||||
|
if (this == o) return true;
|
||||||
|
if (o == null || getClass() != o.getClass()) return false;
|
||||||
|
ImageCacheObject that = (ImageCacheObject) o;
|
||||||
|
return illustId == that.illustId &&
|
||||||
|
Objects.equals(imageCache, that.imageCache) &&
|
||||||
|
Objects.equals(downloadLink, that.downloadLink) &&
|
||||||
|
Objects.equals(storeFile, that.storeFile);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public int hashCode() {
|
||||||
|
return Objects.hash(imageCache, illustId, downloadLink, storeFile);
|
||||||
|
}
|
||||||
|
}
|
28
src/main/java/net/lamgc/cgj/db/RankingDB.java
Normal file
28
src/main/java/net/lamgc/cgj/db/RankingDB.java
Normal file
@ -0,0 +1,28 @@
|
|||||||
|
package net.lamgc.cgj.db;
|
||||||
|
|
||||||
|
import java.sql.Connection;
|
||||||
|
import java.sql.DriverManager;
|
||||||
|
import java.sql.SQLException;
|
||||||
|
import java.sql.Statement;
|
||||||
|
|
||||||
|
public class RankingDB {
|
||||||
|
|
||||||
|
private final Connection dbConnection;
|
||||||
|
|
||||||
|
public RankingDB(String dbUrl, String username, String password) throws SQLException {
|
||||||
|
try {
|
||||||
|
Class.forName("com.mysql.cj.jdbc.Driver");
|
||||||
|
} catch (ClassNotFoundException e) {
|
||||||
|
e.printStackTrace();
|
||||||
|
}
|
||||||
|
dbConnection = DriverManager.getConnection("jdbc:mysql://" + dbUrl + "/pixivRanking?useSSL=false&serverTimezone=UTC",username,password);
|
||||||
|
checkAndFix(dbConnection);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void checkAndFix(Connection connection) throws SQLException {
|
||||||
|
Statement statement = connection.createStatement();
|
||||||
|
statement.execute("");
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
}
|
69
src/main/java/net/lamgc/cgj/pixiv/IllustInfo.java
Normal file
69
src/main/java/net/lamgc/cgj/pixiv/IllustInfo.java
Normal file
@ -0,0 +1,69 @@
|
|||||||
|
package net.lamgc.cgj.pixiv;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 插图(集)的信息
|
||||||
|
*/
|
||||||
|
public class IllustInfo{
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 插图(集)ID
|
||||||
|
*/
|
||||||
|
public final int illustID;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 第几页
|
||||||
|
*/
|
||||||
|
public final int page;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 插图标题
|
||||||
|
*/
|
||||||
|
public final String title;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 插图说明
|
||||||
|
*/
|
||||||
|
public final String description;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 插图标签
|
||||||
|
*/
|
||||||
|
public final String[] tags;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 插图图片长度
|
||||||
|
*/
|
||||||
|
public final int width;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 插图图片高度
|
||||||
|
*/
|
||||||
|
public final int height;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 作者名
|
||||||
|
*/
|
||||||
|
public final String authorName;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 作者用户ID
|
||||||
|
*/
|
||||||
|
public final int authorUserID;
|
||||||
|
|
||||||
|
public IllustInfo(int illustID, String title, String description, String[] tags, int width, int height, String authorName, int authorUserID){
|
||||||
|
this(illustID, 0, title, description, tags, width, height, authorName, authorUserID);
|
||||||
|
}
|
||||||
|
|
||||||
|
public IllustInfo(int illustID, int p, String title, String description, String[] tags, int width, int height, String authorName, int authorUserID){
|
||||||
|
this.illustID = illustID;
|
||||||
|
this.page = p;
|
||||||
|
this.title = title;
|
||||||
|
this.description = description;
|
||||||
|
this.tags = tags;
|
||||||
|
this.width = width;
|
||||||
|
this.height = height;
|
||||||
|
this.authorName = authorName;
|
||||||
|
this.authorUserID = authorUserID;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
267
src/main/java/net/lamgc/cgj/pixiv/Pixiv.java
Normal file
267
src/main/java/net/lamgc/cgj/pixiv/Pixiv.java
Normal file
@ -0,0 +1,267 @@
|
|||||||
|
package net.lamgc.cgj.pixiv;
|
||||||
|
|
||||||
|
import com.google.gson.JsonArray;
|
||||||
|
import com.google.gson.JsonObject;
|
||||||
|
import com.google.gson.JsonParser;
|
||||||
|
import org.apache.http.HttpHost;
|
||||||
|
import org.apache.http.HttpResponse;
|
||||||
|
import org.apache.http.client.HttpClient;
|
||||||
|
import org.apache.http.client.config.RequestConfig;
|
||||||
|
import org.apache.http.client.methods.HttpGet;
|
||||||
|
import org.apache.http.conn.ConnectionPoolTimeoutException;
|
||||||
|
import org.apache.http.impl.client.HttpClientBuilder;
|
||||||
|
import org.apache.http.util.EntityUtils;
|
||||||
|
import org.jsoup.Jsoup;
|
||||||
|
import org.jsoup.nodes.Document;
|
||||||
|
|
||||||
|
import java.io.ByteArrayInputStream;
|
||||||
|
import java.io.ByteArrayOutputStream;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.io.InputStream;
|
||||||
|
import java.nio.charset.StandardCharsets;
|
||||||
|
import java.util.*;
|
||||||
|
|
||||||
|
public class Pixiv {
|
||||||
|
|
||||||
|
/**
|
||||||
|
* illust Link
|
||||||
|
*/
|
||||||
|
public final static String ATTR_LINK = "link";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* illust Id
|
||||||
|
*/
|
||||||
|
public final static String ATTR_ILLUST_ID = "illustId";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* illust Title
|
||||||
|
*/
|
||||||
|
public final static String ATTR_TITLE = "title";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* illust Author Name
|
||||||
|
*/
|
||||||
|
public final static String ATTR_AUTHOR_NAME = "authorName";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* illust Author UserID
|
||||||
|
*/
|
||||||
|
public final static String ATTR_AUTHOR_ID = "authorId";
|
||||||
|
|
||||||
|
private final HttpClient httpClient;
|
||||||
|
|
||||||
|
public Pixiv(HttpClient client){
|
||||||
|
this.httpClient = client;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 使用帐号密码登录Pixiv
|
||||||
|
* @param PixivID Pixiv账户登录名
|
||||||
|
* @param Password Pixiv帐号密码
|
||||||
|
* @throws IOException 当登录连接出现异常时抛出
|
||||||
|
* @deprecated {@link PixivSession#Login(String, String)} 已经废除, 故本方法不可用
|
||||||
|
*/
|
||||||
|
public Pixiv(String PixivID, String Password) throws IOException {
|
||||||
|
this(PixivID, Password, null);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 使用帐号密码登录Pixiv
|
||||||
|
* @param PixivID Pixiv账户登录名
|
||||||
|
* @param Password Pixiv帐号密码
|
||||||
|
* @param proxy 代理设置
|
||||||
|
* @throws IOException 当登录连接出现异常时抛出
|
||||||
|
* @deprecated {@link PixivSession#Login(String, String)} 已经废除, 故本方法不可用
|
||||||
|
*/
|
||||||
|
public Pixiv(String PixivID, String Password, HttpHost proxy) throws IOException {
|
||||||
|
PixivSession pixivSession = new PixivSession(proxy, null);
|
||||||
|
if(pixivSession.Login(PixivID, Password)){
|
||||||
|
System.out.println("P站登录成功!");
|
||||||
|
}else{
|
||||||
|
System.out.println("P站登录失败!错误信息: " + pixivSession.getErrMsg());
|
||||||
|
throw new RuntimeException(pixivSession.getErrMsg());
|
||||||
|
}
|
||||||
|
//httpClient = pixivSession.getHttpClient();
|
||||||
|
httpClient = HttpClientBuilder.create()
|
||||||
|
.setDefaultCookieStore(pixivSession.getCookieStore())
|
||||||
|
.build();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 获取首页推荐列表
|
||||||
|
* @return 首页推荐列表, 一个Map对应一个推荐项, 使用<code>ATTR_</code>开头常量访问即可
|
||||||
|
* @throws IOException
|
||||||
|
*/
|
||||||
|
public List<Map<String, String>> getRecommend() throws IOException {
|
||||||
|
HttpGet getRequest = new HttpGet(PixivURL.PIXIV_INDEX_URL);
|
||||||
|
HttpResponse response = httpClient.execute(getRequest);
|
||||||
|
String pageAsXML = EntityUtils.toString(response.getEntity(),"utf-8");
|
||||||
|
|
||||||
|
//获取推荐图列表(li)
|
||||||
|
//System.out.println(pageAsXML);
|
||||||
|
|
||||||
|
Document document = Jsoup.parse(pageAsXML);
|
||||||
|
|
||||||
|
List<String> links = document.select("._image-items.gtm-illust-recommend-zone>li>.gtm-illust-recommend-thumbnail-link").eachAttr("href");
|
||||||
|
List<String> illustId = document.select("._image-items.gtm-illust-recommend-zone>li>.gtm-illust-recommend-thumbnail-link").eachAttr("data-gtm-recommend-illust-id");
|
||||||
|
List<String> title = document.select("._image-items.gtm-illust-recommend-zone>li>.gtm-illust-recommend-title>h1").eachAttr("title");
|
||||||
|
List<String> authorName = document.select("._image-items.gtm-illust-recommend-zone>li>.gtm-illust-recommend-user-name").eachText();
|
||||||
|
List<String> authorId = document.select("._image-items.gtm-illust-recommend-zone>li>.gtm-illust-recommend-user-name").eachAttr("data-user_id");
|
||||||
|
|
||||||
|
List<Map<String, String>> recommendList = new ArrayList<>();
|
||||||
|
for(int i = 0; i < links.size(); i++){
|
||||||
|
//System.out.println(links.get(i));
|
||||||
|
Map<String, String> info = new HashMap<>();
|
||||||
|
info.put(ATTR_LINK, PixivURL.PIXIV_INDEX_URL + links.get(i));
|
||||||
|
info.put(ATTR_ILLUST_ID, illustId.get(i));
|
||||||
|
info.put(ATTR_TITLE, title.get(i));
|
||||||
|
info.put(ATTR_AUTHOR_NAME, authorName.get(i));
|
||||||
|
info.put(ATTR_AUTHOR_ID, authorId.get(i));
|
||||||
|
recommendList.add(info);
|
||||||
|
}
|
||||||
|
return recommendList;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String[] getAllDownloadLink(int illustID) throws IOException {
|
||||||
|
HttpGet illustPage = new HttpGet(PixivURL.PIXIV_ILLUST_API_URL.replaceAll("\\{illustId}", String.valueOf(illustID)));
|
||||||
|
HttpResponse response = httpClient.execute(illustPage);
|
||||||
|
String pageAsXML = EntityUtils.toString(response.getEntity(),"utf-8");
|
||||||
|
//System.out.println(pageAsXML);
|
||||||
|
JsonObject resultObj = (JsonObject) new JsonParser().parse(pageAsXML);
|
||||||
|
if(!resultObj.get("error").getAsBoolean()){
|
||||||
|
JsonArray bodyArray = resultObj.get("body").getAsJsonArray();
|
||||||
|
int length = bodyArray.size();
|
||||||
|
String[] result = new String[length];
|
||||||
|
for(int i = 0; i < length; i++){
|
||||||
|
JsonObject childObj = bodyArray.get(i).getAsJsonObject();
|
||||||
|
result[i] = childObj.get("urls").getAsJsonObject().get("original").getAsString();
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}else{
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 下载P站图片
|
||||||
|
* @param illustID 插图ID
|
||||||
|
* @return 成功返回图片输入流,失败或为多图则返回null
|
||||||
|
*/
|
||||||
|
public InputStream[] downloadIllustImage(int illustID) throws IOException {
|
||||||
|
String[] links = getAllDownloadLink(illustID);
|
||||||
|
List<InputStream> inputStreamList = new ArrayList<>();
|
||||||
|
int count = 1;
|
||||||
|
boolean retry = false;
|
||||||
|
for(int i = 0; i < links.length; i++){
|
||||||
|
try {
|
||||||
|
long sleepTime = (new Random().nextInt(4) + 2) * 1000;
|
||||||
|
System.out.println("nextTime: " + (float)(sleepTime / 1000));
|
||||||
|
Thread.sleep(sleepTime);
|
||||||
|
} catch (InterruptedException ignored) {}
|
||||||
|
String link = links[i];
|
||||||
|
System.out.print("page:" + count++ + "/" + links.length + " ...");
|
||||||
|
HttpGet imgGet = new HttpGet(link);
|
||||||
|
//关键!如果不加上Referer的话,会返回403
|
||||||
|
imgGet.setHeader("Referer", PixivURL.PIXIV_ILLUST_MEDIUM_URL.replaceAll("\\{illustId}", String.valueOf(illustID)));
|
||||||
|
RequestConfig config = RequestConfig.custom()
|
||||||
|
.setConnectTimeout(20 * 1000)
|
||||||
|
.setConnectionRequestTimeout(20 * 1000)
|
||||||
|
.setSocketTimeout(30 * 1000)
|
||||||
|
.build();
|
||||||
|
imgGet.setConfig(config);
|
||||||
|
HttpResponse response;
|
||||||
|
try {
|
||||||
|
response = httpClient.execute(imgGet);
|
||||||
|
}catch(ConnectionPoolTimeoutException e){
|
||||||
|
if(retry){
|
||||||
|
retry = false;
|
||||||
|
System.out.println("获取失败,跳过...");
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
System.out.println("连接超时,重新获取...");
|
||||||
|
retry = true;
|
||||||
|
i--;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
retry = false;
|
||||||
|
|
||||||
|
ByteArrayOutputStream cacheOutputStream = new ByteArrayOutputStream((int)response.getEntity().getContentLength());
|
||||||
|
InputStream content = response.getEntity().getContent();
|
||||||
|
int readLength;
|
||||||
|
byte[] cache = new byte[4096];
|
||||||
|
while((readLength = content.read(cache)) != -1){
|
||||||
|
cacheOutputStream.write(cache, 0, readLength);
|
||||||
|
}
|
||||||
|
byte[] data = cacheOutputStream.toByteArray();
|
||||||
|
//System.out.println("读到数据: " + data.length);
|
||||||
|
inputStreamList.add(new ByteArrayInputStream(data));
|
||||||
|
System.out.println("done!(length: " + response.getEntity().getContentLength() + ")");
|
||||||
|
|
||||||
|
}
|
||||||
|
return inputStreamList.toArray(new InputStream[0]);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 下载P站图片
|
||||||
|
* @param illustID 插图ID
|
||||||
|
* @return 成功返回图片输入流,失败或为多图则返回null
|
||||||
|
*/
|
||||||
|
public InputStream downloadIllustImages(int illustID){
|
||||||
|
throw new UnsupportedOperationException();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 通过解析插图详情页获取
|
||||||
|
* - 插图标题
|
||||||
|
* - 插图作者(及其UserId)
|
||||||
|
* - 插图上传时间
|
||||||
|
* - 插图标签(原始标签)
|
||||||
|
* ...
|
||||||
|
* @return 成功返回IllustInfo对象,失败返回null
|
||||||
|
*/
|
||||||
|
public IllustInfo[] getIllustInfo(int[] illustIDs) throws IOException {
|
||||||
|
//获取Api
|
||||||
|
HttpGet apiRequest = new HttpGet(PixivURL.getPixivIllustInfoAPI(illustIDs));
|
||||||
|
HttpResponse response = httpClient.execute(apiRequest);
|
||||||
|
String resultText = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8);
|
||||||
|
System.out.println(resultText);
|
||||||
|
JsonObject resultObj = ((JsonObject) new JsonParser().parse(resultText));
|
||||||
|
if(resultObj.get("error").getAsBoolean()){
|
||||||
|
System.err.println("获取失败!");
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
List<IllustInfo> illustInfoList = new ArrayList<>();
|
||||||
|
JsonArray illustArray = resultObj.get("body").getAsJsonObject().get("illusts").getAsJsonArray();
|
||||||
|
illustArray.forEach(jsonElement -> {
|
||||||
|
JsonObject illustInfoObj = jsonElement.getAsJsonObject();
|
||||||
|
JsonArray tagsArray = illustInfoObj.get("tags").getAsJsonArray();
|
||||||
|
String[] tags = new String[tagsArray.size()];
|
||||||
|
for(int i = 0; i < tags.length; i++){
|
||||||
|
tags[i] = tagsArray.get(i).getAsString();
|
||||||
|
}
|
||||||
|
//TODO: 通过不需要作者id就能获取图片信息的api无法获取图片尺寸
|
||||||
|
IllustInfo illustInfo = new IllustInfo(
|
||||||
|
illustInfoObj.get("workId").getAsInt(),
|
||||||
|
illustInfoObj.get("title").getAsString(),
|
||||||
|
null,
|
||||||
|
tags,
|
||||||
|
-1,
|
||||||
|
-1,
|
||||||
|
illustInfoObj.get("userName").getAsString(),
|
||||||
|
illustInfoObj.get("userId").getAsInt()
|
||||||
|
);
|
||||||
|
});
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 获取指定用户的所有插画
|
||||||
|
*/
|
||||||
|
public void getUserAllIllustTest() {
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
}
|
533
src/main/java/net/lamgc/cgj/pixiv/PixivDownload.java
Normal file
533
src/main/java/net/lamgc/cgj/pixiv/PixivDownload.java
Normal file
@ -0,0 +1,533 @@
|
|||||||
|
package net.lamgc.cgj.pixiv;
|
||||||
|
|
||||||
|
import com.google.gson.Gson;
|
||||||
|
import com.google.gson.JsonArray;
|
||||||
|
import com.google.gson.JsonElement;
|
||||||
|
import com.google.gson.JsonObject;
|
||||||
|
import io.netty.handler.codec.http.HttpHeaderNames;
|
||||||
|
import org.apache.http.Header;
|
||||||
|
import org.apache.http.HttpHost;
|
||||||
|
import org.apache.http.HttpRequest;
|
||||||
|
import org.apache.http.HttpResponse;
|
||||||
|
import org.apache.http.client.CookieStore;
|
||||||
|
import org.apache.http.client.HttpClient;
|
||||||
|
import org.apache.http.client.config.RequestConfig;
|
||||||
|
import org.apache.http.client.methods.HttpGet;
|
||||||
|
import org.apache.http.impl.client.HttpClientBuilder;
|
||||||
|
import org.apache.http.message.BasicHeader;
|
||||||
|
import org.apache.http.util.EntityUtils;
|
||||||
|
import org.jsoup.Jsoup;
|
||||||
|
import org.jsoup.nodes.Document;
|
||||||
|
import org.jsoup.select.Elements;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
import java.io.BufferedInputStream;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.io.InputStream;
|
||||||
|
import java.nio.charset.StandardCharsets;
|
||||||
|
import java.util.*;
|
||||||
|
import java.util.concurrent.atomic.AtomicInteger;
|
||||||
|
import java.util.function.BiConsumer;
|
||||||
|
|
||||||
|
public class PixivDownload {
|
||||||
|
|
||||||
|
private final static Logger log = LoggerFactory.getLogger("PixivDownload");
|
||||||
|
|
||||||
|
private final HttpClient httpClient;
|
||||||
|
|
||||||
|
private final CookieStore cookieStore;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 构造一个PixivDownload对象
|
||||||
|
* @param cookieStore 存在已登录Pixiv的CookieStore对象
|
||||||
|
*/
|
||||||
|
public PixivDownload(CookieStore cookieStore) {
|
||||||
|
this(cookieStore, null);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 构造一个PixivDownload对象
|
||||||
|
* @param cookieStore 存在已登录Pixiv的CookieStore对象
|
||||||
|
* @param proxy 访问代理
|
||||||
|
*/
|
||||||
|
public PixivDownload(CookieStore cookieStore, HttpHost proxy) {
|
||||||
|
this.cookieStore = cookieStore;
|
||||||
|
HttpClientBuilder builder = HttpClientBuilder.create();
|
||||||
|
builder.setDefaultCookieStore(cookieStore);
|
||||||
|
// UA: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.122 Safari/537.36
|
||||||
|
ArrayList<Header> defaultHeaders = new ArrayList<>(2);
|
||||||
|
defaultHeaders.add(new BasicHeader("User-Agent",
|
||||||
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.122 Safari/537.36"));
|
||||||
|
builder.setDefaultHeaders(defaultHeaders);
|
||||||
|
builder.setProxy(proxy);
|
||||||
|
httpClient = builder.build();
|
||||||
|
}
|
||||||
|
|
||||||
|
public HttpClient getHttpClient() {
|
||||||
|
return httpClient;
|
||||||
|
}
|
||||||
|
|
||||||
|
public CookieStore getCookieStore() {
|
||||||
|
return cookieStore;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 获取帐号所有的收藏插画,并以输入流形式提供
|
||||||
|
* @param fn 回调函数,函数传进的InputStream无需手动关闭
|
||||||
|
* @throws IOException 当获取时发生异常则直接抛出
|
||||||
|
*/
|
||||||
|
public void getCollectionAsInputStream(PageQuality quality, BiConsumer<String, InputStream> fn) throws IOException {
|
||||||
|
int pageIndex = 0;
|
||||||
|
HttpGet request;
|
||||||
|
Document document;
|
||||||
|
ArrayList<String> linkList = new ArrayList<>();
|
||||||
|
do {
|
||||||
|
request = new HttpGet(PixivURL.PIXIV_USER_COLLECTION_PAGE.replace("{pageIndex}", Integer.toString(++pageIndex)));
|
||||||
|
setCookieInRequest(request, cookieStore);
|
||||||
|
log.info("Request Link: " + request.getURI().toString());
|
||||||
|
HttpResponse response = httpClient.execute(request);
|
||||||
|
// 解析网页内容,获得所有的收藏信息
|
||||||
|
document = Jsoup.parse(EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8));
|
||||||
|
Elements items = document.select(".display_editable_works .image-item a.work");
|
||||||
|
List<String> hrefList = items.eachAttr("href");
|
||||||
|
log.info("第 {} 页获取到的图片项数量: {}", pageIndex, hrefList.size());
|
||||||
|
if(hrefList.size() == 0) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
Gson gson = new Gson();
|
||||||
|
for (String href : hrefList) {
|
||||||
|
HttpGet linkApiRequest = createHttpGetRequest(PixivURL.PIXIV_ILLUST_API_URL.replace("{illustId}", href.substring(href.lastIndexOf("/") + 1)));
|
||||||
|
log.debug(linkApiRequest.getURI().toString());
|
||||||
|
HttpResponse httpResponse = httpClient.execute(linkApiRequest);
|
||||||
|
JsonObject linkResult = gson.fromJson(EntityUtils.toString(httpResponse.getEntity()), JsonObject.class);
|
||||||
|
if(linkResult.get("error").getAsBoolean()) {
|
||||||
|
log.error("接口返回错误信息: {}", linkResult.get("message").getAsString());
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
JsonArray linkArray = linkResult.get("body").getAsJsonArray();
|
||||||
|
for (int i = 0; i < linkArray.size(); i++) {
|
||||||
|
JsonObject linkObject = linkArray.get(i).getAsJsonObject().get("urls").getAsJsonObject();
|
||||||
|
linkList.add(linkObject.get((quality == null ? PageQuality.ORIGINAL : quality).toString().toLowerCase()).getAsString());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} while(!document.select(".pager-container>.next").isEmpty());
|
||||||
|
log.info("获取完成.");
|
||||||
|
AtomicInteger count = new AtomicInteger(1);
|
||||||
|
linkList.forEach(link -> {
|
||||||
|
log.info("Next Link [{}]: {}", count.getAndIncrement(), link);
|
||||||
|
InputStream imageInputStream = null;
|
||||||
|
int tryCount = 0;
|
||||||
|
do {
|
||||||
|
try {
|
||||||
|
imageInputStream = getImageAsInputStream(httpClient, link);
|
||||||
|
} catch (IOException e) {
|
||||||
|
log.error("获取图片数据时发生异常", e);
|
||||||
|
if(++tryCount < 5) {
|
||||||
|
log.warn("即将重试[{} / 5]", tryCount);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} while(imageInputStream == null);
|
||||||
|
|
||||||
|
try(InputStream imageInput = new BufferedInputStream(imageInputStream, 256 * 1024)) {
|
||||||
|
log.debug("调用回调方法...");
|
||||||
|
fn.accept(link, imageInput);
|
||||||
|
log.debug("回调方法调用完成.");
|
||||||
|
} catch (IOException e) {
|
||||||
|
log.error("图片获取失败", e);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 获取推荐插图
|
||||||
|
* @param quality 图片质量
|
||||||
|
* @param fn 回调函数
|
||||||
|
* @throws IOException 当获取时发生异常则直接抛出
|
||||||
|
*/
|
||||||
|
public void getRecommendAsInputStream(PageQuality quality, BiConsumer<String, InputStream> fn) throws IOException {
|
||||||
|
HttpResponse response = httpClient.execute(createHttpGetRequest(PixivURL.PIXIV_INDEX_URL));
|
||||||
|
Document document = Jsoup.parse(EntityUtils.toString(response.getEntity()));
|
||||||
|
|
||||||
|
HttpClient imageClient = HttpClientBuilder.create().build();
|
||||||
|
Elements elements = document.select(".gtm-illust-recommend-zone>.image-item>.gtm-illust-recommend-thumbnail-link");
|
||||||
|
for(int illustIndex = 0; illustIndex < elements.size(); illustIndex++){
|
||||||
|
String href = elements.get(illustIndex).attr("href");
|
||||||
|
int illustId = Integer.parseInt(href.substring(href.lastIndexOf("/") + 1));
|
||||||
|
log.info("({}/{}) Href: {}, IllustID: {}", illustIndex + 1, elements.size(), href, illustId);
|
||||||
|
List<String> pageLinkList = getIllustAllPageDownload(httpClient, this.cookieStore, illustId, quality);
|
||||||
|
for (int linkIndex = 0; linkIndex < pageLinkList.size(); linkIndex++) {
|
||||||
|
String link = pageLinkList.get(linkIndex);
|
||||||
|
String fileName = link.substring(link.lastIndexOf("/") + 1);
|
||||||
|
log.info("({}/{})正在处理 {}", linkIndex, pageLinkList.size(), fileName);
|
||||||
|
InputStream imageInputStream = null;
|
||||||
|
int tryCount = 0;
|
||||||
|
do {
|
||||||
|
try {
|
||||||
|
imageInputStream = getImageAsInputStream(imageClient, link);
|
||||||
|
} catch (IOException e) {
|
||||||
|
log.error("获取图片数据时发生异常", e);
|
||||||
|
if(++tryCount < 5) {
|
||||||
|
log.warn("即将重试[{} / 5]", tryCount);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} while(imageInputStream == null);
|
||||||
|
|
||||||
|
try(InputStream pageInputStream = new BufferedInputStream(imageInputStream, 256 * 1024)) {
|
||||||
|
fn.accept(fileName, pageInputStream);
|
||||||
|
}
|
||||||
|
log.info("Done!");
|
||||||
|
|
||||||
|
}
|
||||||
|
log.info("IllustId {} 处理完成.", illustId);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 获取排行榜
|
||||||
|
* @param contentType 内容类型
|
||||||
|
* @param mode 查询模式
|
||||||
|
* @param time 查询时间
|
||||||
|
* @param range 从第一名开始的范围
|
||||||
|
* @param quality 图片质量
|
||||||
|
* @param fn 回调函数
|
||||||
|
* @throws IOException 当请求发生异常时抛出
|
||||||
|
*/
|
||||||
|
public void getRankingAsInputStream(PixivURL.RankingContentType contentType, PixivURL.RankingMode mode,
|
||||||
|
Date time, int range, PageQuality quality, RankingDownloadFunction fn) throws IOException {
|
||||||
|
getRankingAsInputStream(contentType, mode, time, 1, range, quality, fn);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 获取排行榜
|
||||||
|
* @param contentType 内容类型
|
||||||
|
* @param mode 查询模式
|
||||||
|
* @param time 查询时间
|
||||||
|
* @param rankStart 开始排行位(包括)
|
||||||
|
* @param range 范围
|
||||||
|
* @param quality 图片质量
|
||||||
|
* @param fn 回调函数
|
||||||
|
* @throws IOException 当请求发生异常时抛出
|
||||||
|
*/
|
||||||
|
public void getRankingAsInputStream(PixivURL.RankingContentType contentType, PixivURL.RankingMode mode,
|
||||||
|
Date time, int rankStart, int range, PageQuality quality, RankingDownloadFunction fn) throws IOException {
|
||||||
|
getRanking(contentType, mode, time, rankStart, range).forEach(rankInfo -> {
|
||||||
|
int rank = rankInfo.get("rank").getAsInt();
|
||||||
|
int illustId = rankInfo.get("illust_id").getAsInt();
|
||||||
|
int authorId = rankInfo.get("user_id").getAsInt();
|
||||||
|
String authorName = rankInfo.get("user_name").getAsString();
|
||||||
|
String title = rankInfo.get("title").getAsString();
|
||||||
|
log.info("当前到第 {}/{} 名(总共 {} 名), IllustID: {}, Author: ({}) {}, Title: {}", rank, rankStart + range - 1, range, illustId, authorId, authorName, title);
|
||||||
|
log.info("正在获取PagesLink...");
|
||||||
|
List<String> linkList;
|
||||||
|
try {
|
||||||
|
linkList = getIllustAllPageDownload(httpClient, this.cookieStore, illustId, quality);
|
||||||
|
} catch (IOException e) {
|
||||||
|
if(e.getMessage().equals("該当作品は削除されたか、存在しない作品IDです。")) {
|
||||||
|
log.warn("作品 {} 不存在.", illustId);
|
||||||
|
} else {
|
||||||
|
e.printStackTrace();
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
log.info("PagesLink 获取完成, 总数: {}", linkList.size());
|
||||||
|
for (int pageIndex = 0; pageIndex < linkList.size(); pageIndex++) {
|
||||||
|
String downloadLink = linkList.get(pageIndex);
|
||||||
|
log.info("当前Page: {}/{}", pageIndex + 1, linkList.size());
|
||||||
|
try(InputStream imageInputStream = new BufferedInputStream(getImageAsInputStream(HttpClientBuilder.create().build(), downloadLink), 256 * 1024)) {
|
||||||
|
fn.download(rank, downloadLink, rankInfo.deepCopy(), imageInputStream);
|
||||||
|
} catch(IOException e) {
|
||||||
|
log.error("下载插画时发生异常", e);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
log.info("完成.");
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
public List<JsonObject> getRanking(PixivURL.RankingContentType contentType, PixivURL.RankingMode mode,
|
||||||
|
Date time, int rankStart, int range) throws IOException {
|
||||||
|
if(rankStart <= 0) {
|
||||||
|
throw new IllegalArgumentException("rankStart cannot be less than or equal to zero");
|
||||||
|
}
|
||||||
|
if(range <= 0) {
|
||||||
|
throw new IllegalArgumentException("range cannot be less than or equal to zero");
|
||||||
|
}
|
||||||
|
|
||||||
|
int startPage = (int) Math.ceil(rankStart / 50F);
|
||||||
|
int requestFrequency = (int) Math.ceil((rankStart + (range - 1)) / 50F);
|
||||||
|
int surplusQuantity = range;
|
||||||
|
boolean firstRequest = true;
|
||||||
|
Gson gson = new Gson();
|
||||||
|
ArrayList<JsonObject> results = new ArrayList<>();
|
||||||
|
for (int requestCount = startPage; requestCount <= requestFrequency && requestCount <= 10; requestCount++) {
|
||||||
|
int rangeStart = (requestCount - 1) * 50 + 1;
|
||||||
|
log.info("正在请求第 {} 到 {} 位排名榜 (第{}次请求, 共 {} 次)", rangeStart, rangeStart + 49, requestCount - startPage + 1, requestFrequency - startPage);
|
||||||
|
HttpGet request = createHttpGetRequest(PixivURL.getRankingLink(contentType, mode, time, requestCount, true));
|
||||||
|
log.info("Request URL: {}", request.getURI());
|
||||||
|
HttpResponse response = httpClient.execute(request);
|
||||||
|
String content = EntityUtils.toString(response.getEntity());
|
||||||
|
log.debug("Content: " + content);
|
||||||
|
JsonObject contentObject = gson.fromJson(content, JsonObject.class);
|
||||||
|
if(contentObject.has("error")) {
|
||||||
|
log.warn("接口报错, 返回信息: {}", contentObject.get("error").getAsString());
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
JsonArray rankingArray = contentObject.getAsJsonArray("contents");
|
||||||
|
log.info("正在解析数据...");
|
||||||
|
|
||||||
|
//需要添加一个总量, 否则会完整跑完一次.
|
||||||
|
//检查是否为最后一次请求,和剩余量有多少
|
||||||
|
int firstRequestStartIndex = (rankStart % 50) - 1;
|
||||||
|
for (int rankIndex = firstRequest ? firstRequestStartIndex : 0; rankIndex < rankingArray.size() && surplusQuantity > 0; rankIndex++, surplusQuantity--) {
|
||||||
|
JsonElement jsonElement = rankingArray.get(rankIndex);
|
||||||
|
JsonObject rankInfo = jsonElement.getAsJsonObject();
|
||||||
|
int rank = rankInfo.get("rank").getAsInt();
|
||||||
|
int illustId = rankInfo.get("illust_id").getAsInt();
|
||||||
|
int authorId = rankInfo.get("user_id").getAsInt();
|
||||||
|
String authorName = rankInfo.get("user_name").getAsString();
|
||||||
|
String title = rankInfo.get("title").getAsString();
|
||||||
|
log.info("当前到第 {}/{} 名(总共 {} 名), IllustID: {}, Author: ({}) {}, Title: {}", rank, rankStart + range - 1, range, illustId, authorId, authorName, title);
|
||||||
|
results.add(rankInfo);
|
||||||
|
}
|
||||||
|
firstRequest = false;
|
||||||
|
log.info("第 {} 到 {} 位排名榜完成. (第{}次请求)", rangeStart, rangeStart + 49, requestCount);
|
||||||
|
}
|
||||||
|
|
||||||
|
if(requestFrequency > 10) {
|
||||||
|
log.warn("请求的排名榜范围超出所支持的范围, 已终止请求.");
|
||||||
|
}
|
||||||
|
|
||||||
|
return results;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 获取作品的预加载数据
|
||||||
|
* @param illustId 作品id
|
||||||
|
* @return 如果请求成功返回JsonObject, 失败返回null
|
||||||
|
* @throws IOException 当请求响应非200或请求发生异常时抛出.
|
||||||
|
*/
|
||||||
|
public JsonObject getIllustPreLoadDataById(int illustId) throws IOException {
|
||||||
|
HttpGet request = createHttpGetRequest(PixivURL.getPixivRefererLink(illustId));
|
||||||
|
HttpResponse response = httpClient.execute(request);
|
||||||
|
|
||||||
|
if(response.getStatusLine().getStatusCode() != 200) {
|
||||||
|
throw new IOException("Http响应码非200: " + response.getStatusLine());
|
||||||
|
}
|
||||||
|
|
||||||
|
Document document = Jsoup.parse(EntityUtils.toString(response.getEntity()));
|
||||||
|
Elements selectElements = document.select("#meta-preload-data");
|
||||||
|
if(selectElements.size() == 0) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
return new Gson().fromJson(selectElements.attr("content"), JsonObject.class);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@FunctionalInterface
|
||||||
|
public interface RankingDownloadFunction {
|
||||||
|
/**
|
||||||
|
* 接收图片InputStream
|
||||||
|
* @param rank 当前作品排名
|
||||||
|
* @param link 作品下载链接
|
||||||
|
* @param inputStream 作品下载输入流, InputStream会自动关闭
|
||||||
|
*/
|
||||||
|
void download(int rank, String link, JsonObject rankInfo, InputStream inputStream);
|
||||||
|
}
|
||||||
|
|
||||||
|
public HttpGet createHttpGetRequest(String url) {
|
||||||
|
HttpGet request = new HttpGet(url);
|
||||||
|
setCookieInRequest(request, cookieStore);
|
||||||
|
return request;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 取Illust所有页的原图下载链接
|
||||||
|
* @param httpClient 用于发起请求的HttpClient对象
|
||||||
|
* @param illustId 插画ID
|
||||||
|
* @param quality 页质量, 见{@link PageQuality}
|
||||||
|
* @return 返回该illust所有Page的下载链接
|
||||||
|
* @throws IOException 当HttpClient在请求时发生异常, 或接口报错时抛出, 注意{@link IOException#getMessage()}
|
||||||
|
*/
|
||||||
|
public static List<String> getIllustAllPageDownload(HttpClient httpClient, CookieStore cookieStore, int illustId, PageQuality quality) throws IOException {
|
||||||
|
HttpGet linkApiRequest = new HttpGet(PixivURL.PIXIV_ILLUST_API_URL.replace("{illustId}", Integer.toString(illustId)));
|
||||||
|
setCookieInRequest(linkApiRequest, cookieStore);
|
||||||
|
HttpResponse response = httpClient.execute(linkApiRequest);
|
||||||
|
JsonObject resultObject = new Gson().fromJson(EntityUtils.toString(response.getEntity()), JsonObject.class);
|
||||||
|
|
||||||
|
if(resultObject.get("error").getAsBoolean()) {
|
||||||
|
String message = resultObject.get("message").getAsString();
|
||||||
|
log.info("请求错误, 错误信息: {}", message);
|
||||||
|
throw new IOException(message);
|
||||||
|
}
|
||||||
|
|
||||||
|
JsonArray linkArray = resultObject.getAsJsonArray("body");
|
||||||
|
|
||||||
|
ArrayList<String> resultList = new ArrayList<>();
|
||||||
|
String qualityType = quality == null ? "original" : quality.toString().toLowerCase();
|
||||||
|
log.info("已选择插画类型: {}", qualityType);
|
||||||
|
linkArray.forEach(el -> {
|
||||||
|
JsonObject urlObj = el.getAsJsonObject().getAsJsonObject("urls");
|
||||||
|
resultList.add(urlObj.get(qualityType).getAsString());
|
||||||
|
});
|
||||||
|
|
||||||
|
return resultList;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 插图质量
|
||||||
|
*/
|
||||||
|
public enum PageQuality{
|
||||||
|
/**
|
||||||
|
* 原图画质
|
||||||
|
*/
|
||||||
|
ORIGINAL,
|
||||||
|
/**
|
||||||
|
* 常规画质
|
||||||
|
*/
|
||||||
|
REGULAR,
|
||||||
|
/**
|
||||||
|
* 小图画质
|
||||||
|
*/
|
||||||
|
SMALL,
|
||||||
|
/**
|
||||||
|
* 迷你画质
|
||||||
|
*/
|
||||||
|
THUMB_MINI
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 获取帐号所有的收藏插画,并以输入流形式提供
|
||||||
|
* @return 获取所有链接的InputStream, 请注意关闭InputStream
|
||||||
|
* @throws IOException 当获取时发生异常则直接抛出
|
||||||
|
*/
|
||||||
|
public Set<Map.Entry<String, InputStream>> getCollectionAsInputStream(PageQuality quality) throws IOException {
|
||||||
|
HashSet<Map.Entry<String, InputStream>> illustInputStreamSet = new HashSet<>();
|
||||||
|
getCollectionAsInputStream(quality, (link, inputStream) -> illustInputStreamSet.add(new AbstractMap.SimpleEntry<>(link, inputStream)));
|
||||||
|
return illustInputStreamSet;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 获取Pixiv图片
|
||||||
|
* @param httpClient HttpClient对象
|
||||||
|
* @param link Pixiv图片链接
|
||||||
|
* @return 返回图片InputStream,注意关闭InputStream
|
||||||
|
* @throws IOException 获取失败时抛出
|
||||||
|
* @throws IllegalArgumentException 当链接无法处理时抛出
|
||||||
|
*/
|
||||||
|
public static InputStream getImageAsInputStream(HttpClient httpClient, String link) throws IOException {
|
||||||
|
HttpGet request = new HttpGet(link);
|
||||||
|
int startIndex = link.lastIndexOf("/");
|
||||||
|
int endIndex = link.lastIndexOf("_");
|
||||||
|
if(startIndex == -1 || endIndex == -1) {
|
||||||
|
throw new IllegalArgumentException("无法从链接获取illustID: " + link);
|
||||||
|
}
|
||||||
|
|
||||||
|
String referer = PixivURL.getPixivRefererLink(link.substring(startIndex + 1, endIndex));
|
||||||
|
request.addHeader(HttpHeaderNames.REFERER.toString(), referer);
|
||||||
|
|
||||||
|
HttpResponse response = httpClient.execute(request);
|
||||||
|
log.debug("response: {}", response);
|
||||||
|
log.info("Content Length: {}KB", Float.parseFloat(response.getFirstHeader(HttpHeaderNames.CONTENT_LENGTH.toString()).getValue()) / 1024F);
|
||||||
|
log.info("{}", response.getFirstHeader(HttpHeaderNames.CONTENT_TYPE.toString()));
|
||||||
|
return response.getEntity().getContent();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 登出当前会话.<br/>
|
||||||
|
* 登出成功后, 该Cookies作废.
|
||||||
|
* @return 返回是否成功登出
|
||||||
|
* @throws IOException 当登出请求异常时抛出
|
||||||
|
*/
|
||||||
|
public boolean logOut() throws IOException {
|
||||||
|
HttpGet request = new HttpGet(PixivURL.PIXIV_LOGOUT_URL);
|
||||||
|
request.setConfig(RequestConfig.custom().setRedirectsEnabled(false).build());
|
||||||
|
setCookieInRequest(request, cookieStore);
|
||||||
|
HttpResponse response = httpClient.execute(request);
|
||||||
|
if (response.getStatusLine().getStatusCode() == 302) {
|
||||||
|
cookieStore.clear();
|
||||||
|
return true;
|
||||||
|
} else {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 获取作品信息
|
||||||
|
* @param illustId 作品ID
|
||||||
|
* @return 成功获取返回JsonObject, 失败返回null, <br/>
|
||||||
|
* Json示例: <br/>
|
||||||
|
* <pre>
|
||||||
|
* {
|
||||||
|
* "illustId": "79584670",
|
||||||
|
* "illustTitle": "このヤンキーはウブすぎる",
|
||||||
|
* "id": "79584670",
|
||||||
|
* "title": "このヤンキーはウブすぎる",
|
||||||
|
* "illustType": 1,
|
||||||
|
* "xRestrict": 0,
|
||||||
|
* "restrict": 0,
|
||||||
|
* "sl": 2,
|
||||||
|
* "url": "https://i.pximg.net/c/360x360_70/img-master/img/2020/02/19/00/38/23/79584670_p0_square1200.jpg",
|
||||||
|
* "description": "",
|
||||||
|
* "tags": [
|
||||||
|
* "漫画",
|
||||||
|
* "オリジナル",
|
||||||
|
* "創作",
|
||||||
|
* "創作男女",
|
||||||
|
* "コロさん、ポリさん此方です!",
|
||||||
|
* "恋の予感",
|
||||||
|
* "あまずっぺー",
|
||||||
|
* "交換日記",
|
||||||
|
* "続編希望!!",
|
||||||
|
* "オリジナル10000users入り"
|
||||||
|
* ],
|
||||||
|
* "userId": "4778293",
|
||||||
|
* "userName": "隈浪さえ",
|
||||||
|
* "width": 3288,
|
||||||
|
* "height": 4564,
|
||||||
|
* "pageCount": 4,
|
||||||
|
* "isBookmarkable": true,
|
||||||
|
* "bookmarkData": null,
|
||||||
|
* "alt": "#オリジナル このヤンキーはウブすぎる - 隈浪さえ的漫画",
|
||||||
|
* "isAdContainer": false,
|
||||||
|
* "profileImageUrl": "https://i.pximg.net/user-profile/img/2019/12/04/18/56/19/16639046_fea29ce38ea89b0cb2313b40b3a72f9a_50.jpg",
|
||||||
|
* "type": "illust"
|
||||||
|
* }
|
||||||
|
* </pre>
|
||||||
|
* @throws IOException 当请求发生异常, 或接口返回错误信息时抛出.
|
||||||
|
*/
|
||||||
|
public JsonObject getIllustInfoByIllustId(int illustId) throws IOException {
|
||||||
|
HttpGet request = createHttpGetRequest(PixivURL.getPixivIllustInfoAPI(new int[] {illustId}));;
|
||||||
|
HttpResponse response = httpClient.execute(request);
|
||||||
|
String responseStr = EntityUtils.toString(response.getEntity());
|
||||||
|
log.debug("Response Content: {}", responseStr);
|
||||||
|
JsonObject responseObj = new Gson().fromJson(responseStr, JsonObject.class);
|
||||||
|
|
||||||
|
if(responseObj.get("error").getAsBoolean()) {
|
||||||
|
throw new IOException(responseObj.get("message").getAsString());
|
||||||
|
}
|
||||||
|
|
||||||
|
JsonArray illustsArray = responseObj.getAsJsonObject("body").getAsJsonArray("illusts");
|
||||||
|
if(illustsArray.size() == 1) {
|
||||||
|
return illustsArray.get(0).getAsJsonObject();
|
||||||
|
} else {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
public static void setCookieInRequest(HttpRequest request, CookieStore cookieStore) {
|
||||||
|
StringBuilder builder = new StringBuilder();
|
||||||
|
cookieStore.getCookies().forEach(cookie -> builder.append(cookie.getName()).append("=").append(cookie.getValue()).append("; "));
|
||||||
|
request.setHeader(HttpHeaderNames.COOKIE.toString(), builder.toString());
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
340
src/main/java/net/lamgc/cgj/pixiv/PixivSearchBuilder.java
Normal file
340
src/main/java/net/lamgc/cgj/pixiv/PixivSearchBuilder.java
Normal file
@ -0,0 +1,340 @@
|
|||||||
|
package net.lamgc.cgj.pixiv;
|
||||||
|
|
||||||
|
import com.google.common.base.Strings;
|
||||||
|
|
||||||
|
import java.io.UnsupportedEncodingException;
|
||||||
|
import java.net.URLEncoder;
|
||||||
|
import java.text.SimpleDateFormat;
|
||||||
|
import java.util.Date;
|
||||||
|
import java.util.HashSet;
|
||||||
|
import java.util.Objects;
|
||||||
|
|
||||||
|
public class PixivSearchBuilder {
|
||||||
|
|
||||||
|
private final String content;
|
||||||
|
|
||||||
|
private SearchArea searchArea = SearchArea.ARTWORKS;
|
||||||
|
private SearchMode searchMode = SearchMode.TAG_FULL;
|
||||||
|
private SearchType searchType = SearchType.ILLUST_AND_UGOIRA;
|
||||||
|
private SearchOrder searchOrder = SearchOrder.DATE_D;
|
||||||
|
private SearchContentOption searchContentOption = SearchContentOption.ALL;
|
||||||
|
|
||||||
|
private HashSet<String> includeKeywords = new HashSet<>(0);
|
||||||
|
private HashSet<String> excludeKeywords = new HashSet<>(0);
|
||||||
|
|
||||||
|
private int page = 1;
|
||||||
|
|
||||||
|
private int wgt = 0;
|
||||||
|
private int hgt = 0;
|
||||||
|
|
||||||
|
private int wlt = 0;
|
||||||
|
private int hlt = 0;
|
||||||
|
|
||||||
|
private RatioOption ratioOption = null;
|
||||||
|
|
||||||
|
private Date startDate = null;
|
||||||
|
private Date endDate = null;
|
||||||
|
|
||||||
|
public PixivSearchBuilder(String searchContent) {
|
||||||
|
this.content = Objects.requireNonNull(searchContent);
|
||||||
|
}
|
||||||
|
|
||||||
|
public String buildURL() {
|
||||||
|
StringBuilder builder;
|
||||||
|
try {
|
||||||
|
builder = new StringBuilder(PixivURL.PIXIV_SEARCH_CONTENT_URL.replaceAll("\\{area}", searchArea.name().toLowerCase())
|
||||||
|
.replaceAll("\\{content}",
|
||||||
|
URLEncoder.encode(getSearchCondition(), "UTF-8").replaceAll("\\+", "%20")
|
||||||
|
)
|
||||||
|
);
|
||||||
|
} catch (UnsupportedEncodingException e) {
|
||||||
|
throw new RuntimeException(e);
|
||||||
|
}
|
||||||
|
if(searchArea.equals(SearchArea.TOP)) {
|
||||||
|
return builder.toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
builder.append("&s_mode=").append(searchMode.name().toLowerCase());
|
||||||
|
builder.append("&type=").append(searchType.name().toLowerCase());
|
||||||
|
builder.append("&p=").append(page);
|
||||||
|
builder.append("&order=").append(searchOrder.name().toLowerCase());
|
||||||
|
builder.append("&mode=").append(searchContentOption.name().toLowerCase());
|
||||||
|
|
||||||
|
//可选参数
|
||||||
|
if(wgt > 0 && hgt > 0) {
|
||||||
|
builder.append("&wgt=").append(wgt);
|
||||||
|
builder.append("&hgt").append(hgt);
|
||||||
|
}
|
||||||
|
|
||||||
|
//可选参数
|
||||||
|
if(wlt > 0 && hlt > 0) {
|
||||||
|
builder.append("&wlt=").append(wlt);
|
||||||
|
builder.append("&hlt").append(hlt);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (ratioOption != null) {
|
||||||
|
builder.append("&ratio=").append(ratioOption.value);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd");
|
||||||
|
if (startDate != null) {
|
||||||
|
builder.append("&scd=").append(format.format(startDate));
|
||||||
|
}
|
||||||
|
if (endDate != null) {
|
||||||
|
builder.append("&ecd=").append(format.format(endDate));
|
||||||
|
}
|
||||||
|
|
||||||
|
return builder.toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
public PixivSearchBuilder setSearchArea(SearchArea searchArea) {
|
||||||
|
this.searchArea = searchArea;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 获取搜索区域
|
||||||
|
* @return 返回搜索区域对象
|
||||||
|
*/
|
||||||
|
public SearchArea getSearchArea() {
|
||||||
|
return searchArea;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 获取搜索条件.
|
||||||
|
* @return 搜索条件内容
|
||||||
|
*/
|
||||||
|
public String getSearchCondition() {
|
||||||
|
StringBuilder searchContent = new StringBuilder(Strings.nullToEmpty(this.content));
|
||||||
|
if(searchArea.equals(SearchArea.TOP)) {
|
||||||
|
return searchContent.toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
excludeKeywords.forEach(keyword -> searchContent.append(" -").append(keyword));
|
||||||
|
if(!includeKeywords.isEmpty()) {
|
||||||
|
if(!Strings.isNullOrEmpty(searchContent.toString())) {
|
||||||
|
searchContent.append(" (");
|
||||||
|
}
|
||||||
|
includeKeywords.forEach(keyword -> searchContent.append(keyword).append(" OR "));
|
||||||
|
int deleteStart = searchContent.lastIndexOf(" OR ");
|
||||||
|
if(searchContent.length() >= 4 && deleteStart != -1) {
|
||||||
|
searchContent.delete(deleteStart, searchContent.length());
|
||||||
|
}
|
||||||
|
if(!Strings.isNullOrEmpty(searchContent.toString())) {
|
||||||
|
searchContent.append(")");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return searchContent.toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
public PixivSearchBuilder setSearchMode(SearchMode searchMode) {
|
||||||
|
this.searchMode = Objects.requireNonNull(searchMode);
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
public PixivSearchBuilder setSearchType(SearchType searchType) {
|
||||||
|
this.searchType = Objects.requireNonNull(searchType);
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
public PixivSearchBuilder setSearchOrder(SearchOrder searchOrder) {
|
||||||
|
this.searchOrder = Objects.requireNonNull(searchOrder);
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
public PixivSearchBuilder setSearchContentOption(SearchContentOption searchContentOption) {
|
||||||
|
this.searchContentOption = Objects.requireNonNull(searchContentOption);
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
public PixivSearchBuilder setRatioOption(RatioOption ratioOption) {
|
||||||
|
this.ratioOption = Objects.requireNonNull(ratioOption);
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
public PixivSearchBuilder setDateRange(Date startDate, Date endDate) {
|
||||||
|
this.startDate = startDate;
|
||||||
|
this.endDate = endDate;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
public PixivSearchBuilder setMaxSize(int width, int height) {
|
||||||
|
this.wgt = width;
|
||||||
|
this.hgt = height;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
public PixivSearchBuilder setMinSize(int width, int height) {
|
||||||
|
this.wlt = width;
|
||||||
|
this.hlt = height;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
public PixivSearchBuilder setPage(int pageIndex) {
|
||||||
|
if (pageIndex <= 0) {
|
||||||
|
throw new IllegalArgumentException("Invalid pageIndex: " + pageIndex);
|
||||||
|
}
|
||||||
|
this.page = pageIndex;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
public PixivSearchBuilder addExcludeKeyword(String keyword) {
|
||||||
|
excludeKeywords.add(keyword);
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
public PixivSearchBuilder removeExcludeKeyword(String keyword) {
|
||||||
|
excludeKeywords.remove(keyword);
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
public PixivSearchBuilder addIncludeKeyword(String keyword) {
|
||||||
|
includeKeywords.add(keyword);
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
public PixivSearchBuilder removeIncludeKeyword(String keyword) {
|
||||||
|
includeKeywords.remove(keyword);
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 搜索区域
|
||||||
|
*/
|
||||||
|
public enum SearchArea {
|
||||||
|
/**
|
||||||
|
* 所有(可能是 插画 + 漫画)
|
||||||
|
*/
|
||||||
|
ARTWORKS("illustManga"),
|
||||||
|
/**
|
||||||
|
* 顶部(所有内容)
|
||||||
|
* 同时包含了:
|
||||||
|
* {@link #ILLUSTRATIONS}
|
||||||
|
* {@link #MANGA}
|
||||||
|
* {@link #NOVELS}
|
||||||
|
* 选择此项后, 将直接显示所有与content相关内容, 而忽略所有附加搜索条件.
|
||||||
|
* 因为无法指定pageIndex, 数据只有24项
|
||||||
|
*/
|
||||||
|
TOP(null),
|
||||||
|
/**
|
||||||
|
* 插画
|
||||||
|
*/
|
||||||
|
ILLUSTRATIONS("illust"),
|
||||||
|
/**
|
||||||
|
* 漫画
|
||||||
|
*/
|
||||||
|
MANGA("manga"),
|
||||||
|
/**
|
||||||
|
* 小说
|
||||||
|
*/
|
||||||
|
NOVELS("novel");
|
||||||
|
|
||||||
|
public final String jsonKey;
|
||||||
|
|
||||||
|
SearchArea(String jsonKey) {
|
||||||
|
this.jsonKey = jsonKey;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 搜索模式
|
||||||
|
*/
|
||||||
|
public enum SearchMode {
|
||||||
|
/**
|
||||||
|
* 按标签搜索, 部分一致
|
||||||
|
*/
|
||||||
|
TAG,
|
||||||
|
/**
|
||||||
|
* 按标签搜索, 完全一致
|
||||||
|
*/
|
||||||
|
TAG_FULL,
|
||||||
|
/**
|
||||||
|
* 按标题和说明文字搜索
|
||||||
|
*/
|
||||||
|
TC
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 搜索内容类型
|
||||||
|
*/
|
||||||
|
public enum SearchType {
|
||||||
|
/**
|
||||||
|
* 全部内容(插画、漫画、动图)
|
||||||
|
*/
|
||||||
|
ALL,
|
||||||
|
/**
|
||||||
|
* 插画和动图(不包括漫画)
|
||||||
|
*/
|
||||||
|
ILLUST_AND_UGOIRA,
|
||||||
|
/**
|
||||||
|
* 插图
|
||||||
|
*/
|
||||||
|
ILLUST,
|
||||||
|
/**
|
||||||
|
* 漫画
|
||||||
|
*/
|
||||||
|
MANGA,
|
||||||
|
/**
|
||||||
|
* 动图
|
||||||
|
*/
|
||||||
|
UGOIRA
|
||||||
|
}
|
||||||
|
|
||||||
|
public enum SearchOrder {
|
||||||
|
/**
|
||||||
|
* 按旧排序
|
||||||
|
*/
|
||||||
|
DATE,
|
||||||
|
/**
|
||||||
|
* 按新排序
|
||||||
|
*/
|
||||||
|
DATE_D
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 搜索内容选项
|
||||||
|
*/
|
||||||
|
public enum SearchContentOption {
|
||||||
|
/**
|
||||||
|
* 所有内容
|
||||||
|
*/
|
||||||
|
ALL,
|
||||||
|
/**
|
||||||
|
* 全年龄
|
||||||
|
*/
|
||||||
|
SAFE,
|
||||||
|
/**
|
||||||
|
* R18
|
||||||
|
*/
|
||||||
|
R18
|
||||||
|
}
|
||||||
|
|
||||||
|
public enum RatioOption {
|
||||||
|
/**
|
||||||
|
* 横向
|
||||||
|
*/
|
||||||
|
TRANSVERSE(0.5F),
|
||||||
|
/**
|
||||||
|
* 纵向
|
||||||
|
*/
|
||||||
|
PORTRAIT(-0.5F),
|
||||||
|
/**
|
||||||
|
* 正方形
|
||||||
|
*/
|
||||||
|
SQUARE(0F)
|
||||||
|
;
|
||||||
|
|
||||||
|
public final float value;
|
||||||
|
|
||||||
|
RatioOption(float ratio) {
|
||||||
|
this.value = ratio;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
}
|
171
src/main/java/net/lamgc/cgj/pixiv/PixivSession.java
Normal file
171
src/main/java/net/lamgc/cgj/pixiv/PixivSession.java
Normal file
@ -0,0 +1,171 @@
|
|||||||
|
package net.lamgc.cgj.pixiv;
|
||||||
|
|
||||||
|
import com.google.gson.JsonObject;
|
||||||
|
import com.google.gson.JsonParser;
|
||||||
|
import org.apache.http.Header;
|
||||||
|
import org.apache.http.HttpHost;
|
||||||
|
import org.apache.http.HttpResponse;
|
||||||
|
import org.apache.http.NameValuePair;
|
||||||
|
import org.apache.http.client.CookieStore;
|
||||||
|
import org.apache.http.client.HttpClient;
|
||||||
|
import org.apache.http.client.entity.UrlEncodedFormEntity;
|
||||||
|
import org.apache.http.client.methods.HttpGet;
|
||||||
|
import org.apache.http.client.methods.HttpPost;
|
||||||
|
import org.apache.http.impl.client.BasicCookieStore;
|
||||||
|
import org.apache.http.impl.client.HttpClients;
|
||||||
|
import org.apache.http.message.BasicHeader;
|
||||||
|
import org.apache.http.message.BasicNameValuePair;
|
||||||
|
import org.apache.http.util.EntityUtils;
|
||||||
|
import org.jsoup.Jsoup;
|
||||||
|
import org.jsoup.nodes.Document;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.nio.charset.StandardCharsets;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
public class PixivSession {
|
||||||
|
|
||||||
|
public static final String USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 全登陆过程的关键,
|
||||||
|
* 保存会话用的cookieStore!
|
||||||
|
*/
|
||||||
|
private CookieStore cookieStore = new BasicCookieStore();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 可以直接使用的HttpClient对象
|
||||||
|
*/
|
||||||
|
private HttpClient httpClient;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 最后一次登录的错误信息
|
||||||
|
*/
|
||||||
|
private String errMsg;
|
||||||
|
|
||||||
|
public PixivSession(){
|
||||||
|
this(null);
|
||||||
|
}
|
||||||
|
|
||||||
|
public PixivSession(CookieStore cookieStore){
|
||||||
|
this(null, cookieStore);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 创建一个Pixiv登录会话
|
||||||
|
*/
|
||||||
|
public PixivSession(HttpHost proxy, CookieStore cookieStore) {
|
||||||
|
if(cookieStore != null){
|
||||||
|
this.cookieStore = cookieStore;
|
||||||
|
}
|
||||||
|
List<Header> defaultHeader = new ArrayList<>();
|
||||||
|
defaultHeader.add(new BasicHeader("accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9"));
|
||||||
|
defaultHeader.add(new BasicHeader("user-agent", PixivSession.USER_AGENT));
|
||||||
|
defaultHeader.add(new BasicHeader("accept-encoding", "gzip, deflate, br"));
|
||||||
|
defaultHeader.add(new BasicHeader("accept-language", "zh-CN,zh;q=0.9"));
|
||||||
|
|
||||||
|
/*defaultHeader.add(new BasicHeader("sec-fetch-mode", "navigate"));
|
||||||
|
defaultHeader.add(new BasicHeader("sec-fetch-site", "same-origin"));
|
||||||
|
defaultHeader.add(new BasicHeader("upgrade-insecure-requests", "1"));*/
|
||||||
|
//创建一个Http访问器
|
||||||
|
httpClient = HttpClients.custom()
|
||||||
|
.setDefaultCookieStore(cookieStore)
|
||||||
|
.setDefaultHeaders(defaultHeader)
|
||||||
|
.setProxy(proxy)
|
||||||
|
.build();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 程序自行通过帐号密码登录Pixiv.
|
||||||
|
* @param PixivID Pixiv帐号
|
||||||
|
* @param Password Pixiv密码
|
||||||
|
* @return 登录成功返回true
|
||||||
|
* @throws IOException 当登录抛出异常时返回
|
||||||
|
* @deprecated Pixiv已经新增Google人机验证, 程序已无法自行登录Pixiv
|
||||||
|
*/
|
||||||
|
public boolean Login(String PixivID, String Password) throws IOException {
|
||||||
|
// 获取登录接口所需的PostKey
|
||||||
|
String post_key = getPostKey();
|
||||||
|
HttpPost postRequest = new HttpPost(PixivURL.PIXIV_LOGIN_URL); //https://accounts.pixiv.net/api/login?lang=zh
|
||||||
|
List<NameValuePair> params = new ArrayList<>();
|
||||||
|
params.add(new BasicNameValuePair("pixiv_id", PixivID));
|
||||||
|
params.add(new BasicNameValuePair("password", Password));
|
||||||
|
params.add(new BasicNameValuePair("post_key", post_key));
|
||||||
|
//Form编码表单,作为Post的数据
|
||||||
|
postRequest.setEntity(new UrlEncodedFormEntity(params, StandardCharsets.UTF_8));
|
||||||
|
//启动访问
|
||||||
|
HttpResponse response = httpClient.execute(postRequest);
|
||||||
|
//获取接口返回数据
|
||||||
|
String httpXML = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8);
|
||||||
|
System.out.println(httpXML);
|
||||||
|
JsonObject responseJson = (JsonObject) new JsonParser().parse(httpXML);
|
||||||
|
if(!responseJson.get("error").getAsBoolean() && !responseJson.get("body").getAsJsonObject().has("validation_errors")){
|
||||||
|
errMsg = null;
|
||||||
|
return true;
|
||||||
|
}else{
|
||||||
|
errMsg = responseJson.get("body").getAsJsonObject().get("validation_errors").toString();
|
||||||
|
//System.err.println("登录失败!MSG: " + errMsg);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 登录前准备, 获取PostKey
|
||||||
|
* @return Post_Key
|
||||||
|
*/
|
||||||
|
private String getPostKey() throws IOException {
|
||||||
|
//创建请求,获取PostKey
|
||||||
|
HttpGet getRequest = new HttpGet(PixivURL.PIXIV_LOGIN_PAGE_URL);
|
||||||
|
//设置请求
|
||||||
|
//getRequest.setConfig(config);
|
||||||
|
getRequest.setHeader("User-Agent", USER_AGENT);
|
||||||
|
//启动访问
|
||||||
|
HttpResponse response = httpClient.execute(getRequest);
|
||||||
|
//获取网页内容
|
||||||
|
String pageAsXML = EntityUtils.toString(response.getEntity(),"utf-8");
|
||||||
|
//创建Http解析器
|
||||||
|
Document document = Jsoup.parse(pageAsXML);
|
||||||
|
//获取init-config内容
|
||||||
|
String init_config = document.getElementById("init-config").val();
|
||||||
|
//System.out.println(init_config);
|
||||||
|
//创建Json解析器解析init-config
|
||||||
|
JsonObject initConfigObj = (JsonObject) new JsonParser().parse(init_config);
|
||||||
|
//检查是否有postKey
|
||||||
|
if(!initConfigObj.has("pixivAccount.postKey")){
|
||||||
|
throw new RuntimeException("postKey获取失败!可能是Pixiv修改了登录过程!");
|
||||||
|
}
|
||||||
|
//获取postKey
|
||||||
|
return initConfigObj.get("pixivAccount.postKey").getAsString();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 获取CookieStore
|
||||||
|
* @return CookieStore
|
||||||
|
*/
|
||||||
|
public CookieStore getCookieStore(){
|
||||||
|
return cookieStore;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 获取可直接使用的HttpClient对象
|
||||||
|
* @return 已配置好的HttpClient对象
|
||||||
|
*/
|
||||||
|
public HttpClient getHttpClient(){
|
||||||
|
return this.httpClient;
|
||||||
|
}
|
||||||
|
|
||||||
|
public boolean hasError(){
|
||||||
|
return errMsg == null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 获取错误信息
|
||||||
|
* @return 返回登录错误信息
|
||||||
|
* @deprecated {@link #Login(String, String)}已经废除, 故本接口废除
|
||||||
|
*/
|
||||||
|
public String getErrMsg(){
|
||||||
|
return errMsg;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
16
src/main/java/net/lamgc/cgj/pixiv/PixivTag.java
Normal file
16
src/main/java/net/lamgc/cgj/pixiv/PixivTag.java
Normal file
@ -0,0 +1,16 @@
|
|||||||
|
package net.lamgc.cgj.pixiv;
|
||||||
|
|
||||||
|
public class PixivTag {
|
||||||
|
|
||||||
|
public static PixivTag create(String tagName) {
|
||||||
|
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
private PixivTag() {
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
}
|
290
src/main/java/net/lamgc/cgj/pixiv/PixivURL.java
Normal file
290
src/main/java/net/lamgc/cgj/pixiv/PixivURL.java
Normal file
@ -0,0 +1,290 @@
|
|||||||
|
package net.lamgc.cgj.pixiv;
|
||||||
|
|
||||||
|
import java.text.SimpleDateFormat;
|
||||||
|
import java.util.Calendar;
|
||||||
|
import java.util.Date;
|
||||||
|
import java.util.GregorianCalendar;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 目前已整理的一些Pixiv接口列表
|
||||||
|
*/
|
||||||
|
public class PixivURL {
|
||||||
|
|
||||||
|
|
||||||
|
public static final String PIXIV_INDEX_URL = "https://www.pixiv.net";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* P站预登陆url
|
||||||
|
*/
|
||||||
|
public static final String PIXIV_LOGIN_PAGE_URL = "https://accounts.pixiv.net/login?lang=zh&source=pc&view_type=page&ref=wwwtop_accounts_index";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* P站登录请求url
|
||||||
|
*/
|
||||||
|
public static final String PIXIV_LOGIN_URL = "https://accounts.pixiv.net/api/login?lang=zh";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* P站搜索请求url
|
||||||
|
*/
|
||||||
|
private static final String PIXIV_SEARCH_URL = "https://www.pixiv.net/search.php";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* P站搜索用户url
|
||||||
|
* 需要替换的参数:
|
||||||
|
* {nick} - 用户昵称、部分名称
|
||||||
|
*/
|
||||||
|
public static final String PIXIV_SEARCH_USER_URL = PIXIV_SEARCH_URL + "?s_mode=s_usr&nick={nick}";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* P站搜索插画url
|
||||||
|
* 需要替换的参数:
|
||||||
|
* {word} - 插画相关文本
|
||||||
|
*/
|
||||||
|
public static final String PIXIV_SEARCH_TAG_URL = PIXIV_SEARCH_URL + "?s_mode=s_tag&word={word}";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* P站插图下载链接获取url
|
||||||
|
* 需要替换的文本:
|
||||||
|
* {illustId} - 插画ID
|
||||||
|
*/
|
||||||
|
public static final String PIXIV_ILLUST_API_URL = "https://www.pixiv.net/ajax/illust/{illustId}/pages";
|
||||||
|
|
||||||
|
/*
|
||||||
|
* P站用户插图列表获取API
|
||||||
|
* 需要替换的文本:
|
||||||
|
* {userId} - 用户ID
|
||||||
|
*/
|
||||||
|
//TODO: 所需数据在 body属性内的 illusts(属性名,属性值不重要), manga(多图) pickup(精选)
|
||||||
|
//{"error":false,"message":"","body":{"illusts":{"74369837":null,"70990542":null,"70608653":null,"69755191":null,"69729450":null,"69729416":null,"69503608":null,"69288766":null,"69083882":null,"69051458":null,"68484200":null,"68216927":null,"68216866":null,"68192333":null,"67915106":null,"67914932":null,"67854803":null,"67854745":null,"67854670":null,"67787211":null,"67772199":null,"67770637":null,"67754861":null,"67754804":null,"67754726":null,"67740486":null,"67740480":null,"67740450":null,"67740434":null,"67726337":null,"67499196":null,"67499163":null,"67499145":null,"67499111":null,"67499085":null,"67499038":null,"67498987":null,"67473178":null,"66271465":null,"63682753":null,"63682697":null,"59385148":null,"59383265":null,"59383240":null,"59383227":null,"59383173":null},"manga":[],"novels":[],"mangaSeries":[],"novelSeries":[],"pickup":[],"bookmarkCount":{"public":{"illust":1,"novel":0},"private":{"illust":0,"novel":0}}}}
|
||||||
|
//public static final String PIXIV_USER_ILLUST_LIST_URL = "https://www.pixiv.net/ajax/user/{userId}/profile/all";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 能够同时获取插图信息的用户插图列表获取API
|
||||||
|
*/
|
||||||
|
public static final String PIXIV_USER_ILLUSTINFO_LIST_URL = "https://www.pixiv.net/ajax/user/{userId}/profile/top";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* P站单图详情页url
|
||||||
|
* 需要替换的文本:
|
||||||
|
* {illustId} - 插画ID
|
||||||
|
*/
|
||||||
|
public static final String PIXIV_ILLUST_MEDIUM_URL = "https://www.pixiv.net/member_illust.php?mode=medium&illust_id={illustId}";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* P站多图详情页url
|
||||||
|
* 需要替换的文本:
|
||||||
|
* {illustId} - 插画ID
|
||||||
|
*/
|
||||||
|
public static final String PIXIV_ILLUST_MANGA_URL = "https://www.pixiv.net/member_illust.php?mode=manga&illust_id={illustId}";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* P站用户页面url
|
||||||
|
* 需要替换的文本:
|
||||||
|
* {userId} - 用户ID
|
||||||
|
*/
|
||||||
|
public static final String PIXIV_USER_URL = "https://www.pixiv.net/member.php?id={userId}";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* P站插图信息获取API
|
||||||
|
* 这个API能获取插图基本信息,但不能获取大小
|
||||||
|
* 请使用{@link #getPixivIllustInfoAPI(int[])}获取URL
|
||||||
|
*/
|
||||||
|
private static final String PIXIV_GET_ILLUST_INFO_URL = "https://www.pixiv.net/ajax/illust/recommend/illusts?";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* P站获取用户所有插图ID的Api
|
||||||
|
* 这个API只能获取该用户的插图ID,不能获取图片信息(图片信息要另外获取)
|
||||||
|
* 需要替换的文本:
|
||||||
|
* {userId} - 用户ID
|
||||||
|
*/
|
||||||
|
public static final String PIXIV_GET_USER_ALL_ILLUST_ID_URL = "https://www.pixiv.net/ajax/user/{userId}/profile/all";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* P站标签搜索URL
|
||||||
|
* 可以将Tag的大概内容搜索成P站精确的Tag,以搜索其他接口返回的Tags数组;
|
||||||
|
* 需要替换的文本:
|
||||||
|
* {content} - 大致tag内容
|
||||||
|
*/
|
||||||
|
public static final String PIXIV_TAG_SEARCH_URL = "https://www.pixiv.net/ajax/search/tags/{content}";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 请求时带上需要退出的Cookies
|
||||||
|
* 无论成功与否都会返回302重定向到{@linkplain #PIXIV_LOGIN_PAGE_URL 登录页面}
|
||||||
|
*/
|
||||||
|
public static final String PIXIV_LOGOUT_URL = "https://www.pixiv.net/logout.php";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 构造P站获取插图信息的Api Url
|
||||||
|
* @param illustIds 要查询的插图ID数组
|
||||||
|
* @return 对应查询的API Url
|
||||||
|
*/
|
||||||
|
public static String getPixivIllustInfoAPI(int[] illustIds){
|
||||||
|
StringBuilder strBuilder = new StringBuilder().append(PIXIV_GET_ILLUST_INFO_URL);
|
||||||
|
for(int illustId : illustIds){
|
||||||
|
strBuilder.append("illust_ids[]=").append(illustId).append("&");
|
||||||
|
}
|
||||||
|
return strBuilder.toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 获取用于下载图片时防盗链所需Referer的链接
|
||||||
|
* @param illustId 欲下载图片所属illustId
|
||||||
|
* @return 返回Referer链接, 也可以作为作品链接使用
|
||||||
|
*/
|
||||||
|
public static String getPixivRefererLink(int illustId){
|
||||||
|
return "https://www.pixiv.net/artworks/" + illustId;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 获取用于下载图片时防盗链所需Referer的链接
|
||||||
|
* @param illustId 欲下载图片所属illustId
|
||||||
|
* @return 返回Referer链接, 也可以作为作品链接使用
|
||||||
|
*/
|
||||||
|
public static String getPixivRefererLink(String illustId){
|
||||||
|
return "https://www.pixiv.net/artworks/" + illustId;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 排行榜接口, 需加入"&format=json"
|
||||||
|
*/
|
||||||
|
private final static String PIXIV_RANKING_LINK = "https://www.pixiv.net/ranking.php?";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 查询用户收藏.<br/>
|
||||||
|
* 该URL返回HTML页面,需要进行解析.<br/>
|
||||||
|
* 需要替换的文本:<br/>
|
||||||
|
* {pageIndex} - 页数, 超出了则结果为空<br/>
|
||||||
|
*/
|
||||||
|
public final static String PIXIV_USER_COLLECTION_PAGE = "https://www.pixiv.net/bookmark.php?rest=show&p={pageIndex}";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 获取排名榜
|
||||||
|
* @param mode 查询类型, 详细信息看{@link RankingMode}, 如本参数为null, 则为每天
|
||||||
|
* @param contentType 排名榜类型, 如为null则为综合
|
||||||
|
* @param time 欲查询的时间, 最新只能查询昨天, 根据mode不同:
|
||||||
|
* 每天 - 查询指定日期的排名榜
|
||||||
|
* 每周 - 查询指定时间结束(含)到七天前一段时间内的排名榜
|
||||||
|
* 每月 - 查询指定日期结束(含)到28天时间内的排名榜
|
||||||
|
* 新人 - 与每周相同
|
||||||
|
* 受男性欢迎 - 与每天相同
|
||||||
|
* 受女性欢迎 - 与每天相同
|
||||||
|
* 默认值为昨天
|
||||||
|
* @param pageIndex 页数,一页50位,总共10页
|
||||||
|
* @return 返回构建好的链接
|
||||||
|
*/
|
||||||
|
public static String getRankingLink(RankingContentType contentType, RankingMode mode, Date time, int pageIndex, boolean json){
|
||||||
|
StringBuilder linkBuilder = new StringBuilder(PIXIV_RANKING_LINK);
|
||||||
|
linkBuilder.append("mode=").append(mode == null ? RankingMode.MODE_DAILY.modeParam : mode.modeParam);
|
||||||
|
if(contentType != null){
|
||||||
|
linkBuilder.append("&content=").append(contentType.typeName);
|
||||||
|
}
|
||||||
|
SimpleDateFormat format = new SimpleDateFormat("yyyyMMdd");
|
||||||
|
Date queryDate;
|
||||||
|
if(time == null){
|
||||||
|
GregorianCalendar gregorianCalendar = new GregorianCalendar();
|
||||||
|
gregorianCalendar.setTime(new Date());
|
||||||
|
gregorianCalendar.add(Calendar.DATE, -1);
|
||||||
|
queryDate = gregorianCalendar.getTime();
|
||||||
|
} else {
|
||||||
|
queryDate = time;
|
||||||
|
}
|
||||||
|
linkBuilder.append("&date=").append(format.format(queryDate));
|
||||||
|
if(pageIndex > 0 && pageIndex <= 10) {
|
||||||
|
linkBuilder.append("&p=").append(pageIndex);
|
||||||
|
}
|
||||||
|
if(json) {
|
||||||
|
linkBuilder.append("&format=").append("json");
|
||||||
|
}
|
||||||
|
return linkBuilder.toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 排名榜模式
|
||||||
|
*/
|
||||||
|
public enum RankingMode{
|
||||||
|
/**
|
||||||
|
* 每天
|
||||||
|
*/
|
||||||
|
MODE_DAILY("daily"),
|
||||||
|
/**
|
||||||
|
* 每周
|
||||||
|
*/
|
||||||
|
MODE_WEEKLY("weekly"),
|
||||||
|
/**
|
||||||
|
* 每月
|
||||||
|
*/
|
||||||
|
MODE_MONTHLY("monthly"),
|
||||||
|
/**
|
||||||
|
* 新人
|
||||||
|
*/
|
||||||
|
MODE_ROOKIE("rookie"),
|
||||||
|
/**
|
||||||
|
* 受男性喜欢
|
||||||
|
*/
|
||||||
|
MODE_MALE("male"),
|
||||||
|
/**
|
||||||
|
* 受女性喜欢
|
||||||
|
*/
|
||||||
|
MODE_FEMALE("female"),
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 每天 - R18
|
||||||
|
*/
|
||||||
|
MODE_DAILY_R18("daily_r18"),
|
||||||
|
/**
|
||||||
|
* 每周 - R18
|
||||||
|
*/
|
||||||
|
MODE_WEEKLY_R18("weekly_r18"),
|
||||||
|
/**
|
||||||
|
* 受男性喜欢 - R18
|
||||||
|
*/
|
||||||
|
MODE_MALE_R18("male_r18"),
|
||||||
|
/**
|
||||||
|
* 受女性喜欢 - R18
|
||||||
|
*/
|
||||||
|
MODE_FEMALE_R18("female_r18"),
|
||||||
|
;
|
||||||
|
public String modeParam;
|
||||||
|
|
||||||
|
RankingMode(String modeParamName){
|
||||||
|
this.modeParam = modeParamName;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Pixiv搜索接口.<br/>
|
||||||
|
* 要使用该链接请使用{@link PixivSearchBuilder}构造链接.<br/>
|
||||||
|
* 需要替换的参数: <br/>
|
||||||
|
* content - 搜索内容
|
||||||
|
*/
|
||||||
|
final static String PIXIV_SEARCH_CONTENT_URL = "https://www.pixiv.net/ajax/search/{area}/{content}?word={content}";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 排名榜类型
|
||||||
|
*/
|
||||||
|
public enum RankingContentType{
|
||||||
|
/**
|
||||||
|
* 插画
|
||||||
|
* 支持的时间类型: 每天, 每周, 每月, 新人
|
||||||
|
*/
|
||||||
|
TYPE_ILLUST("illust"),
|
||||||
|
/**
|
||||||
|
* 动图
|
||||||
|
* 支持的时间类型:每天, 每周
|
||||||
|
*/
|
||||||
|
TYPE_UGOIRA("ugoira"),
|
||||||
|
/**
|
||||||
|
* 漫画
|
||||||
|
* 支持的时间类型: 每天, 每周, 每月, 新人
|
||||||
|
*/
|
||||||
|
TYPE_MANGA("manga")
|
||||||
|
;
|
||||||
|
|
||||||
|
String typeName;
|
||||||
|
|
||||||
|
RankingContentType(String typeName){
|
||||||
|
this.typeName = typeName;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
149
src/main/java/net/lamgc/cgj/proxy/PixivAccessProxyServer.java
Normal file
149
src/main/java/net/lamgc/cgj/proxy/PixivAccessProxyServer.java
Normal file
@ -0,0 +1,149 @@
|
|||||||
|
package net.lamgc.cgj.proxy;
|
||||||
|
|
||||||
|
import com.github.monkeywie.proxyee.intercept.HttpProxyIntercept;
|
||||||
|
import com.github.monkeywie.proxyee.intercept.HttpProxyInterceptInitializer;
|
||||||
|
import com.github.monkeywie.proxyee.intercept.HttpProxyInterceptPipeline;
|
||||||
|
import com.github.monkeywie.proxyee.intercept.common.CertDownIntercept;
|
||||||
|
import com.github.monkeywie.proxyee.proxy.ProxyConfig;
|
||||||
|
import com.github.monkeywie.proxyee.server.HttpProxyServer;
|
||||||
|
import com.github.monkeywie.proxyee.server.HttpProxyServerConfig;
|
||||||
|
import io.netty.channel.Channel;
|
||||||
|
import io.netty.handler.codec.http.HttpHeaderNames;
|
||||||
|
import io.netty.handler.codec.http.HttpHeaders;
|
||||||
|
import io.netty.handler.codec.http.HttpRequest;
|
||||||
|
import io.netty.handler.codec.http.HttpResponse;
|
||||||
|
import org.apache.http.client.CookieStore;
|
||||||
|
import org.apache.http.impl.cookie.BasicClientCookie;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
import java.net.HttpCookie;
|
||||||
|
import java.util.Date;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 登录成功后提供CookieStore, 然后由程序自动登录Pixiv
|
||||||
|
* @author LamGC
|
||||||
|
*/
|
||||||
|
public class PixivAccessProxyServer {
|
||||||
|
|
||||||
|
private final Logger log = LoggerFactory.getLogger(this.getClass().getSimpleName());
|
||||||
|
|
||||||
|
private final HttpProxyServer proxyServer;
|
||||||
|
|
||||||
|
private final CookieStore cookieStore;
|
||||||
|
|
||||||
|
public PixivAccessProxyServer(CookieStore cookieStore){
|
||||||
|
this(cookieStore, null);
|
||||||
|
}
|
||||||
|
|
||||||
|
public PixivAccessProxyServer(CookieStore cookieStore, ProxyConfig proxyConfig){
|
||||||
|
HttpProxyServerConfig config = new HttpProxyServerConfig();
|
||||||
|
this.cookieStore = cookieStore;
|
||||||
|
config.setHandleSsl(true);
|
||||||
|
this.proxyServer = new HttpProxyServer();
|
||||||
|
this.proxyServer
|
||||||
|
.serverConfig(config)
|
||||||
|
.proxyConfig(proxyConfig)
|
||||||
|
.proxyInterceptInitializer(new HttpProxyInterceptInitializer(){
|
||||||
|
@Override
|
||||||
|
public void init(HttpProxyInterceptPipeline pipeline) {
|
||||||
|
pipeline.addLast(new CertDownIntercept());
|
||||||
|
pipeline.addLast(new HttpProxyIntercept(){
|
||||||
|
|
||||||
|
private boolean match(HttpRequest request){
|
||||||
|
String host = request.headers().get(HttpHeaderNames.HOST);
|
||||||
|
return host.equalsIgnoreCase("pixiv.net") || host.contains(".pixiv.net");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void beforeRequest(Channel clientChannel, HttpRequest httpRequest, HttpProxyInterceptPipeline pipeline) throws Exception {
|
||||||
|
log.info("URL: " + httpRequest.headers().get(HttpHeaderNames.HOST) + httpRequest.uri());
|
||||||
|
if(!match(httpRequest)){
|
||||||
|
super.beforeRequest(clientChannel, httpRequest, pipeline);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
log.info("正在注入Cookies...");
|
||||||
|
HttpHeaders requestHeaders = httpRequest.headers();
|
||||||
|
if(requestHeaders.contains(HttpHeaderNames.COOKIE)){
|
||||||
|
log.info("原请求存在自带Cookies, 正在清除Cookies...");
|
||||||
|
log.debug("原Cookies: {}", requestHeaders.getAsString(HttpHeaderNames.COOKIE));
|
||||||
|
requestHeaders.remove(HttpHeaderNames.COOKIE);
|
||||||
|
}
|
||||||
|
StringBuilder cookieBuilder = new StringBuilder();
|
||||||
|
cookieStore.getCookies().forEach(cookie -> {
|
||||||
|
if(cookie.isExpired(new Date())){
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
cookieBuilder.append(cookie.getName()).append("=").append(cookie.getValue()).append("; ");
|
||||||
|
});
|
||||||
|
log.info("Cookies构造完成, 结果: " + cookieBuilder.toString());
|
||||||
|
requestHeaders.add(HttpHeaderNames.COOKIE, cookieBuilder.toString());
|
||||||
|
log.info("Cookies注入完成.");
|
||||||
|
|
||||||
|
super.beforeRequest(clientChannel, httpRequest, pipeline);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void afterResponse(Channel clientChannel, Channel proxyChannel, HttpResponse httpResponse, HttpProxyInterceptPipeline pipeline) throws Exception {
|
||||||
|
if(!match(pipeline.getHttpRequest())){
|
||||||
|
super.afterResponse(clientChannel, proxyChannel, httpResponse, pipeline);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
log.info("正在更新Response Cookie...(Header Name: " + HttpHeaderNames.SET_COOKIE + ")");
|
||||||
|
List<String> responseCookies = httpResponse.headers().getAll(HttpHeaderNames.SET_COOKIE);
|
||||||
|
responseCookies.forEach(value -> {
|
||||||
|
/*if(check(value)){
|
||||||
|
log.info("黑名单Cookie, 已忽略: " + value);
|
||||||
|
return;
|
||||||
|
}*/
|
||||||
|
log.info("Response Cookie: " + value);
|
||||||
|
BasicClientCookie cookie = parseRawCookie(value);
|
||||||
|
cookieStore.addCookie(null);
|
||||||
|
});
|
||||||
|
httpResponse.headers().remove(HttpHeaderNames.SET_COOKIE);
|
||||||
|
super.afterResponse(clientChannel, proxyChannel, httpResponse, pipeline);
|
||||||
|
}
|
||||||
|
|
||||||
|
protected BasicClientCookie parseRawCookie(String rawCookie) {
|
||||||
|
List<HttpCookie> cookies = HttpCookie.parse(rawCookie);
|
||||||
|
if (cookies.size() < 1)
|
||||||
|
return null;
|
||||||
|
HttpCookie httpCookie = cookies.get(0);
|
||||||
|
BasicClientCookie cookie = new BasicClientCookie(httpCookie.getName(), httpCookie.getValue());
|
||||||
|
if (httpCookie.getMaxAge() >= 0) {
|
||||||
|
Date expiryDate = new Date(System.currentTimeMillis() + httpCookie.getMaxAge() * 1000);
|
||||||
|
cookie.setExpiryDate(expiryDate);
|
||||||
|
}
|
||||||
|
if (httpCookie.getDomain() != null)
|
||||||
|
cookie.setDomain(httpCookie.getDomain());
|
||||||
|
if (httpCookie.getPath() != null)
|
||||||
|
cookie.setPath(httpCookie.getPath());
|
||||||
|
if (httpCookie.getComment() != null)
|
||||||
|
cookie.setComment(httpCookie.getComment());
|
||||||
|
cookie.setSecure(httpCookie.getSecure());
|
||||||
|
return cookie;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
public void start(int port){
|
||||||
|
this.proxyServer.start(port);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void close(){
|
||||||
|
this.proxyServer.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 导出CookieStore.
|
||||||
|
* 注意!该方法导出的CookieStore不适用于ApacheHttpClient, 如需使用则需要进行转换.
|
||||||
|
* @return CookieStore对象
|
||||||
|
*/
|
||||||
|
public CookieStore getCookieStore(){
|
||||||
|
return this.cookieStore;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
157
src/main/java/net/lamgc/cgj/proxy/PixivLoginProxyServer.java
Normal file
157
src/main/java/net/lamgc/cgj/proxy/PixivLoginProxyServer.java
Normal file
@ -0,0 +1,157 @@
|
|||||||
|
package net.lamgc.cgj.proxy;
|
||||||
|
|
||||||
|
import com.github.monkeywie.proxyee.intercept.HttpProxyInterceptInitializer;
|
||||||
|
import com.github.monkeywie.proxyee.intercept.HttpProxyInterceptPipeline;
|
||||||
|
import com.github.monkeywie.proxyee.intercept.common.CertDownIntercept;
|
||||||
|
import com.github.monkeywie.proxyee.intercept.common.FullResponseIntercept;
|
||||||
|
import com.github.monkeywie.proxyee.proxy.ProxyConfig;
|
||||||
|
import com.github.monkeywie.proxyee.server.HttpProxyServer;
|
||||||
|
import com.github.monkeywie.proxyee.server.HttpProxyServerConfig;
|
||||||
|
import com.google.gson.Gson;
|
||||||
|
import com.google.gson.JsonObject;
|
||||||
|
import io.netty.handler.codec.http.FullHttpResponse;
|
||||||
|
import io.netty.handler.codec.http.HttpHeaderNames;
|
||||||
|
import io.netty.handler.codec.http.HttpRequest;
|
||||||
|
import io.netty.handler.codec.http.HttpResponse;
|
||||||
|
import org.apache.commons.text.StringEscapeUtils;
|
||||||
|
import org.apache.http.client.CookieStore;
|
||||||
|
import org.apache.http.impl.client.BasicCookieStore;
|
||||||
|
import org.apache.http.impl.cookie.BasicClientCookie;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
import java.net.HttpCookie;
|
||||||
|
import java.nio.ByteBuffer;
|
||||||
|
import java.nio.charset.StandardCharsets;
|
||||||
|
import java.util.Date;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.concurrent.atomic.AtomicInteger;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 由用户介入, 让用户手动登录Pixiv的方式, 再通过代理服务器捕获Cookie来绕过Google人机验证
|
||||||
|
* @author LamGC
|
||||||
|
*/
|
||||||
|
public class PixivLoginProxyServer {
|
||||||
|
|
||||||
|
private final Logger log = LoggerFactory.getLogger(this.getClass().getSimpleName());
|
||||||
|
|
||||||
|
private boolean login = false;
|
||||||
|
|
||||||
|
private final HttpProxyServer proxyServer;
|
||||||
|
|
||||||
|
private final CookieStore cookieStore = new BasicCookieStore();
|
||||||
|
|
||||||
|
public PixivLoginProxyServer(){
|
||||||
|
this(null);
|
||||||
|
}
|
||||||
|
|
||||||
|
public PixivLoginProxyServer(ProxyConfig proxyConfig){
|
||||||
|
HttpProxyServerConfig config = new HttpProxyServerConfig();
|
||||||
|
config.setHandleSsl(true);
|
||||||
|
this.proxyServer = new HttpProxyServer();
|
||||||
|
this.proxyServer
|
||||||
|
.serverConfig(config)
|
||||||
|
.proxyConfig(proxyConfig)
|
||||||
|
.proxyInterceptInitializer(new HttpProxyInterceptInitializer(){
|
||||||
|
@Override
|
||||||
|
public void init(HttpProxyInterceptPipeline pipeline) {
|
||||||
|
pipeline.addLast(new CertDownIntercept());
|
||||||
|
pipeline.addLast(new FullResponseIntercept() {
|
||||||
|
@Override
|
||||||
|
public boolean match(HttpRequest httpRequest, HttpResponse httpResponse, HttpProxyInterceptPipeline httpProxyInterceptPipeline) {
|
||||||
|
String host = httpRequest.headers().get(HttpHeaderNames.HOST);
|
||||||
|
return host.equalsIgnoreCase("pixiv.net") || host.contains(".pixiv.net");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void handelResponse(HttpRequest httpRequest, FullHttpResponse fullHttpResponse, HttpProxyInterceptPipeline httpProxyInterceptPipeline) {
|
||||||
|
String url = httpRequest.headers().get(HttpHeaderNames.HOST) + httpRequest.uri();
|
||||||
|
log.info("拦截到Pixiv请求, URL: " + url);
|
||||||
|
|
||||||
|
log.info("正在导出Response Cookie...(Header Name: " + HttpHeaderNames.SET_COOKIE + ")");
|
||||||
|
List<String> responseCookies = fullHttpResponse.headers().getAll(HttpHeaderNames.SET_COOKIE);
|
||||||
|
AtomicInteger responseCookieCount = new AtomicInteger();
|
||||||
|
responseCookies.forEach(value -> {
|
||||||
|
log.debug("Response Cookie: " + value);
|
||||||
|
cookieStore.addCookie(parseRawCookie(value));
|
||||||
|
responseCookieCount.incrementAndGet();
|
||||||
|
});
|
||||||
|
log.info("Cookie导出完成(已导出 " + responseCookieCount.get() + " 条Cookie)");
|
||||||
|
|
||||||
|
//登录检查
|
||||||
|
// 如果用户在登录界面登录成功后反复刷新,会出现登录返回不对但已经成功登录的情况,
|
||||||
|
// 故此处在登录完成后不再判断是否成功登录
|
||||||
|
if(!isLogin() && url.contains("accounts.pixiv.net/api/login")){
|
||||||
|
log.info("正在检查登录结果...");
|
||||||
|
//拷贝一份以防止对原响应造成影响
|
||||||
|
FullHttpResponse copyResponse = fullHttpResponse.copy();
|
||||||
|
ByteBuffer buffer = ByteBuffer.allocate(copyResponse.content().capacity());
|
||||||
|
String contentStr;
|
||||||
|
copyResponse.content().readBytes(buffer);
|
||||||
|
contentStr = new String(buffer.array(), StandardCharsets.UTF_8);
|
||||||
|
log.debug("Login Result: " + contentStr);
|
||||||
|
|
||||||
|
JsonObject resultObject = new Gson().fromJson(contentStr, JsonObject.class);
|
||||||
|
//只要error:false, body存在(应该是会存在的)且success字段存在, 即为登录成功
|
||||||
|
login = !resultObject.get("error").getAsBoolean() &&
|
||||||
|
resultObject.has("body") &&
|
||||||
|
resultObject.get("body").getAsJsonObject().has("success");
|
||||||
|
log.info("登录状态确认: " + (login ? "登录成功" : "登录失败"));
|
||||||
|
|
||||||
|
fullHttpResponse.content().clear().writeBytes(
|
||||||
|
("{\"error\":false,\"message\":\"\",\"body\":{\"validation_errors\":{\"etc\":\"" +
|
||||||
|
StringEscapeUtils.escapeJava("Pixiv登录代理器已确认登录") + "\"}}}")
|
||||||
|
.getBytes(StandardCharsets.UTF_8));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
protected BasicClientCookie parseRawCookie(String rawCookie) {
|
||||||
|
List<HttpCookie> cookies = HttpCookie.parse(rawCookie);
|
||||||
|
if (cookies.size() < 1)
|
||||||
|
return null;
|
||||||
|
HttpCookie httpCookie = cookies.get(0);
|
||||||
|
BasicClientCookie cookie = new BasicClientCookie(httpCookie.getName(), httpCookie.getValue());
|
||||||
|
if (httpCookie.getMaxAge() >= 0) {
|
||||||
|
Date expiryDate = new Date(System.currentTimeMillis() + httpCookie.getMaxAge() * 1000);
|
||||||
|
cookie.setExpiryDate(expiryDate);
|
||||||
|
}
|
||||||
|
if (httpCookie.getDomain() != null)
|
||||||
|
cookie.setDomain(httpCookie.getDomain());
|
||||||
|
if (httpCookie.getPath() != null)
|
||||||
|
cookie.setPath(httpCookie.getPath());
|
||||||
|
if (httpCookie.getComment() != null)
|
||||||
|
cookie.setComment(httpCookie.getComment());
|
||||||
|
cookie.setSecure(httpCookie.getSecure());
|
||||||
|
return cookie;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
public void start(int port){
|
||||||
|
this.proxyServer.start(port);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void close(){
|
||||||
|
this.proxyServer.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 是否已登录Pixiv
|
||||||
|
* @return 如已登录返回true
|
||||||
|
*/
|
||||||
|
public boolean isLogin(){
|
||||||
|
return login;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 导出CookieStore.
|
||||||
|
* 注意!该方法导出的CookieStore不适用于ApacheHttpClient, 如需使用则需要进行转换.
|
||||||
|
* @return CookieStore对象
|
||||||
|
*/
|
||||||
|
public CookieStore getCookieStore(){
|
||||||
|
return this.cookieStore;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
185
src/main/java/net/lamgc/cgj/proxy/PixivLoginProxyServer_Old.java
Normal file
185
src/main/java/net/lamgc/cgj/proxy/PixivLoginProxyServer_Old.java
Normal file
@ -0,0 +1,185 @@
|
|||||||
|
package net.lamgc.cgj.proxy;
|
||||||
|
|
||||||
|
import com.github.monkeywie.proxyee.intercept.HttpProxyInterceptInitializer;
|
||||||
|
import com.github.monkeywie.proxyee.intercept.HttpProxyInterceptPipeline;
|
||||||
|
import com.github.monkeywie.proxyee.intercept.common.CertDownIntercept;
|
||||||
|
import com.github.monkeywie.proxyee.intercept.common.FullResponseIntercept;
|
||||||
|
import com.github.monkeywie.proxyee.proxy.ProxyConfig;
|
||||||
|
import com.github.monkeywie.proxyee.server.HttpProxyServer;
|
||||||
|
import com.github.monkeywie.proxyee.server.HttpProxyServerConfig;
|
||||||
|
import com.google.gson.Gson;
|
||||||
|
import com.google.gson.JsonObject;
|
||||||
|
import io.netty.handler.codec.http.*;
|
||||||
|
import org.apache.http.client.CookieStore;
|
||||||
|
import org.apache.http.impl.client.BasicCookieStore;
|
||||||
|
import org.apache.http.impl.cookie.BasicClientCookie;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
import java.net.HttpCookie;
|
||||||
|
import java.nio.ByteBuffer;
|
||||||
|
import java.nio.charset.StandardCharsets;
|
||||||
|
import java.util.*;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 由用户介入, 让用户手动登录Pixiv的方式, 再通过代理服务器捕获Cookie来绕过Google人机验证
|
||||||
|
* @author LamGC
|
||||||
|
*/
|
||||||
|
public class PixivLoginProxyServer_Old {
|
||||||
|
|
||||||
|
private final Logger log = LoggerFactory.getLogger(this.getClass().getSimpleName());
|
||||||
|
|
||||||
|
private boolean login = false;
|
||||||
|
|
||||||
|
private final HttpProxyServer proxyServer;
|
||||||
|
|
||||||
|
//private final CookieManager cookieManager = new CookieManager();
|
||||||
|
|
||||||
|
private final CookieStore cookieStore = new BasicCookieStore();
|
||||||
|
|
||||||
|
public PixivLoginProxyServer_Old(){
|
||||||
|
this(null);
|
||||||
|
}
|
||||||
|
|
||||||
|
public PixivLoginProxyServer_Old(ProxyConfig proxyConfig){
|
||||||
|
HttpProxyServerConfig config = new HttpProxyServerConfig();
|
||||||
|
config.setHandleSsl(true);
|
||||||
|
this.proxyServer = new HttpProxyServer();
|
||||||
|
this.proxyServer
|
||||||
|
.serverConfig(config)
|
||||||
|
.proxyConfig(proxyConfig)
|
||||||
|
.proxyInterceptInitializer(new HttpProxyInterceptInitializer(){
|
||||||
|
@Override
|
||||||
|
public void init(HttpProxyInterceptPipeline pipeline) {
|
||||||
|
pipeline.addLast(new CertDownIntercept());
|
||||||
|
pipeline.addLast(new FullResponseIntercept() {
|
||||||
|
@Override
|
||||||
|
public boolean match(HttpRequest httpRequest, HttpResponse httpResponse, HttpProxyInterceptPipeline httpProxyInterceptPipeline) {
|
||||||
|
//log.info("Match: " + httpRequest.headers().get(HttpHeaderNames.HOST) + httpRequest.uri());
|
||||||
|
//return HttpUtil.checkUrl(httpRequest, ".*pixiv\\.net");
|
||||||
|
String host = httpRequest.headers().get(HttpHeaderNames.HOST);
|
||||||
|
return host.equalsIgnoreCase("pixiv.net") || host.contains(".pixiv.net");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void handelResponse(HttpRequest httpRequest, FullHttpResponse fullHttpResponse, HttpProxyInterceptPipeline httpProxyInterceptPipeline) {
|
||||||
|
log.info("拦截到Pixiv请求, 正在导出Response Cookie...");
|
||||||
|
String url = httpRequest.headers().get(HttpHeaderNames.HOST) + httpRequest.uri();
|
||||||
|
log.info("URL: " + url);
|
||||||
|
/*URI requestURI = URI.create(httpRequest.headers().get(HttpHeaderNames.HOST) + httpRequest.uri());
|
||||||
|
log.info("正在导出Request Cookie...(Header Name: " + HttpHeaderNames.COOKIE + ")");
|
||||||
|
List<String> requestCookie = httpRequest.headers().getAll(HttpHeaderNames.COOKIE);
|
||||||
|
//requestCookie.forEach(value -> log.info("Request Cookie: " + value));
|
||||||
|
requestCookie.forEach(value -> {
|
||||||
|
String[] items = value.split(";");
|
||||||
|
for(String item : items){
|
||||||
|
log.info("Raw Request Cookie: " + item);
|
||||||
|
String[] keyValueSet = item.split("=");
|
||||||
|
if(keyValueSet.length == 2){
|
||||||
|
//TODO: 会出现重复的情况
|
||||||
|
log.info("Request Cookie: " + keyValueSet[0].trim() + "=" + keyValueSet[1].trim());
|
||||||
|
cookieStore.addCookie(new BasicClientCookie(keyValueSet[0].trim(), keyValueSet[1].trim()));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
//cookieStore.addCookie(parseRawCookie(value));
|
||||||
|
});*/
|
||||||
|
log.info("正在导出Response Cookie...(Header Name: " + HttpHeaderNames.SET_COOKIE + ")");
|
||||||
|
List<String> responseCookies = fullHttpResponse.headers().getAll(HttpHeaderNames.SET_COOKIE);
|
||||||
|
responseCookies.forEach(value -> {
|
||||||
|
/*if(check(value)){
|
||||||
|
log.info("黑名单Cookie, 已忽略: " + value);
|
||||||
|
return;
|
||||||
|
}*/
|
||||||
|
log.info("Response Cookie: " + value);
|
||||||
|
cookieStore.addCookie(parseRawCookie(value));
|
||||||
|
});
|
||||||
|
|
||||||
|
log.info("Cookie导出完成");
|
||||||
|
|
||||||
|
if(url.contains("accounts.pixiv.net/api/login")){
|
||||||
|
log.info("正在检查登录结果...");
|
||||||
|
FullHttpResponse copyResponse = fullHttpResponse.copy();
|
||||||
|
//ByteArrayOutputStream contentOS = new ByteArrayOutputStream(copyResponse.content().capacity());
|
||||||
|
ByteBuffer buffer = ByteBuffer.allocate(copyResponse.content().capacity());
|
||||||
|
String contentStr;
|
||||||
|
copyResponse.content().readBytes(buffer);
|
||||||
|
contentStr = new String(buffer.array(), StandardCharsets.UTF_8);
|
||||||
|
//log.info("Login Result: " + contentStr);
|
||||||
|
JsonObject resultObject = new Gson().fromJson(contentStr, JsonObject.class);
|
||||||
|
login = !resultObject.get("error").getAsBoolean() &&
|
||||||
|
resultObject.has("body") &&
|
||||||
|
resultObject.get("body").getAsJsonObject().has("success");
|
||||||
|
if(login) {
|
||||||
|
log.info("登录状态确认: 登录成功");
|
||||||
|
} else {
|
||||||
|
log.info("登录状态确认: 登录失败");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
protected BasicClientCookie parseRawCookie(String rawCookie) {
|
||||||
|
List<HttpCookie> cookies = HttpCookie.parse(rawCookie);
|
||||||
|
if (cookies.size() < 1)
|
||||||
|
return null;
|
||||||
|
HttpCookie httpCookie = cookies.get(0);
|
||||||
|
BasicClientCookie cookie = new BasicClientCookie(httpCookie.getName(), httpCookie.getValue());
|
||||||
|
if (httpCookie.getMaxAge() >= 0) {
|
||||||
|
Date expiryDate = new Date(System.currentTimeMillis() + httpCookie.getMaxAge() * 1000);
|
||||||
|
cookie.setExpiryDate(expiryDate);
|
||||||
|
}
|
||||||
|
if (httpCookie.getDomain() != null)
|
||||||
|
cookie.setDomain(httpCookie.getDomain());
|
||||||
|
if (httpCookie.getPath() != null)
|
||||||
|
cookie.setPath(httpCookie.getPath());
|
||||||
|
if (httpCookie.getComment() != null)
|
||||||
|
cookie.setComment(httpCookie.getComment());
|
||||||
|
cookie.setSecure(httpCookie.getSecure());
|
||||||
|
return cookie;
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean check(String cookieValue){
|
||||||
|
for(String blackItem : new String[]{
|
||||||
|
"a_type",
|
||||||
|
"b_type",
|
||||||
|
"c_type",
|
||||||
|
"is_sensei_service_user",
|
||||||
|
"module_orders_mypage",
|
||||||
|
}){
|
||||||
|
if(cookieValue.startsWith(blackItem)){
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
});
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
public void start(int port){
|
||||||
|
this.proxyServer.start(port);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void close(){
|
||||||
|
this.proxyServer.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 是否已登录Pixiv
|
||||||
|
* @return 如已登录返回true
|
||||||
|
*/
|
||||||
|
public boolean isLogin(){
|
||||||
|
return login;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 导出CookieStore.
|
||||||
|
* 注意!该方法导出的CookieStore不适用于ApacheHttpClient, 如需使用则需要进行转换.
|
||||||
|
* @return CookieStore对象
|
||||||
|
*/
|
||||||
|
public CookieStore getCookieStore(){
|
||||||
|
return this.cookieStore;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
31
src/main/java/net/lamgc/cgj/util/CookieUtil.java
Normal file
31
src/main/java/net/lamgc/cgj/util/CookieUtil.java
Normal file
@ -0,0 +1,31 @@
|
|||||||
|
package net.lamgc.cgj.util;
|
||||||
|
|
||||||
|
import org.apache.http.client.CookieStore;
|
||||||
|
import org.apache.http.impl.client.BasicCookieStore;
|
||||||
|
import org.apache.http.impl.cookie.BasicClientCookie;
|
||||||
|
|
||||||
|
import java.util.Date;
|
||||||
|
|
||||||
|
public class CookieUtil {
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 将{@link java.net.CookieStore}转换到{@link CookieStore}
|
||||||
|
* @param netCookieStore java.net.CookieStore
|
||||||
|
* @return org.apache.http.client.CookieStore
|
||||||
|
*/
|
||||||
|
public static CookieStore NAParse(java.net.CookieStore netCookieStore){
|
||||||
|
CookieStore apacheCookieStore = new BasicCookieStore();
|
||||||
|
netCookieStore.getCookies().forEach(netCookie -> {
|
||||||
|
BasicClientCookie aCookie = new BasicClientCookie(netCookie.getName(), netCookie.getValue());
|
||||||
|
aCookie.setComment(netCookie.getComment());
|
||||||
|
aCookie.setDomain(netCookie.getDomain());
|
||||||
|
aCookie.setExpiryDate(new Date(netCookie.getMaxAge()));
|
||||||
|
aCookie.setPath(netCookie.getPath());
|
||||||
|
aCookie.setSecure(netCookie.getSecure());
|
||||||
|
aCookie.setVersion(netCookie.getVersion());
|
||||||
|
apacheCookieStore.addCookie(aCookie);
|
||||||
|
});
|
||||||
|
return apacheCookieStore;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
25
src/main/java/net/lamgc/cgj/util/DateParser.java
Normal file
25
src/main/java/net/lamgc/cgj/util/DateParser.java
Normal file
@ -0,0 +1,25 @@
|
|||||||
|
package net.lamgc.cgj.util;
|
||||||
|
|
||||||
|
import net.lamgc.utils.base.runner.StringParameterParser;
|
||||||
|
|
||||||
|
import java.text.DateFormat;
|
||||||
|
import java.util.Date;
|
||||||
|
|
||||||
|
public class DateParser implements StringParameterParser<Date> {
|
||||||
|
|
||||||
|
private final DateFormat dateFormat;
|
||||||
|
|
||||||
|
public DateParser(DateFormat dateFormat) {
|
||||||
|
this.dateFormat = dateFormat;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Date parse(String strValue) throws Exception {
|
||||||
|
return dateFormat.parse(strValue);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Date defaultValue() {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
18
src/main/java/net/lamgc/cgj/util/PagesQualityParser.java
Normal file
18
src/main/java/net/lamgc/cgj/util/PagesQualityParser.java
Normal file
@ -0,0 +1,18 @@
|
|||||||
|
package net.lamgc.cgj.util;
|
||||||
|
|
||||||
|
import net.lamgc.cgj.pixiv.PixivDownload;
|
||||||
|
import net.lamgc.cgj.pixiv.PixivURL;
|
||||||
|
import net.lamgc.utils.base.runner.StringParameterParser;
|
||||||
|
|
||||||
|
public class PagesQualityParser implements StringParameterParser<PixivDownload.PageQuality> {
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public PixivDownload.PageQuality parse(String strValue) throws Exception {
|
||||||
|
return PixivDownload.PageQuality.valueOf(strValue.toUpperCase());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public PixivDownload.PageQuality defaultValue() {
|
||||||
|
return PixivDownload.PageQuality.ORIGINAL;
|
||||||
|
}
|
||||||
|
}
|
2
src/main/resources/application.properties
Normal file
2
src/main/resources/application.properties
Normal file
@ -0,0 +1,2 @@
|
|||||||
|
server.port=8081
|
||||||
|
server.tomcat.max-threads=10
|
15
src/main/resources/log4j.properties
Normal file
15
src/main/resources/log4j.properties
Normal file
@ -0,0 +1,15 @@
|
|||||||
|
log4j.rootLogger = info,console,logfile
|
||||||
|
|
||||||
|
#\u8BBE\u7F6Edebug\u65E5\u5FD7\u4FE1\u606F
|
||||||
|
log4j.appender.console=org.apache.log4j.ConsoleAppender
|
||||||
|
log4j.appender.console.Target=System.out
|
||||||
|
log4j.appender.console.layout=org.apache.log4j.PatternLayout
|
||||||
|
#[\u5E74.\u6708.\u65E5 \u65F6:\u5206:\u79D2,\u6BEB\u79D2 \u4F18\u5148\u7EA7] [\u7C7B\u5168\u540D.\u65B9\u6CD5\u540D:\u884C\u53F7][\u7EBF\u7A0B\u540D] \u65E5\u5FD7\u5185\u5BB9
|
||||||
|
log4j.appender.console.layout.ConversionPattern=[%d{HH:mm:ss,SSS} %5p][%c.%M():%-3L][%t]: %m%n
|
||||||
|
|
||||||
|
log4j.appender.logfile=org.apache.log4j.DailyRollingFileAppender
|
||||||
|
log4j.appender.logfile.File=logs/running.log.
|
||||||
|
log4j.appender.logfile.DatePattern=yyyy-MM-dd
|
||||||
|
log4j.appender.logfile.layout=org.apache.log4j.PatternLayout
|
||||||
|
#[\u65F6:\u5206:\u79D2,\u6BEB\u79D2 \u4F18\u5148\u7EA7] [\u7C7B\u5168\u540D.\u65B9\u6CD5\u540D:\u884C\u53F7][\u7EBF\u7A0B\u540D] \u65E5\u5FD7\u5185\u5BB9
|
||||||
|
log4j.appender.logfile.layout.ConversionPattern=[%d{HH:mm:ss,SSS} %5p][%c.%M():%-3L][%t]: %m%n
|
9
src/main/resources/simple.properties
Normal file
9
src/main/resources/simple.properties
Normal file
@ -0,0 +1,9 @@
|
|||||||
|
#用于访问Pixiv的代理服务器
|
||||||
|
reptile.proxy.type=socks5/socks4/http
|
||||||
|
reptile.proxy.host=127.0.0.1
|
||||||
|
reptile.proxy.port=1080
|
||||||
|
reptile.proxy.username=
|
||||||
|
reptile.proxy.password=
|
||||||
|
#登录用代理, 需要让浏览器使用该代理, 访问Pixiv并登录
|
||||||
|
login.proxy.host=127.0.0.1
|
||||||
|
login.proxy.port=1080
|
202
src/test/java/net/lamgc/cgj/pixiv/PixivDownloadTest.java
Normal file
202
src/test/java/net/lamgc/cgj/pixiv/PixivDownloadTest.java
Normal file
@ -0,0 +1,202 @@
|
|||||||
|
package net.lamgc.cgj.pixiv;
|
||||||
|
|
||||||
|
import org.apache.commons.io.IOUtils;
|
||||||
|
import org.apache.http.HttpHost;
|
||||||
|
import org.apache.http.client.CookieStore;
|
||||||
|
import org.junit.Assert;
|
||||||
|
import org.junit.BeforeClass;
|
||||||
|
import org.junit.Ignore;
|
||||||
|
import org.junit.Test;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
import java.io.*;
|
||||||
|
import java.text.SimpleDateFormat;
|
||||||
|
import java.util.Calendar;
|
||||||
|
import java.util.Date;
|
||||||
|
import java.util.GregorianCalendar;
|
||||||
|
import java.util.Objects;
|
||||||
|
import java.util.zip.ZipEntry;
|
||||||
|
import java.util.zip.ZipOutputStream;
|
||||||
|
|
||||||
|
@Ignore
|
||||||
|
public class PixivDownloadTest {
|
||||||
|
|
||||||
|
private static CookieStore cookieStore;
|
||||||
|
|
||||||
|
private final static Logger log = LoggerFactory.getLogger("PixivDownloadTest");
|
||||||
|
|
||||||
|
private static HttpHost proxy = new HttpHost("127.0.0.1", 1001);
|
||||||
|
|
||||||
|
@BeforeClass
|
||||||
|
public static void before() throws IOException, ClassNotFoundException {
|
||||||
|
ObjectInputStream ois = new ObjectInputStream(new FileInputStream(new File("cookies.store")));
|
||||||
|
cookieStore = (CookieStore) ois.readObject();
|
||||||
|
ois.close();
|
||||||
|
log.info("已载入CookieStore");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void logOutTest() throws IOException {
|
||||||
|
PixivDownload pixivDownload = new PixivDownload(Objects.requireNonNull(cookieStore), proxy);
|
||||||
|
Assert.assertTrue(pixivDownload.logOut());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void collectionDownloadTest() throws IOException {
|
||||||
|
PixivDownload pixivDownload = new PixivDownload(Objects.requireNonNull(cookieStore), proxy);
|
||||||
|
File outputFile = new File("collection.zip");
|
||||||
|
if(!outputFile.exists() && !outputFile.createNewFile()) {
|
||||||
|
Assert.fail("文件创建失败: " + outputFile.getAbsolutePath());
|
||||||
|
}
|
||||||
|
ZipOutputStream zos = new ZipOutputStream(new FileOutputStream(outputFile));
|
||||||
|
zos.setLevel(9);
|
||||||
|
log.info("正在调用方法...");
|
||||||
|
pixivDownload.getCollectionAsInputStream(PixivDownload.PageQuality.ORIGINAL, (link, inputStream) -> {
|
||||||
|
try {
|
||||||
|
ZipEntry entry = new ZipEntry(link.substring(link.lastIndexOf("/") + 1));
|
||||||
|
log.info("正在写入: " + entry.getName());
|
||||||
|
zos.putNextEntry(entry);
|
||||||
|
IOUtils.copy(inputStream, zos);
|
||||||
|
zos.flush();
|
||||||
|
} catch (IOException e) {
|
||||||
|
log.error("写入文件项时发生异常", e);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
log.info("调用完成.");
|
||||||
|
zos.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void getRecommendsTest() throws IOException {
|
||||||
|
PixivDownload pixivDownload = new PixivDownload(Objects.requireNonNull(cookieStore), proxy);
|
||||||
|
String date = new SimpleDateFormat("yyyyMMdd").format(new Date());
|
||||||
|
int id = 1;
|
||||||
|
File outputFile = new File("recommends-" + date + "-" + id + ".zip");
|
||||||
|
while(outputFile.exists()) {
|
||||||
|
id++;
|
||||||
|
outputFile = new File("recommends-" + date + "-" + id + ".zip");
|
||||||
|
}
|
||||||
|
|
||||||
|
if(!outputFile.createNewFile()) {
|
||||||
|
Assert.fail("文件创建失败: " + outputFile.getAbsolutePath());
|
||||||
|
}
|
||||||
|
ZipOutputStream zos = new ZipOutputStream(new FileOutputStream(outputFile));
|
||||||
|
zos.setLevel(9);
|
||||||
|
log.info("正在调用方法...");
|
||||||
|
pixivDownload.getRecommendAsInputStream(PixivDownload.PageQuality.ORIGINAL, (link, inputStream) -> {
|
||||||
|
try {
|
||||||
|
ZipEntry entry = new ZipEntry(link.substring(link.lastIndexOf("/") + 1));
|
||||||
|
log.info("正在写入: " + entry.getName());
|
||||||
|
zos.putNextEntry(entry);
|
||||||
|
IOUtils.copy(inputStream, zos);
|
||||||
|
zos.flush();
|
||||||
|
log.info("已成功写入 {}", entry.getName());
|
||||||
|
} catch (IOException e) {
|
||||||
|
log.error("写入文件项时发生异常", e);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
log.info("调用完成.");
|
||||||
|
zos.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void getRankingIllustByRangeTest() throws IOException {
|
||||||
|
PixivDownload pixivDownload = new PixivDownload(cookieStore, proxy);
|
||||||
|
|
||||||
|
GregorianCalendar gregorianCalendar = new GregorianCalendar();
|
||||||
|
gregorianCalendar.setTime(new Date());
|
||||||
|
gregorianCalendar.add(Calendar.DATE, -1);
|
||||||
|
Date queryDate = gregorianCalendar.getTime();
|
||||||
|
String date = new SimpleDateFormat("yyyyMMdd").format(queryDate);
|
||||||
|
log.info("查询时间: {}", date);
|
||||||
|
|
||||||
|
|
||||||
|
int id = 1;
|
||||||
|
File outputFile = new File("ranking-" + date + "-" + id + ".zip");
|
||||||
|
while(outputFile.exists()) {
|
||||||
|
id++;
|
||||||
|
outputFile = new File("ranking-" + date + "-" + id + ".zip");
|
||||||
|
}
|
||||||
|
|
||||||
|
if(!outputFile.createNewFile()) {
|
||||||
|
Assert.fail("文件创建失败: " + outputFile.getAbsolutePath());
|
||||||
|
}
|
||||||
|
ZipOutputStream zos = new ZipOutputStream(new FileOutputStream(outputFile));
|
||||||
|
zos.setLevel(9);
|
||||||
|
log.info("正在调用方法...");
|
||||||
|
try {
|
||||||
|
pixivDownload.getRankingAsInputStream(null, PixivURL.RankingMode.MODE_DAILY_R18, queryDate, 15, PixivDownload.PageQuality.ORIGINAL, (rank, link, rankInfo, inputStream) -> {
|
||||||
|
try {
|
||||||
|
ZipEntry entry = new ZipEntry("Rank" + rank + "-" + link.substring(link.lastIndexOf("/") + 1));
|
||||||
|
entry.setComment(rankInfo.toString());
|
||||||
|
log.info("正在写入: " + entry.getName());
|
||||||
|
zos.putNextEntry(entry);
|
||||||
|
IOUtils.copy(inputStream, zos);
|
||||||
|
zos.flush();
|
||||||
|
log.info("已成功写入 {}", entry.getName());
|
||||||
|
} catch (IOException e) {
|
||||||
|
log.error("写入文件项时发生异常", e);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
} finally {
|
||||||
|
zos.finish();
|
||||||
|
zos.flush();
|
||||||
|
zos.close();
|
||||||
|
}
|
||||||
|
log.info("调用完成.");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void getRankingIllustTest() throws IOException {
|
||||||
|
PixivDownload pixivDownload = new PixivDownload(cookieStore, proxy);
|
||||||
|
|
||||||
|
GregorianCalendar gregorianCalendar = new GregorianCalendar();
|
||||||
|
gregorianCalendar.setTime(new Date());
|
||||||
|
gregorianCalendar.add(Calendar.DATE, -1);
|
||||||
|
Date queryDate = gregorianCalendar.getTime();
|
||||||
|
String date = new SimpleDateFormat("yyyyMMdd").format(queryDate);
|
||||||
|
log.info("查询时间: {}", date);
|
||||||
|
|
||||||
|
int id = 1;
|
||||||
|
File outputFile = new File("ranking-" + date + "-" + id + ".zip");
|
||||||
|
while(outputFile.exists()) {
|
||||||
|
id++;
|
||||||
|
outputFile = new File("ranking-" + date + "-" + id + ".zip");
|
||||||
|
}
|
||||||
|
|
||||||
|
if(!outputFile.createNewFile()) {
|
||||||
|
Assert.fail("文件创建失败: " + outputFile.getAbsolutePath());
|
||||||
|
}
|
||||||
|
ZipOutputStream zos = new ZipOutputStream(new FileOutputStream(outputFile));
|
||||||
|
zos.setLevel(9);
|
||||||
|
|
||||||
|
log.info("正在调用方法...");
|
||||||
|
try {
|
||||||
|
pixivDownload.getRankingAsInputStream(null, null, queryDate, 65, 55, PixivDownload.PageQuality.ORIGINAL, (rank, link, rankInfo, inputStream) -> {
|
||||||
|
try {
|
||||||
|
ZipEntry entry = new ZipEntry("Rank" + rank + "-" + link.substring(link.lastIndexOf("/") + 1));
|
||||||
|
entry.setComment(rankInfo.toString());
|
||||||
|
log.info("正在写入: " + entry.getName());
|
||||||
|
zos.putNextEntry(entry);
|
||||||
|
IOUtils.copy(inputStream, zos);
|
||||||
|
zos.flush();
|
||||||
|
log.info("已成功写入 {}", entry.getName());
|
||||||
|
} catch (IOException e) {
|
||||||
|
log.error("写入文件项时发生异常", e);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
} finally {
|
||||||
|
zos.finish();
|
||||||
|
zos.flush();
|
||||||
|
zos.close();
|
||||||
|
}
|
||||||
|
log.info("调用完成.");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void getIllustPreLoadDataByIdTest() throws IOException {
|
||||||
|
log.info(new PixivDownload(cookieStore, proxy).getIllustPreLoadDataById(64076261).toString());
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
@ -0,0 +1,15 @@
|
|||||||
|
package net.lamgc.cgj.pixiv;
|
||||||
|
|
||||||
|
import org.junit.Test;
|
||||||
|
|
||||||
|
public class PixivSearchBuilderTest {
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void buildTest() {
|
||||||
|
PixivSearchBuilder builder = new PixivSearchBuilder("hololive");
|
||||||
|
//builder.addExcludeKeyword("fubuki").addExcludeKeyword("minato");
|
||||||
|
builder.addIncludeKeyword("35").addIncludeKeyword("okayu").addIncludeKeyword("百鬼あやめ");
|
||||||
|
System.out.println(builder.buildURL());
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
Loading…
Reference in New Issue
Block a user