initial commit

This commit is contained in:
LamGC 2020-03-26 14:39:59 +08:00
commit 532d7371d8
31 changed files with 4224 additions and 0 deletions

6
.gitignore vendored Normal file
View File

@ -0,0 +1,6 @@
/pluginData/
/logs/
/.idea/
/CGJ_2.iml
/cookies.store
/target/

150
pom.xml Normal file
View File

@ -0,0 +1,150 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>org.example</groupId>
<artifactId>CGJ_2</artifactId>
<version>1.0-SNAPSHOT</version>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<maven.compiler.encoding>UTF-8</maven.compiler.encoding>
</properties>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<source>8</source>
<target>8</target>
</configuration>
</plugin>
<!--<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>3.2.2</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<transformers>
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
<mainClass>net.lamgc.cgj.Main</mainClass>
</transformer>
</transformers>
<filters>
<filter>
<artifact>*:*</artifact>
<excludes>
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.RSA</exclude>
</excludes>
</filter>
</filters>
</configuration>
</execution>
</executions>
</plugin>-->
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<configuration>
<mainClass>net.lamgc.cgj.Main</mainClass>
<classifier>exec</classifier>
</configuration>
<executions>
<execution>
<goals>
<goal>repackage</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
<dependencies>
<dependency>
<groupId>com.github.monkeywie</groupId>
<artifactId>proxyee</artifactId>
<version>1.0.4</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>1.7.30</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
<version>1.7.30</version>
</dependency>
<dependency>
<groupId>net.lamgc</groupId>
<artifactId>java-utils</artifactId>
<version>1.1.0_5-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.12</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>4.5.12</version>
</dependency>
<dependency>
<groupId>org.jsoup</groupId>
<artifactId>jsoup</artifactId>
<version>1.11.3</version>
</dependency>
<dependency>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
<version>2.8.5</version>
</dependency>
<dependency>
<groupId>net.sourceforge.htmlunit</groupId>
<artifactId>htmlunit</artifactId>
<version>2.29</version>
</dependency>
<dependency>
<groupId>com.github.rholder</groupId>
<artifactId>guava-retrying</artifactId>
<version>2.0.0</version>
</dependency>
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>8.0.19</version>
</dependency>
<dependency>
<groupId>net.lz1998</groupId>
<artifactId>spring-cq</artifactId>
<version>4.14.0.6</version>
<exclusions>
<exclusion>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
</exclusion>
<exclusion>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-core</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies>
</project>

24
search.txt Normal file
View File

@ -0,0 +1,24 @@
搜索标签信息https://www.pixiv.net/ajax/search/tags/标签名
搜索接口:
https://www.pixiv.net/ajax/search/{Type}/搜索内容
Type = illustrations(插画) / top(顶部?) / manga(漫画) / novels(小说)
word=搜索内容 [参数可能不是必须的]
s_mode=s_tag(标签-部分一致) / s_tag_full(标签-完全一致) / s_tc(标题、说明文字)
type= all(插画、漫画、动图_动态插图) / illust_and_ugoira(插画、动图) / illust(插画) / manga(漫画) / ugoira(动图)
p=页数 [超出页数的情况下将获取不到数据(即"body.illust.data"是空数组)]
order=date(按旧排序) / date_d(按新排序) / Unknown(按热门度排序, 需要会员)
mode= all(全部) / safe(全年龄) / r18(咳咳)
可选参数:
wlt=最小宽度像素
wgt=最高宽度像素
hlt=最小高度像素
hgt=最高高度像素
ratio=0.5(横图) / -0.5(纵图) / 0(正方形) [可能不能改变参数, 三个值是固定的]
tool=使用工具, 不是很重要晚些再加
scd=开始时间(yyyy-MM-dd)
ecd=结束时间(yyyy-MM-dd)
最小收藏数 = 收藏数限定参数为会员功能, 无法获取

View File

@ -0,0 +1,58 @@
package net.lamgc.cgj;
import com.google.common.base.Strings;
import net.lamgc.utils.base.runner.Argument;
import net.lamgc.utils.base.runner.Command;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
public class CQBotAdminProcess {
private final static Logger log = LoggerFactory.getLogger("CQBotAdminProcess");
@Command
public String clearCache() {
CQProcess.clearCache();
return "操作已完成.";
}
@Command
public String setGlobalProperty(@Argument(name = "key") String key, @Argument(name = "value") String value, @Argument(name = "save", force = false) boolean saveNow) {
String lastValue = CQPluginMain.globalProp.getProperty(key);
CQPluginMain.globalProp.setProperty(key, Strings.nullToEmpty(value));
if(saveNow) {
saveGlobalProperties();
}
return "全局配置项 " + key + " 现已设置为: " + value + " (设置前的值: " + lastValue + ")";
}
@Command
public String getGlobalProperty(@Argument(name = "key") String key) {
return "全局配置项 " + key + " 当前值: " + CQPluginMain.globalProp.getProperty(key, "(Empty)");
}
@Command
public String saveGlobalProperties() {
log.info("正在保存全局配置文件...");
File globalPropFile = new File("./global.properties");
try {
if(!globalPropFile.exists()) {
if(!globalPropFile.createNewFile()) {
log.error("全局配置项文件保存失败!({})", "文件创建失败");
return "全局配置项文件保存失败!";
}
}
CQPluginMain.globalProp.store(new FileOutputStream(globalPropFile), "");
log.info("全局配置文件保存成功!");
return "保存全局配置文件 - 操作已完成.";
} catch (IOException e) {
log.error("全局配置项文件保存失败!", e);
return "全局配置项文件保存失败!";
}
}
}

View File

@ -0,0 +1,13 @@
package net.lamgc.cgj;
import net.lz1998.cq.CQGlobal;
import net.lz1998.cq.EnableCQ;
@EnableCQ
public class CQConfig {
public static void init() {
CQGlobal.pluginList.add(CQPluginMain.class);
}
}

View File

@ -0,0 +1,147 @@
package net.lamgc.cgj;
import com.google.common.base.Strings;
import net.lamgc.cgj.util.DateParser;
import net.lamgc.cgj.util.PagesQualityParser;
import net.lamgc.utils.base.runner.ArgumentsRunner;
import net.lamgc.utils.base.runner.ArgumentsRunnerConfig;
import net.lamgc.utils.base.runner.exception.DeveloperRunnerException;
import net.lamgc.utils.base.runner.exception.NoSuchCommandException;
import net.lamgc.utils.base.runner.exception.ParameterNoFoundException;
import net.lz1998.cq.event.message.CQDiscussMessageEvent;
import net.lz1998.cq.event.message.CQGroupMessageEvent;
import net.lz1998.cq.event.message.CQMessageEvent;
import net.lz1998.cq.event.message.CQPrivateMessageEvent;
import net.lz1998.cq.robot.CQPlugin;
import net.lz1998.cq.robot.CoolQ;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Objects;
import java.util.Properties;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@Component
public class CQPluginMain extends CQPlugin {
private final static String COMMAND_PREFIX = ".cgj";
private final Logger log = LoggerFactory.getLogger("CQPluginMain@" + Integer.toHexString(this.hashCode()));
private final ArgumentsRunnerConfig runnerConfig = new ArgumentsRunnerConfig();
public final static Properties globalProp = new Properties();
public CQPluginMain() {
runnerConfig.setUseDefaultValueInsteadOfException(true);
runnerConfig.setCommandIgnoreCase(true);
runnerConfig.addStringParameterParser(new DateParser(new SimpleDateFormat("yyyy-MM-dd")));
runnerConfig.addStringParameterParser(new PagesQualityParser());
File globalPropFile = new File("./global.properties");
if(globalPropFile.exists() && globalPropFile.isFile()) {
log.info("正在加载全局配置文件...");
try {
globalProp.load(new FileInputStream(globalPropFile));
log.info("全局配置文件加载完成.");
} catch (IOException e) {
log.error("加载全局配置文件时发生异常", e);
}
} else {
log.info("未找到全局配置文件,跳过加载.");
}
}
@Override
public int onPrivateMessage(CoolQ cq, CQPrivateMessageEvent event) {
//log.info("私聊消息到达: 发送者[{}], 消息内容: {}", event.getSender().getUserId(), event.getMessage());
return processMessage(cq, event);
}
@Override
public int onGroupMessage(CoolQ cq, CQGroupMessageEvent event) {
//log.info("群消息到达: 群[{}], 发送者[{}], 消息内容: {}", event.getGroupId(), event.getSender().getUserId(), event.getMessage());
return processMessage(cq, event);
}
@Override
public int onDiscussMessage(CoolQ cq, CQDiscussMessageEvent event) {
//log.info("讨论组消息到达: 群[{}], 发送者[{}], 消息内容: {}", event.getDiscussId(), event.getSender().getUserId(), event.getMessage());
return processMessage(cq, event);
}
public int processMessage(CoolQ cq, CQMessageEvent event) {
String msg = event.getMessage();
if(!msg.startsWith(COMMAND_PREFIX)) {
return MESSAGE_IGNORE;
}
Pattern pattern = Pattern.compile("/\\s*(\".+?\"|[^:\\s])+((\\s*:\\s*(\".+?\"|[^\\s])+)|)|(\".+?\"|[^\"\\s])+");
Matcher matcher = pattern.matcher(Strings.nullToEmpty(msg));
ArrayList<String> argsList = new ArrayList<>();
while (matcher.find()) {
argsList.add(matcher.group());
}
String[] args = new String[argsList.size()];
argsList.toArray(args);
log.info("正在处理命令...");
long time = System.currentTimeMillis();
Object result;
try {
if(msg.toLowerCase().startsWith(COMMAND_PREFIX + "admin")) {
if(!String.valueOf(event.getUserId()).equals(globalProp.getProperty("admin.adminId"))) {
sendMessage(cq, event, "你没有执行该命令的权限!", false);
return MESSAGE_BLOCK;
} else {
result = new ArgumentsRunner(CQBotAdminProcess.class, runnerConfig)
.run(new CQBotAdminProcess(), args.length <= 1 ? new String[0] : Arrays.copyOfRange(args, 1, args.length));
}
} else {
result = new ArgumentsRunner(CQProcess.class, runnerConfig).run(args.length <= 1 ? new String[0] : Arrays.copyOfRange(args, 1, args.length));
}
} catch(NoSuchCommandException e) {
result = "没有这个命令!请使用“.cgj”查看帮助说明";
} catch(ParameterNoFoundException e) {
result = "命令缺少参数: " + e.getParameterName();
} catch(DeveloperRunnerException e) {
log.error("执行命令时发生异常", e);
result = "命令执行时发生错误,无法完成!";
}
log.info("命令处理完成(耗时: {}ms)", System.currentTimeMillis() - time);
if(Objects.requireNonNull(result) instanceof String) {
sendMessage(cq, event, (String) result, false);
}
return MESSAGE_BLOCK;
}
private final static Logger msgLog = LoggerFactory.getLogger("SendMsg");
/**
* 发送消息
* @param cq CoolQ对象
* @param event 消息事件对象
* @param message 消息内容
* @param auto_escape 消息内容是否作为纯文本发送即不解析 CQ 只在 message 字段是字符串时有效.
*/
public static void sendMessage(CoolQ cq, CQMessageEvent event, String message, boolean auto_escape) {
msgLog.debug("发送消息:{}", message);
if(event instanceof CQPrivateMessageEvent) {
CQPrivateMessageEvent _event = (CQPrivateMessageEvent) event;
cq.sendPrivateMsg(_event.getSender().getUserId(), message, auto_escape);
} else if(event instanceof CQGroupMessageEvent) {
CQGroupMessageEvent _event = (CQGroupMessageEvent) event;
cq.sendGroupMsg(_event.getGroupId(), message, auto_escape);
} else if(event instanceof CQDiscussMessageEvent) {
CQDiscussMessageEvent _event = (CQDiscussMessageEvent) event;
cq.sendGroupMsg(_event.getDiscussId(), message, auto_escape).getData().getMessageId();
}
}
}

View File

@ -0,0 +1,553 @@
package net.lamgc.cgj;
import com.google.common.base.Strings;
import com.google.gson.*;
import io.netty.handler.codec.http.HttpHeaderNames;
import net.lamgc.cgj.cache.CacheObject;
import net.lamgc.cgj.cache.ImageCacheHandler;
import net.lamgc.cgj.cache.ImageCacheObject;
import net.lamgc.cgj.pixiv.PixivDownload;
import net.lamgc.cgj.pixiv.PixivSearchBuilder;
import net.lamgc.cgj.pixiv.PixivURL;
import net.lamgc.utils.base.runner.Argument;
import net.lamgc.utils.base.runner.Command;
import net.lamgc.utils.event.EventExecutor;
import net.lz1998.cq.utils.CQCode;
import org.apache.http.HttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpHead;
import org.apache.http.util.EntityUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.*;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.StandardOpenOption;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
public class CQProcess {
private final static PixivDownload pixivDownload = new PixivDownload(Main.cookieStore, Main.proxy);
private final static Logger log = LoggerFactory.getLogger("CQProcess");
private final static File imageStoreDir = new File(System.getProperty("cgj.cqRootDir") + "data/image/cgj/");
private final static Hashtable<String, File> imageCache = new Hashtable<>();
private final static Hashtable<Integer, JsonObject> illustInfoCache = new Hashtable<>();
private final static Hashtable<Integer, CacheObject<JsonObject>> illustPreLoadDataCache = new Hashtable<>();
private final static Hashtable<String, CacheObject<JsonObject>> searchBodyCache = new Hashtable<>();
private final static Hashtable<String, List<String>> pagesCache = new Hashtable<>();
private final static Object searchCacheLock = new Object();
private final static Gson gson = new GsonBuilder()
.serializeNulls()
.create();
private final static EventExecutor imageCacheExecutor = new EventExecutor(new ThreadPoolExecutor(
1,
(int) Math.ceil(Runtime.getRuntime().availableProcessors() / 2F),
15L, TimeUnit.SECONDS,
new ArrayBlockingQueue<>(30),
new ThreadPoolExecutor.DiscardOldestPolicy()
));
static {
try {
imageCacheExecutor.addHandler(new ImageCacheHandler());
} catch (IllegalAccessException e) {
e.printStackTrace();
}
}
@Command(defaultCommand = true)
public static String help() {
StringBuilder helpStrBuilder = new StringBuilder();
helpStrBuilder.append("CGJ Bot使用指南").append("\n");
helpStrBuilder.append("目前可用的命令:").append("\n");
helpStrBuilder.append("\t").append("ranking - 获取今天或指定日期排行榜的前10名作品").append("\n");
helpStrBuilder.append("\t\t").append("date - 指定查询日期(年年年年-月月-日日)").append("\n");
helpStrBuilder.append("\t").append("search - 搜索指定关键词并显示前10个作品").append("\n");
helpStrBuilder.append("\t\t").append("content - 搜索内容").append("\n");
helpStrBuilder.append("\t").append("artworks - 获取作品的Pixiv页面").append("\n");
helpStrBuilder.append("\t\t").append("id - 作品id").append("\n");
return helpStrBuilder.toString();
}
@Command
public static String ranking(
@Argument(force = false, name = "date") Date queryTime,
@Argument(force = false, name = "contentMode", defaultValue = "DAILY") String contentMode
) {
Date queryDate = queryTime;
if (queryDate == null) {
queryDate = new Date();
GregorianCalendar gregorianCalendar = new GregorianCalendar();
log.info("CurrentDate: {}", queryDate);
gregorianCalendar.setTime(queryDate);
if (gregorianCalendar.get(Calendar.HOUR_OF_DAY) < 12) {
gregorianCalendar.add(Calendar.DATE, -2);
} else {
gregorianCalendar.add(Calendar.DATE, -1);
}
queryDate = gregorianCalendar.getTime();
}
PixivURL.RankingMode mode = PixivURL.RankingMode.MODE_DAILY;
try {
mode = PixivURL.RankingMode.valueOf("MODE_" + contentMode.toUpperCase());
} catch (IllegalArgumentException e) {
log.warn("无效的RankingMode值: {}", contentMode);
}
StringBuilder resultBuilder = new StringBuilder(mode.name() + " - 以下是 ").append(new SimpleDateFormat("yyyy-MM-dd").format(queryDate)).append(" 的Pixiv插画排名榜前十名\n");
try {
int index = 0;
for (JsonObject rankInfo : pixivDownload.getRanking(PixivURL.RankingContentType.TYPE_ILLUST, mode, queryDate, 1, 10)) {
index++;
int rank = rankInfo.get("rank").getAsInt();
int illustId = rankInfo.get("illust_id").getAsInt();
int authorId = rankInfo.get("user_id").getAsInt();
String authorName = rankInfo.get("user_name").getAsString();
String title = rankInfo.get("title").getAsString();
resultBuilder.append(rank).append(". (id: ").append(illustId).append(") ").append(title)
.append("(Author: ").append(authorName).append(",").append(authorId).append(")\n");
if (index < 4) {
resultBuilder.append(getImageById(illustId, PixivDownload.PageQuality.REGULAR, 1)).append("\n");
}
}
} catch (IOException e) {
log.error("消息处理异常", e);
return "排名榜获取失败!详情请查看机器人控制台。";
}
return resultBuilder.append("如查询当前时间获取到昨天时间,则今日排名榜未更新。").toString();
}
@Command(commandName = "userArt")
public static String userArtworks() {
return "功能未完成";
}
@Command
public static String search(@Argument(name = "content") String content,
@Argument(name = "type", force = false) String type,
@Argument(name = "area", force = false) String area,
@Argument(name = "in", force = false) String includeKeywords,
@Argument(name = "ex", force = false) String excludeKeywords,
@Argument(name = "contentOption", force = false) String contentOption,
@Argument(name = "page", force = false, defaultValue = "1") int pagesIndex
) throws IOException {
PixivSearchBuilder searchBuilder = new PixivSearchBuilder(Strings.isNullOrEmpty(content) ? "" : content);
if (type != null) {
try {
searchBuilder.setSearchType(PixivSearchBuilder.SearchType.valueOf(type.toUpperCase()));
} catch (IllegalArgumentException e) {
log.warn("不支持的SearchType: {}", type);
}
}
if (area != null) {
try {
searchBuilder.setSearchArea(PixivSearchBuilder.SearchArea.valueOf(area));
} catch (IllegalArgumentException e) {
log.warn("不支持的SearchArea: {}", area);
}
}
if (contentOption != null) {
try {
searchBuilder.setSearchContentOption(PixivSearchBuilder.SearchContentOption.valueOf(contentOption));
} catch (IllegalArgumentException e) {
log.warn("不支持的SearchContentOption: {}", contentOption);
}
}
if (!Strings.isNullOrEmpty(includeKeywords)) {
for (String keyword : includeKeywords.split(";")) {
searchBuilder.removeExcludeKeyword(keyword);
searchBuilder.addIncludeKeyword(keyword);
log.info("已添加关键字: {}", keyword);
}
}
if (!Strings.isNullOrEmpty(excludeKeywords)) {
for (String keyword : excludeKeywords.split(";")) {
searchBuilder.removeIncludeKeyword(keyword);
searchBuilder.addExcludeKeyword(keyword);
log.info("已添加排除关键字: {}", keyword);
}
}
log.info("正在搜索作品, 条件: {}", searchBuilder.getSearchCondition());
String requestUrl = searchBuilder.buildURL();
log.info("RequestUrl: {}", requestUrl);
CacheObject<JsonObject> cacheObject = new CacheObject<>();
if(!searchBodyCache.containsKey(requestUrl) || (cacheObject = searchBodyCache.get(requestUrl)).isExpire(new Date())) {
synchronized (searchCacheLock) {
if (!searchBodyCache.containsKey(requestUrl) || (cacheObject = searchBodyCache.get(requestUrl)).isExpire(new Date())) {
log.info("searchBody缓存失效, 正在更新...");
JsonObject jsonObject;
HttpGet httpGetRequest = pixivDownload.createHttpGetRequest(requestUrl);
HttpResponse response = pixivDownload.getHttpClient().execute(httpGetRequest);
String responseBody = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8);
log.info("ResponseBody: {}", responseBody);
jsonObject = gson.fromJson(responseBody, JsonObject.class);
if (jsonObject.get("error").getAsBoolean()) {
log.error("接口请求错误, 错误信息: {}", jsonObject.get("message").getAsString());
return "处理命令时发生错误!";
}
Date newExpireDate = new Date();
long expire = 7200 * 1000;
String propValue = CQPluginMain.globalProp.getProperty("cache.searchBody.expire", "7200000");
try {
expire = Long.parseLong(propValue);
} catch (Exception e) {
log.warn("全局配置项 \"{}\" 值非法, 已使用默认值: {}", propValue, expire);
}
newExpireDate.setTime(newExpireDate.getTime() + expire);
cacheObject.update(jsonObject, newExpireDate);
searchBodyCache.put(requestUrl, cacheObject);
log.info("searchBody缓存已更新(到期时间: {})", newExpireDate);
}
}
} else {
log.info("搜索缓存命中.");
}
JsonObject resultBody = searchBodyCache.get(requestUrl).get().getAsJsonObject("body");
StringBuilder result = new StringBuilder("搜索结果:");
log.info("正在处理信息...");
int limit = 8;
try {
limit = Integer.parseInt(CQPluginMain.globalProp.getProperty("search.ItemCountLimit", "8"));
} catch (Exception e) {
log.warn("参数转换异常!将使用默认值(" + limit + ")", e);
}
for (PixivSearchBuilder.SearchArea searchArea : PixivSearchBuilder.SearchArea.values()) {
if (!resultBody.has(searchArea.jsonKey) || resultBody.getAsJsonObject(searchArea.jsonKey).getAsJsonArray("data").size() == 0) {
log.debug("返回数据不包含 {}", searchArea.jsonKey);
continue;
}
JsonArray illustsArray = resultBody
.getAsJsonObject(searchArea.jsonKey).getAsJsonArray("data");
ArrayList<JsonElement> illustsList = new ArrayList<>();
illustsArray.forEach(illustsList::add);
illustsList.sort((o1, o2) -> {
try {
int illustLikeCount1 = getIllustPreLoadData(o1.getAsJsonObject().get("illustId").getAsInt()).get("likeCount").getAsInt();
int illustLikeCount2 = getIllustPreLoadData(o2.getAsJsonObject().get("illustId").getAsInt()).get("likeCount").getAsInt();
return Integer.compare(illustLikeCount2, illustLikeCount1);
} catch (IOException e) {
e.printStackTrace();
return 0;
}
});
log.info("已找到与 {} 相关插图信息({})", content, searchArea.name().toLowerCase());
int count = 1;
for (JsonElement jsonElement : illustsList) {
if (count > limit) {
break;
}
JsonObject illustObj = jsonElement.getAsJsonObject();
if (!illustObj.has("illustId")) {
continue;
}
int illustId = illustObj.get("illustId").getAsInt();
StringBuilder builder = new StringBuilder("[");
illustObj.get("tags").getAsJsonArray().forEach(el -> builder.append(el.getAsString()).append(", "));
builder.replace(builder.length() - 2, builder.length(), "]");
log.debug("{} ({} / {})\n\t作品id: {}, \n\t作者名(作者id): {} ({}), \n\t作品标题: {}, \n\t作品Tags: {}, \n\t作品链接: {}",
searchArea.name(),
count,
illustsList.size(),
illustId,
illustObj.get("userName").getAsString(),
illustObj.get("userId").getAsInt(),
illustObj.get("illustTitle").getAsString(),
builder,
PixivURL.getPixivRefererLink(illustId)
);
String imageMsg = getImageById(illustId, PixivDownload.PageQuality.REGULAR, 1);
if (isNoSafe(illustId, CQPluginMain.globalProp, true)) {
log.warn("作品Id {} 为R-18作品, 跳过.", illustId);
count--;
continue;
}
result.append(searchArea.name()).append(" (").append(count).append(" / ").append(illustsList.size()).append(")\n\t作品id: ").append(illustId)
.append(", \n\t作者名: ").append(illustObj.get("userName").getAsString())
.append("\n\t作品标题: ").append(illustObj.get("illustTitle").getAsString()).append("\n").append(imageMsg).append("\n");
count++;
}
if (count > limit) {
break;
}
}
return Strings.nullToEmpty(result.toString()) + "预览图片并非原图,使用“.cgj image -id 作品id”获取原图";
}
@Command(commandName = "pages")
public static String getPagesList(@Argument(name = "id") int illustId, @Argument(name = "quality", force = false) PixivDownload.PageQuality quality) {
try {
List<String> pagesList = PixivDownload.getIllustAllPageDownload(pixivDownload.getHttpClient(), pixivDownload.getCookieStore(), illustId, quality);
StringBuilder builder = new StringBuilder("作品ID ").append(illustId).append(" 共有").append(pagesList.size()).append("页:").append("\n");
int index = 0;
for (String link : pagesList) {
builder.append("Page ").append(++index).append(": ").append(link).append("\n");
}
return builder.toString();
} catch (IOException e) {
log.error("获取作品所有页面下载链接失败!", e);
return "发生错误,无法完成命令";
}
}
@Command(commandName = "artworks")
public static String artworksLink(@Argument(name = "id") int illustId) {
try {
if (isNoSafe(illustId, CQPluginMain.globalProp, false)) {
log.warn("作品Id {} 已被屏蔽.", illustId);
return "由于相关设置,该作品已被屏蔽!";
}
} catch (IOException e) {
log.error("获取作品信息失败!", e);
return "作品信息无法获取!";
}
return PixivURL.getPixivRefererLink(illustId);
}
@Command(commandName = "image")
public static String getImageById(@Argument(name = "id") int illustId,
@Argument(name = "quality", force = false) PixivDownload.PageQuality quality,
@Argument(name = "page", force = false, defaultValue = "1") int pageIndex) {
log.info("IllustId: {}, Quality: {}, PageIndex: {}", illustId, quality.name(), pageIndex);
List<String> pagesList;
try {
pagesList = getIllustPages(illustId, quality);
} catch (IOException e) {
log.error("获取下载链接列表时发生异常", e);
return "发生网络异常,无法获取图片!";
}
if (pagesList.size() < pageIndex || pageIndex <= 0) {
log.warn("指定的页数超出了总页数({} / {})", pageIndex, pagesList.size());
return "指定的页数超出了范围(总共 " + pagesList.size() + " 页)";
}
try {
if (isNoSafe(illustId, CQPluginMain.globalProp, false)) {
log.warn("作品 {} 存在R-18内容且设置\"image.allowR18\"为false将屏蔽该作品不发送.", illustId);
pageIndex = -1;
}
} catch (IOException e) {
log.warn("作品信息无法获取!", e);
return "发生网络异常,无法获取图片!";
}
int index = 0;
String targetLink = null;
File targetFile;
File currentImageFile;
for (String link : pagesList) {
index++;
if (index == pageIndex) {
targetLink = link;
}
currentImageFile = new File(getImageStoreDir(), link.substring(link.lastIndexOf("/") + 1));
if (!imageCache.containsKey(link)) {
HttpHead headRequest = new HttpHead(link);
headRequest.addHeader("Referer", PixivURL.getPixivRefererLink(illustId));
HttpResponse headResponse;
try {
headResponse = pixivDownload.getHttpClient().execute(headRequest);
} catch (IOException e) {
log.error("获取图片大小失败!", e);
return "图片获取失败!";
}
String contentLengthStr = headResponse.getFirstHeader(HttpHeaderNames.CONTENT_LENGTH.toString()).getValue();
if (currentImageFile.exists() && currentImageFile.length() == Long.parseLong(contentLengthStr)) {
imageCache.put(link, currentImageFile);
log.info("作品Id {} 第 {} 页缓存已补充.", illustId, index);
continue;
}
if (index == pageIndex) {
try {
imageCacheExecutor.executorSync(new ImageCacheObject(imageCache, illustId, link, currentImageFile));
} catch (InterruptedException e) {
log.warn("图片下载遭到中断!", e);
}
} else {
imageCacheExecutor.executor(
new ImageCacheObject(imageCache, illustId, link, currentImageFile));
}
}
}
if (pageIndex == -1) {
return "(根据设置,该作品已被屏蔽!)";
}
if (targetLink == null) {
return "未找到对应的图片!";
}
targetFile = imageCache.get(targetLink);
return targetFile == null ? "未找到对应的图片!" : CQCode.image(getImageStoreDir().getName() + "/" + targetFile.getName());
}
static void clearCache() {
log.warn("正在清除所有图片缓存...");
imageCache.clear();
File imageStoreDir = new File(System.getProperty("cgj.cqRootDir") + "data/image/cgj/");
File[] listFiles = imageStoreDir.listFiles();
if (listFiles == null) {
log.info("图片缓存目录为空或内部文件获取失败!");
} else {
for (File file : listFiles) {
log.info("图片文件 {} 删除: {}", file.getName(), file.delete());
}
}
log.info("图片缓存目录删除: {}", imageStoreDir.delete());
log.warn("缓存删除完成.");
}
/*
下一目标
添加定时发图
定时发图支持设置关注标签
标签....标签支持搜索吧
*/
private static boolean isNoSafe(int illustId, Properties settingProp, boolean returnRaw) throws IOException {
boolean rawValue = getIllustInfo(illustId).getAsJsonArray("tags").contains(new JsonPrimitive("R-18"));
return returnRaw || settingProp == null ? rawValue : rawValue && !settingProp.getProperty("image.allowR18", "false").equalsIgnoreCase("true");
}
private final static Object illustInfoLock = new Object();
private static JsonObject getIllustInfo(int illustId) throws IOException {
synchronized (illustInfoLock) {
File cacheFile = new File(getImageStoreDir(), illustId + ".illustInfo.json");
if (!illustInfoCache.containsKey(illustId)) {
log.info("IllustInfoFileName: {}", cacheFile.getName());
JsonObject illustInfoObj;
if (!cacheFile.exists()) {
try {
cacheFile.createNewFile();
illustInfoObj = pixivDownload.getIllustInfoByIllustId(illustId);
Files.write(cacheFile.toPath(), gson.toJson(illustInfoObj).getBytes(StandardCharsets.UTF_8), StandardOpenOption.CREATE);
} catch(IOException e) {
cacheFile.delete();
throw e;
}
} else {
illustInfoObj = gson.fromJson(new FileReader(cacheFile), JsonObject.class);
}
illustInfoCache.put(illustId, illustInfoObj);
}
return illustInfoCache.get(illustId);
}
}
private final static Object illustPreLoadDataLock = new Object();
public static JsonObject getIllustPreLoadData(int illustId) throws IOException {
synchronized (illustPreLoadDataLock) {
File cacheFile = new File(getImageStoreDir(), illustId + ".illustPreLoadData.json");
CacheObject<JsonObject> cacheObject = new CacheObject<>();
Date currentDate = new Date();
if (!illustPreLoadDataCache.containsKey(illustId) || (cacheObject = illustPreLoadDataCache.get(illustId)).isExpire(currentDate)) {
log.info("因为到期而失效: {}", cacheObject.isExpire(new Date()));
log.info("因为缓存文件不存在而失效: {}", !cacheFile.exists());
log.info("缓存失效, 正在更新...");
log.info("illustPreLoadDataFileName: {}", cacheFile.getName());
JsonObject preLoadDataObj;
if (!cacheFile.exists()) {
try {
cacheFile.createNewFile();
preLoadDataObj = pixivDownload.getIllustPreLoadDataById(illustId)
.getAsJsonObject("illust")
.getAsJsonObject(Integer.toString(illustId));
Files.write(cacheFile.toPath(), gson.toJson(preLoadDataObj).getBytes(StandardCharsets.UTF_8), StandardOpenOption.CREATE);
} catch(IOException e) {
cacheFile.delete();
throw e;
}
} else {
preLoadDataObj = gson.fromJson(new FileReader(cacheFile), JsonObject.class);
}
long expire = 7200 * 1000;
String propValue = CQPluginMain.globalProp.getProperty("cache.illustPreLoadData.expire", "7200000");
log.info("PreLoadData有效时间设定: {}", propValue);
try {
expire = Long.parseLong(propValue);
} catch (Exception e) {
log.warn("全局配置项 \"{}\" 值非法, 已使用默认值: {}", propValue, expire);
}
Date newExpire = new Date();
newExpire.setTime(newExpire.getTime() + expire);
cacheObject.update(preLoadDataObj, newExpire);
illustPreLoadDataCache.put(illustId, cacheObject);
log.info("作品Id {} preLoadData缓存已更新(到期时间: {})", illustId, newExpire);
}
return illustPreLoadDataCache.get(illustId).get();
}
}
private final static Object illustPagesLock = new Object();
public static List<String> getIllustPages(int illustId, PixivDownload.PageQuality quality) throws IOException {
synchronized (illustPagesLock) {
File cacheFile = new File(getImageStoreDir(), illustId + "." + quality.name() + ".illustPages.json");
if (!pagesCache.containsKey(illustId + "." + quality.name())) {
log.info("illustPagesFileName: {}", cacheFile.getName());
List<String> linkList;
if (!cacheFile.exists()) {
try {
cacheFile.createNewFile();
linkList = PixivDownload.getIllustAllPageDownload(pixivDownload.getHttpClient(), pixivDownload.getCookieStore(), illustId, quality);
JsonArray jsonArray = new JsonArray(linkList.size());
linkList.forEach(jsonArray::add);
Files.write(cacheFile.toPath(), gson.toJson(jsonArray).getBytes(StandardCharsets.UTF_8), StandardOpenOption.CREATE);
} catch(IOException e) {
cacheFile.delete();
throw e;
}
} else {
JsonArray jsonArray = gson.fromJson(new FileReader(cacheFile), JsonArray.class);
linkList = new ArrayList<>(jsonArray.size());
jsonArray.forEach(jsonElement -> linkList.add(jsonElement.getAsString()));
}
pagesCache.put(illustId + "." + quality.name(), linkList);
}
return pagesCache.get(illustId + "." + quality.name());
}
}
private static File getImageStoreDir() {
if(!imageStoreDir.exists() && !imageStoreDir.mkdirs()) {
log.warn("酷Q图片缓存目录失效(Path: {} )", imageStoreDir.getAbsolutePath());
throw new RuntimeException(new IOException("文件夹创建失败!"));
}
return imageStoreDir;
}
}

View File

@ -0,0 +1,518 @@
package net.lamgc.cgj;
import com.github.monkeywie.proxyee.proxy.ProxyConfig;
import com.github.monkeywie.proxyee.proxy.ProxyType;
import com.google.common.base.Strings;
import com.google.gson.Gson;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import net.lamgc.cgj.pixiv.*;
import net.lamgc.cgj.proxy.PixivAccessProxyServer;
import net.lamgc.cgj.proxy.PixivLoginProxyServer;
import net.lamgc.utils.base.ArgumentsProperties;
import net.lamgc.utils.base.runner.Argument;
import net.lamgc.utils.base.runner.ArgumentsRunner;
import net.lamgc.utils.base.runner.Command;
import org.apache.commons.io.IOUtils;
import org.apache.http.HttpHost;
import org.apache.http.HttpResponse;
import org.apache.http.client.CookieStore;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.message.BasicHeader;
import org.apache.http.util.EntityUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import java.io.*;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
@SpringBootApplication
public class Main {
private final static Logger log = LoggerFactory.getLogger("Main");
private final static File storeDir = new File("store/");
public static CookieStore cookieStore;
public static HttpHost proxy;
static {
if(!storeDir.exists() && !storeDir.mkdirs()) {
log.error("创建文件夹失败!");
}
}
public static void main(String[] args) throws IOException, ClassNotFoundException {
ArgumentsProperties argsProp = new ArgumentsProperties(args);
if(argsProp.containsKey("proxy")) {
URL proxyUrl = new URL(argsProp.getValue("proxy"));
proxy = new HttpHost(proxyUrl.getHost(), proxyUrl.getPort());
log.info("已启用Http协议代理{}", proxy.toHostString());
} else {
proxy = null;
}
if(argsProp.containsKey("cqRootDir")) {
log.info("cqRootDir: {}", argsProp.getValue("cqRootDir"));
System.setProperty("cgj.cqRootDir", argsProp.getValue("cqRootDir"));
} else {
log.info("未设置cqRootDir");
}
File cookieStoreFile = new File("cookies.store");
if(!cookieStoreFile.exists()) {
log.error("未找到cookies.store文件, 请确保文件存在于运行目录下!");
System.exit(1);
return;
}
ObjectInputStream ois = new ObjectInputStream(new FileInputStream(cookieStoreFile));
cookieStore = (CookieStore) ois.readObject();
ois.close();
log.info("已载入CookieStore");
log.debug(Arrays.toString(args));
ArgumentsRunner.run(Main.class, args);
}
@Command
public static void pluginMode(@Argument(name = "args", force = false) String argsStr) {
if(!System.getProperty("cgj.cqRootDir").endsWith("\\") && !System.getProperty("cgj.cqRootDir").endsWith("/")) {
System.setProperty("cgj.cqRootDir", System.getProperty("cgj.cqRootDir") + "/");
}
log.info("酷Q机器人根目录: {}", System.getProperty("cgj.cqRootDir"));
CQConfig.init();
Pattern pattern = Pattern.compile("/\\s*(\".+?\"|[^:\\s])+((\\s*:\\s*(\".+?\"|[^\\s])+)|)|(\".+?\"|[^\"\\s])+");
Matcher matcher = pattern.matcher(Strings.nullToEmpty(argsStr));
ArrayList<String> argsList = new ArrayList<>();
while (matcher.find()) {
argsList.add(matcher.group());
}
String[] args = new String[argsList.size()];
argsList.toArray(args);
SpringApplication.run(Main.class, args);
}
@Command
public static void collectionDownload() throws IOException {
PixivDownload pixivDownload = new PixivDownload(Objects.requireNonNull(cookieStore), proxy);
File outputFile = new File(storeDir, "collection.zip");
if(!outputFile.exists() && !outputFile.createNewFile()) {
log.error("文件创建失败: " + outputFile.getAbsolutePath());
}
ZipOutputStream zos = new ZipOutputStream(new FileOutputStream(outputFile));
zos.setLevel(9);
log.info("正在调用方法...");
pixivDownload.getCollectionAsInputStream(PixivDownload.PageQuality.ORIGINAL, (link, inputStream) -> {
try {
ZipEntry entry = new ZipEntry(link.substring(link.lastIndexOf("/") + 1));
log.info("正在写入: " + entry.getName());
zos.putNextEntry(entry);
IOUtils.copy(inputStream, zos);
zos.flush();
} catch (IOException e) {
log.error("写入文件项时发生异常", e);
}
});
log.info("调用完成.");
zos.close();
}
@Command
public static void getRecommends() throws IOException {
PixivDownload pixivDownload = new PixivDownload(Objects.requireNonNull(cookieStore), proxy);
String date = new SimpleDateFormat("yyyyMMdd").format(new Date());
int id = 1;
File outputFile = new File(storeDir, "recommends-" + date + "-" + id + ".zip");
while(outputFile.exists()) {
id++;
outputFile = new File(storeDir, "recommends-" + date + "-" + id + ".zip");
}
if(!outputFile.createNewFile()) {
log.error("文件创建失败: " + outputFile.getAbsolutePath());
}
ZipOutputStream zos = new ZipOutputStream(new FileOutputStream(outputFile));
zos.setLevel(9);
log.info("正在调用方法...");
pixivDownload.getRecommendAsInputStream(PixivDownload.PageQuality.ORIGINAL, (link, inputStream) -> {
try {
ZipEntry entry = new ZipEntry(link.substring(link.lastIndexOf("/") + 1));
log.info("正在写入: " + entry.getName());
zos.putNextEntry(entry);
IOUtils.copy(inputStream, zos);
zos.flush();
log.info("已成功写入 {}", entry.getName());
} catch (IOException e) {
log.error("写入文件项时发生异常", e);
}
});
log.info("调用完成.");
zos.close();
}
@Command
public static void getRankingIllust(@Argument(name = "range", force = false, defaultValue = "100") int range,
@Argument(name = "mode", force = false) String mode,
@Argument(name = "content", force = false) String content,
@Argument(name = "queryTime", force = false) String queryTime) throws IOException, ParseException {
PixivDownload pixivDownload = new PixivDownload(cookieStore, proxy);
SimpleDateFormat format = new SimpleDateFormat("yyyyMMdd");
Date queryDate;
String date;
if (queryTime == null) {
GregorianCalendar gregorianCalendar = new GregorianCalendar();
gregorianCalendar.setTime(new Date());
gregorianCalendar.add(Calendar.DATE, -1);
queryDate = gregorianCalendar.getTime();
} else {
queryDate = format.parse(queryTime);
}
date = format.format(queryDate);
log.info("查询时间: {}", date);
PixivURL.RankingMode rankingMode = PixivURL.RankingMode.MODE_DAILY;
PixivURL.RankingContentType contentType = null;
if(mode != null) {
try {
rankingMode = PixivURL.RankingMode.valueOf(mode);
} catch (IllegalArgumentException e) {
log.warn("不支持的RankingMode: {}", mode);
}
}
if(content != null) {
try {
contentType = PixivURL.RankingContentType.valueOf(content);
} catch (IllegalArgumentException e) {
log.warn("不支持的RankingContentType: {}", content);
}
}
int id = 1;
File outputFile = new File(storeDir, "ranking" + rankingMode.modeParam + "-" + date + "-" + id + ".zip");
while(outputFile.exists()) {
id++;
outputFile = new File(storeDir, "ranking" + rankingMode.modeParam + "-" + date + "-" + id + ".zip");
}
if(!outputFile.createNewFile()) {
log.error("文件创建失败: " + outputFile.getAbsolutePath());
return;
}
ZipOutputStream zos = new ZipOutputStream(new FileOutputStream(outputFile));
zos.setLevel(9);
log.info("正在调用方法...");
try {
pixivDownload.getRankingAsInputStream(contentType, rankingMode, queryDate, range, PixivDownload.PageQuality.ORIGINAL, (rank, link, rankInfo, inputStream) -> {
try {
ZipEntry entry = new ZipEntry("Rank" + rank + "-" + link.substring(link.lastIndexOf("/") + 1));
entry.setComment(rankInfo.toString());
log.info("正在写入: " + entry.getName());
zos.putNextEntry(entry);
IOUtils.copy(inputStream, zos);
zos.flush();
log.info("已成功写入 {}", entry.getName());
} catch (IOException e) {
log.error("写入文件项时发生异常", e);
}
});
} finally {
zos.finish();
zos.flush();
zos.close();
}
log.info("调用完成.");
}
@Command
public static void search(
@Argument(name = "content") String content,
@Argument(name = "type", force = false) String type,
@Argument(name = "area", force = false) String area,
@Argument(name = "includeKeywords", force = false) String includeKeywords,
@Argument(name = "excludeKeywords", force = false) String excludeKeywords,
@Argument(name = "contentOption", force = false) String contentOption
) throws IOException {
PixivSearchBuilder searchBuilder = new PixivSearchBuilder(Strings.isNullOrEmpty(content) ? "" : content);
if (type != null) {
try {
searchBuilder.setSearchType(PixivSearchBuilder.SearchType.valueOf(type.toUpperCase()));
} catch (IllegalArgumentException e) {
log.warn("不支持的SearchType: {}", type);
}
}
if(area != null) {
try {
searchBuilder.setSearchArea(PixivSearchBuilder.SearchArea.valueOf(area));
} catch (IllegalArgumentException e) {
log.warn("不支持的SearchArea: {}", area);
}
}
if(contentOption != null) {
try {
searchBuilder.setSearchContentOption(PixivSearchBuilder.SearchContentOption.valueOf(contentOption));
} catch (IllegalArgumentException e) {
log.warn("不支持的SearchContentOption: {}", contentOption);
}
}
if(!Strings.isNullOrEmpty(includeKeywords)) {
for (String keyword : includeKeywords.split(";")) {
searchBuilder.removeExcludeKeyword(keyword);
searchBuilder.addIncludeKeyword(keyword);
log.info("已添加关键字: {}", keyword);
}
}
if(!Strings.isNullOrEmpty(excludeKeywords)) {
for (String keyword : excludeKeywords.split(";")) {
searchBuilder.removeIncludeKeyword(keyword);
searchBuilder.addExcludeKeyword(keyword);
log.info("已添加排除关键字: {}", keyword);
}
}
log.info("正在搜索作品, 条件: {}", searchBuilder.getSearchCondition());
String requestUrl = searchBuilder.buildURL();
log.info("RequestUrl: {}", requestUrl);
PixivDownload pixivDownload = new PixivDownload(cookieStore, proxy);
HttpGet httpGetRequest = pixivDownload.createHttpGetRequest(requestUrl);
HttpResponse response = pixivDownload.getHttpClient().execute(httpGetRequest);
String responseBody = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8);
log.info("ResponseBody: {}", responseBody);
JsonObject jsonObject = new Gson().fromJson(responseBody, JsonObject.class);
if(jsonObject.get("error").getAsBoolean()) {
log.error("接口请求错误, 错误信息: {}", jsonObject.get("message").getAsString());
return;
}
JsonObject resultBody = jsonObject.getAsJsonObject("body");
for(PixivSearchBuilder.SearchArea searchArea : PixivSearchBuilder.SearchArea.values()) {
if(!resultBody.has(searchArea.jsonKey) || resultBody.getAsJsonObject(searchArea.jsonKey).getAsJsonArray("data").size() == 0) {
//log.info("返回数据不包含 {}", searchArea.jsonKey);
continue;
}
JsonArray illustsArray = resultBody
.getAsJsonObject(searchArea.jsonKey).getAsJsonArray("data");
log.info("已找到与 {} 相关插图信息({})", content, searchArea.name().toLowerCase());
int count = 1;
for (JsonElement jsonElement : illustsArray) {
JsonObject illustObj = jsonElement.getAsJsonObject();
if(!illustObj.has("illustId")) {
continue;
}
int illustId = illustObj.get("illustId").getAsInt();
StringBuilder builder = new StringBuilder("[");
illustObj.get("tags").getAsJsonArray().forEach(el -> builder.append(el.getAsString()).append(", "));
builder.replace(builder.length() - 2, builder.length(), "]");
log.info("{} ({} / {})\n\t作品id: {}, \n\t作者名(作者id): {} ({}), \n\t作品标题: {}, \n\t作品Tags: {}, \n\t作品链接: {}",
searchArea.name(),
count++,
illustsArray.size(),
illustId,
illustObj.get("userName").getAsString(),
illustObj.get("userId").getAsInt(),
illustObj.get("illustTitle").getAsString(),
builder,
PixivURL.getPixivRefererLink(illustId)
);
/*log.info("正在下载...");
List<String> list = PixivDownload.getIllustAllPageDownload(
HttpClientBuilder.create()
.setProxy(proxy)
.build(),
illustId, PixivDownload.PageQuality.ORIGINAL);*/
}
}
/*
if(searchBuilder.getSearchArea().equals(PixivSearchBuilder.SearchArea.TOP)) {
} else {
JsonArray illustsArray = resultBody
.getAsJsonObject(searchBuilder.getSearchArea().jsonKey).getAsJsonArray("data");
log.info("已找到与 {} 相关插图信息:", content);
int count = 1;
for (JsonElement jsonElement : illustsArray) {
JsonObject illustObj = jsonElement.getAsJsonObject();
//TODO: 防止数据内混入无效内容, 需要检查对象是否有illustId
if(!illustObj.has("illustId")) {
continue;
}
int illustId = illustObj.get("illustId").getAsInt();
StringBuilder builder = new StringBuilder("[");
illustObj.get("tags").getAsJsonArray().forEach(el -> builder.append(el.getAsString()).append(", "));
builder.replace(builder.length() - 2, builder.length(), "]");
log.info("({} / {})\n\t作品id: {}, \n\t作者名(作者id): {} ({}), \n\t作品标题: {}, \n\t作品Tags: {}, \n\t作品链接: {}",
count++,
illustsArray.size(),
illustId,
illustObj.get("userName").getAsString(),
illustObj.get("userId").getAsInt(),
illustObj.get("illustTitle").getAsString(),
builder,
PixivURL.getPixivRefererLink(illustId)
);
}
}
*/
}
@Command(defaultCommand = true)
public static void testRun() throws IOException {
/*loadCookieStoreFromFile();
if(cookieStore == null){
startPixivLoginProxyServer();
}*/
//accessPixivToFile();
//startPixivAccessProxyServer();
//saveCookieStoreToFile();
log.info("这里啥都没有哟w");
}
private static void loadCookieStoreFromFile() throws IOException {
log.info("正在加载CookieStore...");
File storeFile = new File("./cookies.store");
if(!storeFile.exists()){
log.info("未找到CookieStore, 跳过加载.");
return;
}
ObjectInputStream stream = new ObjectInputStream(new FileInputStream(storeFile));
Object result;
try {
result = stream.readObject();
} catch (ClassNotFoundException e) {
log.error("加载出错", e);
return;
}
cookieStore = (CookieStore) result;
cookieStore.getCookies().forEach(cookie -> log.debug(cookie.getName() + ": " + cookie.getValue() + ", isExpired: " + cookie.isExpired(new Date())));
log.info("CookieStore加载完成.");
}
private static void saveCookieStoreToFile() throws IOException {
log.info("正在保存CookieStore...");
File outputFile = new File("./cookies.store");
if(!outputFile.exists() && !outputFile.delete() && !outputFile.createNewFile()){
log.error("保存CookieStore失败.");
return;
}
ObjectOutputStream stream = new ObjectOutputStream(new FileOutputStream(outputFile));
stream.writeObject(cookieStore);
stream.flush();
stream.close();
log.info("CookieStore保存成功.");
}
private static void startPixivLoginProxyServer(){
ProxyConfig proxyConfig = new ProxyConfig(ProxyType.SOCKS5, "127.0.0.1", 1080);
PixivLoginProxyServer proxyServer = new PixivLoginProxyServer(proxyConfig);
Thread proxyServerStartThread = new Thread(() -> {
log.info("启动代理服务器...");
proxyServer.start(1006);
log.info("代理服务器已关闭.");
});
proxyServerStartThread.setName("LoginProxyServerThread");
proxyServerStartThread.start();
//System.console().readLine();
new Scanner(System.in).nextLine();
log.info("关闭PLPS服务器...");
proxyServer.close();
cookieStore = proxyServer.getCookieStore();
}
private static void accessPixivToFile() throws IOException {
File cookieStoreFile = new File("./cookie.txt");
if (!cookieStoreFile.exists() && !cookieStoreFile.createNewFile()) {
log.info("Cookie文件存储失败");
}
/*log.info("正在写出Cookie, Cookie count: " + cookieStore.getCookies().size());
FileWriter cookieWriter = new FileWriter(cookieStoreFile);
cookieStore.getCookies().forEach(cookie -> {
try {
StringBuilder sb = new StringBuilder().append(cookie.getName()).append(" = ").append(cookie.getValue());
log.info("正在导出Cookie: " + sb.toString());
cookieWriter.append(sb.toString()).append("\n").flush();
} catch (IOException e) {
e.printStackTrace();
}
});
log.info("Cookie写出完成");*/
log.info("尝试通过捕获的Cookie访问Pixiv...");
HttpClient httpClient = new PixivSession(new HttpHost("127.0.0.1", 1080), cookieStore).getHttpClient();
HttpGet request = new HttpGet("https://www.pixiv.net");
request.addHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:73.0) Gecko/20100101 Firefox/73.0");
request.addHeader(new BasicHeader("accept-encoding", "gzip, deflate, br"));
request.addHeader(new BasicHeader("accept-language", "zh-CN,zh;q=0.9"));
StringBuilder cookieBuilder = new StringBuilder();
cookieStore.getCookies().forEach(cookie -> {
if(cookie.isExpired(new Date())){
return;
}
cookieBuilder.append(cookie.getName()).append("=").append(cookie.getValue()).append("; ");
});
request.addHeader("cookie", cookieBuilder.toString());
HttpResponse response = httpClient.execute(request);
log.info("正在写入文件...");
File outFile = new File("./pixiv.html");
if (outFile.createNewFile() && !outFile.exists()) {
log.info("文件创建失败!");
}else {
new FileWriter(outFile).append(EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8)).flush();
}
Pixiv pixiv = new Pixiv(httpClient);
pixiv.getRecommend().forEach(illustMap -> {
StringBuilder builder = new StringBuilder();
illustMap.forEach((key, value) -> builder.append(key).append(": ").append(value).append("\n"));
try {
builder.append("download Link: ").append(Arrays.toString(pixiv.getAllDownloadLink(Integer.parseInt(illustMap.get(Pixiv.ATTR_ILLUST_ID)))));
} catch (IOException e) {
log.error("获取下载链接时出错!", e);
}
log.info(builder.append("\n").toString());
});
}
private static void startPixivAccessProxyServer(){
log.info("正在启动访问代理服务器, 将浏览器相关缓存清空后, 使用浏览器进行访问以尝试Cookie正确性.");
final PixivAccessProxyServer accessProxyServer = new PixivAccessProxyServer(cookieStore, new ProxyConfig(ProxyType.SOCKS5, "127.0.0.1", 1080));
Thread accessProxyServerThread = new Thread(() -> {
log.info("正在启动PAPS...");
accessProxyServer.start(1007);
log.info("PAPS已关闭.");
});
accessProxyServerThread.setName("AccessProxyServerThread");
accessProxyServerThread.start();
new Scanner(System.in).nextLine();
log.info("关闭PAPS服务器...");
accessProxyServer.close();
}
}

View File

@ -0,0 +1,57 @@
package net.lamgc.cgj.bot;
import net.lz1998.cq.robot.CoolQ;
import org.apache.http.client.methods.HttpGet;
import java.util.Random;
import java.util.Timer;
import java.util.TimerTask;
public class AutoArtworksSender {
private final CoolQ CQ;
private final ReceiveType receiveType;
private final long targetReceiveId;
private Timer timer = new Timer();
private TimerTask task = new TimerTask() {
@Override
public void run() {
HttpGet request = new HttpGet();
// https://api.imjad.cn/pixiv/v2/?type=tags
}
};
public AutoArtworksSender(CoolQ cq, ReceiveType receiveType, long receiveId) {
this.CQ = cq;
this.receiveType = receiveType;
this.targetReceiveId = receiveId;
}
public void reset(long time) {
if(time <= 0) {
timer.schedule(task, new Random().nextInt(10 * 60 * 60 * 1000) + 7200000L); //2H ~ 12H
} else {
timer.schedule(task, time);
}
}
public void sendMessage(String message, boolean auto_escape) {
switch (receiveType) {
case GROUP:
CQ.sendGroupMsg(targetReceiveId, message, auto_escape);
break;
case Discuss:
CQ.sendDiscussMsg(targetReceiveId, message, auto_escape);
break;
case PRIVATE:
CQ.sendPrivateMsg(targetReceiveId, message, auto_escape);
break;
}
}
public enum ReceiveType {
PRIVATE, GROUP, Discuss
}
}

View File

@ -0,0 +1,40 @@
package net.lamgc.cgj.cache;
import java.util.Date;
import java.util.concurrent.atomic.AtomicReference;
public class CacheObject<T> {
private AtomicReference<T> value;
private AtomicReference<Date> expire;
public CacheObject() {
this(null, null);
}
public CacheObject(T value, Date expire) {
this.value = new AtomicReference<>(value);
this.expire = new AtomicReference<>(expire);
}
public synchronized void update(T value, Date newExpire) {
if(new Date().after(newExpire)) {
throw new IllegalArgumentException("Due earlier than current time");
}
this.expire.set(newExpire);
this.value.set(value);
}
public synchronized T get() {
return value.get();
}
public Date getExpireDate() {
return expire.get();
}
public boolean isExpire(Date time) {
Date expireDate = getExpireDate();
return expireDate != null && expireDate.before(time);
}
}

View File

@ -0,0 +1,79 @@
package net.lamgc.cgj.cache;
import net.lamgc.cgj.Main;
import net.lamgc.cgj.pixiv.PixivURL;
import net.lamgc.utils.event.EventHandler;
import org.apache.commons.io.IOUtils;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.HttpClientBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.Collections;
import java.util.HashSet;
import java.util.Set;
public class ImageCacheHandler implements EventHandler {
private final static Logger log = LoggerFactory.getLogger("ImageCacheHandler");
private final static HttpClient httpClient = HttpClientBuilder.create().setProxy(Main.proxy).build();
private final static Set<ImageCacheObject> cacheQueue = Collections.synchronizedSet(new HashSet<>());
public void getImageToCache(ImageCacheObject event) {
if(cacheQueue.contains(event)) {
log.info("图片 {} 已存在相同缓存任务, 跳过.", event.getStoreFile().getName());
return;
} else {
cacheQueue.add(event);
}
try {
log.info("图片 {} Event正在进行...({})", event.getStoreFile().getName(), Integer.toHexString(event.hashCode()));
File storeFile = event.getStoreFile();
log.info("正在缓存图片 {} (Path: {})", storeFile.getName(), storeFile.getAbsolutePath());
try {
if(!storeFile.exists() && !storeFile.createNewFile()) {
log.error("无法创建文件(Path: {})", storeFile.getAbsolutePath());
return;
}
} catch (IOException e) {
log.error("无法创建文件(Path: {})", storeFile.getAbsolutePath());
e.printStackTrace();
}
HttpGet request = new HttpGet(event.getDownloadLink());
request.addHeader("Referer", PixivURL.getPixivRefererLink(event.getIllustId()));
HttpResponse response;
try {
response = httpClient.execute(request);
} catch (IOException e) {
log.error("Http请求时发生异常", e);
return;
}
if(response.getStatusLine().getStatusCode() != 200) {
log.warn("Http请求异常{}", response.getStatusLine());
return;
}
log.info("正在下载...(Content-Length: {}KB)", response.getEntity().getContentLength() / 1024);
try(FileOutputStream fos = new FileOutputStream(storeFile)) {
IOUtils.copy(response.getEntity().getContent(), fos);
} catch (IOException e) {
log.error("下载图片时发生异常", e);
return;
}
event.getImageCache().put(event.getDownloadLink(), storeFile);
} finally {
log.info("图片 {} Event结束({})", event.getStoreFile().getName(), Integer.toHexString(event.hashCode()));
cacheQueue.remove(event);
}
}
}

View File

@ -0,0 +1,57 @@
package net.lamgc.cgj.cache;
import net.lamgc.utils.event.EventObject;
import java.io.File;
import java.util.Map;
import java.util.Objects;
public class ImageCacheObject implements EventObject {
private final Map<String, File> imageCache;
private final int illustId;
private final String downloadLink;
private final File storeFile;
public ImageCacheObject(Map<String, File> imageCache, int illustId, String downloadLink, File storeFile) {
this.imageCache = imageCache;
this.illustId = illustId;
this.downloadLink = downloadLink;
this.storeFile = storeFile;
}
public Map<String, File> getImageCache() {
return imageCache;
}
public String getDownloadLink() {
return downloadLink;
}
public File getStoreFile() {
return storeFile;
}
public int getIllustId() {
return illustId;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
ImageCacheObject that = (ImageCacheObject) o;
return illustId == that.illustId &&
Objects.equals(imageCache, that.imageCache) &&
Objects.equals(downloadLink, that.downloadLink) &&
Objects.equals(storeFile, that.storeFile);
}
@Override
public int hashCode() {
return Objects.hash(imageCache, illustId, downloadLink, storeFile);
}
}

View File

@ -0,0 +1,28 @@
package net.lamgc.cgj.db;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.sql.Statement;
public class RankingDB {
private final Connection dbConnection;
public RankingDB(String dbUrl, String username, String password) throws SQLException {
try {
Class.forName("com.mysql.cj.jdbc.Driver");
} catch (ClassNotFoundException e) {
e.printStackTrace();
}
dbConnection = DriverManager.getConnection("jdbc:mysql://" + dbUrl + "/pixivRanking?useSSL=false&serverTimezone=UTC",username,password);
checkAndFix(dbConnection);
}
private static void checkAndFix(Connection connection) throws SQLException {
Statement statement = connection.createStatement();
statement.execute("");
}
}

View File

@ -0,0 +1,69 @@
package net.lamgc.cgj.pixiv;
/**
* 插图()的信息
*/
public class IllustInfo{
/**
* 插图()ID
*/
public final int illustID;
/**
* 第几页
*/
public final int page;
/**
* 插图标题
*/
public final String title;
/**
* 插图说明
*/
public final String description;
/**
* 插图标签
*/
public final String[] tags;
/**
* 插图图片长度
*/
public final int width;
/**
* 插图图片高度
*/
public final int height;
/**
* 作者名
*/
public final String authorName;
/**
* 作者用户ID
*/
public final int authorUserID;
public IllustInfo(int illustID, String title, String description, String[] tags, int width, int height, String authorName, int authorUserID){
this(illustID, 0, title, description, tags, width, height, authorName, authorUserID);
}
public IllustInfo(int illustID, int p, String title, String description, String[] tags, int width, int height, String authorName, int authorUserID){
this.illustID = illustID;
this.page = p;
this.title = title;
this.description = description;
this.tags = tags;
this.width = width;
this.height = height;
this.authorName = authorName;
this.authorUserID = authorUserID;
}
}

View File

@ -0,0 +1,267 @@
package net.lamgc.cgj.pixiv;
import com.google.gson.JsonArray;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import org.apache.http.HttpHost;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.conn.ConnectionPoolTimeoutException;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.util.EntityUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.util.*;
public class Pixiv {
/**
* illust Link
*/
public final static String ATTR_LINK = "link";
/**
* illust Id
*/
public final static String ATTR_ILLUST_ID = "illustId";
/**
* illust Title
*/
public final static String ATTR_TITLE = "title";
/**
* illust Author Name
*/
public final static String ATTR_AUTHOR_NAME = "authorName";
/**
* illust Author UserID
*/
public final static String ATTR_AUTHOR_ID = "authorId";
private final HttpClient httpClient;
public Pixiv(HttpClient client){
this.httpClient = client;
}
/**
* 使用帐号密码登录Pixiv
* @param PixivID Pixiv账户登录名
* @param Password Pixiv帐号密码
* @throws IOException 当登录连接出现异常时抛出
* @deprecated {@link PixivSession#Login(String, String)} 已经废除, 故本方法不可用
*/
public Pixiv(String PixivID, String Password) throws IOException {
this(PixivID, Password, null);
}
/**
* 使用帐号密码登录Pixiv
* @param PixivID Pixiv账户登录名
* @param Password Pixiv帐号密码
* @param proxy 代理设置
* @throws IOException 当登录连接出现异常时抛出
* @deprecated {@link PixivSession#Login(String, String)} 已经废除, 故本方法不可用
*/
public Pixiv(String PixivID, String Password, HttpHost proxy) throws IOException {
PixivSession pixivSession = new PixivSession(proxy, null);
if(pixivSession.Login(PixivID, Password)){
System.out.println("P站登录成功!");
}else{
System.out.println("P站登录失败!错误信息: " + pixivSession.getErrMsg());
throw new RuntimeException(pixivSession.getErrMsg());
}
//httpClient = pixivSession.getHttpClient();
httpClient = HttpClientBuilder.create()
.setDefaultCookieStore(pixivSession.getCookieStore())
.build();
}
/**
* 获取首页推荐列表
* @return 首页推荐列表, 一个Map对应一个推荐项, 使用<code>ATTR_</code>开头常量访问即可
* @throws IOException
*/
public List<Map<String, String>> getRecommend() throws IOException {
HttpGet getRequest = new HttpGet(PixivURL.PIXIV_INDEX_URL);
HttpResponse response = httpClient.execute(getRequest);
String pageAsXML = EntityUtils.toString(response.getEntity(),"utf-8");
//获取推荐图列表(li)
//System.out.println(pageAsXML);
Document document = Jsoup.parse(pageAsXML);
List<String> links = document.select("._image-items.gtm-illust-recommend-zone>li>.gtm-illust-recommend-thumbnail-link").eachAttr("href");
List<String> illustId = document.select("._image-items.gtm-illust-recommend-zone>li>.gtm-illust-recommend-thumbnail-link").eachAttr("data-gtm-recommend-illust-id");
List<String> title = document.select("._image-items.gtm-illust-recommend-zone>li>.gtm-illust-recommend-title>h1").eachAttr("title");
List<String> authorName = document.select("._image-items.gtm-illust-recommend-zone>li>.gtm-illust-recommend-user-name").eachText();
List<String> authorId = document.select("._image-items.gtm-illust-recommend-zone>li>.gtm-illust-recommend-user-name").eachAttr("data-user_id");
List<Map<String, String>> recommendList = new ArrayList<>();
for(int i = 0; i < links.size(); i++){
//System.out.println(links.get(i));
Map<String, String> info = new HashMap<>();
info.put(ATTR_LINK, PixivURL.PIXIV_INDEX_URL + links.get(i));
info.put(ATTR_ILLUST_ID, illustId.get(i));
info.put(ATTR_TITLE, title.get(i));
info.put(ATTR_AUTHOR_NAME, authorName.get(i));
info.put(ATTR_AUTHOR_ID, authorId.get(i));
recommendList.add(info);
}
return recommendList;
}
public String[] getAllDownloadLink(int illustID) throws IOException {
HttpGet illustPage = new HttpGet(PixivURL.PIXIV_ILLUST_API_URL.replaceAll("\\{illustId}", String.valueOf(illustID)));
HttpResponse response = httpClient.execute(illustPage);
String pageAsXML = EntityUtils.toString(response.getEntity(),"utf-8");
//System.out.println(pageAsXML);
JsonObject resultObj = (JsonObject) new JsonParser().parse(pageAsXML);
if(!resultObj.get("error").getAsBoolean()){
JsonArray bodyArray = resultObj.get("body").getAsJsonArray();
int length = bodyArray.size();
String[] result = new String[length];
for(int i = 0; i < length; i++){
JsonObject childObj = bodyArray.get(i).getAsJsonObject();
result[i] = childObj.get("urls").getAsJsonObject().get("original").getAsString();
}
return result;
}else{
return null;
}
}
/**
* 下载P站图片
* @param illustID 插图ID
* @return 成功返回图片输入流,失败或为多图则返回null
*/
public InputStream[] downloadIllustImage(int illustID) throws IOException {
String[] links = getAllDownloadLink(illustID);
List<InputStream> inputStreamList = new ArrayList<>();
int count = 1;
boolean retry = false;
for(int i = 0; i < links.length; i++){
try {
long sleepTime = (new Random().nextInt(4) + 2) * 1000;
System.out.println("nextTime: " + (float)(sleepTime / 1000));
Thread.sleep(sleepTime);
} catch (InterruptedException ignored) {}
String link = links[i];
System.out.print("page:" + count++ + "/" + links.length + " ...");
HttpGet imgGet = new HttpGet(link);
//关键!如果不加上Referer的话,会返回403
imgGet.setHeader("Referer", PixivURL.PIXIV_ILLUST_MEDIUM_URL.replaceAll("\\{illustId}", String.valueOf(illustID)));
RequestConfig config = RequestConfig.custom()
.setConnectTimeout(20 * 1000)
.setConnectionRequestTimeout(20 * 1000)
.setSocketTimeout(30 * 1000)
.build();
imgGet.setConfig(config);
HttpResponse response;
try {
response = httpClient.execute(imgGet);
}catch(ConnectionPoolTimeoutException e){
if(retry){
retry = false;
System.out.println("获取失败,跳过...");
continue;
}
System.out.println("连接超时,重新获取...");
retry = true;
i--;
continue;
}
retry = false;
ByteArrayOutputStream cacheOutputStream = new ByteArrayOutputStream((int)response.getEntity().getContentLength());
InputStream content = response.getEntity().getContent();
int readLength;
byte[] cache = new byte[4096];
while((readLength = content.read(cache)) != -1){
cacheOutputStream.write(cache, 0, readLength);
}
byte[] data = cacheOutputStream.toByteArray();
//System.out.println("读到数据: " + data.length);
inputStreamList.add(new ByteArrayInputStream(data));
System.out.println("done!length: " + response.getEntity().getContentLength() + ")");
}
return inputStreamList.toArray(new InputStream[0]);
}
/**
* 下载P站图片
* @param illustID 插图ID
* @return 成功返回图片输入流,失败或为多图则返回null
*/
public InputStream downloadIllustImages(int illustID){
throw new UnsupportedOperationException();
}
/**
* 通过解析插图详情页获取
* - 插图标题
* - 插图作者(及其UserId)
* - 插图上传时间
* - 插图标签(原始标签)
* ...
* @return 成功返回IllustInfo对象,失败返回null
*/
public IllustInfo[] getIllustInfo(int[] illustIDs) throws IOException {
//获取Api
HttpGet apiRequest = new HttpGet(PixivURL.getPixivIllustInfoAPI(illustIDs));
HttpResponse response = httpClient.execute(apiRequest);
String resultText = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8);
System.out.println(resultText);
JsonObject resultObj = ((JsonObject) new JsonParser().parse(resultText));
if(resultObj.get("error").getAsBoolean()){
System.err.println("获取失败!");
return null;
}
List<IllustInfo> illustInfoList = new ArrayList<>();
JsonArray illustArray = resultObj.get("body").getAsJsonObject().get("illusts").getAsJsonArray();
illustArray.forEach(jsonElement -> {
JsonObject illustInfoObj = jsonElement.getAsJsonObject();
JsonArray tagsArray = illustInfoObj.get("tags").getAsJsonArray();
String[] tags = new String[tagsArray.size()];
for(int i = 0; i < tags.length; i++){
tags[i] = tagsArray.get(i).getAsString();
}
//TODO: 通过不需要作者id就能获取图片信息的api无法获取图片尺寸
IllustInfo illustInfo = new IllustInfo(
illustInfoObj.get("workId").getAsInt(),
illustInfoObj.get("title").getAsString(),
null,
tags,
-1,
-1,
illustInfoObj.get("userName").getAsString(),
illustInfoObj.get("userId").getAsInt()
);
});
return null;
}
/**
* 获取指定用户的所有插画
*/
public void getUserAllIllustTest() {
}
}

View File

@ -0,0 +1,533 @@
package net.lamgc.cgj.pixiv;
import com.google.gson.Gson;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import io.netty.handler.codec.http.HttpHeaderNames;
import org.apache.http.Header;
import org.apache.http.HttpHost;
import org.apache.http.HttpRequest;
import org.apache.http.HttpResponse;
import org.apache.http.client.CookieStore;
import org.apache.http.client.HttpClient;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.message.BasicHeader;
import org.apache.http.util.EntityUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.BufferedInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.util.*;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.BiConsumer;
public class PixivDownload {
private final static Logger log = LoggerFactory.getLogger("PixivDownload");
private final HttpClient httpClient;
private final CookieStore cookieStore;
/**
* 构造一个PixivDownload对象
* @param cookieStore 存在已登录Pixiv的CookieStore对象
*/
public PixivDownload(CookieStore cookieStore) {
this(cookieStore, null);
}
/**
* 构造一个PixivDownload对象
* @param cookieStore 存在已登录Pixiv的CookieStore对象
* @param proxy 访问代理
*/
public PixivDownload(CookieStore cookieStore, HttpHost proxy) {
this.cookieStore = cookieStore;
HttpClientBuilder builder = HttpClientBuilder.create();
builder.setDefaultCookieStore(cookieStore);
// UA: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.122 Safari/537.36
ArrayList<Header> defaultHeaders = new ArrayList<>(2);
defaultHeaders.add(new BasicHeader("User-Agent",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.122 Safari/537.36"));
builder.setDefaultHeaders(defaultHeaders);
builder.setProxy(proxy);
httpClient = builder.build();
}
public HttpClient getHttpClient() {
return httpClient;
}
public CookieStore getCookieStore() {
return cookieStore;
}
/**
* 获取帐号所有的收藏插画并以输入流形式提供
* @param fn 回调函数函数传进的InputStream无需手动关闭
* @throws IOException 当获取时发生异常则直接抛出
*/
public void getCollectionAsInputStream(PageQuality quality, BiConsumer<String, InputStream> fn) throws IOException {
int pageIndex = 0;
HttpGet request;
Document document;
ArrayList<String> linkList = new ArrayList<>();
do {
request = new HttpGet(PixivURL.PIXIV_USER_COLLECTION_PAGE.replace("{pageIndex}", Integer.toString(++pageIndex)));
setCookieInRequest(request, cookieStore);
log.info("Request Link: " + request.getURI().toString());
HttpResponse response = httpClient.execute(request);
// 解析网页内容获得所有的收藏信息
document = Jsoup.parse(EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8));
Elements items = document.select(".display_editable_works .image-item a.work");
List<String> hrefList = items.eachAttr("href");
log.info("第 {} 页获取到的图片项数量: {}", pageIndex, hrefList.size());
if(hrefList.size() == 0) {
break;
}
Gson gson = new Gson();
for (String href : hrefList) {
HttpGet linkApiRequest = createHttpGetRequest(PixivURL.PIXIV_ILLUST_API_URL.replace("{illustId}", href.substring(href.lastIndexOf("/") + 1)));
log.debug(linkApiRequest.getURI().toString());
HttpResponse httpResponse = httpClient.execute(linkApiRequest);
JsonObject linkResult = gson.fromJson(EntityUtils.toString(httpResponse.getEntity()), JsonObject.class);
if(linkResult.get("error").getAsBoolean()) {
log.error("接口返回错误信息: {}", linkResult.get("message").getAsString());
continue;
}
JsonArray linkArray = linkResult.get("body").getAsJsonArray();
for (int i = 0; i < linkArray.size(); i++) {
JsonObject linkObject = linkArray.get(i).getAsJsonObject().get("urls").getAsJsonObject();
linkList.add(linkObject.get((quality == null ? PageQuality.ORIGINAL : quality).toString().toLowerCase()).getAsString());
}
}
} while(!document.select(".pager-container>.next").isEmpty());
log.info("获取完成.");
AtomicInteger count = new AtomicInteger(1);
linkList.forEach(link -> {
log.info("Next Link [{}]: {}", count.getAndIncrement(), link);
InputStream imageInputStream = null;
int tryCount = 0;
do {
try {
imageInputStream = getImageAsInputStream(httpClient, link);
} catch (IOException e) {
log.error("获取图片数据时发生异常", e);
if(++tryCount < 5) {
log.warn("即将重试[{} / 5]", tryCount);
}
}
} while(imageInputStream == null);
try(InputStream imageInput = new BufferedInputStream(imageInputStream, 256 * 1024)) {
log.debug("调用回调方法...");
fn.accept(link, imageInput);
log.debug("回调方法调用完成.");
} catch (IOException e) {
log.error("图片获取失败", e);
}
});
}
/**
* 获取推荐插图
* @param quality 图片质量
* @param fn 回调函数
* @throws IOException 当获取时发生异常则直接抛出
*/
public void getRecommendAsInputStream(PageQuality quality, BiConsumer<String, InputStream> fn) throws IOException {
HttpResponse response = httpClient.execute(createHttpGetRequest(PixivURL.PIXIV_INDEX_URL));
Document document = Jsoup.parse(EntityUtils.toString(response.getEntity()));
HttpClient imageClient = HttpClientBuilder.create().build();
Elements elements = document.select(".gtm-illust-recommend-zone>.image-item>.gtm-illust-recommend-thumbnail-link");
for(int illustIndex = 0; illustIndex < elements.size(); illustIndex++){
String href = elements.get(illustIndex).attr("href");
int illustId = Integer.parseInt(href.substring(href.lastIndexOf("/") + 1));
log.info("({}/{}) Href: {}, IllustID: {}", illustIndex + 1, elements.size(), href, illustId);
List<String> pageLinkList = getIllustAllPageDownload(httpClient, this.cookieStore, illustId, quality);
for (int linkIndex = 0; linkIndex < pageLinkList.size(); linkIndex++) {
String link = pageLinkList.get(linkIndex);
String fileName = link.substring(link.lastIndexOf("/") + 1);
log.info("({}/{})正在处理 {}", linkIndex, pageLinkList.size(), fileName);
InputStream imageInputStream = null;
int tryCount = 0;
do {
try {
imageInputStream = getImageAsInputStream(imageClient, link);
} catch (IOException e) {
log.error("获取图片数据时发生异常", e);
if(++tryCount < 5) {
log.warn("即将重试[{} / 5]", tryCount);
}
}
} while(imageInputStream == null);
try(InputStream pageInputStream = new BufferedInputStream(imageInputStream, 256 * 1024)) {
fn.accept(fileName, pageInputStream);
}
log.info("Done!");
}
log.info("IllustId {} 处理完成.", illustId);
}
}
/**
* 获取排行榜
* @param contentType 内容类型
* @param mode 查询模式
* @param time 查询时间
* @param range 从第一名开始的范围
* @param quality 图片质量
* @param fn 回调函数
* @throws IOException 当请求发生异常时抛出
*/
public void getRankingAsInputStream(PixivURL.RankingContentType contentType, PixivURL.RankingMode mode,
Date time, int range, PageQuality quality, RankingDownloadFunction fn) throws IOException {
getRankingAsInputStream(contentType, mode, time, 1, range, quality, fn);
}
/**
* 获取排行榜
* @param contentType 内容类型
* @param mode 查询模式
* @param time 查询时间
* @param rankStart 开始排行位(包括)
* @param range 范围
* @param quality 图片质量
* @param fn 回调函数
* @throws IOException 当请求发生异常时抛出
*/
public void getRankingAsInputStream(PixivURL.RankingContentType contentType, PixivURL.RankingMode mode,
Date time, int rankStart, int range, PageQuality quality, RankingDownloadFunction fn) throws IOException {
getRanking(contentType, mode, time, rankStart, range).forEach(rankInfo -> {
int rank = rankInfo.get("rank").getAsInt();
int illustId = rankInfo.get("illust_id").getAsInt();
int authorId = rankInfo.get("user_id").getAsInt();
String authorName = rankInfo.get("user_name").getAsString();
String title = rankInfo.get("title").getAsString();
log.info("当前到第 {}/{} 名(总共 {} 名), IllustID: {}, Author: ({}) {}, Title: {}", rank, rankStart + range - 1, range, illustId, authorId, authorName, title);
log.info("正在获取PagesLink...");
List<String> linkList;
try {
linkList = getIllustAllPageDownload(httpClient, this.cookieStore, illustId, quality);
} catch (IOException e) {
if(e.getMessage().equals("該当作品は削除されたか、存在しない作品IDです。")) {
log.warn("作品 {} 不存在.", illustId);
} else {
e.printStackTrace();
}
return;
}
log.info("PagesLink 获取完成, 总数: {}", linkList.size());
for (int pageIndex = 0; pageIndex < linkList.size(); pageIndex++) {
String downloadLink = linkList.get(pageIndex);
log.info("当前Page: {}/{}", pageIndex + 1, linkList.size());
try(InputStream imageInputStream = new BufferedInputStream(getImageAsInputStream(HttpClientBuilder.create().build(), downloadLink), 256 * 1024)) {
fn.download(rank, downloadLink, rankInfo.deepCopy(), imageInputStream);
} catch(IOException e) {
log.error("下载插画时发生异常", e);
return;
}
log.info("完成.");
}
});
}
public List<JsonObject> getRanking(PixivURL.RankingContentType contentType, PixivURL.RankingMode mode,
Date time, int rankStart, int range) throws IOException {
if(rankStart <= 0) {
throw new IllegalArgumentException("rankStart cannot be less than or equal to zero");
}
if(range <= 0) {
throw new IllegalArgumentException("range cannot be less than or equal to zero");
}
int startPage = (int) Math.ceil(rankStart / 50F);
int requestFrequency = (int) Math.ceil((rankStart + (range - 1)) / 50F);
int surplusQuantity = range;
boolean firstRequest = true;
Gson gson = new Gson();
ArrayList<JsonObject> results = new ArrayList<>();
for (int requestCount = startPage; requestCount <= requestFrequency && requestCount <= 10; requestCount++) {
int rangeStart = (requestCount - 1) * 50 + 1;
log.info("正在请求第 {} 到 {} 位排名榜 (第{}次请求, 共 {} 次)", rangeStart, rangeStart + 49, requestCount - startPage + 1, requestFrequency - startPage);
HttpGet request = createHttpGetRequest(PixivURL.getRankingLink(contentType, mode, time, requestCount, true));
log.info("Request URL: {}", request.getURI());
HttpResponse response = httpClient.execute(request);
String content = EntityUtils.toString(response.getEntity());
log.debug("Content: " + content);
JsonObject contentObject = gson.fromJson(content, JsonObject.class);
if(contentObject.has("error")) {
log.warn("接口报错, 返回信息: {}", contentObject.get("error").getAsString());
break;
}
JsonArray rankingArray = contentObject.getAsJsonArray("contents");
log.info("正在解析数据...");
//需要添加一个总量, 否则会完整跑完一次.
//检查是否为最后一次请求和剩余量有多少
int firstRequestStartIndex = (rankStart % 50) - 1;
for (int rankIndex = firstRequest ? firstRequestStartIndex : 0; rankIndex < rankingArray.size() && surplusQuantity > 0; rankIndex++, surplusQuantity--) {
JsonElement jsonElement = rankingArray.get(rankIndex);
JsonObject rankInfo = jsonElement.getAsJsonObject();
int rank = rankInfo.get("rank").getAsInt();
int illustId = rankInfo.get("illust_id").getAsInt();
int authorId = rankInfo.get("user_id").getAsInt();
String authorName = rankInfo.get("user_name").getAsString();
String title = rankInfo.get("title").getAsString();
log.info("当前到第 {}/{} 名(总共 {} 名), IllustID: {}, Author: ({}) {}, Title: {}", rank, rankStart + range - 1, range, illustId, authorId, authorName, title);
results.add(rankInfo);
}
firstRequest = false;
log.info("第 {} 到 {} 位排名榜完成. (第{}次请求)", rangeStart, rangeStart + 49, requestCount);
}
if(requestFrequency > 10) {
log.warn("请求的排名榜范围超出所支持的范围, 已终止请求.");
}
return results;
}
/**
* 获取作品的预加载数据
* @param illustId 作品id
* @return 如果请求成功返回JsonObject, 失败返回null
* @throws IOException 当请求响应非200或请求发生异常时抛出.
*/
public JsonObject getIllustPreLoadDataById(int illustId) throws IOException {
HttpGet request = createHttpGetRequest(PixivURL.getPixivRefererLink(illustId));
HttpResponse response = httpClient.execute(request);
if(response.getStatusLine().getStatusCode() != 200) {
throw new IOException("Http响应码非200: " + response.getStatusLine());
}
Document document = Jsoup.parse(EntityUtils.toString(response.getEntity()));
Elements selectElements = document.select("#meta-preload-data");
if(selectElements.size() == 0) {
return null;
}
return new Gson().fromJson(selectElements.attr("content"), JsonObject.class);
}
@FunctionalInterface
public interface RankingDownloadFunction {
/**
* 接收图片InputStream
* @param rank 当前作品排名
* @param link 作品下载链接
* @param inputStream 作品下载输入流, InputStream会自动关闭
*/
void download(int rank, String link, JsonObject rankInfo, InputStream inputStream);
}
public HttpGet createHttpGetRequest(String url) {
HttpGet request = new HttpGet(url);
setCookieInRequest(request, cookieStore);
return request;
}
/**
* 取Illust所有页的原图下载链接
* @param httpClient 用于发起请求的HttpClient对象
* @param illustId 插画ID
* @param quality 页质量, {@link PageQuality}
* @return 返回该illust所有Page的下载链接
* @throws IOException 当HttpClient在请求时发生异常, 或接口报错时抛出, 注意{@link IOException#getMessage()}
*/
public static List<String> getIllustAllPageDownload(HttpClient httpClient, CookieStore cookieStore, int illustId, PageQuality quality) throws IOException {
HttpGet linkApiRequest = new HttpGet(PixivURL.PIXIV_ILLUST_API_URL.replace("{illustId}", Integer.toString(illustId)));
setCookieInRequest(linkApiRequest, cookieStore);
HttpResponse response = httpClient.execute(linkApiRequest);
JsonObject resultObject = new Gson().fromJson(EntityUtils.toString(response.getEntity()), JsonObject.class);
if(resultObject.get("error").getAsBoolean()) {
String message = resultObject.get("message").getAsString();
log.info("请求错误, 错误信息: {}", message);
throw new IOException(message);
}
JsonArray linkArray = resultObject.getAsJsonArray("body");
ArrayList<String> resultList = new ArrayList<>();
String qualityType = quality == null ? "original" : quality.toString().toLowerCase();
log.info("已选择插画类型: {}", qualityType);
linkArray.forEach(el -> {
JsonObject urlObj = el.getAsJsonObject().getAsJsonObject("urls");
resultList.add(urlObj.get(qualityType).getAsString());
});
return resultList;
}
/**
* 插图质量
*/
public enum PageQuality{
/**
* 原图画质
*/
ORIGINAL,
/**
* 常规画质
*/
REGULAR,
/**
* 小图画质
*/
SMALL,
/**
* 迷你画质
*/
THUMB_MINI
}
/**
* 获取帐号所有的收藏插画并以输入流形式提供
* @return 获取所有链接的InputStream, 请注意关闭InputStream
* @throws IOException 当获取时发生异常则直接抛出
*/
public Set<Map.Entry<String, InputStream>> getCollectionAsInputStream(PageQuality quality) throws IOException {
HashSet<Map.Entry<String, InputStream>> illustInputStreamSet = new HashSet<>();
getCollectionAsInputStream(quality, (link, inputStream) -> illustInputStreamSet.add(new AbstractMap.SimpleEntry<>(link, inputStream)));
return illustInputStreamSet;
}
/**
* 获取Pixiv图片
* @param httpClient HttpClient对象
* @param link Pixiv图片链接
* @return 返回图片InputStream注意关闭InputStream
* @throws IOException 获取失败时抛出
* @throws IllegalArgumentException 当链接无法处理时抛出
*/
public static InputStream getImageAsInputStream(HttpClient httpClient, String link) throws IOException {
HttpGet request = new HttpGet(link);
int startIndex = link.lastIndexOf("/");
int endIndex = link.lastIndexOf("_");
if(startIndex == -1 || endIndex == -1) {
throw new IllegalArgumentException("无法从链接获取illustID: " + link);
}
String referer = PixivURL.getPixivRefererLink(link.substring(startIndex + 1, endIndex));
request.addHeader(HttpHeaderNames.REFERER.toString(), referer);
HttpResponse response = httpClient.execute(request);
log.debug("response: {}", response);
log.info("Content Length: {}KB", Float.parseFloat(response.getFirstHeader(HttpHeaderNames.CONTENT_LENGTH.toString()).getValue()) / 1024F);
log.info("{}", response.getFirstHeader(HttpHeaderNames.CONTENT_TYPE.toString()));
return response.getEntity().getContent();
}
/**
* 登出当前会话.<br/>
* 登出成功后, 该Cookies作废.
* @return 返回是否成功登出
* @throws IOException 当登出请求异常时抛出
*/
public boolean logOut() throws IOException {
HttpGet request = new HttpGet(PixivURL.PIXIV_LOGOUT_URL);
request.setConfig(RequestConfig.custom().setRedirectsEnabled(false).build());
setCookieInRequest(request, cookieStore);
HttpResponse response = httpClient.execute(request);
if (response.getStatusLine().getStatusCode() == 302) {
cookieStore.clear();
return true;
} else {
return false;
}
}
/**
* 获取作品信息
* @param illustId 作品ID
* @return 成功获取返回JsonObject, 失败返回null, <br/>
* Json示例: <br/>
* <pre>
* {
* "illustId": "79584670",
* "illustTitle": "このヤンキーはウブすぎる",
* "id": "79584670",
* "title": "このヤンキーはウブすぎる",
* "illustType": 1,
* "xRestrict": 0,
* "restrict": 0,
* "sl": 2,
* "url": "https://i.pximg.net/c/360x360_70/img-master/img/2020/02/19/00/38/23/79584670_p0_square1200.jpg",
* "description": "",
* "tags": [
* "漫画",
* "オリジナル",
* "創作",
* "創作男女",
* "コロさん、ポリさん此方です!",
* "恋の予感",
* "あまずっぺー",
* "交換日記",
* "続編希望!!",
* "オリジナル10000users入り"
* ],
* "userId": "4778293",
* "userName": "隈浪さえ",
* "width": 3288,
* "height": 4564,
* "pageCount": 4,
* "isBookmarkable": true,
* "bookmarkData": null,
* "alt": "#オリジナル このヤンキーはウブすぎる - 隈浪さえ的漫画",
* "isAdContainer": false,
* "profileImageUrl": "https://i.pximg.net/user-profile/img/2019/12/04/18/56/19/16639046_fea29ce38ea89b0cb2313b40b3a72f9a_50.jpg",
* "type": "illust"
* }
* </pre>
* @throws IOException 当请求发生异常, 或接口返回错误信息时抛出.
*/
public JsonObject getIllustInfoByIllustId(int illustId) throws IOException {
HttpGet request = createHttpGetRequest(PixivURL.getPixivIllustInfoAPI(new int[] {illustId}));;
HttpResponse response = httpClient.execute(request);
String responseStr = EntityUtils.toString(response.getEntity());
log.debug("Response Content: {}", responseStr);
JsonObject responseObj = new Gson().fromJson(responseStr, JsonObject.class);
if(responseObj.get("error").getAsBoolean()) {
throw new IOException(responseObj.get("message").getAsString());
}
JsonArray illustsArray = responseObj.getAsJsonObject("body").getAsJsonArray("illusts");
if(illustsArray.size() == 1) {
return illustsArray.get(0).getAsJsonObject();
} else {
return null;
}
}
public static void setCookieInRequest(HttpRequest request, CookieStore cookieStore) {
StringBuilder builder = new StringBuilder();
cookieStore.getCookies().forEach(cookie -> builder.append(cookie.getName()).append("=").append(cookie.getValue()).append("; "));
request.setHeader(HttpHeaderNames.COOKIE.toString(), builder.toString());
}
}

View File

@ -0,0 +1,340 @@
package net.lamgc.cgj.pixiv;
import com.google.common.base.Strings;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.HashSet;
import java.util.Objects;
public class PixivSearchBuilder {
private final String content;
private SearchArea searchArea = SearchArea.ARTWORKS;
private SearchMode searchMode = SearchMode.TAG_FULL;
private SearchType searchType = SearchType.ILLUST_AND_UGOIRA;
private SearchOrder searchOrder = SearchOrder.DATE_D;
private SearchContentOption searchContentOption = SearchContentOption.ALL;
private HashSet<String> includeKeywords = new HashSet<>(0);
private HashSet<String> excludeKeywords = new HashSet<>(0);
private int page = 1;
private int wgt = 0;
private int hgt = 0;
private int wlt = 0;
private int hlt = 0;
private RatioOption ratioOption = null;
private Date startDate = null;
private Date endDate = null;
public PixivSearchBuilder(String searchContent) {
this.content = Objects.requireNonNull(searchContent);
}
public String buildURL() {
StringBuilder builder;
try {
builder = new StringBuilder(PixivURL.PIXIV_SEARCH_CONTENT_URL.replaceAll("\\{area}", searchArea.name().toLowerCase())
.replaceAll("\\{content}",
URLEncoder.encode(getSearchCondition(), "UTF-8").replaceAll("\\+", "%20")
)
);
} catch (UnsupportedEncodingException e) {
throw new RuntimeException(e);
}
if(searchArea.equals(SearchArea.TOP)) {
return builder.toString();
}
builder.append("&s_mode=").append(searchMode.name().toLowerCase());
builder.append("&type=").append(searchType.name().toLowerCase());
builder.append("&p=").append(page);
builder.append("&order=").append(searchOrder.name().toLowerCase());
builder.append("&mode=").append(searchContentOption.name().toLowerCase());
//可选参数
if(wgt > 0 && hgt > 0) {
builder.append("&wgt=").append(wgt);
builder.append("&hgt").append(hgt);
}
//可选参数
if(wlt > 0 && hlt > 0) {
builder.append("&wlt=").append(wlt);
builder.append("&hlt").append(hlt);
}
if (ratioOption != null) {
builder.append("&ratio=").append(ratioOption.value);
}
SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd");
if (startDate != null) {
builder.append("&scd=").append(format.format(startDate));
}
if (endDate != null) {
builder.append("&ecd=").append(format.format(endDate));
}
return builder.toString();
}
public PixivSearchBuilder setSearchArea(SearchArea searchArea) {
this.searchArea = searchArea;
return this;
}
/**
* 获取搜索区域
* @return 返回搜索区域对象
*/
public SearchArea getSearchArea() {
return searchArea;
}
/**
* 获取搜索条件.
* @return 搜索条件内容
*/
public String getSearchCondition() {
StringBuilder searchContent = new StringBuilder(Strings.nullToEmpty(this.content));
if(searchArea.equals(SearchArea.TOP)) {
return searchContent.toString();
}
excludeKeywords.forEach(keyword -> searchContent.append(" -").append(keyword));
if(!includeKeywords.isEmpty()) {
if(!Strings.isNullOrEmpty(searchContent.toString())) {
searchContent.append(" (");
}
includeKeywords.forEach(keyword -> searchContent.append(keyword).append(" OR "));
int deleteStart = searchContent.lastIndexOf(" OR ");
if(searchContent.length() >= 4 && deleteStart != -1) {
searchContent.delete(deleteStart, searchContent.length());
}
if(!Strings.isNullOrEmpty(searchContent.toString())) {
searchContent.append(")");
}
}
return searchContent.toString();
}
public PixivSearchBuilder setSearchMode(SearchMode searchMode) {
this.searchMode = Objects.requireNonNull(searchMode);
return this;
}
public PixivSearchBuilder setSearchType(SearchType searchType) {
this.searchType = Objects.requireNonNull(searchType);
return this;
}
public PixivSearchBuilder setSearchOrder(SearchOrder searchOrder) {
this.searchOrder = Objects.requireNonNull(searchOrder);
return this;
}
public PixivSearchBuilder setSearchContentOption(SearchContentOption searchContentOption) {
this.searchContentOption = Objects.requireNonNull(searchContentOption);
return this;
}
public PixivSearchBuilder setRatioOption(RatioOption ratioOption) {
this.ratioOption = Objects.requireNonNull(ratioOption);
return this;
}
public PixivSearchBuilder setDateRange(Date startDate, Date endDate) {
this.startDate = startDate;
this.endDate = endDate;
return this;
}
public PixivSearchBuilder setMaxSize(int width, int height) {
this.wgt = width;
this.hgt = height;
return this;
}
public PixivSearchBuilder setMinSize(int width, int height) {
this.wlt = width;
this.hlt = height;
return this;
}
public PixivSearchBuilder setPage(int pageIndex) {
if (pageIndex <= 0) {
throw new IllegalArgumentException("Invalid pageIndex: " + pageIndex);
}
this.page = pageIndex;
return this;
}
public PixivSearchBuilder addExcludeKeyword(String keyword) {
excludeKeywords.add(keyword);
return this;
}
public PixivSearchBuilder removeExcludeKeyword(String keyword) {
excludeKeywords.remove(keyword);
return this;
}
public PixivSearchBuilder addIncludeKeyword(String keyword) {
includeKeywords.add(keyword);
return this;
}
public PixivSearchBuilder removeIncludeKeyword(String keyword) {
includeKeywords.remove(keyword);
return this;
}
/**
* 搜索区域
*/
public enum SearchArea {
/**
* 所有(可能是 插画 + 漫画)
*/
ARTWORKS("illustManga"),
/**
* 顶部(所有内容)
* 同时包含了:
* {@link #ILLUSTRATIONS}
* {@link #MANGA}
* {@link #NOVELS}
* 选择此项后, 将直接显示所有与content相关内容, 而忽略所有附加搜索条件.
* 因为无法指定pageIndex, 数据只有24项
*/
TOP(null),
/**
* 插画
*/
ILLUSTRATIONS("illust"),
/**
* 漫画
*/
MANGA("manga"),
/**
* 小说
*/
NOVELS("novel");
public final String jsonKey;
SearchArea(String jsonKey) {
this.jsonKey = jsonKey;
}
}
/**
* 搜索模式
*/
public enum SearchMode {
/**
* 按标签搜索, 部分一致
*/
TAG,
/**
* 按标签搜索, 完全一致
*/
TAG_FULL,
/**
* 按标题和说明文字搜索
*/
TC
}
/**
* 搜索内容类型
*/
public enum SearchType {
/**
* 全部内容(插画漫画动图)
*/
ALL,
/**
* 插画和动图(不包括漫画)
*/
ILLUST_AND_UGOIRA,
/**
* 插图
*/
ILLUST,
/**
* 漫画
*/
MANGA,
/**
* 动图
*/
UGOIRA
}
public enum SearchOrder {
/**
* 按旧排序
*/
DATE,
/**
* 按新排序
*/
DATE_D
}
/**
* 搜索内容选项
*/
public enum SearchContentOption {
/**
* 所有内容
*/
ALL,
/**
* 全年龄
*/
SAFE,
/**
* R18
*/
R18
}
public enum RatioOption {
/**
* 横向
*/
TRANSVERSE(0.5F),
/**
* 纵向
*/
PORTRAIT(-0.5F),
/**
* 正方形
*/
SQUARE(0F)
;
public final float value;
RatioOption(float ratio) {
this.value = ratio;
}
}
}

View File

@ -0,0 +1,171 @@
package net.lamgc.cgj.pixiv;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import org.apache.http.Header;
import org.apache.http.HttpHost;
import org.apache.http.HttpResponse;
import org.apache.http.NameValuePair;
import org.apache.http.client.CookieStore;
import org.apache.http.client.HttpClient;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.impl.client.BasicCookieStore;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.message.BasicHeader;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
public class PixivSession {
public static final String USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36";
/**
* 全登陆过程的关键,
* 保存会话用的cookieStore!
*/
private CookieStore cookieStore = new BasicCookieStore();
/**
* 可以直接使用的HttpClient对象
*/
private HttpClient httpClient;
/**
* 最后一次登录的错误信息
*/
private String errMsg;
public PixivSession(){
this(null);
}
public PixivSession(CookieStore cookieStore){
this(null, cookieStore);
}
/**
* 创建一个Pixiv登录会话
*/
public PixivSession(HttpHost proxy, CookieStore cookieStore) {
if(cookieStore != null){
this.cookieStore = cookieStore;
}
List<Header> defaultHeader = new ArrayList<>();
defaultHeader.add(new BasicHeader("accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9"));
defaultHeader.add(new BasicHeader("user-agent", PixivSession.USER_AGENT));
defaultHeader.add(new BasicHeader("accept-encoding", "gzip, deflate, br"));
defaultHeader.add(new BasicHeader("accept-language", "zh-CN,zh;q=0.9"));
/*defaultHeader.add(new BasicHeader("sec-fetch-mode", "navigate"));
defaultHeader.add(new BasicHeader("sec-fetch-site", "same-origin"));
defaultHeader.add(new BasicHeader("upgrade-insecure-requests", "1"));*/
//创建一个Http访问器
httpClient = HttpClients.custom()
.setDefaultCookieStore(cookieStore)
.setDefaultHeaders(defaultHeader)
.setProxy(proxy)
.build();
}
/**
* 程序自行通过帐号密码登录Pixiv.
* @param PixivID Pixiv帐号
* @param Password Pixiv密码
* @return 登录成功返回true
* @throws IOException 当登录抛出异常时返回
* @deprecated Pixiv已经新增Google人机验证, 程序已无法自行登录Pixiv
*/
public boolean Login(String PixivID, String Password) throws IOException {
// 获取登录接口所需的PostKey
String post_key = getPostKey();
HttpPost postRequest = new HttpPost(PixivURL.PIXIV_LOGIN_URL); //https://accounts.pixiv.net/api/login?lang=zh
List<NameValuePair> params = new ArrayList<>();
params.add(new BasicNameValuePair("pixiv_id", PixivID));
params.add(new BasicNameValuePair("password", Password));
params.add(new BasicNameValuePair("post_key", post_key));
//Form编码表单,作为Post的数据
postRequest.setEntity(new UrlEncodedFormEntity(params, StandardCharsets.UTF_8));
//启动访问
HttpResponse response = httpClient.execute(postRequest);
//获取接口返回数据
String httpXML = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8);
System.out.println(httpXML);
JsonObject responseJson = (JsonObject) new JsonParser().parse(httpXML);
if(!responseJson.get("error").getAsBoolean() && !responseJson.get("body").getAsJsonObject().has("validation_errors")){
errMsg = null;
return true;
}else{
errMsg = responseJson.get("body").getAsJsonObject().get("validation_errors").toString();
//System.err.println("登录失败MSG: " + errMsg);
return false;
}
}
/**
* 登录前准备, 获取PostKey
* @return Post_Key
*/
private String getPostKey() throws IOException {
//创建请求,获取PostKey
HttpGet getRequest = new HttpGet(PixivURL.PIXIV_LOGIN_PAGE_URL);
//设置请求
//getRequest.setConfig(config);
getRequest.setHeader("User-Agent", USER_AGENT);
//启动访问
HttpResponse response = httpClient.execute(getRequest);
//获取网页内容
String pageAsXML = EntityUtils.toString(response.getEntity(),"utf-8");
//创建Http解析器
Document document = Jsoup.parse(pageAsXML);
//获取init-config内容
String init_config = document.getElementById("init-config").val();
//System.out.println(init_config);
//创建Json解析器解析init-config
JsonObject initConfigObj = (JsonObject) new JsonParser().parse(init_config);
//检查是否有postKey
if(!initConfigObj.has("pixivAccount.postKey")){
throw new RuntimeException("postKey获取失败!可能是Pixiv修改了登录过程!");
}
//获取postKey
return initConfigObj.get("pixivAccount.postKey").getAsString();
}
/**
* 获取CookieStore
* @return CookieStore
*/
public CookieStore getCookieStore(){
return cookieStore;
}
/**
* 获取可直接使用的HttpClient对象
* @return 已配置好的HttpClient对象
*/
public HttpClient getHttpClient(){
return this.httpClient;
}
public boolean hasError(){
return errMsg == null;
}
/**
* 获取错误信息
* @return 返回登录错误信息
* @deprecated {@link #Login(String, String)}已经废除, 故本接口废除
*/
public String getErrMsg(){
return errMsg;
}
}

View File

@ -0,0 +1,16 @@
package net.lamgc.cgj.pixiv;
public class PixivTag {
public static PixivTag create(String tagName) {
return null;
}
private PixivTag() {
}
}

View File

@ -0,0 +1,290 @@
package net.lamgc.cgj.pixiv;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;
import java.util.GregorianCalendar;
/**
* 目前已整理的一些Pixiv接口列表
*/
public class PixivURL {
public static final String PIXIV_INDEX_URL = "https://www.pixiv.net";
/**
* P站预登陆url
*/
public static final String PIXIV_LOGIN_PAGE_URL = "https://accounts.pixiv.net/login?lang=zh&source=pc&view_type=page&ref=wwwtop_accounts_index";
/**
* P站登录请求url
*/
public static final String PIXIV_LOGIN_URL = "https://accounts.pixiv.net/api/login?lang=zh";
/**
* P站搜索请求url
*/
private static final String PIXIV_SEARCH_URL = "https://www.pixiv.net/search.php";
/**
* P站搜索用户url
* 需要替换的参数:
* {nick} - 用户昵称部分名称
*/
public static final String PIXIV_SEARCH_USER_URL = PIXIV_SEARCH_URL + "?s_mode=s_usr&nick={nick}";
/**
* P站搜索插画url
* 需要替换的参数:
* {word} - 插画相关文本
*/
public static final String PIXIV_SEARCH_TAG_URL = PIXIV_SEARCH_URL + "?s_mode=s_tag&word={word}";
/**
* P站插图下载链接获取url
* 需要替换的文本:
* {illustId} - 插画ID
*/
public static final String PIXIV_ILLUST_API_URL = "https://www.pixiv.net/ajax/illust/{illustId}/pages";
/*
* P站用户插图列表获取API
* 需要替换的文本:
* {userId} - 用户ID
*/
//TODO: 所需数据在 body属性内的 illusts(属性名,属性值不重要), manga(多图) pickup(精选)
//{"error":false,"message":"","body":{"illusts":{"74369837":null,"70990542":null,"70608653":null,"69755191":null,"69729450":null,"69729416":null,"69503608":null,"69288766":null,"69083882":null,"69051458":null,"68484200":null,"68216927":null,"68216866":null,"68192333":null,"67915106":null,"67914932":null,"67854803":null,"67854745":null,"67854670":null,"67787211":null,"67772199":null,"67770637":null,"67754861":null,"67754804":null,"67754726":null,"67740486":null,"67740480":null,"67740450":null,"67740434":null,"67726337":null,"67499196":null,"67499163":null,"67499145":null,"67499111":null,"67499085":null,"67499038":null,"67498987":null,"67473178":null,"66271465":null,"63682753":null,"63682697":null,"59385148":null,"59383265":null,"59383240":null,"59383227":null,"59383173":null},"manga":[],"novels":[],"mangaSeries":[],"novelSeries":[],"pickup":[],"bookmarkCount":{"public":{"illust":1,"novel":0},"private":{"illust":0,"novel":0}}}}
//public static final String PIXIV_USER_ILLUST_LIST_URL = "https://www.pixiv.net/ajax/user/{userId}/profile/all";
/**
* 能够同时获取插图信息的用户插图列表获取API
*/
public static final String PIXIV_USER_ILLUSTINFO_LIST_URL = "https://www.pixiv.net/ajax/user/{userId}/profile/top";
/**
* P站单图详情页url
* 需要替换的文本:
* {illustId} - 插画ID
*/
public static final String PIXIV_ILLUST_MEDIUM_URL = "https://www.pixiv.net/member_illust.php?mode=medium&illust_id={illustId}";
/**
* P站多图详情页url
* 需要替换的文本:
* {illustId} - 插画ID
*/
public static final String PIXIV_ILLUST_MANGA_URL = "https://www.pixiv.net/member_illust.php?mode=manga&illust_id={illustId}";
/**
* P站用户页面url
* 需要替换的文本:
* {userId} - 用户ID
*/
public static final String PIXIV_USER_URL = "https://www.pixiv.net/member.php?id={userId}";
/**
* P站插图信息获取API
* 这个API能获取插图基本信息但不能获取大小
* 请使用{@link #getPixivIllustInfoAPI(int[])}获取URL
*/
private static final String PIXIV_GET_ILLUST_INFO_URL = "https://www.pixiv.net/ajax/illust/recommend/illusts?";
/**
* P站获取用户所有插图ID的Api
* 这个API只能获取该用户的插图ID不能获取图片信息(图片信息要另外获取)
* 需要替换的文本:
* {userId} - 用户ID
*/
public static final String PIXIV_GET_USER_ALL_ILLUST_ID_URL = "https://www.pixiv.net/ajax/user/{userId}/profile/all";
/**
* P站标签搜索URL
* 可以将Tag的大概内容搜索成P站精确的Tag以搜索其他接口返回的Tags数组;
* 需要替换的文本:
* {content} - 大致tag内容
*/
public static final String PIXIV_TAG_SEARCH_URL = "https://www.pixiv.net/ajax/search/tags/{content}";
/**
* 请求时带上需要退出的Cookies
* 无论成功与否都会返回302重定向到{@linkplain #PIXIV_LOGIN_PAGE_URL 登录页面}
*/
public static final String PIXIV_LOGOUT_URL = "https://www.pixiv.net/logout.php";
/**
* 构造P站获取插图信息的Api Url
* @param illustIds 要查询的插图ID数组
* @return 对应查询的API Url
*/
public static String getPixivIllustInfoAPI(int[] illustIds){
StringBuilder strBuilder = new StringBuilder().append(PIXIV_GET_ILLUST_INFO_URL);
for(int illustId : illustIds){
strBuilder.append("illust_ids[]=").append(illustId).append("&");
}
return strBuilder.toString();
}
/**
* 获取用于下载图片时防盗链所需Referer的链接
* @param illustId 欲下载图片所属illustId
* @return 返回Referer链接, 也可以作为作品链接使用
*/
public static String getPixivRefererLink(int illustId){
return "https://www.pixiv.net/artworks/" + illustId;
}
/**
* 获取用于下载图片时防盗链所需Referer的链接
* @param illustId 欲下载图片所属illustId
* @return 返回Referer链接, 也可以作为作品链接使用
*/
public static String getPixivRefererLink(String illustId){
return "https://www.pixiv.net/artworks/" + illustId;
}
/**
* 排行榜接口, 需加入"&format=json"
*/
private final static String PIXIV_RANKING_LINK = "https://www.pixiv.net/ranking.php?";
/**
* 查询用户收藏.<br/>
* 该URL返回HTML页面需要进行解析.<br/>
* 需要替换的文本:<br/>
* {pageIndex} - 页数, 超出了则结果为空<br/>
*/
public final static String PIXIV_USER_COLLECTION_PAGE = "https://www.pixiv.net/bookmark.php?rest=show&p={pageIndex}";
/**
* 获取排名榜
* @param mode 查询类型, 详细信息看{@link RankingMode}, 如本参数为null, 则为每天
* @param contentType 排名榜类型, 如为null则为综合
* @param time 欲查询的时间, 最新只能查询昨天, 根据mode不同:
* 每天 - 查询指定日期的排名榜
* 每周 - 查询指定时间结束()到七天前一段时间内的排名榜
* 每月 - 查询指定日期结束()到28天时间内的排名榜
* 新人 - 与每周相同
* 受男性欢迎 - 与每天相同
* 受女性欢迎 - 与每天相同
* 默认值为昨天
* @param pageIndex 页数一页50位总共10页
* @return 返回构建好的链接
*/
public static String getRankingLink(RankingContentType contentType, RankingMode mode, Date time, int pageIndex, boolean json){
StringBuilder linkBuilder = new StringBuilder(PIXIV_RANKING_LINK);
linkBuilder.append("mode=").append(mode == null ? RankingMode.MODE_DAILY.modeParam : mode.modeParam);
if(contentType != null){
linkBuilder.append("&content=").append(contentType.typeName);
}
SimpleDateFormat format = new SimpleDateFormat("yyyyMMdd");
Date queryDate;
if(time == null){
GregorianCalendar gregorianCalendar = new GregorianCalendar();
gregorianCalendar.setTime(new Date());
gregorianCalendar.add(Calendar.DATE, -1);
queryDate = gregorianCalendar.getTime();
} else {
queryDate = time;
}
linkBuilder.append("&date=").append(format.format(queryDate));
if(pageIndex > 0 && pageIndex <= 10) {
linkBuilder.append("&p=").append(pageIndex);
}
if(json) {
linkBuilder.append("&format=").append("json");
}
return linkBuilder.toString();
}
/**
* 排名榜模式
*/
public enum RankingMode{
/**
* 每天
*/
MODE_DAILY("daily"),
/**
* 每周
*/
MODE_WEEKLY("weekly"),
/**
* 每月
*/
MODE_MONTHLY("monthly"),
/**
* 新人
*/
MODE_ROOKIE("rookie"),
/**
* 受男性喜欢
*/
MODE_MALE("male"),
/**
* 受女性喜欢
*/
MODE_FEMALE("female"),
/**
* 每天 - R18
*/
MODE_DAILY_R18("daily_r18"),
/**
* 每周 - R18
*/
MODE_WEEKLY_R18("weekly_r18"),
/**
* 受男性喜欢 - R18
*/
MODE_MALE_R18("male_r18"),
/**
* 受女性喜欢 - R18
*/
MODE_FEMALE_R18("female_r18"),
;
public String modeParam;
RankingMode(String modeParamName){
this.modeParam = modeParamName;
}
}
/**
* Pixiv搜索接口.<br/>
* 要使用该链接请使用{@link PixivSearchBuilder}构造链接.<br/>
* 需要替换的参数: <br/>
* content - 搜索内容
*/
final static String PIXIV_SEARCH_CONTENT_URL = "https://www.pixiv.net/ajax/search/{area}/{content}?word={content}";
/**
* 排名榜类型
*/
public enum RankingContentType{
/**
* 插画
* 支持的时间类型: 每天, 每周, 每月, 新人
*/
TYPE_ILLUST("illust"),
/**
* 动图
* 支持的时间类型:每天, 每周
*/
TYPE_UGOIRA("ugoira"),
/**
* 漫画
* 支持的时间类型: 每天, 每周, 每月, 新人
*/
TYPE_MANGA("manga")
;
String typeName;
RankingContentType(String typeName){
this.typeName = typeName;
}
}
}

View File

@ -0,0 +1,149 @@
package net.lamgc.cgj.proxy;
import com.github.monkeywie.proxyee.intercept.HttpProxyIntercept;
import com.github.monkeywie.proxyee.intercept.HttpProxyInterceptInitializer;
import com.github.monkeywie.proxyee.intercept.HttpProxyInterceptPipeline;
import com.github.monkeywie.proxyee.intercept.common.CertDownIntercept;
import com.github.monkeywie.proxyee.proxy.ProxyConfig;
import com.github.monkeywie.proxyee.server.HttpProxyServer;
import com.github.monkeywie.proxyee.server.HttpProxyServerConfig;
import io.netty.channel.Channel;
import io.netty.handler.codec.http.HttpHeaderNames;
import io.netty.handler.codec.http.HttpHeaders;
import io.netty.handler.codec.http.HttpRequest;
import io.netty.handler.codec.http.HttpResponse;
import org.apache.http.client.CookieStore;
import org.apache.http.impl.cookie.BasicClientCookie;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.net.HttpCookie;
import java.util.Date;
import java.util.List;
/**
* 登录成功后提供CookieStore, 然后由程序自动登录Pixiv
* @author LamGC
*/
public class PixivAccessProxyServer {
private final Logger log = LoggerFactory.getLogger(this.getClass().getSimpleName());
private final HttpProxyServer proxyServer;
private final CookieStore cookieStore;
public PixivAccessProxyServer(CookieStore cookieStore){
this(cookieStore, null);
}
public PixivAccessProxyServer(CookieStore cookieStore, ProxyConfig proxyConfig){
HttpProxyServerConfig config = new HttpProxyServerConfig();
this.cookieStore = cookieStore;
config.setHandleSsl(true);
this.proxyServer = new HttpProxyServer();
this.proxyServer
.serverConfig(config)
.proxyConfig(proxyConfig)
.proxyInterceptInitializer(new HttpProxyInterceptInitializer(){
@Override
public void init(HttpProxyInterceptPipeline pipeline) {
pipeline.addLast(new CertDownIntercept());
pipeline.addLast(new HttpProxyIntercept(){
private boolean match(HttpRequest request){
String host = request.headers().get(HttpHeaderNames.HOST);
return host.equalsIgnoreCase("pixiv.net") || host.contains(".pixiv.net");
}
@Override
public void beforeRequest(Channel clientChannel, HttpRequest httpRequest, HttpProxyInterceptPipeline pipeline) throws Exception {
log.info("URL: " + httpRequest.headers().get(HttpHeaderNames.HOST) + httpRequest.uri());
if(!match(httpRequest)){
super.beforeRequest(clientChannel, httpRequest, pipeline);
return;
}
log.info("正在注入Cookies...");
HttpHeaders requestHeaders = httpRequest.headers();
if(requestHeaders.contains(HttpHeaderNames.COOKIE)){
log.info("原请求存在自带Cookies, 正在清除Cookies...");
log.debug("原Cookies: {}", requestHeaders.getAsString(HttpHeaderNames.COOKIE));
requestHeaders.remove(HttpHeaderNames.COOKIE);
}
StringBuilder cookieBuilder = new StringBuilder();
cookieStore.getCookies().forEach(cookie -> {
if(cookie.isExpired(new Date())){
return;
}
cookieBuilder.append(cookie.getName()).append("=").append(cookie.getValue()).append("; ");
});
log.info("Cookies构造完成, 结果: " + cookieBuilder.toString());
requestHeaders.add(HttpHeaderNames.COOKIE, cookieBuilder.toString());
log.info("Cookies注入完成.");
super.beforeRequest(clientChannel, httpRequest, pipeline);
}
@Override
public void afterResponse(Channel clientChannel, Channel proxyChannel, HttpResponse httpResponse, HttpProxyInterceptPipeline pipeline) throws Exception {
if(!match(pipeline.getHttpRequest())){
super.afterResponse(clientChannel, proxyChannel, httpResponse, pipeline);
return;
}
log.info("正在更新Response Cookie...(Header Name: " + HttpHeaderNames.SET_COOKIE + ")");
List<String> responseCookies = httpResponse.headers().getAll(HttpHeaderNames.SET_COOKIE);
responseCookies.forEach(value -> {
/*if(check(value)){
log.info("黑名单Cookie, 已忽略: " + value);
return;
}*/
log.info("Response Cookie: " + value);
BasicClientCookie cookie = parseRawCookie(value);
cookieStore.addCookie(null);
});
httpResponse.headers().remove(HttpHeaderNames.SET_COOKIE);
super.afterResponse(clientChannel, proxyChannel, httpResponse, pipeline);
}
protected BasicClientCookie parseRawCookie(String rawCookie) {
List<HttpCookie> cookies = HttpCookie.parse(rawCookie);
if (cookies.size() < 1)
return null;
HttpCookie httpCookie = cookies.get(0);
BasicClientCookie cookie = new BasicClientCookie(httpCookie.getName(), httpCookie.getValue());
if (httpCookie.getMaxAge() >= 0) {
Date expiryDate = new Date(System.currentTimeMillis() + httpCookie.getMaxAge() * 1000);
cookie.setExpiryDate(expiryDate);
}
if (httpCookie.getDomain() != null)
cookie.setDomain(httpCookie.getDomain());
if (httpCookie.getPath() != null)
cookie.setPath(httpCookie.getPath());
if (httpCookie.getComment() != null)
cookie.setComment(httpCookie.getComment());
cookie.setSecure(httpCookie.getSecure());
return cookie;
}
});
}
});
}
public void start(int port){
this.proxyServer.start(port);
}
public void close(){
this.proxyServer.close();
}
/**
* 导出CookieStore.
* 注意!该方法导出的CookieStore不适用于ApacheHttpClient, 如需使用则需要进行转换.
* @return CookieStore对象
*/
public CookieStore getCookieStore(){
return this.cookieStore;
}
}

View File

@ -0,0 +1,157 @@
package net.lamgc.cgj.proxy;
import com.github.monkeywie.proxyee.intercept.HttpProxyInterceptInitializer;
import com.github.monkeywie.proxyee.intercept.HttpProxyInterceptPipeline;
import com.github.monkeywie.proxyee.intercept.common.CertDownIntercept;
import com.github.monkeywie.proxyee.intercept.common.FullResponseIntercept;
import com.github.monkeywie.proxyee.proxy.ProxyConfig;
import com.github.monkeywie.proxyee.server.HttpProxyServer;
import com.github.monkeywie.proxyee.server.HttpProxyServerConfig;
import com.google.gson.Gson;
import com.google.gson.JsonObject;
import io.netty.handler.codec.http.FullHttpResponse;
import io.netty.handler.codec.http.HttpHeaderNames;
import io.netty.handler.codec.http.HttpRequest;
import io.netty.handler.codec.http.HttpResponse;
import org.apache.commons.text.StringEscapeUtils;
import org.apache.http.client.CookieStore;
import org.apache.http.impl.client.BasicCookieStore;
import org.apache.http.impl.cookie.BasicClientCookie;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.net.HttpCookie;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.util.Date;
import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;
/**
* 由用户介入, 让用户手动登录Pixiv的方式, 再通过代理服务器捕获Cookie来绕过Google人机验证
* @author LamGC
*/
public class PixivLoginProxyServer {
private final Logger log = LoggerFactory.getLogger(this.getClass().getSimpleName());
private boolean login = false;
private final HttpProxyServer proxyServer;
private final CookieStore cookieStore = new BasicCookieStore();
public PixivLoginProxyServer(){
this(null);
}
public PixivLoginProxyServer(ProxyConfig proxyConfig){
HttpProxyServerConfig config = new HttpProxyServerConfig();
config.setHandleSsl(true);
this.proxyServer = new HttpProxyServer();
this.proxyServer
.serverConfig(config)
.proxyConfig(proxyConfig)
.proxyInterceptInitializer(new HttpProxyInterceptInitializer(){
@Override
public void init(HttpProxyInterceptPipeline pipeline) {
pipeline.addLast(new CertDownIntercept());
pipeline.addLast(new FullResponseIntercept() {
@Override
public boolean match(HttpRequest httpRequest, HttpResponse httpResponse, HttpProxyInterceptPipeline httpProxyInterceptPipeline) {
String host = httpRequest.headers().get(HttpHeaderNames.HOST);
return host.equalsIgnoreCase("pixiv.net") || host.contains(".pixiv.net");
}
@Override
public void handelResponse(HttpRequest httpRequest, FullHttpResponse fullHttpResponse, HttpProxyInterceptPipeline httpProxyInterceptPipeline) {
String url = httpRequest.headers().get(HttpHeaderNames.HOST) + httpRequest.uri();
log.info("拦截到Pixiv请求, URL: " + url);
log.info("正在导出Response Cookie...(Header Name: " + HttpHeaderNames.SET_COOKIE + ")");
List<String> responseCookies = fullHttpResponse.headers().getAll(HttpHeaderNames.SET_COOKIE);
AtomicInteger responseCookieCount = new AtomicInteger();
responseCookies.forEach(value -> {
log.debug("Response Cookie: " + value);
cookieStore.addCookie(parseRawCookie(value));
responseCookieCount.incrementAndGet();
});
log.info("Cookie导出完成(已导出 " + responseCookieCount.get() + " 条Cookie)");
//登录检查
// 如果用户在登录界面登录成功后反复刷新会出现登录返回不对但已经成功登录的情况
// 故此处在登录完成后不再判断是否成功登录
if(!isLogin() && url.contains("accounts.pixiv.net/api/login")){
log.info("正在检查登录结果...");
//拷贝一份以防止对原响应造成影响
FullHttpResponse copyResponse = fullHttpResponse.copy();
ByteBuffer buffer = ByteBuffer.allocate(copyResponse.content().capacity());
String contentStr;
copyResponse.content().readBytes(buffer);
contentStr = new String(buffer.array(), StandardCharsets.UTF_8);
log.debug("Login Result: " + contentStr);
JsonObject resultObject = new Gson().fromJson(contentStr, JsonObject.class);
//只要error:false, body存在(应该是会存在的)且success字段存在, 即为登录成功
login = !resultObject.get("error").getAsBoolean() &&
resultObject.has("body") &&
resultObject.get("body").getAsJsonObject().has("success");
log.info("登录状态确认: " + (login ? "登录成功" : "登录失败"));
fullHttpResponse.content().clear().writeBytes(
("{\"error\":false,\"message\":\"\",\"body\":{\"validation_errors\":{\"etc\":\"" +
StringEscapeUtils.escapeJava("Pixiv登录代理器已确认登录") + "\"}}}")
.getBytes(StandardCharsets.UTF_8));
}
}
protected BasicClientCookie parseRawCookie(String rawCookie) {
List<HttpCookie> cookies = HttpCookie.parse(rawCookie);
if (cookies.size() < 1)
return null;
HttpCookie httpCookie = cookies.get(0);
BasicClientCookie cookie = new BasicClientCookie(httpCookie.getName(), httpCookie.getValue());
if (httpCookie.getMaxAge() >= 0) {
Date expiryDate = new Date(System.currentTimeMillis() + httpCookie.getMaxAge() * 1000);
cookie.setExpiryDate(expiryDate);
}
if (httpCookie.getDomain() != null)
cookie.setDomain(httpCookie.getDomain());
if (httpCookie.getPath() != null)
cookie.setPath(httpCookie.getPath());
if (httpCookie.getComment() != null)
cookie.setComment(httpCookie.getComment());
cookie.setSecure(httpCookie.getSecure());
return cookie;
}
});
}
});
}
public void start(int port){
this.proxyServer.start(port);
}
public void close(){
this.proxyServer.close();
}
/**
* 是否已登录Pixiv
* @return 如已登录返回true
*/
public boolean isLogin(){
return login;
}
/**
* 导出CookieStore.
* 注意!该方法导出的CookieStore不适用于ApacheHttpClient, 如需使用则需要进行转换.
* @return CookieStore对象
*/
public CookieStore getCookieStore(){
return this.cookieStore;
}
}

View File

@ -0,0 +1,185 @@
package net.lamgc.cgj.proxy;
import com.github.monkeywie.proxyee.intercept.HttpProxyInterceptInitializer;
import com.github.monkeywie.proxyee.intercept.HttpProxyInterceptPipeline;
import com.github.monkeywie.proxyee.intercept.common.CertDownIntercept;
import com.github.monkeywie.proxyee.intercept.common.FullResponseIntercept;
import com.github.monkeywie.proxyee.proxy.ProxyConfig;
import com.github.monkeywie.proxyee.server.HttpProxyServer;
import com.github.monkeywie.proxyee.server.HttpProxyServerConfig;
import com.google.gson.Gson;
import com.google.gson.JsonObject;
import io.netty.handler.codec.http.*;
import org.apache.http.client.CookieStore;
import org.apache.http.impl.client.BasicCookieStore;
import org.apache.http.impl.cookie.BasicClientCookie;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.net.HttpCookie;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.util.*;
/**
* 由用户介入, 让用户手动登录Pixiv的方式, 再通过代理服务器捕获Cookie来绕过Google人机验证
* @author LamGC
*/
public class PixivLoginProxyServer_Old {
private final Logger log = LoggerFactory.getLogger(this.getClass().getSimpleName());
private boolean login = false;
private final HttpProxyServer proxyServer;
//private final CookieManager cookieManager = new CookieManager();
private final CookieStore cookieStore = new BasicCookieStore();
public PixivLoginProxyServer_Old(){
this(null);
}
public PixivLoginProxyServer_Old(ProxyConfig proxyConfig){
HttpProxyServerConfig config = new HttpProxyServerConfig();
config.setHandleSsl(true);
this.proxyServer = new HttpProxyServer();
this.proxyServer
.serverConfig(config)
.proxyConfig(proxyConfig)
.proxyInterceptInitializer(new HttpProxyInterceptInitializer(){
@Override
public void init(HttpProxyInterceptPipeline pipeline) {
pipeline.addLast(new CertDownIntercept());
pipeline.addLast(new FullResponseIntercept() {
@Override
public boolean match(HttpRequest httpRequest, HttpResponse httpResponse, HttpProxyInterceptPipeline httpProxyInterceptPipeline) {
//log.info("Match: " + httpRequest.headers().get(HttpHeaderNames.HOST) + httpRequest.uri());
//return HttpUtil.checkUrl(httpRequest, ".*pixiv\\.net");
String host = httpRequest.headers().get(HttpHeaderNames.HOST);
return host.equalsIgnoreCase("pixiv.net") || host.contains(".pixiv.net");
}
@Override
public void handelResponse(HttpRequest httpRequest, FullHttpResponse fullHttpResponse, HttpProxyInterceptPipeline httpProxyInterceptPipeline) {
log.info("拦截到Pixiv请求, 正在导出Response Cookie...");
String url = httpRequest.headers().get(HttpHeaderNames.HOST) + httpRequest.uri();
log.info("URL: " + url);
/*URI requestURI = URI.create(httpRequest.headers().get(HttpHeaderNames.HOST) + httpRequest.uri());
log.info("正在导出Request Cookie...(Header Name: " + HttpHeaderNames.COOKIE + ")");
List<String> requestCookie = httpRequest.headers().getAll(HttpHeaderNames.COOKIE);
//requestCookie.forEach(value -> log.info("Request Cookie: " + value));
requestCookie.forEach(value -> {
String[] items = value.split(";");
for(String item : items){
log.info("Raw Request Cookie: " + item);
String[] keyValueSet = item.split("=");
if(keyValueSet.length == 2){
//TODO: 会出现重复的情况
log.info("Request Cookie: " + keyValueSet[0].trim() + "=" + keyValueSet[1].trim());
cookieStore.addCookie(new BasicClientCookie(keyValueSet[0].trim(), keyValueSet[1].trim()));
}
}
//cookieStore.addCookie(parseRawCookie(value));
});*/
log.info("正在导出Response Cookie...(Header Name: " + HttpHeaderNames.SET_COOKIE + ")");
List<String> responseCookies = fullHttpResponse.headers().getAll(HttpHeaderNames.SET_COOKIE);
responseCookies.forEach(value -> {
/*if(check(value)){
log.info("黑名单Cookie, 已忽略: " + value);
return;
}*/
log.info("Response Cookie: " + value);
cookieStore.addCookie(parseRawCookie(value));
});
log.info("Cookie导出完成");
if(url.contains("accounts.pixiv.net/api/login")){
log.info("正在检查登录结果...");
FullHttpResponse copyResponse = fullHttpResponse.copy();
//ByteArrayOutputStream contentOS = new ByteArrayOutputStream(copyResponse.content().capacity());
ByteBuffer buffer = ByteBuffer.allocate(copyResponse.content().capacity());
String contentStr;
copyResponse.content().readBytes(buffer);
contentStr = new String(buffer.array(), StandardCharsets.UTF_8);
//log.info("Login Result: " + contentStr);
JsonObject resultObject = new Gson().fromJson(contentStr, JsonObject.class);
login = !resultObject.get("error").getAsBoolean() &&
resultObject.has("body") &&
resultObject.get("body").getAsJsonObject().has("success");
if(login) {
log.info("登录状态确认: 登录成功");
} else {
log.info("登录状态确认: 登录失败");
}
}
}
protected BasicClientCookie parseRawCookie(String rawCookie) {
List<HttpCookie> cookies = HttpCookie.parse(rawCookie);
if (cookies.size() < 1)
return null;
HttpCookie httpCookie = cookies.get(0);
BasicClientCookie cookie = new BasicClientCookie(httpCookie.getName(), httpCookie.getValue());
if (httpCookie.getMaxAge() >= 0) {
Date expiryDate = new Date(System.currentTimeMillis() + httpCookie.getMaxAge() * 1000);
cookie.setExpiryDate(expiryDate);
}
if (httpCookie.getDomain() != null)
cookie.setDomain(httpCookie.getDomain());
if (httpCookie.getPath() != null)
cookie.setPath(httpCookie.getPath());
if (httpCookie.getComment() != null)
cookie.setComment(httpCookie.getComment());
cookie.setSecure(httpCookie.getSecure());
return cookie;
}
private boolean check(String cookieValue){
for(String blackItem : new String[]{
"a_type",
"b_type",
"c_type",
"is_sensei_service_user",
"module_orders_mypage",
}){
if(cookieValue.startsWith(blackItem)){
return true;
}
}
return false;
}
});
}
});
}
public void start(int port){
this.proxyServer.start(port);
}
public void close(){
this.proxyServer.close();
}
/**
* 是否已登录Pixiv
* @return 如已登录返回true
*/
public boolean isLogin(){
return login;
}
/**
* 导出CookieStore.
* 注意!该方法导出的CookieStore不适用于ApacheHttpClient, 如需使用则需要进行转换.
* @return CookieStore对象
*/
public CookieStore getCookieStore(){
return this.cookieStore;
}
}

View File

@ -0,0 +1,31 @@
package net.lamgc.cgj.util;
import org.apache.http.client.CookieStore;
import org.apache.http.impl.client.BasicCookieStore;
import org.apache.http.impl.cookie.BasicClientCookie;
import java.util.Date;
public class CookieUtil {
/**
* {@link java.net.CookieStore}转换到{@link CookieStore}
* @param netCookieStore java.net.CookieStore
* @return org.apache.http.client.CookieStore
*/
public static CookieStore NAParse(java.net.CookieStore netCookieStore){
CookieStore apacheCookieStore = new BasicCookieStore();
netCookieStore.getCookies().forEach(netCookie -> {
BasicClientCookie aCookie = new BasicClientCookie(netCookie.getName(), netCookie.getValue());
aCookie.setComment(netCookie.getComment());
aCookie.setDomain(netCookie.getDomain());
aCookie.setExpiryDate(new Date(netCookie.getMaxAge()));
aCookie.setPath(netCookie.getPath());
aCookie.setSecure(netCookie.getSecure());
aCookie.setVersion(netCookie.getVersion());
apacheCookieStore.addCookie(aCookie);
});
return apacheCookieStore;
}
}

View File

@ -0,0 +1,25 @@
package net.lamgc.cgj.util;
import net.lamgc.utils.base.runner.StringParameterParser;
import java.text.DateFormat;
import java.util.Date;
public class DateParser implements StringParameterParser<Date> {
private final DateFormat dateFormat;
public DateParser(DateFormat dateFormat) {
this.dateFormat = dateFormat;
}
@Override
public Date parse(String strValue) throws Exception {
return dateFormat.parse(strValue);
}
@Override
public Date defaultValue() {
return null;
}
}

View File

@ -0,0 +1,18 @@
package net.lamgc.cgj.util;
import net.lamgc.cgj.pixiv.PixivDownload;
import net.lamgc.cgj.pixiv.PixivURL;
import net.lamgc.utils.base.runner.StringParameterParser;
public class PagesQualityParser implements StringParameterParser<PixivDownload.PageQuality> {
@Override
public PixivDownload.PageQuality parse(String strValue) throws Exception {
return PixivDownload.PageQuality.valueOf(strValue.toUpperCase());
}
@Override
public PixivDownload.PageQuality defaultValue() {
return PixivDownload.PageQuality.ORIGINAL;
}
}

View File

@ -0,0 +1,2 @@
server.port=8081
server.tomcat.max-threads=10

View File

@ -0,0 +1,15 @@
log4j.rootLogger = info,console,logfile
#\u8BBE\u7F6Edebug\u65E5\u5FD7\u4FE1\u606F
log4j.appender.console=org.apache.log4j.ConsoleAppender
log4j.appender.console.Target=System.out
log4j.appender.console.layout=org.apache.log4j.PatternLayout
#[\u5E74.\u6708.\u65E5 \u65F6:\u5206:\u79D2,\u6BEB\u79D2 \u4F18\u5148\u7EA7] [\u7C7B\u5168\u540D.\u65B9\u6CD5\u540D:\u884C\u53F7][\u7EBF\u7A0B\u540D] \u65E5\u5FD7\u5185\u5BB9
log4j.appender.console.layout.ConversionPattern=[%d{HH:mm:ss,SSS} %5p][%c.%M():%-3L][%t]: %m%n
log4j.appender.logfile=org.apache.log4j.DailyRollingFileAppender
log4j.appender.logfile.File=logs/running.log.
log4j.appender.logfile.DatePattern=yyyy-MM-dd
log4j.appender.logfile.layout=org.apache.log4j.PatternLayout
#[\u65F6:\u5206:\u79D2,\u6BEB\u79D2 \u4F18\u5148\u7EA7] [\u7C7B\u5168\u540D.\u65B9\u6CD5\u540D:\u884C\u53F7][\u7EBF\u7A0B\u540D] \u65E5\u5FD7\u5185\u5BB9
log4j.appender.logfile.layout.ConversionPattern=[%d{HH:mm:ss,SSS} %5p][%c.%M():%-3L][%t]: %m%n

View File

@ -0,0 +1,9 @@
#用于访问Pixiv的代理服务器
reptile.proxy.type=socks5/socks4/http
reptile.proxy.host=127.0.0.1
reptile.proxy.port=1080
reptile.proxy.username=
reptile.proxy.password=
#登录用代理, 需要让浏览器使用该代理, 访问Pixiv并登录
login.proxy.host=127.0.0.1
login.proxy.port=1080

View File

@ -0,0 +1,202 @@
package net.lamgc.cgj.pixiv;
import org.apache.commons.io.IOUtils;
import org.apache.http.HttpHost;
import org.apache.http.client.CookieStore;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.*;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;
import java.util.GregorianCalendar;
import java.util.Objects;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
@Ignore
public class PixivDownloadTest {
private static CookieStore cookieStore;
private final static Logger log = LoggerFactory.getLogger("PixivDownloadTest");
private static HttpHost proxy = new HttpHost("127.0.0.1", 1001);
@BeforeClass
public static void before() throws IOException, ClassNotFoundException {
ObjectInputStream ois = new ObjectInputStream(new FileInputStream(new File("cookies.store")));
cookieStore = (CookieStore) ois.readObject();
ois.close();
log.info("已载入CookieStore");
}
@Test
public void logOutTest() throws IOException {
PixivDownload pixivDownload = new PixivDownload(Objects.requireNonNull(cookieStore), proxy);
Assert.assertTrue(pixivDownload.logOut());
}
@Test
public void collectionDownloadTest() throws IOException {
PixivDownload pixivDownload = new PixivDownload(Objects.requireNonNull(cookieStore), proxy);
File outputFile = new File("collection.zip");
if(!outputFile.exists() && !outputFile.createNewFile()) {
Assert.fail("文件创建失败: " + outputFile.getAbsolutePath());
}
ZipOutputStream zos = new ZipOutputStream(new FileOutputStream(outputFile));
zos.setLevel(9);
log.info("正在调用方法...");
pixivDownload.getCollectionAsInputStream(PixivDownload.PageQuality.ORIGINAL, (link, inputStream) -> {
try {
ZipEntry entry = new ZipEntry(link.substring(link.lastIndexOf("/") + 1));
log.info("正在写入: " + entry.getName());
zos.putNextEntry(entry);
IOUtils.copy(inputStream, zos);
zos.flush();
} catch (IOException e) {
log.error("写入文件项时发生异常", e);
}
});
log.info("调用完成.");
zos.close();
}
@Test
public void getRecommendsTest() throws IOException {
PixivDownload pixivDownload = new PixivDownload(Objects.requireNonNull(cookieStore), proxy);
String date = new SimpleDateFormat("yyyyMMdd").format(new Date());
int id = 1;
File outputFile = new File("recommends-" + date + "-" + id + ".zip");
while(outputFile.exists()) {
id++;
outputFile = new File("recommends-" + date + "-" + id + ".zip");
}
if(!outputFile.createNewFile()) {
Assert.fail("文件创建失败: " + outputFile.getAbsolutePath());
}
ZipOutputStream zos = new ZipOutputStream(new FileOutputStream(outputFile));
zos.setLevel(9);
log.info("正在调用方法...");
pixivDownload.getRecommendAsInputStream(PixivDownload.PageQuality.ORIGINAL, (link, inputStream) -> {
try {
ZipEntry entry = new ZipEntry(link.substring(link.lastIndexOf("/") + 1));
log.info("正在写入: " + entry.getName());
zos.putNextEntry(entry);
IOUtils.copy(inputStream, zos);
zos.flush();
log.info("已成功写入 {}", entry.getName());
} catch (IOException e) {
log.error("写入文件项时发生异常", e);
}
});
log.info("调用完成.");
zos.close();
}
@Test
public void getRankingIllustByRangeTest() throws IOException {
PixivDownload pixivDownload = new PixivDownload(cookieStore, proxy);
GregorianCalendar gregorianCalendar = new GregorianCalendar();
gregorianCalendar.setTime(new Date());
gregorianCalendar.add(Calendar.DATE, -1);
Date queryDate = gregorianCalendar.getTime();
String date = new SimpleDateFormat("yyyyMMdd").format(queryDate);
log.info("查询时间: {}", date);
int id = 1;
File outputFile = new File("ranking-" + date + "-" + id + ".zip");
while(outputFile.exists()) {
id++;
outputFile = new File("ranking-" + date + "-" + id + ".zip");
}
if(!outputFile.createNewFile()) {
Assert.fail("文件创建失败: " + outputFile.getAbsolutePath());
}
ZipOutputStream zos = new ZipOutputStream(new FileOutputStream(outputFile));
zos.setLevel(9);
log.info("正在调用方法...");
try {
pixivDownload.getRankingAsInputStream(null, PixivURL.RankingMode.MODE_DAILY_R18, queryDate, 15, PixivDownload.PageQuality.ORIGINAL, (rank, link, rankInfo, inputStream) -> {
try {
ZipEntry entry = new ZipEntry("Rank" + rank + "-" + link.substring(link.lastIndexOf("/") + 1));
entry.setComment(rankInfo.toString());
log.info("正在写入: " + entry.getName());
zos.putNextEntry(entry);
IOUtils.copy(inputStream, zos);
zos.flush();
log.info("已成功写入 {}", entry.getName());
} catch (IOException e) {
log.error("写入文件项时发生异常", e);
}
});
} finally {
zos.finish();
zos.flush();
zos.close();
}
log.info("调用完成.");
}
@Test
public void getRankingIllustTest() throws IOException {
PixivDownload pixivDownload = new PixivDownload(cookieStore, proxy);
GregorianCalendar gregorianCalendar = new GregorianCalendar();
gregorianCalendar.setTime(new Date());
gregorianCalendar.add(Calendar.DATE, -1);
Date queryDate = gregorianCalendar.getTime();
String date = new SimpleDateFormat("yyyyMMdd").format(queryDate);
log.info("查询时间: {}", date);
int id = 1;
File outputFile = new File("ranking-" + date + "-" + id + ".zip");
while(outputFile.exists()) {
id++;
outputFile = new File("ranking-" + date + "-" + id + ".zip");
}
if(!outputFile.createNewFile()) {
Assert.fail("文件创建失败: " + outputFile.getAbsolutePath());
}
ZipOutputStream zos = new ZipOutputStream(new FileOutputStream(outputFile));
zos.setLevel(9);
log.info("正在调用方法...");
try {
pixivDownload.getRankingAsInputStream(null, null, queryDate, 65, 55, PixivDownload.PageQuality.ORIGINAL, (rank, link, rankInfo, inputStream) -> {
try {
ZipEntry entry = new ZipEntry("Rank" + rank + "-" + link.substring(link.lastIndexOf("/") + 1));
entry.setComment(rankInfo.toString());
log.info("正在写入: " + entry.getName());
zos.putNextEntry(entry);
IOUtils.copy(inputStream, zos);
zos.flush();
log.info("已成功写入 {}", entry.getName());
} catch (IOException e) {
log.error("写入文件项时发生异常", e);
}
});
} finally {
zos.finish();
zos.flush();
zos.close();
}
log.info("调用完成.");
}
@Test
public void getIllustPreLoadDataByIdTest() throws IOException {
log.info(new PixivDownload(cookieStore, proxy).getIllustPreLoadDataById(64076261).toString());
}
}

View File

@ -0,0 +1,15 @@
package net.lamgc.cgj.pixiv;
import org.junit.Test;
public class PixivSearchBuilderTest {
@Test
public void buildTest() {
PixivSearchBuilder builder = new PixivSearchBuilder("hololive");
//builder.addExcludeKeyword("fubuki").addExcludeKeyword("minato");
builder.addIncludeKeyword("35").addIncludeKeyword("okayu").addIncludeKeyword("百鬼あやめ");
System.out.println(builder.buildURL());
}
}