mirror of
https://github.com/LamGC/ContentGrabbingJi.git
synced 2025-04-30 06:37:36 +00:00
[Delete] 删除不再使用的类和方法;
[Update] 优化Javadoc并设置编译注解, 优化代码; [Change] 调整部分日志的输出级别; [Change] RedisPoolCacheStore clear方法将根据返回信息确定是否清空成功; [Change] Dockerfile.sample 调整镜像构建步骤;
This commit is contained in:
parent
0075446412
commit
fba6d3532c
@ -4,4 +4,5 @@ ENV jarFileName=ContentGrabbingJi-exec.jar
|
||||
COPY ${jarFileName} /program/CGJ.jar
|
||||
RUN mkdir /program/data
|
||||
|
||||
CMD cd /program/data && java -jar /program/CGJ.jar botMode
|
||||
WORKDIR /program/data
|
||||
CMD java -jar /program/CGJ.jar botMode
|
||||
@ -9,8 +9,9 @@ import com.google.gson.JsonElement;
|
||||
import com.google.gson.JsonObject;
|
||||
import net.lamgc.cgj.bot.framework.coolq.CQConfig;
|
||||
import net.lamgc.cgj.bot.framework.mirai.MiraiMain;
|
||||
import net.lamgc.cgj.pixiv.*;
|
||||
import net.lamgc.cgj.proxy.PixivAccessProxyServer;
|
||||
import net.lamgc.cgj.pixiv.PixivDownload;
|
||||
import net.lamgc.cgj.pixiv.PixivSearchBuilder;
|
||||
import net.lamgc.cgj.pixiv.PixivURL;
|
||||
import net.lamgc.plps.PixivLoginProxyServer;
|
||||
import net.lamgc.utils.base.ArgumentsProperties;
|
||||
import net.lamgc.utils.base.runner.Argument;
|
||||
@ -20,9 +21,7 @@ import org.apache.commons.io.IOUtils;
|
||||
import org.apache.http.HttpHost;
|
||||
import org.apache.http.HttpResponse;
|
||||
import org.apache.http.client.CookieStore;
|
||||
import org.apache.http.client.HttpClient;
|
||||
import org.apache.http.client.methods.HttpGet;
|
||||
import org.apache.http.message.BasicHeader;
|
||||
import org.apache.http.util.EntityUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
@ -355,86 +354,16 @@ public class Main {
|
||||
builder,
|
||||
PixivURL.getPixivRefererLink(illustId)
|
||||
);
|
||||
|
||||
/*log.info("正在下载...");
|
||||
List<String> list = PixivDownload.getIllustAllPageDownload(
|
||||
HttpClientBuilder.create()
|
||||
.setProxy(proxy)
|
||||
.build(),
|
||||
illustId, PixivDownload.PageQuality.ORIGINAL);*/
|
||||
}
|
||||
}
|
||||
/*
|
||||
|
||||
if(searchBuilder.getSearchArea().equals(PixivSearchBuilder.SearchArea.TOP)) {
|
||||
|
||||
} else {
|
||||
JsonArray illustsArray = resultBody
|
||||
.getAsJsonObject(searchBuilder.getSearchArea().jsonKey).getAsJsonArray("data");
|
||||
log.info("已找到与 {} 相关插图信息：", content);
|
||||
int count = 1;
|
||||
for (JsonElement jsonElement : illustsArray) {
|
||||
JsonObject illustObj = jsonElement.getAsJsonObject();
|
||||
//TODO: 防止数据内混入无效内容, 需要检查对象是否有illustId
|
||||
if(!illustObj.has("illustId")) {
|
||||
continue;
|
||||
}
|
||||
int illustId = illustObj.get("illustId").getAsInt();
|
||||
StringBuilder builder = new StringBuilder("[");
|
||||
illustObj.get("tags").getAsJsonArray().forEach(el -> builder.append(el.getAsString()).append(", "));
|
||||
builder.replace(builder.length() - 2, builder.length(), "]");
|
||||
log.info("({} / {})\n\t作品id: {}, \n\t作者名(作者id): {} ({}), \n\t作品标题: {}, \n\t作品Tags: {}, \n\t作品链接: {}",
|
||||
count++,
|
||||
illustsArray.size(),
|
||||
illustId,
|
||||
illustObj.get("userName").getAsString(),
|
||||
illustObj.get("userId").getAsInt(),
|
||||
illustObj.get("illustTitle").getAsString(),
|
||||
builder,
|
||||
PixivURL.getPixivRefererLink(illustId)
|
||||
);
|
||||
}
|
||||
}
|
||||
*/
|
||||
}
|
||||
|
||||
|
||||
@Command(defaultCommand = true)
|
||||
public static void testRun() {
|
||||
/*loadCookieStoreFromFile();
|
||||
|
||||
if(cookieStore == null){
|
||||
startPixivLoginProxyServer();
|
||||
}*/
|
||||
|
||||
//accessPixivToFile();
|
||||
|
||||
//startPixivAccessProxyServer();
|
||||
|
||||
//saveCookieStoreToFile();
|
||||
log.info("这里啥都没有哟w");
|
||||
}
|
||||
|
||||
private static void loadCookieStoreFromFile() throws IOException {
|
||||
log.info("正在加载CookieStore...");
|
||||
File storeFile = new File("./cookies.store");
|
||||
if(!storeFile.exists()){
|
||||
log.info("未找到CookieStore, 跳过加载.");
|
||||
return;
|
||||
}
|
||||
ObjectInputStream stream = new ObjectInputStream(new FileInputStream(storeFile));
|
||||
Object result;
|
||||
try {
|
||||
result = stream.readObject();
|
||||
} catch (ClassNotFoundException e) {
|
||||
log.error("加载出错", e);
|
||||
return;
|
||||
}
|
||||
cookieStore = (CookieStore) result;
|
||||
cookieStore.getCookies().forEach(cookie -> log.debug(cookie.getName() + ": " + cookie.getValue() + ", isExpired: " + cookie.isExpired(new Date())));
|
||||
log.info("CookieStore加载完成.");
|
||||
}
|
||||
|
||||
private static void saveCookieStoreToFile() throws IOException {
|
||||
log.info("正在保存CookieStore...");
|
||||
File outputFile = new File("./cookies.store");
|
||||
@ -484,75 +413,4 @@ public class Main {
|
||||
}
|
||||
}
|
||||
|
||||
private static void accessPixivToFile() throws IOException {
|
||||
File cookieStoreFile = new File("./cookie.txt");
|
||||
if (!cookieStoreFile.exists() && !cookieStoreFile.createNewFile()) {
|
||||
log.info("Cookie文件存储失败");
|
||||
}
|
||||
/*log.info("正在写出Cookie, Cookie count: " + cookieStore.getCookies().size());
|
||||
FileWriter cookieWriter = new FileWriter(cookieStoreFile);
|
||||
cookieStore.getCookies().forEach(cookie -> {
|
||||
try {
|
||||
StringBuilder sb = new StringBuilder().append(cookie.getName()).append(" = ").append(cookie.getValue());
|
||||
log.info("正在导出Cookie: " + sb.toString());
|
||||
cookieWriter.append(sb.toString()).append("\n").flush();
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
});
|
||||
log.info("Cookie写出完成");*/
|
||||
|
||||
log.info("尝试通过捕获的Cookie访问Pixiv...");
|
||||
HttpClient httpClient = new PixivSession(new HttpHost("127.0.0.1", 1080), cookieStore).getHttpClient();
|
||||
HttpGet request = new HttpGet("https://www.pixiv.net");
|
||||
request.addHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:73.0) Gecko/20100101 Firefox/73.0");
|
||||
request.addHeader(new BasicHeader("accept-encoding", "gzip, deflate, br"));
|
||||
request.addHeader(new BasicHeader("accept-language", "zh-CN,zh;q=0.9"));
|
||||
StringBuilder cookieBuilder = new StringBuilder();
|
||||
cookieStore.getCookies().forEach(cookie -> {
|
||||
if(cookie.isExpired(new Date())){
|
||||
return;
|
||||
}
|
||||
cookieBuilder.append(cookie.getName()).append("=").append(cookie.getValue()).append("; ");
|
||||
});
|
||||
request.addHeader("cookie", cookieBuilder.toString());
|
||||
|
||||
HttpResponse response = httpClient.execute(request);
|
||||
log.info("正在写入文件...");
|
||||
File outFile = new File("./pixiv.html");
|
||||
if (outFile.createNewFile() && !outFile.exists()) {
|
||||
log.info("文件创建失败!");
|
||||
}else {
|
||||
new FileWriter(outFile).append(EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8)).flush();
|
||||
}
|
||||
|
||||
Pixiv pixiv = new Pixiv(httpClient);
|
||||
pixiv.getRecommend().forEach(illustMap -> {
|
||||
StringBuilder builder = new StringBuilder();
|
||||
illustMap.forEach((key, value) -> builder.append(key).append(": ").append(value).append("\n"));
|
||||
try {
|
||||
builder.append("download Link: ").append(Arrays.toString(pixiv.getAllDownloadLink(Integer.parseInt(illustMap.get(Pixiv.ATTR_ILLUST_ID)))));
|
||||
} catch (IOException e) {
|
||||
log.error("获取下载链接时出错!", e);
|
||||
}
|
||||
log.info(builder.append("\n").toString());
|
||||
});
|
||||
|
||||
}
|
||||
|
||||
private static void startPixivAccessProxyServer(){
|
||||
log.info("正在启动访问代理服务器, 将浏览器相关缓存清空后, 使用浏览器进行访问以尝试Cookie正确性.");
|
||||
final PixivAccessProxyServer accessProxyServer = new PixivAccessProxyServer(cookieStore, new ProxyConfig(ProxyType.SOCKS5, "127.0.0.1", 1080));
|
||||
Thread accessProxyServerThread = new Thread(() -> {
|
||||
log.info("正在启动PAPS...");
|
||||
accessProxyServer.start(1007);
|
||||
log.info("PAPS已关闭.");
|
||||
});
|
||||
accessProxyServerThread.setName("AccessProxyServerThread");
|
||||
accessProxyServerThread.start();
|
||||
new Scanner(System.in).nextLine();
|
||||
log.info("关闭PAPS服务器...");
|
||||
accessProxyServer.close();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -684,7 +684,7 @@ public class BotCommandProcess {
|
||||
if (!illustPreLoadDataCache.exists(illustIdStr) || flushCache) {
|
||||
synchronized (illustIdStr) {
|
||||
if (!illustPreLoadDataCache.exists(illustIdStr) || flushCache) {
|
||||
log.info("IllustId {} 缓存失效, 正在更新...", illustId);
|
||||
log.debug("IllustId {} 缓存失效, 正在更新...", illustId);
|
||||
JsonObject preLoadDataObj = pixivDownload.getIllustPreLoadDataById(illustId)
|
||||
.getAsJsonObject("illust")
|
||||
.getAsJsonObject(Integer.toString(illustId));
|
||||
@ -700,7 +700,7 @@ public class BotCommandProcess {
|
||||
|
||||
result = preLoadDataObj;
|
||||
illustPreLoadDataCache.update(illustIdStr, preLoadDataObj, expire);
|
||||
log.info("作品Id {} preLoadData缓存已更新(有效时间: {})", illustId, expire);
|
||||
log.debug("作品Id {} preLoadData缓存已更新(有效时间: {})", illustId, expire);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -772,15 +772,16 @@ public class BotCommandProcess {
|
||||
if(!rankingCache.exists(requestSign) || flushCache) {
|
||||
synchronized(requestSign) {
|
||||
if(!rankingCache.exists(requestSign) || flushCache) {
|
||||
log.info("Ranking缓存失效, 正在更新...(RequestSign: {})", requestSign);
|
||||
log.debug("Ranking缓存失效, 正在更新...(RequestSign: {})", requestSign);
|
||||
List<JsonObject> rankingResult = pixivDownload.getRanking(contentType, mode, queryDate, 1, 500);
|
||||
long expireTime = 0;
|
||||
if(rankingResult.size() == 0) {
|
||||
log.info("数据获取失败, 将设置浮动有效时间以准备下次更新.");
|
||||
expireTime = 5400000 + expireTimeFloatRandom.nextInt(1800000);
|
||||
log.warn("数据获取失败, 将设置浮动有效时间以准备下次更新. (ExpireTime: {}ms)", expireTime);
|
||||
}
|
||||
result = new ArrayList<>(rankingResult).subList(start - 1, start + range - 1);
|
||||
rankingCache.update(requestSign, rankingResult,
|
||||
rankingResult.size() == 0 ? 5400000 + expireTimeFloatRandom.nextInt(1800000) : 0);
|
||||
log.info("Ranking缓存更新完成.(RequestSign: {})", requestSign);
|
||||
rankingCache.update(requestSign, rankingResult, expireTime);
|
||||
log.debug("Ranking缓存更新完成.(RequestSign: {})", requestSign);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -69,7 +69,7 @@ public class RandomRankingArtworksSender extends AutoSender {
|
||||
selectRanking,
|
||||
1, false);
|
||||
|
||||
log.info("RankingResult.size: {}", rankingList.size());
|
||||
log.debug("RankingResult.size: {}", rankingList.size());
|
||||
if(rankingList.size() != 1) {
|
||||
log.error("排行榜选取失败!(获取到了多个结果)");
|
||||
return;
|
||||
|
||||
@ -30,7 +30,7 @@ public class ImageCacheHandler implements EventHandler {
|
||||
@SuppressWarnings("unused")
|
||||
public void getImageToCache(ImageCacheObject event) {
|
||||
if(cacheQueue.contains(event)) {
|
||||
log.info("图片 {} 已存在相同缓存任务, 跳过.", event.getStoreFile().getName());
|
||||
log.debug("图片 {} 已存在相同缓存任务, 跳过.", event.getStoreFile().getName());
|
||||
return;
|
||||
} else {
|
||||
cacheQueue.add(event);
|
||||
@ -39,7 +39,7 @@ public class ImageCacheHandler implements EventHandler {
|
||||
try {
|
||||
log.info("图片 {} Event正在进行...({})", event.getStoreFile().getName(), Integer.toHexString(event.hashCode()));
|
||||
File storeFile = event.getStoreFile();
|
||||
log.info("正在缓存图片 {} (Path: {})", storeFile.getName(), storeFile.getAbsolutePath());
|
||||
log.debug("正在缓存图片 {} (Path: {})", storeFile.getName(), storeFile.getAbsolutePath());
|
||||
try {
|
||||
if(!storeFile.exists() && !storeFile.createNewFile()) {
|
||||
log.error("无法创建文件(Path: {})", storeFile.getAbsolutePath());
|
||||
@ -64,7 +64,7 @@ public class ImageCacheHandler implements EventHandler {
|
||||
return;
|
||||
}
|
||||
|
||||
log.info("正在下载...(Content-Length: {}KB)", response.getEntity().getContentLength() / 1024);
|
||||
log.debug("正在下载...(Content-Length: {}KB)", response.getEntity().getContentLength() / 1024);
|
||||
try(FileOutputStream fos = new FileOutputStream(storeFile)) {
|
||||
IOUtils.copy(response.getEntity().getContent(), fos);
|
||||
} catch (IOException e) {
|
||||
|
||||
@ -82,9 +82,7 @@ public abstract class RedisPoolCacheStore<T> implements CacheStore<T> {
|
||||
@Override
|
||||
public boolean clear() {
|
||||
try (Jedis jedis = jedisPool.getResource()) {
|
||||
String result = jedis.flushDB();
|
||||
log.info("flushDB返回结果: {}", result);
|
||||
return true;
|
||||
return jedis.flushDB().equalsIgnoreCase("ok");
|
||||
}
|
||||
}
|
||||
|
||||
@ -158,13 +156,4 @@ public abstract class RedisPoolCacheStore<T> implements CacheStore<T> {
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* 替换原本的分隔符(.)为(:).<br/>
|
||||
* 即将启用
|
||||
* @param key 要处理的键名
|
||||
* @return 处理后的键名
|
||||
*/
|
||||
public static String replaceKey(String key) {
|
||||
return key.replaceAll("\\.", ":");
|
||||
}
|
||||
}
|
||||
|
||||
@ -24,19 +24,16 @@ public class CQPluginMain extends CQPlugin implements EventHandler {
|
||||
|
||||
@Override
|
||||
public int onPrivateMessage(CoolQ cq, CQPrivateMessageEvent event) {
|
||||
//log.info("私聊消息到达: 发送者[{}], 消息内容: {}", event.getSender().getUserId(), event.getMessage());
|
||||
return processMessage(cq, event);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int onGroupMessage(CoolQ cq, CQGroupMessageEvent event) {
|
||||
//log.info("群消息到达: 群[{}], 发送者[{}], 消息内容: {}", event.getGroupId(), event.getSender().getUserId(), event.getMessage());
|
||||
return processMessage(cq, event);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int onDiscussMessage(CoolQ cq, CQDiscussMessageEvent event) {
|
||||
//log.info("讨论组消息到达: 群[{}], 发送者[{}], 消息内容: {}", event.getDiscussId(), event.getSender().getUserId(), event.getMessage());
|
||||
return processMessage(cq, event);
|
||||
}
|
||||
|
||||
|
||||
@ -156,7 +156,7 @@ public class MiraiMessageSender implements MessageSender {
|
||||
}
|
||||
}
|
||||
imageIdCache.update(imageName, image.getImageId(), expireTime);
|
||||
log.info("imageName [{}] 缓存更新完成.(有效时间: {})", imageName, expireTime);
|
||||
log.debug("imageName [{}] 缓存更新完成.(有效时间: {})", imageName, expireTime);
|
||||
} else {
|
||||
log.debug("ImageName: [{}] 缓存命中.", imageName);
|
||||
}
|
||||
|
||||
@ -1,69 +0,0 @@
|
||||
package net.lamgc.cgj.pixiv;
|
||||
|
||||
/**
|
||||
* 插图(集)的信息
|
||||
*/
|
||||
public class IllustInfo{
|
||||
|
||||
/**
|
||||
* 插图(集)ID
|
||||
*/
|
||||
public final int illustID;
|
||||
|
||||
/**
|
||||
* 第几页
|
||||
*/
|
||||
public final int page;
|
||||
|
||||
/**
|
||||
* 插图标题
|
||||
*/
|
||||
public final String title;
|
||||
|
||||
/**
|
||||
* 插图说明
|
||||
*/
|
||||
public final String description;
|
||||
|
||||
/**
|
||||
* 插图标签
|
||||
*/
|
||||
public final String[] tags;
|
||||
|
||||
/**
|
||||
* 插图图片长度
|
||||
*/
|
||||
public final int width;
|
||||
|
||||
/**
|
||||
* 插图图片高度
|
||||
*/
|
||||
public final int height;
|
||||
|
||||
/**
|
||||
* 作者名
|
||||
*/
|
||||
public final String authorName;
|
||||
|
||||
/**
|
||||
* 作者用户ID
|
||||
*/
|
||||
public final int authorUserID;
|
||||
|
||||
public IllustInfo(int illustID, String title, String description, String[] tags, int width, int height, String authorName, int authorUserID){
|
||||
this(illustID, 0, title, description, tags, width, height, authorName, authorUserID);
|
||||
}
|
||||
|
||||
public IllustInfo(int illustID, int p, String title, String description, String[] tags, int width, int height, String authorName, int authorUserID){
|
||||
this.illustID = illustID;
|
||||
this.page = p;
|
||||
this.title = title;
|
||||
this.description = description;
|
||||
this.tags = tags;
|
||||
this.width = width;
|
||||
this.height = height;
|
||||
this.authorName = authorName;
|
||||
this.authorUserID = authorUserID;
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,267 +0,0 @@
|
||||
package net.lamgc.cgj.pixiv;
|
||||
|
||||
import com.google.gson.JsonArray;
|
||||
import com.google.gson.JsonObject;
|
||||
import com.google.gson.JsonParser;
|
||||
import org.apache.http.HttpHost;
|
||||
import org.apache.http.HttpResponse;
|
||||
import org.apache.http.client.HttpClient;
|
||||
import org.apache.http.client.config.RequestConfig;
|
||||
import org.apache.http.client.methods.HttpGet;
|
||||
import org.apache.http.conn.ConnectionPoolTimeoutException;
|
||||
import org.apache.http.impl.client.HttpClientBuilder;
|
||||
import org.apache.http.util.EntityUtils;
|
||||
import org.jsoup.Jsoup;
|
||||
import org.jsoup.nodes.Document;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.*;
|
||||
|
||||
public class Pixiv {
|
||||
|
||||
/**
|
||||
* illust Link
|
||||
*/
|
||||
public final static String ATTR_LINK = "link";
|
||||
|
||||
/**
|
||||
* illust Id
|
||||
*/
|
||||
public final static String ATTR_ILLUST_ID = "illustId";
|
||||
|
||||
/**
|
||||
* illust Title
|
||||
*/
|
||||
public final static String ATTR_TITLE = "title";
|
||||
|
||||
/**
|
||||
* illust Author Name
|
||||
*/
|
||||
public final static String ATTR_AUTHOR_NAME = "authorName";
|
||||
|
||||
/**
|
||||
* illust Author UserID
|
||||
*/
|
||||
public final static String ATTR_AUTHOR_ID = "authorId";
|
||||
|
||||
private final HttpClient httpClient;
|
||||
|
||||
public Pixiv(HttpClient client){
|
||||
this.httpClient = client;
|
||||
}
|
||||
|
||||
/**
|
||||
* 使用帐号密码登录Pixiv
|
||||
* @param PixivID Pixiv账户登录名
|
||||
* @param Password Pixiv帐号密码
|
||||
* @throws IOException 当登录连接出现异常时抛出
|
||||
* @deprecated {@link PixivSession#Login(String, String)} 已经废除, 故本方法不可用
|
||||
*/
|
||||
public Pixiv(String PixivID, String Password) throws IOException {
|
||||
this(PixivID, Password, null);
|
||||
}
|
||||
|
||||
/**
|
||||
* 使用帐号密码登录Pixiv
|
||||
* @param PixivID Pixiv账户登录名
|
||||
* @param Password Pixiv帐号密码
|
||||
* @param proxy 代理设置
|
||||
* @throws IOException 当登录连接出现异常时抛出
|
||||
* @deprecated {@link PixivSession#Login(String, String)} 已经废除, 故本方法不可用
|
||||
*/
|
||||
public Pixiv(String PixivID, String Password, HttpHost proxy) throws IOException {
|
||||
PixivSession pixivSession = new PixivSession(proxy, null);
|
||||
if(pixivSession.Login(PixivID, Password)){
|
||||
System.out.println("P站登录成功!");
|
||||
}else{
|
||||
System.out.println("P站登录失败!错误信息: " + pixivSession.getErrMsg());
|
||||
throw new RuntimeException(pixivSession.getErrMsg());
|
||||
}
|
||||
//httpClient = pixivSession.getHttpClient();
|
||||
httpClient = HttpClientBuilder.create()
|
||||
.setDefaultCookieStore(pixivSession.getCookieStore())
|
||||
.build();
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取首页推荐列表
|
||||
* @return 首页推荐列表, 一个Map对应一个推荐项, 使用<code>ATTR_</code>开头常量访问即可
|
||||
* @throws IOException
|
||||
*/
|
||||
public List<Map<String, String>> getRecommend() throws IOException {
|
||||
HttpGet getRequest = new HttpGet(PixivURL.PIXIV_INDEX_URL);
|
||||
HttpResponse response = httpClient.execute(getRequest);
|
||||
String pageAsXML = EntityUtils.toString(response.getEntity(),"utf-8");
|
||||
|
||||
//获取推荐图列表(li)
|
||||
//System.out.println(pageAsXML);
|
||||
|
||||
Document document = Jsoup.parse(pageAsXML);
|
||||
|
||||
List<String> links = document.select("._image-items.gtm-illust-recommend-zone>li>.gtm-illust-recommend-thumbnail-link").eachAttr("href");
|
||||
List<String> illustId = document.select("._image-items.gtm-illust-recommend-zone>li>.gtm-illust-recommend-thumbnail-link").eachAttr("data-gtm-recommend-illust-id");
|
||||
List<String> title = document.select("._image-items.gtm-illust-recommend-zone>li>.gtm-illust-recommend-title>h1").eachAttr("title");
|
||||
List<String> authorName = document.select("._image-items.gtm-illust-recommend-zone>li>.gtm-illust-recommend-user-name").eachText();
|
||||
List<String> authorId = document.select("._image-items.gtm-illust-recommend-zone>li>.gtm-illust-recommend-user-name").eachAttr("data-user_id");
|
||||
|
||||
List<Map<String, String>> recommendList = new ArrayList<>();
|
||||
for(int i = 0; i < links.size(); i++){
|
||||
//System.out.println(links.get(i));
|
||||
Map<String, String> info = new HashMap<>();
|
||||
info.put(ATTR_LINK, PixivURL.PIXIV_INDEX_URL + links.get(i));
|
||||
info.put(ATTR_ILLUST_ID, illustId.get(i));
|
||||
info.put(ATTR_TITLE, title.get(i));
|
||||
info.put(ATTR_AUTHOR_NAME, authorName.get(i));
|
||||
info.put(ATTR_AUTHOR_ID, authorId.get(i));
|
||||
recommendList.add(info);
|
||||
}
|
||||
return recommendList;
|
||||
}
|
||||
|
||||
public String[] getAllDownloadLink(int illustID) throws IOException {
|
||||
HttpGet illustPage = new HttpGet(PixivURL.PIXIV_ILLUST_API_URL.replaceAll("\\{illustId}", String.valueOf(illustID)));
|
||||
HttpResponse response = httpClient.execute(illustPage);
|
||||
String pageAsXML = EntityUtils.toString(response.getEntity(),"utf-8");
|
||||
//System.out.println(pageAsXML);
|
||||
JsonObject resultObj = (JsonObject) new JsonParser().parse(pageAsXML);
|
||||
if(!resultObj.get("error").getAsBoolean()){
|
||||
JsonArray bodyArray = resultObj.get("body").getAsJsonArray();
|
||||
int length = bodyArray.size();
|
||||
String[] result = new String[length];
|
||||
for(int i = 0; i < length; i++){
|
||||
JsonObject childObj = bodyArray.get(i).getAsJsonObject();
|
||||
result[i] = childObj.get("urls").getAsJsonObject().get("original").getAsString();
|
||||
}
|
||||
return result;
|
||||
}else{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* 下载P站图片
|
||||
* @param illustID 插图ID
|
||||
* @return 成功返回图片输入流,失败或为多图则返回null
|
||||
*/
|
||||
public InputStream[] downloadIllustImage(int illustID) throws IOException {
|
||||
String[] links = getAllDownloadLink(illustID);
|
||||
List<InputStream> inputStreamList = new ArrayList<>();
|
||||
int count = 1;
|
||||
boolean retry = false;
|
||||
for(int i = 0; i < links.length; i++){
|
||||
try {
|
||||
long sleepTime = (new Random().nextInt(4) + 2) * 1000;
|
||||
System.out.println("nextTime: " + (float)(sleepTime / 1000));
|
||||
Thread.sleep(sleepTime);
|
||||
} catch (InterruptedException ignored) {}
|
||||
String link = links[i];
|
||||
System.out.print("page:" + count++ + "/" + links.length + " ...");
|
||||
HttpGet imgGet = new HttpGet(link);
|
||||
//关键!如果不加上Referer的话,会返回403
|
||||
imgGet.setHeader("Referer", PixivURL.PIXIV_ILLUST_MEDIUM_URL.replaceAll("\\{illustId}", String.valueOf(illustID)));
|
||||
RequestConfig config = RequestConfig.custom()
|
||||
.setConnectTimeout(20 * 1000)
|
||||
.setConnectionRequestTimeout(20 * 1000)
|
||||
.setSocketTimeout(30 * 1000)
|
||||
.build();
|
||||
imgGet.setConfig(config);
|
||||
HttpResponse response;
|
||||
try {
|
||||
response = httpClient.execute(imgGet);
|
||||
}catch(ConnectionPoolTimeoutException e){
|
||||
if(retry){
|
||||
retry = false;
|
||||
System.out.println("获取失败,跳过...");
|
||||
continue;
|
||||
}
|
||||
System.out.println("连接超时,重新获取...");
|
||||
retry = true;
|
||||
i--;
|
||||
continue;
|
||||
}
|
||||
retry = false;
|
||||
|
||||
ByteArrayOutputStream cacheOutputStream = new ByteArrayOutputStream((int)response.getEntity().getContentLength());
|
||||
InputStream content = response.getEntity().getContent();
|
||||
int readLength;
|
||||
byte[] cache = new byte[4096];
|
||||
while((readLength = content.read(cache)) != -1){
|
||||
cacheOutputStream.write(cache, 0, readLength);
|
||||
}
|
||||
byte[] data = cacheOutputStream.toByteArray();
|
||||
//System.out.println("读到数据: " + data.length);
|
||||
inputStreamList.add(new ByteArrayInputStream(data));
|
||||
System.out.println("done!（length: " + response.getEntity().getContentLength() + ")");
|
||||
|
||||
}
|
||||
return inputStreamList.toArray(new InputStream[0]);
|
||||
}
|
||||
|
||||
/**
|
||||
* 下载P站图片
|
||||
* @param illustID 插图ID
|
||||
* @return 成功返回图片输入流,失败或为多图则返回null
|
||||
*/
|
||||
public InputStream downloadIllustImages(int illustID){
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
/**
|
||||
* 通过解析插图详情页获取
|
||||
* - 插图标题
|
||||
* - 插图作者(及其UserId)
|
||||
* - 插图上传时间
|
||||
* - 插图标签(原始标签)
|
||||
* ...
|
||||
* @return 成功返回IllustInfo对象,失败返回null
|
||||
*/
|
||||
public IllustInfo[] getIllustInfo(int[] illustIDs) throws IOException {
|
||||
//获取Api
|
||||
HttpGet apiRequest = new HttpGet(PixivURL.getPixivIllustInfoAPI(illustIDs));
|
||||
HttpResponse response = httpClient.execute(apiRequest);
|
||||
String resultText = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8);
|
||||
System.out.println(resultText);
|
||||
JsonObject resultObj = ((JsonObject) new JsonParser().parse(resultText));
|
||||
if(resultObj.get("error").getAsBoolean()){
|
||||
System.err.println("获取失败!");
|
||||
return null;
|
||||
}
|
||||
List<IllustInfo> illustInfoList = new ArrayList<>();
|
||||
JsonArray illustArray = resultObj.get("body").getAsJsonObject().get("illusts").getAsJsonArray();
|
||||
illustArray.forEach(jsonElement -> {
|
||||
JsonObject illustInfoObj = jsonElement.getAsJsonObject();
|
||||
JsonArray tagsArray = illustInfoObj.get("tags").getAsJsonArray();
|
||||
String[] tags = new String[tagsArray.size()];
|
||||
for(int i = 0; i < tags.length; i++){
|
||||
tags[i] = tagsArray.get(i).getAsString();
|
||||
}
|
||||
//TODO: 通过不需要作者id就能获取图片信息的api无法获取图片尺寸
|
||||
IllustInfo illustInfo = new IllustInfo(
|
||||
illustInfoObj.get("workId").getAsInt(),
|
||||
illustInfoObj.get("title").getAsString(),
|
||||
null,
|
||||
tags,
|
||||
-1,
|
||||
-1,
|
||||
illustInfoObj.get("userName").getAsString(),
|
||||
illustInfoObj.get("userId").getAsInt()
|
||||
);
|
||||
});
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取指定用户的所有插画
|
||||
*/
|
||||
public void getUserAllIllustTest() {
|
||||
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@ -9,6 +9,13 @@ import java.util.Date;
|
||||
import java.util.HashSet;
|
||||
import java.util.Objects;
|
||||
|
||||
/**
|
||||
* Pixiv搜索URL构造器
|
||||
* <p>该构造器通过分析Pixiv搜索链接可用的参数而开发, 对搜索链接的构造有高度自定义能力.</p>
|
||||
* @author LamGC
|
||||
* @see PixivURL#PIXIV_SEARCH_CONTENT_URL
|
||||
*/
|
||||
@SuppressWarnings("ALL")
|
||||
public class PixivSearchBuilder {
|
||||
|
||||
private final String content;
|
||||
|
||||
@ -1,171 +0,0 @@
|
||||
package net.lamgc.cgj.pixiv;
|
||||
|
||||
import com.google.gson.JsonObject;
|
||||
import com.google.gson.JsonParser;
|
||||
import org.apache.http.Header;
|
||||
import org.apache.http.HttpHost;
|
||||
import org.apache.http.HttpResponse;
|
||||
import org.apache.http.NameValuePair;
|
||||
import org.apache.http.client.CookieStore;
|
||||
import org.apache.http.client.HttpClient;
|
||||
import org.apache.http.client.entity.UrlEncodedFormEntity;
|
||||
import org.apache.http.client.methods.HttpGet;
|
||||
import org.apache.http.client.methods.HttpPost;
|
||||
import org.apache.http.impl.client.BasicCookieStore;
|
||||
import org.apache.http.impl.client.HttpClients;
|
||||
import org.apache.http.message.BasicHeader;
|
||||
import org.apache.http.message.BasicNameValuePair;
|
||||
import org.apache.http.util.EntityUtils;
|
||||
import org.jsoup.Jsoup;
|
||||
import org.jsoup.nodes.Document;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
public class PixivSession {
|
||||
|
||||
public static final String USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36";
|
||||
|
||||
/**
|
||||
* 全登陆过程的关键,
|
||||
* 保存会话用的cookieStore!
|
||||
*/
|
||||
private CookieStore cookieStore = new BasicCookieStore();
|
||||
|
||||
/**
|
||||
* 可以直接使用的HttpClient对象
|
||||
*/
|
||||
private HttpClient httpClient;
|
||||
|
||||
/**
|
||||
* 最后一次登录的错误信息
|
||||
*/
|
||||
private String errMsg;
|
||||
|
||||
public PixivSession(){
|
||||
this(null);
|
||||
}
|
||||
|
||||
public PixivSession(CookieStore cookieStore){
|
||||
this(null, cookieStore);
|
||||
}
|
||||
|
||||
/**
|
||||
* 创建一个Pixiv登录会话
|
||||
*/
|
||||
public PixivSession(HttpHost proxy, CookieStore cookieStore) {
|
||||
if(cookieStore != null){
|
||||
this.cookieStore = cookieStore;
|
||||
}
|
||||
List<Header> defaultHeader = new ArrayList<>();
|
||||
defaultHeader.add(new BasicHeader("accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9"));
|
||||
defaultHeader.add(new BasicHeader("user-agent", PixivSession.USER_AGENT));
|
||||
defaultHeader.add(new BasicHeader("accept-encoding", "gzip, deflate, br"));
|
||||
defaultHeader.add(new BasicHeader("accept-language", "zh-CN,zh;q=0.9"));
|
||||
|
||||
/*defaultHeader.add(new BasicHeader("sec-fetch-mode", "navigate"));
|
||||
defaultHeader.add(new BasicHeader("sec-fetch-site", "same-origin"));
|
||||
defaultHeader.add(new BasicHeader("upgrade-insecure-requests", "1"));*/
|
||||
//创建一个Http访问器
|
||||
httpClient = HttpClients.custom()
|
||||
.setDefaultCookieStore(cookieStore)
|
||||
.setDefaultHeaders(defaultHeader)
|
||||
.setProxy(proxy)
|
||||
.build();
|
||||
}
|
||||
|
||||
/**
|
||||
* 程序自行通过帐号密码登录Pixiv.
|
||||
* @param PixivID Pixiv帐号
|
||||
* @param Password Pixiv密码
|
||||
* @return 登录成功返回true
|
||||
* @throws IOException 当登录抛出异常时返回
|
||||
* @deprecated Pixiv已经新增Google人机验证, 程序已无法自行登录Pixiv
|
||||
*/
|
||||
public boolean Login(String PixivID, String Password) throws IOException {
|
||||
// 获取登录接口所需的PostKey
|
||||
String post_key = getPostKey();
|
||||
HttpPost postRequest = new HttpPost(PixivURL.PIXIV_LOGIN_URL); //https://accounts.pixiv.net/api/login?lang=zh
|
||||
List<NameValuePair> params = new ArrayList<>();
|
||||
params.add(new BasicNameValuePair("pixiv_id", PixivID));
|
||||
params.add(new BasicNameValuePair("password", Password));
|
||||
params.add(new BasicNameValuePair("post_key", post_key));
|
||||
//Form编码表单,作为Post的数据
|
||||
postRequest.setEntity(new UrlEncodedFormEntity(params, StandardCharsets.UTF_8));
|
||||
//启动访问
|
||||
HttpResponse response = httpClient.execute(postRequest);
|
||||
//获取接口返回数据
|
||||
String httpXML = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8);
|
||||
System.out.println(httpXML);
|
||||
JsonObject responseJson = (JsonObject) new JsonParser().parse(httpXML);
|
||||
if(!responseJson.get("error").getAsBoolean() && !responseJson.get("body").getAsJsonObject().has("validation_errors")){
|
||||
errMsg = null;
|
||||
return true;
|
||||
}else{
|
||||
errMsg = responseJson.get("body").getAsJsonObject().get("validation_errors").toString();
|
||||
//System.err.println("登录失败！MSG: " + errMsg);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 登录前准备, 获取PostKey
|
||||
* @return Post_Key
|
||||
*/
|
||||
private String getPostKey() throws IOException {
|
||||
//创建请求,获取PostKey
|
||||
HttpGet getRequest = new HttpGet(PixivURL.PIXIV_LOGIN_PAGE_URL);
|
||||
//设置请求
|
||||
//getRequest.setConfig(config);
|
||||
getRequest.setHeader("User-Agent", USER_AGENT);
|
||||
//启动访问
|
||||
HttpResponse response = httpClient.execute(getRequest);
|
||||
//获取网页内容
|
||||
String pageAsXML = EntityUtils.toString(response.getEntity(),"utf-8");
|
||||
//创建Http解析器
|
||||
Document document = Jsoup.parse(pageAsXML);
|
||||
//获取init-config内容
|
||||
String init_config = document.getElementById("init-config").val();
|
||||
//System.out.println(init_config);
|
||||
//创建Json解析器解析init-config
|
||||
JsonObject initConfigObj = (JsonObject) new JsonParser().parse(init_config);
|
||||
//检查是否有postKey
|
||||
if(!initConfigObj.has("pixivAccount.postKey")){
|
||||
throw new RuntimeException("postKey获取失败!可能是Pixiv修改了登录过程!");
|
||||
}
|
||||
//获取postKey
|
||||
return initConfigObj.get("pixivAccount.postKey").getAsString();
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取CookieStore
|
||||
* @return CookieStore
|
||||
*/
|
||||
public CookieStore getCookieStore(){
|
||||
return cookieStore;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取可直接使用的HttpClient对象
|
||||
* @return 已配置好的HttpClient对象
|
||||
*/
|
||||
public HttpClient getHttpClient(){
|
||||
return this.httpClient;
|
||||
}
|
||||
|
||||
public boolean hasError(){
|
||||
return errMsg == null;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取错误信息
|
||||
* @return 返回登录错误信息
|
||||
* @deprecated {@link #Login(String, String)}已经废除, 故本接口废除
|
||||
*/
|
||||
public String getErrMsg(){
|
||||
return errMsg;
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,16 +0,0 @@
|
||||
package net.lamgc.cgj.pixiv;
|
||||
|
||||
public class PixivTag {
|
||||
|
||||
public static PixivTag create(String tagName) {
|
||||
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private PixivTag() {
|
||||
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@ -9,6 +9,7 @@ import java.util.GregorianCalendar;
|
||||
/**
|
||||
* 目前已整理的一些Pixiv接口列表
|
||||
*/
|
||||
@SuppressWarnings("unused")
|
||||
public class PixivURL {
|
||||
|
||||
|
||||
|
||||
@ -26,8 +26,7 @@ import java.util.zip.ZipEntry;
|
||||
import java.util.zip.ZipInputStream;
|
||||
|
||||
/**
|
||||
* Pixiv动图构建器.
|
||||
* 可便捷的接收并处理动图.
|
||||
* Pixiv动图构建器
|
||||
*/
|
||||
public final class PixivUgoiraBuilder {
|
||||
|
||||
|
||||
@ -1,31 +0,0 @@
|
||||
package net.lamgc.cgj.util;
|
||||
|
||||
import org.apache.http.client.CookieStore;
|
||||
import org.apache.http.impl.client.BasicCookieStore;
|
||||
import org.apache.http.impl.cookie.BasicClientCookie;
|
||||
|
||||
import java.util.Date;
|
||||
|
||||
public class CookieUtil {
|
||||
|
||||
/**
|
||||
* 将{@link java.net.CookieStore}转换到{@link CookieStore}
|
||||
* @param netCookieStore java.net.CookieStore
|
||||
* @return org.apache.http.client.CookieStore
|
||||
*/
|
||||
public static CookieStore NAParse(java.net.CookieStore netCookieStore){
|
||||
CookieStore apacheCookieStore = new BasicCookieStore();
|
||||
netCookieStore.getCookies().forEach(netCookie -> {
|
||||
BasicClientCookie aCookie = new BasicClientCookie(netCookie.getName(), netCookie.getValue());
|
||||
aCookie.setComment(netCookie.getComment());
|
||||
aCookie.setDomain(netCookie.getDomain());
|
||||
aCookie.setExpiryDate(new Date(netCookie.getMaxAge()));
|
||||
aCookie.setPath(netCookie.getPath());
|
||||
aCookie.setSecure(netCookie.getSecure());
|
||||
aCookie.setVersion(netCookie.getVersion());
|
||||
apacheCookieStore.addCookie(aCookie);
|
||||
});
|
||||
return apacheCookieStore;
|
||||
}
|
||||
|
||||
}
|
||||
@ -7,7 +7,7 @@ import net.lamgc.utils.base.runner.StringParameterParser;
|
||||
public class PagesQualityParser implements StringParameterParser<PixivDownload.PageQuality> {
|
||||
|
||||
@Override
|
||||
public PixivDownload.PageQuality parse(String strValue) throws Exception {
|
||||
public PixivDownload.PageQuality parse(String strValue) {
|
||||
return PixivDownload.PageQuality.valueOf(strValue.toUpperCase());
|
||||
}
|
||||
|
||||
|
||||
@ -181,19 +181,6 @@ public class PixivDownloadTest {
|
||||
log.info("正在调用方法...");
|
||||
try {
|
||||
pixivDownload.getRankingAsInputStream(null, null, queryDate, 5, 50, PixivDownload.PageQuality.ORIGINAL, (rank, link, rankInfo, inputStream) -> {
|
||||
/*try {
|
||||
|
||||
ZipEntry entry = new ZipEntry("Rank" + rank + "-" + link.substring(link.lastIndexOf("/") + 1));
|
||||
entry.setComment(rankInfo.toString());
|
||||
log.info("正在写入: " + entry.getName());
|
||||
zos.putNextEntry(entry);
|
||||
IOUtils.copy(inputStream, zos);
|
||||
zos.flush();
|
||||
log.info("已成功写入 {}", entry.getName());
|
||||
inputStream.close();
|
||||
} catch (IOException e) {
|
||||
log.error("写入文件项时发生异常", e);
|
||||
}*/
|
||||
log.info("空操作");
|
||||
});
|
||||
} finally {
|
||||
|
||||
Loading…
Reference in New Issue
Block a user