LamGC/ContentGrabbingJi
1
0
Fork 0
mirror of https://github.com/LamGC/ContentGrabbingJi.git synced 2025-04-30 06:37:36 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master'

Browse Source
This commit is contained in:
LamGC 2020-05-22 20:34:32 +08:00
commit 33d18cef6b
13 changed files with 202 additions and 92 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
.gitignore vendored
View File

@ -1,6 +1,16 @@
# Ignore test date folder
/pluginData/
/logs/
/.idea/
/CGJ_2.iml
/cookies.store
/target/
# Ignore Idea files
/.idea/
/CGJ_2.iml
# Ignore Visual Studio Code files
.classpath
.factorypath
.project
/.settings/
/.vscode/

35
README.md Normal file
View File

@ -0,0 +1,35 @@
# ContentGrabbingJi
Pixiv爬虫一只，同时也是一个机器人/插件！
## 支持的机器人平台 ##
- [Mirai](https://github.com/mamoe/mirai)
- [CoolQ](https://cqp.cc)(基于`SpringCQ`, 不支持多账号使用, 需要使用`CQHttp`插件)
## Usage ##
> 注意: 运行色图姬前, 你需要准备一个Pixiv账号的会话Cookie存储文件, 否则色图姬将无法运行.
### Arguments ###
- 通用参数
- `proxy` / `CGJ_PROXY`: 设置代理
- 格式: `协议://地址:端口`
- 示例: `socks5://127.0.0.1:1080`
- 机器人参数
- `botDataDir` / `CGJ_BOT_DATA_DIR`: 设置`botMode`运行模式下机器人数据存储目录
- 格式: `路径`
- 示例: `./data`
- 默认: `./`
- `redisAddress` / `CGJ_REDIS_URI`: Redis服务器地址
- 格式: `地址:端口`
- 示例: `127.0.0.1:6379`
### Commands ###
- `pluginMode`: CoolQ插件模式
- `botMode`: Mirai独立模式
- `collectionDownload`: 收藏下载, 以原图画质下载Cookie所属账号的所有收藏作品
- `getRecommends`: 将访问主页获得的推荐作品全部以原图画质下载
- `getRankingIllust`: 以原图画质下载指定排行榜类型的全部作品
- `search`: 搜索指定内容并获取相关作品信息(不下载)

2
pom.xml
View File

@ -6,7 +6,7 @@
<groupId>net.lamgc</groupId>
<artifactId>ContentGrabbingJi</artifactId>
<version>2.5.2-20200517.1-SNAPSHOT</version>
<version>2.5.2-20200520.1-SNAPSHOT</version>
<repositories>
<repository>

50
src/main/java/net/lamgc/cgj/Main.java
View File

@ -86,13 +86,15 @@ public class Main {
File cookieStoreFile = new File(System.getProperty("cgj.botDataDir"), "cookies.store");
if(!cookieStoreFile.exists()) {
log.warn("未找到cookies.store文件, 是否启动PixivLoginProxyServer? (yes/no)");
Scanner scanner = new Scanner(System.in);
if(scanner.nextLine().trim().equalsIgnoreCase("yes")) {
startPixivLoginProxyServer();
} else {
System.exit(1);
return;
try(Scanner scanner = new Scanner(System.in)) {
if(scanner.nextLine().trim().equalsIgnoreCase("yes")) {
startPixivLoginProxyServer();
} else {
System.exit(1);
return;
}
}
}
ObjectInputStream ois = new ObjectInputStream(new FileInputStream(cookieStoreFile));
cookieStore = (CookieStore) ois.readObject();
@ -137,7 +139,9 @@ public class Main {
@Command
public static void botMode(@Argument(name = "args", force = false) String argsStr) {
new MiraiMain().init();
MiraiMain main = new MiraiMain();
main.init();
main.close();
}
@Command
@ -423,23 +427,25 @@ public class Main {
proxyServerStartThread.setName("LoginProxyServerThread");
proxyServerStartThread.start();
//System.console().readLine();
Scanner scanner = new Scanner(System.in);
log.info("登录完成后, 使用\"done\"命令结束登录过程.");
while(true) {
if (scanner.nextLine().equalsIgnoreCase("done")) {
log.info("关闭PLPS服务器...");
proxyServer.close();
cookieStore = proxyServer.getCookieStore();
try {
log.info("正在保存CookieStore...");
saveCookieStoreToFile();
log.info("CookieStore保存完成.");
} catch (IOException e) {
log.error("CookieStore保存时发生异常, 本次CookieStore仅可在本次运行使用.", e);
try(Scanner scanner = new Scanner(System.in)) {
while(true) {
if (scanner.nextLine().equalsIgnoreCase("done")) {
log.info("关闭PLPS服务器...");
proxyServer.close();
cookieStore = proxyServer.getCookieStore();
try {
log.info("正在保存CookieStore...");
saveCookieStoreToFile();
log.info("CookieStore保存完成.");
} catch (IOException e) {
log.error("CookieStore保存时发生异常, 本次CookieStore仅可在本次运行使用.", e);
}
break;
} else {
log.warn("要结束登录过程, 请使用\"done\"命令.");
}
break;
} else {
log.warn("要结束登录过程, 请使用\"done\"命令.");
}
}
}

57
src/main/java/net/lamgc/cgj/bot/AutoArtworksSender.java
View File

@ -1,57 +0,0 @@
package net.lamgc.cgj.bot;
import net.lz1998.cq.robot.CoolQ;
import org.apache.http.client.methods.HttpGet;
import java.util.Random;
import java.util.Timer;
import java.util.TimerTask;
public class AutoArtworksSender {
private final CoolQ CQ;
private final ReceiveType receiveType;
private final long targetReceiveId;
private Timer timer = new Timer();
private TimerTask task = new TimerTask() {
@Override
public void run() {
HttpGet request = new HttpGet();
// https://api.imjad.cn/pixiv/v2/?type=tags
}
};
public AutoArtworksSender(CoolQ cq, ReceiveType receiveType, long receiveId) {
this.CQ = cq;
this.receiveType = receiveType;
this.targetReceiveId = receiveId;
}
public void reset(long time) {
if(time <= 0) {
timer.schedule(task, new Random().nextInt(10 * 60 * 60 * 1000) + 7200000L); //2H ~ 12H
} else {
timer.schedule(task, time);
}
}
public void sendMessage(String message, boolean auto_escape) {
switch (receiveType) {
case GROUP:
CQ.sendGroupMsg(targetReceiveId, message, auto_escape);
break;
case Discuss:
CQ.sendDiscussMsg(targetReceiveId, message, auto_escape);
break;
case PRIVATE:
CQ.sendPrivateMsg(targetReceiveId, message, auto_escape);
break;
}
}
public enum ReceiveType {
PRIVATE, GROUP, Discuss
}
}

16
src/main/java/net/lamgc/cgj/bot/BotCommandProcess.java
View File

@ -7,10 +7,14 @@ import io.netty.handler.codec.http.HttpHeaderNames;
import net.lamgc.cgj.Main;
import net.lamgc.cgj.bot.cache.*;
import net.lamgc.cgj.bot.event.BotEventHandler;
import net.lamgc.cgj.bot.event.BufferMessageEvent;
import net.lamgc.cgj.bot.sort.PreLoadDataComparator;
import net.lamgc.cgj.pixiv.PixivDownload;
import net.lamgc.cgj.pixiv.PixivSearchBuilder;
import net.lamgc.cgj.pixiv.PixivURL;
import net.lamgc.cgj.pixiv.PixivDownload.PageQuality;
import net.lamgc.cgj.pixiv.PixivURL.RankingContentType;
import net.lamgc.cgj.pixiv.PixivURL.RankingMode;
import net.lamgc.cgj.util.URLs;
import net.lamgc.utils.base.runner.Argument;
import net.lamgc.utils.base.runner.Command;
@ -296,6 +300,18 @@ public class BotCommandProcess {
return "功能未完成";
}
@Command(commandName = "st")
public static String r18Image() {
BufferMessageEvent event = new BufferMessageEvent();
RandomRankingArtworksSender artworksSender =
new RandomRankingArtworksSender(event, 1, 200,
RankingMode.MODE_MALE,
RankingContentType.TYPE_ALL,
PageQuality.ORIGINAL);
artworksSender.send();
return event.getBufferMessage();
}
/**
* 搜索命令
* @param fromGroup 来源群(系统提供)

12
src/main/java/net/lamgc/cgj/bot/MessageEventExecutionDebugger.java
View File

@ -26,13 +26,19 @@ public enum MessageEventExecutionDebugger {
try {
rotation = Integer.parseInt(properties.getProperty("debug.pm.rotation", "5"));
} catch(NumberFormatException ignored) {}
} catch(NumberFormatException e) {
log.warn("配置项 {} 值无效, 将使用默认值.({})", "debug.pm.rotation", rotation);
}
try {
number = Integer.parseInt(properties.getProperty("debug.pm.number", "50"));
} catch(NumberFormatException ignored) {}
} catch(NumberFormatException e) {
log.warn("配置项 {} 值无效, 将使用默认值.({})", "debug.pm.number", number);
}
try {
interval = Integer.parseInt(properties.getProperty("debug.pm.interval", "2500"));
} catch(NumberFormatException ignored) {}
} catch(NumberFormatException e) {
log.warn("配置项 {} 值无效, 将使用默认值.({})", "debug.pm.interval", interval);
}
boolean interrupted = false;
Thread currentThread = Thread.currentThread();

3
src/main/java/net/lamgc/cgj/bot/cache/ImageCacheStore.java vendored
View File

@ -8,7 +8,6 @@ import java.util.Hashtable;
import java.util.Map;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicReference;
import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.ReentrantLock;
public final class ImageCacheStore {
@ -102,8 +101,6 @@ public final class ImageCacheStore {
public final AtomicReference<TaskState> taskState = new AtomicReference<>(TaskState.READY);
public final Condition condition = lock.newCondition();
}
}

41
src/main/java/net/lamgc/cgj/bot/cache/LocalHashCacheStore.java vendored
View File

@ -12,19 +12,45 @@ import java.util.concurrent.atomic.AtomicReference;
* 基于Hashtable的本地缓存库
* @param <T> 缓存类型
*/
public class LocalHashCacheStore<T> implements CacheStore<T> {
public class LocalHashCacheStore<T> implements CacheStore<T>, Cleanable {
private final Hashtable<String, CacheObject<T>> cache;
/**
* 构造一个基于Hashtable的本地缓存库
* @see Hashtable
*/
public LocalHashCacheStore() {
this(0);
}
/**
* 构造一个基于Hashtable的本地缓存库
* @param initialCapacity 初始容量
* @see Hashtable
*/
public LocalHashCacheStore(int initialCapacity) {
this(initialCapacity, 0F);
}
/**
* 构造一个基于Hashtable的本地缓存库
* @param initialCapacity 初始容量
* @param loadFactor 重载因子
* @see Hashtable
*/
public LocalHashCacheStore(int initialCapacity, float loadFactor) {
this(initialCapacity, loadFactor, false);
}
/**
* 构造一个基于Hashtable的本地缓存库
* @param initialCapacity 初始容量
* @param loadFactor 重载因子
* @param autoClean 是否自动清理
* @see Hashtable
*/
public LocalHashCacheStore(int initialCapacity, float loadFactor, boolean autoClean) {
if(initialCapacity != 0) {
if(loadFactor <= 0F) {
cache = new Hashtable<>(initialCapacity);
@ -34,6 +60,10 @@ public class LocalHashCacheStore<T> implements CacheStore<T> {
} else {
cache = new Hashtable<>();
}
if(autoClean) {
AutoCleanTimer.add(this);
}
}
@Override
@ -118,6 +148,15 @@ public class LocalHashCacheStore<T> implements CacheStore<T> {
return false;
}
@Override
public void clean() throws Exception {
Date currentDate = new Date();
cache.forEach((key, value) -> {
if(value.isExpire(currentDate)) {
cache.remove(key);
}
});
}
public static class CacheObject<T> implements Comparable<CacheObject<T>> {

2
src/main/java/net/lamgc/cgj/bot/event/BotEventHandler.java
View File

@ -224,7 +224,7 @@ public class BotEventHandler implements EventHandler {
}
}
long processTime = System.currentTimeMillis() - time;
if(Objects.requireNonNull(result) instanceof String && !isMute(event.getFromGroup())) {
if(!Objects.isNull(result) && result instanceof String && !isMute(event.getFromGroup())) {
try {
event.sendMessage((String) result);
} catch (Exception e) {

59
src/main/java/net/lamgc/cgj/bot/event/BufferMessageEvent.java Normal file
View File

@ -0,0 +1,59 @@
package net.lamgc.cgj.bot.event;
import java.util.Objects;
public class BufferMessageEvent extends MessageEvent {
StringBuffer buffer = new StringBuffer();
public final MessageEvent parent;
/**
* 以空消息空Id生成BufferMessageEvent
*/
public BufferMessageEvent() {
super(0, 0, "");
parent = null;
}
/**
* 提供消息内容构造BufferMessageEvent
* @param message 传入的消息内容
*/
public BufferMessageEvent(String message) {
super(0, 0, message);
parent = null;
}
/**
* 使用事件构造BufferMessageEvent
* @param parentEvent 父级消息事件对象
*/
public BufferMessageEvent(MessageEvent parentEvent) {
super(parentEvent.getFromGroup(), parentEvent.getFromQQ(), parentEvent.getMessage());
parent = parentEvent;
}
@Override
public int sendMessage(String message) {
buffer.append(message);
return 0;
}
/**
* 当提供了父级消息事件时, 本方法调用父级消息事件对象的{@code getImageUrl(String)}, 如果没有, 返回{@code null}
*/
@Override
public String getImageUrl(String image) {
return Objects.isNull(this.parent) ? null : this.parent.getImageUrl(image);
}
/**
* 获取缓冲区消息内容
* @return 消息内容
*/
public String getBufferMessage() {
return buffer.toString();
}
}

2
src/main/java/net/lamgc/cgj/proxy/PixivAccessProxyServer.java
View File

@ -99,7 +99,7 @@ public class PixivAccessProxyServer {
}*/
log.info("Response Cookie: " + value);
BasicClientCookie cookie = parseRawCookie(value);
cookieStore.addCookie(null);
cookieStore.addCookie(cookie);
});
httpResponse.headers().remove(HttpHeaderNames.SET_COOKIE);
super.afterResponse(clientChannel, proxyChannel, httpResponse, pipeline);

1
src/main/java/net/lamgc/cgj/util/PagesQualityParser.java
View File

@ -1,7 +1,6 @@
package net.lamgc.cgj.util;
import net.lamgc.cgj.pixiv.PixivDownload;
import net.lamgc.cgj.pixiv.PixivURL;
import net.lamgc.utils.base.runner.StringParameterParser;
public class PagesQualityParser implements StringParameterParser<PixivDownload.PageQuality> {