LamGC/ContentGrabbingJi
1
0
Fork 0
mirror of https://github.com/LamGC/ContentGrabbingJi.git synced 2025-04-30 06:37:36 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[Change] log4j2.xml 调整日志输出到文件的限制;

Browse Source 
[Change] 将内容较长的日志设为TRACE级别, 以减少日志占用;
[CLear] 整理代码;
This commit is contained in:
LamGC 2020-06-11 09:49:19 +08:00
parent 87f2535b48
commit 26e377a2c7
11 changed files with 55 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/main/java/net/lamgc/cgj/bot/BotCode.java
View File

@ -64,7 +64,7 @@ public class BotCode {
private String platformName;
private String functionName;
private Hashtable<String, String> parameter = new Hashtable<>();
private final Hashtable<String, String> parameter = new Hashtable<>();
/**
* 构造一个机器功能码

37
src/main/java/net/lamgc/cgj/bot/cache/CacheStoreCentral.java vendored
View File

@ -164,10 +164,10 @@ public final class CacheStoreCentral {
ImageChecksum imageChecksum = getImageChecksum(illustId, pageIndex);
if(imageChecksum != null) {
try {
log.debug("正在检查作品Id {} 第 {} 页图片文件 {} ...", illustId, pageIndex, imageFile.getName());
log.trace("正在检查作品Id {} 第 {} 页图片文件 {} ...", illustId, pageIndex, imageFile.getName());
if (ImageChecksum.checkFile(imageChecksum, Files.readAllBytes(imageFile.toPath()))) {
imageCache.put(URLs.getResourceName(downloadLink), imageFile);
log.debug("作品Id {} 第 {} 页缓存已补充.", illustId, pageIndex);
log.trace("作品Id {} 第 {} 页缓存已补充.", illustId, pageIndex);
return getImageToBotCode(imageFile, false).toString();
} else {
log.warn("图片文件 {} 校验失败, 重新下载图片...", imageFile.getName());
@ -194,7 +194,7 @@ public final class CacheStoreCentral {
return "(错误: 图片获取出错)";
}
} else {
log.debug("图片 {} 缓存命中.", fileName);
log.trace("图片 {} 缓存命中.", fileName);
}
return getImageToBotCode(imageCache.get(fileName), false).toString();
@ -240,7 +240,7 @@ public final class CacheStoreCentral {
if(Objects.isNull(illustInfoObj)) {
illustInfoObj = illustInfoCache.getCache(illustIdStr).getAsJsonObject();
log.debug("作品Id {} IllustInfo缓存命中.", illustId);
log.trace("作品Id {} IllustInfo缓存命中.", illustId);
}
return illustInfoObj;
}
@ -259,7 +259,7 @@ public final class CacheStoreCentral {
if (!illustPreLoadDataCache.exists(illustIdStr) || flushCache) {
synchronized (illustIdStr) {
if (!illustPreLoadDataCache.exists(illustIdStr) || flushCache) {
log.debug("IllustId {} 缓存失效, 正在更新...", illustId);
log.trace("IllustId {} 缓存失效, 正在更新...", illustId);
JsonObject preLoadDataObj = BotGlobal.getGlobal().getPixivDownload()
.getIllustPreLoadDataById(illustId)
.getAsJsonObject("illust")
@ -277,14 +277,14 @@ public final class CacheStoreCentral {
result = preLoadDataObj;
illustPreLoadDataCache.update(illustIdStr, preLoadDataObj, expire);
log.debug("作品Id {} preLoadData缓存已更新(有效时间: {})", illustId, expire);
log.trace("作品Id {} preLoadData缓存已更新(有效时间: {})", illustId, expire);
}
}
}
if(Objects.isNull(result)) {
result = illustPreLoadDataCache.getCache(illustIdStr).getAsJsonObject();
log.debug("作品Id {} PreLoadData缓存命中.", illustId);
log.trace("作品Id {} PreLoadData缓存命中.", illustId);
}
return result;
}
@ -307,7 +307,7 @@ public final class CacheStoreCentral {
if(Objects.isNull(result)) {
result = pagesCache.getCache(pagesSign);
log.debug("作品Id {} Pages缓存命中.", illustId);
log.trace("作品Id {} Pages缓存命中.", illustId);
}
return result;
}
@ -346,7 +346,7 @@ public final class CacheStoreCentral {
if(!rankingCache.exists(requestSign) || flushCache) {
synchronized(requestSign) {
if(!rankingCache.exists(requestSign) || flushCache) {
log.debug("Ranking缓存失效, 正在更新...(RequestSign: {})", requestSign);
log.trace("Ranking缓存失效, 正在更新...(RequestSign: {})", requestSign);
List<JsonObject> rankingResult = BotGlobal.getGlobal().getPixivDownload()
.getRanking(contentType, mode, queryDate, 1, 500);
long expireTime = 0;
@ -356,16 +356,15 @@ public final class CacheStoreCentral {
}
result = new ArrayList<>(rankingResult).subList(start - 1, start + range - 1);
rankingCache.update(requestSign, rankingResult, expireTime);
log.debug("Ranking缓存更新完成.(RequestSign: {})", requestSign);
log.trace("Ranking缓存更新完成.(RequestSign: {})", requestSign);
}
}
}
if (Objects.isNull(result)) {
result = rankingCache.getCache(requestSign, start - 1, range);
log.debug("RequestSign [{}] 缓存命中.", requestSign);
log.trace("RequestSign [{}] 缓存命中.", requestSign);
}
log.debug("Result-Length: {}", result.size());
return PixivDownload.getRanking(result, start - 1, range);
}
@ -414,14 +413,14 @@ public final class CacheStoreCentral {
for (String keyword : includeKeywords.split(";")) {
searchBuilder.removeExcludeKeyword(keyword.trim());
searchBuilder.addIncludeKeyword(keyword.trim());
log.debug("已添加关键字: {}", keyword);
log.trace("已添加关键字: {}", keyword);
}
}
if (!Strings.isNullOrEmpty(excludeKeywords)) {
for (String keyword : excludeKeywords.split(";")) {
searchBuilder.removeIncludeKeyword(keyword.trim());
searchBuilder.addExcludeKeyword(keyword.trim());
log.debug("已添加排除关键字: {}", keyword);
log.trace("已添加排除关键字: {}", keyword);
}
}
@ -433,7 +432,7 @@ public final class CacheStoreCentral {
if(!searchBodyCache.exists(requestUrl)) {
synchronized (requestUrl) {
if (!searchBodyCache.exists(requestUrl)) {
log.debug("searchBody缓存失效, 正在更新...");
log.trace("searchBody缓存失效, 正在更新...");
JsonObject jsonObject;
HttpGet httpGetRequest = BotGlobal.getGlobal().getPixivDownload().
createHttpGetRequest(requestUrl);
@ -441,7 +440,7 @@ public final class CacheStoreCentral {
getHttpClient().execute(httpGetRequest);
String responseBody = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8);
log.debug("ResponseBody: {}", responseBody);
log.trace("ResponseBody: {}", responseBody);
jsonObject = BotGlobal.getGlobal().getGson().fromJson(responseBody, JsonObject.class);
if (jsonObject.get("error").getAsBoolean()) {
@ -459,13 +458,13 @@ public final class CacheStoreCentral {
}
resultBody = jsonObject.getAsJsonObject().getAsJsonObject("body");
searchBodyCache.update(requestUrl, jsonObject, expire);
log.debug("searchBody缓存已更新(有效时间: {})", expire);
log.trace("searchBody缓存已更新(有效时间: {})", expire);
} else {
log.debug("搜索缓存命中.");
log.trace("搜索缓存命中.");
}
}
} else {
log.debug("搜索缓存命中.");
log.trace("搜索缓存命中.");
}
if(Objects.isNull(resultBody)) {

2
src/main/java/net/lamgc/cgj/bot/cache/ImageCacheHandler.java vendored
View File

@ -66,7 +66,7 @@ public class ImageCacheHandler implements EventHandler {
throw requestException;
}
log.debug("正在下载...(Content-Length: {}KB)", response.getEntity().getContentLength() / 1024);
log.trace("正在下载...(Content-Length: {}KB)", response.getEntity().getContentLength() / 1024);
ByteArrayOutputStream bufferOutputStream = new ByteArrayOutputStream();
try(FileOutputStream fileOutputStream = new FileOutputStream(storeFile)) {
Streams.copy(response.getEntity().getContent(), bufferOutputStream, false);

6
src/main/java/net/lamgc/cgj/bot/cache/LocalHashCacheStore.java vendored
View File

@ -149,7 +149,7 @@ public class LocalHashCacheStore<T> implements CacheStore<T>, Cleanable {
}
@Override
public void clean() throws Exception {
public void clean() {
Date currentDate = new Date();
cache.forEach((key, value) -> {
if(value.isExpire(currentDate)) {
@ -160,8 +160,8 @@ public class LocalHashCacheStore<T> implements CacheStore<T>, Cleanable {
public static class CacheObject<T> implements Comparable<CacheObject<T>> {
private AtomicReference<T> value;
private AtomicReference<Date> expire;
private final AtomicReference<T> value;
private final AtomicReference<Date> expire;
public CacheObject(T value, Date expire) {
this.value = new AtomicReference<>(value);

2
src/main/java/net/lamgc/cgj/bot/event/BotEventHandler.java
View File

@ -105,8 +105,6 @@ public class BotEventHandler implements EventHandler {
runnerConfig.addStringParameterParser(new DateParser(new SimpleDateFormat("yyyy-MM-dd")));
runnerConfig.addStringParameterParser(new PagesQualityParser());
log.debug("DateParser添加情况: {}", runnerConfig.hasStringParameterParser(Date.class));
processRunner = new ArgumentsRunner(BotCommandProcess.class, runnerConfig);
adminRunner = new ArgumentsRunner(BotAdminCommandProcess.class, runnerConfig);

6
src/main/java/net/lamgc/cgj/bot/framework/coolq/message/SpringCQMessageSender.java
View File

@ -6,9 +6,9 @@ import net.lz1998.cq.robot.CoolQ;
public class SpringCQMessageSender implements MessageSender {
private CoolQ coolQ;
private MessageSource source;
private long target;
private final CoolQ coolQ;
private final MessageSource source;
private final long target;
public SpringCQMessageSender(CoolQ coolQ, MessageSource source, long target) {
this.coolQ = coolQ;

8
src/main/java/net/lamgc/cgj/bot/framework/mirai/message/MiraiMessageSender.java
View File

@ -151,7 +151,7 @@ public class MiraiMessageSender implements MessageSender {
synchronized (imageName) {
if(!imageIdCache.exists(imageName) ||
Strings.nullToEmpty(code.getParameter("updateCache")) .equalsIgnoreCase("true")) {
log.debug("imageName [{}] 缓存失效或强制更新, 正在更新缓存...", imageName);
log.trace("imageName [{}] 缓存失效或强制更新, 正在更新缓存...", imageName);
image = uploadImage0(new File(absolutePath));
String cacheExpireAt;
long expireTime = 864000000; // 10d
@ -163,13 +163,13 @@ public class MiraiMessageSender implements MessageSender {
}
}
imageIdCache.update(imageName, image.getImageId(), expireTime);
log.debug("imageName [{}] 缓存更新完成.(有效时间: {})", imageName, expireTime);
log.trace("imageName [{}] 缓存更新完成.(有效时间: {})", imageName, expireTime);
} else {
log.debug("ImageName: [{}] 缓存命中.", imageName);
log.trace("ImageName: [{}] 缓存命中.", imageName);
}
}
} else {
log.debug("ImageName: [{}] 缓存命中.", imageName);
log.trace("ImageName: [{}] 缓存命中.", imageName);
}
if(image == null) {

35
src/main/java/net/lamgc/cgj/pixiv/PixivDownload.java
View File

@ -31,6 +31,7 @@ import java.util.*;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.BiConsumer;
@SuppressWarnings("ALL")
public class PixivDownload {
private final static Logger log = LoggerFactory.getLogger(PixivDownload.class);
@ -115,10 +116,10 @@ public class PixivDownload {
}
}
} while(!document.select(".pager-container>.next").isEmpty());
log.debug("获取完成.");
log.trace("获取完成.");
AtomicInteger count = new AtomicInteger(1);
linkList.forEach(link -> {
log.debug("Next Link [{}]: {}", count.getAndIncrement(), link);
log.trace("Next Link [{}]: {}", count.getAndIncrement(), link);
InputStream imageInputStream = null;
int tryCount = 0;
do {
@ -133,9 +134,9 @@ public class PixivDownload {
} while(imageInputStream == null);
try(InputStream imageInput = new BufferedInputStream(imageInputStream, 256 * 1024)) {
log.debug("调用回调方法...");
log.trace("调用回调方法...");
fn.accept(link, imageInput);
log.debug("回调方法调用完成.");
log.trace("回调方法调用完成.");
} catch (IOException e) {
log.error("图片获取失败", e);
}
@ -219,8 +220,8 @@ public class PixivDownload {
int authorId = rankInfo.get("user_id").getAsInt();
String authorName = rankInfo.get("user_name").getAsString();
String title = rankInfo.get("title").getAsString();
log.debug("当前到第 {}/{} 名(总共 {} 名), IllustID: {}, Author: ({}) {}, Title: {}", rank, rankStart + range - 1, range, illustId, authorId, authorName, title);
log.debug("正在获取PagesLink...");
log.trace("当前到第 {}/{} 名(总共 {} 名), IllustID: {}, Author: ({}) {}, Title: {}", rank, rankStart + range - 1, range, illustId, authorId, authorName, title);
log.trace("正在获取PagesLink...");
List<String> linkList;
try {
linkList = getIllustAllPageDownload(httpClient, this.cookieStore, illustId, quality);
@ -235,14 +236,14 @@ public class PixivDownload {
log.debug("PagesLink 获取完成, 总数: {}", linkList.size());
for (int pageIndex = 0; pageIndex < linkList.size(); pageIndex++) {
String downloadLink = linkList.get(pageIndex);
log.debug("当前Page: {}/{}", pageIndex + 1, linkList.size());
log.trace("当前Page: {}/{}", pageIndex + 1, linkList.size());
try(InputStream imageInputStream = new BufferedInputStream(getImageAsInputStream(HttpClientBuilder.create().build(), downloadLink), 256 * 1024)) {
fn.download(rank, downloadLink, rankInfo.deepCopy(), imageInputStream);
} catch(IOException e) {
log.error("下载插画时发生异常", e);
return;
}
log.debug("完成.");
log.trace("完成.");
}
});
}
@ -265,7 +266,7 @@ public class PixivDownload {
int authorId = rankInfo.get("user_id").getAsInt();
String authorName = rankInfo.get("user_name").getAsString();
String title = rankInfo.get("title").getAsString();
log.debug("Array-当前到第 {}/{} 名(总共 {} 名), IllustID: {}, Author: ({}) {}, Title: {}", rank, rankStart + range, range, illustId, authorId, authorName, title);
log.trace("Array-当前到第 {}/{} 名(总共 {} 名), IllustID: {}, Author: ({}) {}, Title: {}", rank, rankStart + range, range, illustId, authorId, authorName, title);
results.add(rankInfo);
}
log.debug("JsonArray读取完成.");
@ -320,10 +321,10 @@ public class PixivDownload {
boolean canNext = true;
for (int pageIndex = startPages; canNext && pageIndex <= endPages && count < range; pageIndex++) {
HttpGet request = createHttpGetRequest(PixivURL.getRankingLink(contentType, mode, time, pageIndex, true));
log.debug("RequestUri: {}", request.getURI());
log.trace("RequestUri: {}", request.getURI());
HttpResponse response = httpClient.execute(request);
String responseBody = EntityUtils.toString(response.getEntity());
log.debug("ResponseBody: {}", responseBody);
log.trace("ResponseBody: {}", responseBody);
if(response.getStatusLine().getStatusCode() != 200) {
throw new IOException("Http Response Error: '" + response.getStatusLine() + "', ResponseBody: '" + responseBody + '\'');
}
@ -399,7 +400,7 @@ public class PixivDownload {
if(resultObject.get("error").getAsBoolean()) {
String message = resultObject.get("message").getAsString();
log.debug("请求错误, 错误信息: {}", message);
log.warn("作品页面接口请求错误, 错误信息: {}", message);
throw new IOException(message);
}
@ -473,9 +474,11 @@ public class PixivDownload {
request.addHeader(HttpHeaderNames.REFERER.toString(), referer);
HttpResponse response = httpClient.execute(request);
log.debug("response: {}", response);
log.debug("Content Length: {}KB", Float.parseFloat(response.getFirstHeader(HttpHeaderNames.CONTENT_LENGTH.toString()).getValue()) / 1024F);
log.debug("{}", response.getFirstHeader(HttpHeaderNames.CONTENT_TYPE.toString()));
log.trace("response: {}", response);
log.trace("Content Length: {}KB",
Float.parseFloat(response.getFirstHeader(HttpHeaderNames.CONTENT_LENGTH.toString()).getValue()) / 1024F
);
log.trace(response.getFirstHeader(HttpHeaderNames.CONTENT_TYPE.toString()).toString());
return response.getEntity().getContent();
}
@ -548,7 +551,7 @@ public class PixivDownload {
HttpGet request = createHttpGetRequest(PixivURL.getPixivIllustInfoAPI(illustId));
HttpResponse response = httpClient.execute(request);
String responseStr = EntityUtils.toString(response.getEntity());
log.debug("Response Content: {}", responseStr);
log.trace("Response Content: {}", responseStr);
JsonObject responseObj = new Gson().fromJson(responseStr, JsonObject.class);
if(responseObj.get("error").getAsBoolean()) {

6
src/main/java/net/lamgc/cgj/pixiv/PixivUgoiraBuilder.java
View File

@ -46,7 +46,7 @@ public final class PixivUgoiraBuilder {
log.debug("Request Url: {}", request.getURI());
HttpResponse response = httpClient.execute(request);
String bodyStr = EntityUtils.toString(response.getEntity());
log.debug("JsonBodyStr: {}", bodyStr);
log.trace("JsonBodyStr: {}", bodyStr);
JsonObject resultObject = new Gson().fromJson(bodyStr, JsonObject.class);
if(resultObject.get("error").getAsBoolean()) {
String message = resultObject.get("message").getAsString();
@ -173,10 +173,10 @@ public final class PixivUgoiraBuilder {
private void getUgoiraImageSize() throws IOException {
log.debug("正在从Pixiv获取动图尺寸...");
HttpGet request = new HttpGet(PixivURL.getPixivIllustInfoAPI(illustId));
log.debug("Request Url: {}", request.getURI());
log.trace("Request Url: {}", request.getURI());
HttpResponse response = httpClient.execute(request);
String responseBody = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8);
log.debug("ResponseBody: {}", responseBody);
log.trace("ResponseBody: {}", responseBody);
JsonObject resultObject = new Gson().fromJson(responseBody, JsonObject.class);
if(resultObject.get("error").getAsBoolean()) {
String message = resultObject.get("message").getAsString();

3
src/main/resources/log4j2.xml
View File

@ -34,6 +34,9 @@
</Console>
<RollingFile name="rollingFile" fileName="${logsDir}/latest.log" filePattern="${logsDir}/running.%-d{yyyy-MM-dd_HH-mm-ss}.log.gz">
<Filters>
<ThresholdFilter level="DEBUG" onMatch="ACCEPT" onMismatch="DENY"/>
</Filters>
<PatternLayout charset="${charset}">
<MarkerPatternSelector defaultPattern="${standard_pattern}">
<PatternMatch key="mirai" pattern="${mirai_pattern}" />

1
src/test/java/net/lamgc/cgj/pixiv/PixivSearchBuilderTest.java
View File

@ -8,7 +8,6 @@ public class PixivSearchBuilderTest {
@Test
public void buildTest() {
PixivSearchBuilder builder = new PixivSearchBuilder("hololive");
//builder.addExcludeKeyword("fubuki").addExcludeKeyword("minato");
builder.addIncludeKeyword("35").addIncludeKeyword("okayu").addIncludeKeyword("百鬼あやめ");
System.out.println(builder.buildURL());
}