[Change] BotCommandProcess, CacheStoreCentral 将ImageFile缓存管理转移到CacheStoreCentral;

[Change] RandomRankingArtworksSender 适配修改;
[Change] BotCommandProcess, BotGlobal 将'imageStoreDir'由BotCommandProcess转移到BotGlobal;
This commit is contained in:
LamGC 2020-06-05 10:07:13 +08:00
parent bcc21149b9
commit 4bbed5fd55
4 changed files with 152 additions and 148 deletions

View File

@ -1,13 +1,13 @@
package net.lamgc.cgj.bot;
import com.google.common.base.Strings;
import com.google.common.base.Throwables;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import io.netty.handler.codec.http.HttpHeaderNames;
import net.lamgc.cgj.bot.boot.BotGlobal;
import net.lamgc.cgj.bot.cache.*;
import net.lamgc.cgj.bot.cache.CacheStore;
import net.lamgc.cgj.bot.cache.CacheStoreCentral;
import net.lamgc.cgj.bot.cache.JsonRedisCacheStore;
import net.lamgc.cgj.bot.event.BufferMessageEvent;
import net.lamgc.cgj.bot.sort.PreLoadDataComparator;
import net.lamgc.cgj.pixiv.PixivDownload;
@ -16,33 +16,21 @@ import net.lamgc.cgj.pixiv.PixivSearchBuilder;
import net.lamgc.cgj.pixiv.PixivURL;
import net.lamgc.cgj.pixiv.PixivURL.RankingContentType;
import net.lamgc.cgj.pixiv.PixivURL.RankingMode;
import net.lamgc.cgj.util.URLs;
import net.lamgc.utils.base.runner.Argument;
import net.lamgc.utils.base.runner.Command;
import net.lz1998.cq.utils.CQCode;
import org.apache.http.HttpResponse;
import org.apache.http.client.methods.HttpHead;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.concurrent.atomic.AtomicInteger;
@SuppressWarnings({"SameParameterValue"})
public class BotCommandProcess {
private final static Logger log = LoggerFactory.getLogger(BotCommandProcess.class);
private final static File imageStoreDir = new File(BotGlobal.getGlobal().getDataStoreDir(), "data/image/cgj/");
/* -------------------- 缓存 -------------------- */
private final static Hashtable<String, File> imageCache = new Hashtable<>();
/**
* 作品报告存储 - 不过期
*/
@ -128,7 +116,7 @@ public class BotCommandProcess {
"\n页数" + illustPreLoadData.get(PreLoadDataComparator.Attribute.PAGE.attrName).getAsInt() + "" +
"\n作品链接" + artworksLink(fromGroup, illustId) + "\n" +
"---------------- 作品图片 ----------------\n" +
getImageById(fromGroup, illustId, PageQuality.REGULAR, 1) + "\n" +
CacheStoreCentral.getImageById(fromGroup, illustId, PageQuality.REGULAR, 1) + "\n" +
"使用 \".cgj image -id " +
illustId +
"\" 获取原图。\n如有不当作品可使用\".cgj report -id " +
@ -242,7 +230,8 @@ public class BotCommandProcess {
.append(pagesCount).append("p.\n");
if (index <= imageLimit) {
resultBuilder
.append(getImageById(fromGroup, illustId, PixivDownload.PageQuality.REGULAR, 1))
.append(CacheStoreCentral
.getImageById(fromGroup, illustId, PixivDownload.PageQuality.REGULAR, 1))
.append("\n");
}
}
@ -356,7 +345,8 @@ public class BotCommandProcess {
PixivURL.getPixivRefererLink(illustId)
);
String imageMsg = getImageById(fromGroup, illustId, PixivDownload.PageQuality.REGULAR, 1);
String imageMsg =
CacheStoreCentral.getImageById(fromGroup, illustId, PixivDownload.PageQuality.REGULAR, 1);
if (isNoSafe(illustId, SettingProperties.getProperties(fromGroup), true)) {
log.warn("作品Id {} 为R-18作品, 跳过.", illustId);
continue;
@ -450,120 +440,8 @@ public class BotCommandProcess {
return PixivURL.getPixivRefererLink(illustId);
}
/**
* 通过illustId获取作品图片
* @param fromGroup 来源群(系统提供)
* @param illustId 作品Id
* @param quality 图片质量
* @param pageIndex 指定页面索引, 从1开始
* @return 如果成功, 返回BotCode, 否则返回错误信息.
*/
@Command(commandName = "image")
public static String getImageById(
@Argument(name = "$fromGroup") long fromGroup,
@Argument(name = "id") int illustId,
@Argument(name = "quality", force = false) PixivDownload.PageQuality quality,
@Argument(name = "page", force = false, defaultValue = "1") int pageIndex) {
log.debug("IllustId: {}, Quality: {}, PageIndex: {}", illustId, quality.name(), pageIndex);
try {
if (isNoSafe(illustId, SettingProperties.getProperties(fromGroup), false)) {
log.warn("作品 {} 存在R-18内容且设置\"image.allowR18\"为false将屏蔽该作品不发送.", illustId);
return "(根据设置,该作品已被屏蔽!)";
} else if(isReported(illustId)) {
log.warn("作品Id {} 被报告, 正在等待审核, 跳过该作品.", illustId);
return "(该作品已被封印)";
}
} catch (IOException e) {
log.warn("作品信息无法获取!", e);
return "(发生网络异常,无法获取图片!)";
}
List<String> pagesList;
try {
pagesList = CacheStoreCentral.getIllustPages(illustId, quality, false);
} catch (IOException e) {
log.error("获取下载链接列表时发生异常", e);
return "发生网络异常,无法获取图片!";
}
if(log.isDebugEnabled()) {
StringBuilder logBuilder = new StringBuilder("作品Id " + illustId + " 所有页面下载链接: \n");
AtomicInteger index = new AtomicInteger();
pagesList.forEach(item ->
logBuilder.append(index.incrementAndGet()).append(". ").append(item).append("\n"));
log.debug(logBuilder.toString());
}
if (pagesList.size() < pageIndex || pageIndex <= 0) {
log.warn("指定的页数超出了总页数({} / {})", pageIndex, pagesList.size());
return "指定的页数超出了范围(总共 " + pagesList.size() + " 页)";
}
String downloadLink = pagesList.get(pageIndex - 1);
String fileName = URLs.getResourceName(Strings.nullToEmpty(downloadLink));
File imageFile = new File(getImageStoreDir(), downloadLink.substring(downloadLink.lastIndexOf("/") + 1));
log.debug("FileName: {}, DownloadLink: {}", fileName, downloadLink);
if(!imageCache.containsKey(fileName)) {
if(imageFile.exists()) {
HttpHead headRequest = new HttpHead(downloadLink);
headRequest.addHeader("Referer", PixivURL.getPixivRefererLink(illustId));
HttpResponse headResponse;
try {
headResponse = BotGlobal.getGlobal().getPixivDownload().getHttpClient().execute(headRequest);
} catch (IOException e) {
log.error("获取图片大小失败!", e);
return "图片获取失败!";
}
String contentLengthStr = headResponse
.getFirstHeader(HttpHeaderNames.CONTENT_LENGTH.toString())
.getValue();
log.debug("图片大小: {}B", contentLengthStr);
if (imageFile.length() == Long.parseLong(contentLengthStr)) {
imageCache.put(URLs.getResourceName(downloadLink), imageFile);
log.debug("作品Id {} 第 {} 页缓存已补充.", illustId, pageIndex);
return getImageToBotCode(imageFile, false).toString();
}
}
try {
Throwable throwable = ImageCacheStore.executeCacheRequest(
new ImageCacheObject(imageCache, illustId, downloadLink, imageFile));
if(throwable != null) {
throw throwable;
}
} catch (InterruptedException e) {
log.warn("图片缓存被中断", e);
return "(错误:图片获取超时)";
} catch (Throwable e) {
log.error("图片 {} 获取失败:\n{}", illustId + "p" + pageIndex, Throwables.getStackTraceAsString(e));
return "(错误: 图片获取出错)";
}
} else {
log.debug("图片 {} 缓存命中.", fileName);
}
return getImageToBotCode(imageCache.get(fileName), false).toString();
}
/**
* 通过文件获取图片的BotCode代码
* @param targetFile 图片文件
* @param updateCache 是否刷新缓存(只是让机器人重新上传, 如果上传接口有重复检测的话是无法处理的)
* @return 返回设定好参数的BotCode
*/
private static BotCode getImageToBotCode(File targetFile, boolean updateCache) {
String fileName = Objects.requireNonNull(targetFile, "targetFile is null").getName();
BotCode code = BotCode.parse(CQCode.image(getImageStoreDir().getName() + "/" + fileName));
code.addParameter("absolutePath", targetFile.getAbsolutePath());
code.addParameter("imageName", fileName.substring(0, fileName.lastIndexOf(".")));
code.addParameter("updateCache", updateCache ? "true" : "false");
return code;
}
static void clearCache() {
log.warn("正在清除所有缓存...");
imageCache.clear();
CacheStoreCentral.clearCache();
File imageStoreDir = new File(BotGlobal.getGlobal().getDataStoreDir(), "data/image/cgj/");
File[] listFiles = imageStoreDir.listFiles();
@ -639,20 +517,4 @@ public class BotCommandProcess {
.equalsIgnoreCase("true");
}
/**
* 获取图片存储目录.
* <p>每次调用都会检查目录是否存在, 如不存在则会抛出异常</p>
* @return 返回File对象
* @throws RuntimeException 当目录创建失败时将包装{@link IOException}异常并抛出.
*/
private static File getImageStoreDir() {
if(!imageStoreDir.exists() && !Files.isSymbolicLink(imageStoreDir.toPath())) {
if(!imageStoreDir.mkdirs()) {
log.warn("酷Q图片缓存目录失效(Path: {} )", imageStoreDir.getAbsolutePath());
throw new RuntimeException(new IOException("文件夹创建失败!"));
}
}
return imageStoreDir;
}
}

View File

@ -89,7 +89,7 @@ public class RandomRankingArtworksSender extends AutoSender {
String message = "#美图推送 - 今日排行榜 第 " + rankingInfo.get("rank").getAsInt() + "\n" +
"标题:" + rankingInfo.get("title").getAsString() + "(" + illustId + ")\n" +
"作者:" + rankingInfo.get("user_name").getAsString() + "\n" +
BotCommandProcess.getImageById(0, illustId, quality, 1) +
CacheStoreCentral.getImageById(0, illustId, quality, 1) +
"\n如有不当作品可使用\".cgj report -id " + illustId + "\"向色图姬反馈。";
getMessageSender().sendMessage(message);
} catch (Exception e) {

View File

@ -11,9 +11,11 @@ import org.slf4j.LoggerFactory;
import redis.clients.jedis.JedisPool;
import java.io.File;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URL;
import java.nio.file.Files;
public final class BotGlobal {
@ -47,6 +49,8 @@ public final class BotGlobal {
private PixivDownload pixivDownload;
private final File imageStoreDir;
private BotGlobal() {
this.redisUri = URI.create("redis://" + System.getProperty("cgj.redisAddress"));
this.redisServer = new JedisPool(
@ -56,6 +60,8 @@ public final class BotGlobal {
this.dataStoreDir = new File((!dataStoreDirPath.endsWith("/") || !dataStoreDirPath.endsWith("\\")) ?
dataStoreDirPath + System.getProperty("file.separator") : dataStoreDirPath);
this.imageStoreDir = new File(BotGlobal.getGlobal().getDataStoreDir(), "data/image/cgj/");
String proxyAddress = System.getProperty("cgj.proxy");
HttpHost temp = null;
if(!Strings.isNullOrEmpty(proxyAddress)) {
@ -89,7 +95,6 @@ public final class BotGlobal {
return proxy;
}
public CookieStore getCookieStore() {
if(pixivDownload == null) {
throw new IllegalStateException("CookieStore needs to be set before PixivDownload can be obtained");
@ -113,4 +118,14 @@ public final class BotGlobal {
public PixivDownload getPixivDownload() {
return pixivDownload;
}
public File getImageStoreDir() {
if(!imageStoreDir.exists() && !Files.isSymbolicLink(imageStoreDir.toPath())) {
if(!imageStoreDir.mkdirs()) {
log.warn("酷Q图片缓存目录失效(Path: {} )", imageStoreDir.getAbsolutePath());
throw new RuntimeException(new IOException("文件夹创建失败!"));
}
}
return imageStoreDir;
}
}

View File

@ -4,21 +4,31 @@ import com.google.common.base.Strings;
import com.google.common.base.Throwables;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import io.netty.handler.codec.http.HttpHeaderNames;
import net.lamgc.cgj.bot.BotCode;
import net.lamgc.cgj.bot.BotCommandProcess;
import net.lamgc.cgj.bot.SettingProperties;
import net.lamgc.cgj.bot.boot.BotGlobal;
import net.lamgc.cgj.pixiv.PixivDownload;
import net.lamgc.cgj.pixiv.PixivSearchBuilder;
import net.lamgc.cgj.pixiv.PixivURL;
import net.lamgc.cgj.util.URLs;
import net.lamgc.utils.base.runner.Argument;
import net.lamgc.utils.base.runner.Command;
import net.lz1998.cq.utils.CQCode;
import org.apache.http.HttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpHead;
import org.apache.http.util.EntityUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.concurrent.atomic.AtomicInteger;
@SuppressWarnings("SynchronizationOnLocalVariableOrMethodParameter")
public final class CacheStoreCentral {
@ -27,6 +37,8 @@ public final class CacheStoreCentral {
private final static Logger log = LoggerFactory.getLogger(CacheStoreCentral.class);
private final static Hashtable<String, File> imageCache = new Hashtable<>();
/**
* 作品信息缓存 - 不过期
*/
@ -66,6 +78,7 @@ public final class CacheStoreCentral {
* 清空所有缓存
*/
public static void clearCache() {
imageCache.clear();
illustInfoCache.clear();
illustPreLoadDataCache.clear();
searchBodyCache.clear();
@ -73,6 +86,120 @@ public final class CacheStoreCentral {
pagesCache.clear();
}
/**
* 通过illustId获取作品图片
* @param fromGroup 来源群(系统提供)
* @param illustId 作品Id
* @param quality 图片质量
* @param pageIndex 指定页面索引, 从1开始
* @return 如果成功, 返回BotCode, 否则返回错误信息.
*/
@Command(commandName = "image")
public static String getImageById(
@Argument(name = "$fromGroup") long fromGroup,
@Argument(name = "id") int illustId,
@Argument(name = "quality", force = false) PixivDownload.PageQuality quality,
@Argument(name = "page", force = false, defaultValue = "1") int pageIndex) {
log.debug("IllustId: {}, Quality: {}, PageIndex: {}", illustId, quality.name(), pageIndex);
try {
if (BotCommandProcess.isNoSafe(illustId, SettingProperties.getProperties(fromGroup), false)) {
log.warn("作品 {} 存在R-18内容且设置\"image.allowR18\"为false将屏蔽该作品不发送.", illustId);
return "(根据设置,该作品已被屏蔽!)";
} else if(BotCommandProcess.isReported(illustId)) {
log.warn("作品Id {} 被报告, 正在等待审核, 跳过该作品.", illustId);
return "(该作品已被封印)";
}
} catch (IOException e) {
log.warn("作品信息无法获取!", e);
return "(发生网络异常,无法获取图片!)";
}
List<String> pagesList;
try {
pagesList = CacheStoreCentral.getIllustPages(illustId, quality, false);
} catch (IOException e) {
log.error("获取下载链接列表时发生异常", e);
return "发生网络异常,无法获取图片!";
}
if(log.isDebugEnabled()) {
StringBuilder logBuilder = new StringBuilder("作品Id " + illustId + " 所有页面下载链接: \n");
AtomicInteger index = new AtomicInteger();
pagesList.forEach(item ->
logBuilder.append(index.incrementAndGet()).append(". ").append(item).append("\n"));
log.debug(logBuilder.toString());
}
if (pagesList.size() < pageIndex || pageIndex <= 0) {
log.warn("指定的页数超出了总页数({} / {})", pageIndex, pagesList.size());
return "指定的页数超出了范围(总共 " + pagesList.size() + " 页)";
}
String downloadLink = pagesList.get(pageIndex - 1);
String fileName = URLs.getResourceName(Strings.nullToEmpty(downloadLink));
File imageFile = new File(BotGlobal.getGlobal().getImageStoreDir(),
downloadLink.substring(downloadLink.lastIndexOf("/") + 1));
log.debug("FileName: {}, DownloadLink: {}", fileName, downloadLink);
if(!imageCache.containsKey(fileName)) {
if(imageFile.exists()) {
HttpHead headRequest = new HttpHead(downloadLink);
headRequest.addHeader("Referer", PixivURL.getPixivRefererLink(illustId));
HttpResponse headResponse;
try {
headResponse = BotGlobal.getGlobal().getPixivDownload().getHttpClient().execute(headRequest);
} catch (IOException e) {
log.error("获取图片大小失败!", e);
return "图片获取失败!";
}
String contentLengthStr = headResponse
.getFirstHeader(HttpHeaderNames.CONTENT_LENGTH.toString())
.getValue();
log.debug("图片大小: {}B", contentLengthStr);
if (imageFile.length() == Long.parseLong(contentLengthStr)) {
imageCache.put(URLs.getResourceName(downloadLink), imageFile);
log.debug("作品Id {} 第 {} 页缓存已补充.", illustId, pageIndex);
return getImageToBotCode(imageFile, false).toString();
}
}
try {
Throwable throwable = ImageCacheStore.executeCacheRequest(
new ImageCacheObject(imageCache, illustId, downloadLink, imageFile));
if(throwable != null) {
throw throwable;
}
} catch (InterruptedException e) {
log.warn("图片缓存被中断", e);
return "(错误:图片获取超时)";
} catch (Throwable e) {
log.error("图片 {} 获取失败:\n{}", illustId + "p" + pageIndex, Throwables.getStackTraceAsString(e));
return "(错误: 图片获取出错)";
}
} else {
log.debug("图片 {} 缓存命中.", fileName);
}
return getImageToBotCode(imageCache.get(fileName), false).toString();
}
/**
* 通过文件获取图片的BotCode代码
* @param targetFile 图片文件
* @param updateCache 是否刷新缓存(只是让机器人重新上传, 如果上传接口有重复检测的话是无法处理的)
* @return 返回设定好参数的BotCode
*/
@SuppressWarnings("SameParameterValue")
private static BotCode getImageToBotCode(File targetFile, boolean updateCache) {
String fileName = Objects.requireNonNull(targetFile, "targetFile is null").getName();
BotCode code = BotCode.parse(
CQCode.image(BotGlobal.getGlobal().getImageStoreDir().getName() + "/" + fileName));
code.addParameter("absolutePath", targetFile.getAbsolutePath());
code.addParameter("imageName", fileName.substring(0, fileName.lastIndexOf(".")));
code.addParameter("updateCache", updateCache ? "true" : "false");
return code;
}
/**
* 获取作品信息
* @param illustId 作品Id