LamGC/ContentGrabbingJi
1
0
Fork 0
mirror of https://github.com/LamGC/ContentGrabbingJi.git synced 2025-04-30 06:37:36 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[Change #5] 优化'getImageById'从文件读入缓存时, 检查图片完整性的速度;

Browse Source
This commit is contained in:
LamGC 2020-06-05 17:02:56 +08:00
parent ef5651be47
commit 223d78dbd6
3 changed files with 215 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

209
src/main/java/net/lamgc/cgj/bot/cache/CacheStoreCentral.java vendored
View File

@ -4,7 +4,6 @@ import com.google.common.base.Strings;
import com.google.common.base.Throwables;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import io.netty.handler.codec.http.HttpHeaderNames;
import net.lamgc.cgj.bot.BotCode;
import net.lamgc.cgj.bot.BotCommandProcess;
import net.lamgc.cgj.bot.SettingProperties;
@ -15,17 +14,18 @@ import net.lamgc.cgj.pixiv.PixivURL;
import net.lamgc.cgj.util.URLs;
import net.lamgc.utils.base.runner.Argument;
import net.lamgc.utils.base.runner.Command;
import net.lamgc.utils.encrypt.MessageDigestUtils;
import net.lz1998.cq.utils.CQCode;
import org.apache.http.HttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpHead;
import org.apache.http.util.EntityUtils;
import org.apache.tomcat.util.http.fileupload.IOUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
import java.io.*;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.concurrent.atomic.AtomicInteger;
@ -39,6 +39,10 @@ public final class CacheStoreCentral {
private final static Hashtable<String, File> imageCache = new Hashtable<>();
private final static JsonRedisCacheStore imageChecksumCache =
new JsonRedisCacheStore(BotGlobal.getGlobal().getRedisServer(),
"imageChecksum", BotGlobal.getGlobal().getGson());
/**
* 作品信息缓存 - 不过期
*/
@ -142,30 +146,29 @@ public final class CacheStoreCentral {
downloadLink.substring(downloadLink.lastIndexOf("/") + 1));
log.debug("FileName: {}, DownloadLink: {}", fileName, downloadLink);
if(!imageCache.containsKey(fileName)) {
if(imageFile.exists()) {
HttpHead headRequest = new HttpHead(downloadLink);
headRequest.addHeader("Referer", PixivURL.getPixivRefererLink(illustId));
HttpResponse headResponse;
try {
headResponse = BotGlobal.getGlobal().getPixivDownload().getHttpClient().execute(headRequest);
} catch (IOException e) {
log.error("获取图片大小失败！", e);
return "图片获取失败!";
}
String contentLengthStr = headResponse
.getFirstHeader(HttpHeaderNames.CONTENT_LENGTH.toString())
.getValue();
log.debug("图片大小: {}B", contentLengthStr);
if (imageFile.length() == Long.parseLong(contentLengthStr)) {
imageCache.put(URLs.getResourceName(downloadLink), imageFile);
log.debug("作品Id {} 第 {} 页缓存已补充.", illustId, pageIndex);
return getImageToBotCode(imageFile, false).toString();
if(imageFile.exists() && imageFile.isFile()) {
ImageChecksum imageChecksum = getImageChecksum(illustId, pageIndex);
if(imageChecksum != null) {
try {
log.debug("正在检查作品Id {} 第 {} 页图片文件 {} ...", illustId, pageIndex, imageFile.getName());
if (ImageChecksum.checkFile(imageChecksum, Files.readAllBytes(imageFile.toPath()))) {
imageCache.put(URLs.getResourceName(downloadLink), imageFile);
log.debug("作品Id {} 第 {} 页缓存已补充.", illustId, pageIndex);
return getImageToBotCode(imageFile, false).toString();
} else {
log.warn("图片文件 {} 校验失败, 重新下载图片...", imageFile.getName());
}
} catch(IOException e) {
log.error("文件检验时读取失败, 重新下载文件...(file: {})", imageFile.getPath());
}
} else {
log.warn("图片存在但校验不存在, 重新下载图片...");
}
}
try {
Throwable throwable = ImageCacheStore.executeCacheRequest(
new ImageCacheObject(imageCache, illustId, downloadLink, imageFile));
new ImageCacheObject(imageCache, illustId, pageIndex, downloadLink, imageFile));
if(throwable != null) {
throw throwable;
}
@ -457,6 +460,20 @@ public final class CacheStoreCentral {
return resultBody;
}
protected static ImageChecksum getImageChecksum(int illustId, int pageIndex) {
String cacheKey = illustId + ":" + pageIndex;
if(!imageChecksumCache.exists(cacheKey)) {
return null;
} else {
return ImageChecksum.fromJsonObject(imageChecksumCache.getCache(cacheKey).getAsJsonObject());
}
}
protected static void setImageChecksum(ImageChecksum checksum) {
String cacheKey = checksum.getIllustId() + ":" + checksum.getPage();
imageChecksumCache.update(cacheKey, ImageChecksum.toJsonObject(checksum), 0);
}
/**
* 合并String并存取到常量池, 以保证对象一致
* @param keys String对象
@ -469,4 +486,150 @@ public final class CacheStoreCentral {
}
return sb.toString().intern();
}
/**
* 图片检验信息
*/
public static class ImageChecksum implements Serializable {
private final static MessageDigestUtils.Algorithm ALGORITHM = MessageDigestUtils.Algorithm.SHA256;
private ImageChecksum() {}
private int illustId;
private int page;
private String fileName;
private long size;
private byte[] checksum;
public long getSize() {
return size;
}
public void setSize(long size) {
this.size = size;
}
public byte[] getChecksum() {
return checksum;
}
public void setChecksum(byte[] checksum) {
this.checksum = checksum;
}
public int getIllustId() {
return illustId;
}
public void setIllustId(int illustId) {
this.illustId = illustId;
}
public String getFileName() {
return fileName;
}
public void setFileName(String fileName) {
this.fileName = fileName;
}
public int getPage() {
return page;
}
public void setPage(int page) {
this.page = page;
}
public static ImageChecksum buildImageChecksumFromStream(
int illustId, int pageIndex,
String fileName, InputStream imageStream) throws IOException {
ImageChecksum checksum = new ImageChecksum();
checksum.setIllustId(illustId);
checksum.setPage(pageIndex);
checksum.setFileName(fileName);
ByteArrayOutputStream bufferStream = new ByteArrayOutputStream();
checksum.setSize(IOUtils.copyLarge(imageStream, bufferStream));
checksum.setChecksum(
MessageDigestUtils.encrypt(bufferStream.toByteArray(), ALGORITHM));
return checksum;
}
/**
* 将图片检验信息转换成JsonObject
* @param checksum 检验信息对象
* @return 转换后的JsonObject对象
*/
public static JsonObject toJsonObject(ImageChecksum checksum) {
JsonObject result = new JsonObject();
result.addProperty("illustId", checksum.getIllustId());
result.addProperty("page", checksum.getPage());
result.addProperty("fileName", checksum.getFileName());
result.addProperty("size", checksum.getSize());
result.addProperty("checksum", Base64.getEncoder().encodeToString(checksum.getChecksum()));
return result;
}
/**
* 从JsonObject转换到图片检验信息
* @param checksumObject JsonObject对象
* @return 转换后的图片检验信息对象
*/
public static ImageChecksum fromJsonObject(JsonObject checksumObject) {
ImageChecksum checksum = new ImageChecksum();
checksum.setIllustId(checksumObject.get("illustId").getAsInt());
checksum.setPage(checksumObject.get("page").getAsInt());
checksum.setFileName(checksumObject.get("fileName").getAsString());
checksum.setSize(checksumObject.get("size").getAsLong());
checksum.setChecksum(Base64.getDecoder().decode(checksumObject.get("checksum").getAsString()));
return checksum;
}
/**
* 比对图片文件是否完整.
* @param checksum 图片检验信息
* @param imageData 图片数据
* @return 如果检验成功, 则返回true
*/
public static boolean checkFile(ImageChecksum checksum, byte[] imageData) {
byte[] sha256Checksum = MessageDigestUtils.encrypt(imageData, ALGORITHM);
return checksum.getSize() == imageData.length &&
Arrays.equals(checksum.getChecksum(), sha256Checksum);
}
@Override
public String toString() {
return "ImageChecksum{" +
"illustId=" + illustId +
", page=" + page +
", fileName='" + fileName + '\'' +
", size=" + size +
", checksum=" + Base64.getEncoder().encodeToString(getChecksum()) +
'}';
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
ImageChecksum checksum1 = (ImageChecksum) o;
return illustId == checksum1.illustId &&
page == checksum1.page &&
size == checksum1.size &&
Objects.equals(fileName, checksum1.fileName) &&
Arrays.equals(checksum, checksum1.checksum);
}
@Override
public int hashCode() {
int result = Objects.hash(illustId, page, fileName, size);
result = 31 * result + Arrays.hashCode(checksum);
return result;
}
}
}

20
src/main/java/net/lamgc/cgj/bot/cache/ImageCacheHandler.java vendored
View File

@ -13,9 +13,7 @@ import org.apache.http.impl.client.HttpClientBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.*;
import java.util.Collections;
import java.util.HashSet;
import java.util.Set;
@ -69,8 +67,20 @@ public class ImageCacheHandler implements EventHandler {
}
log.debug("正在下载...(Content-Length: {}KB)", response.getEntity().getContentLength() / 1024);
try(FileOutputStream fos = new FileOutputStream(storeFile)) {
IOUtils.copy(response.getEntity().getContent(), fos);
ByteArrayOutputStream bufferOutputStream = new ByteArrayOutputStream();
try(FileOutputStream fileOutputStream = new FileOutputStream(storeFile)) {
IOUtils.copy(response.getEntity().getContent(), bufferOutputStream);
ByteArrayInputStream bufferInputStream = new ByteArrayInputStream(bufferOutputStream.toByteArray());
CacheStoreCentral.ImageChecksum imageChecksum = CacheStoreCentral.ImageChecksum
.buildImageChecksumFromStream(
event.getIllustId(),
event.getPageIndex(),
event.getStoreFile().getName(),
bufferInputStream
);
bufferInputStream.reset();
IOUtils.copy(bufferInputStream, fileOutputStream);
CacheStoreCentral.setImageChecksum(imageChecksum);
} catch (IOException e) {
log.error("下载图片时发生异常", e);
throw e;

18
src/main/java/net/lamgc/cgj/bot/cache/ImageCacheObject.java vendored
View File

@ -12,13 +12,16 @@ public class ImageCacheObject implements EventObject {
private final int illustId;
private final int pageIndex;
private final String downloadLink;
private final File storeFile;
public ImageCacheObject(Map<String, File> imageCache, int illustId, String downloadLink, File storeFile) {
public ImageCacheObject(Map<String, File> imageCache, int illustId, int pageIndex, String downloadLink, File storeFile) {
this.imageCache = imageCache;
this.illustId = illustId;
this.pageIndex = pageIndex;
this.downloadLink = downloadLink;
this.storeFile = storeFile;
}
@ -39,12 +42,17 @@ public class ImageCacheObject implements EventObject {
return illustId;
}
public int getPageIndex() {
return pageIndex;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
ImageCacheObject that = (ImageCacheObject) o;
return illustId == that.illustId &&
pageIndex == that.pageIndex &&
Objects.equals(imageCache, that.imageCache) &&
Objects.equals(downloadLink, that.downloadLink) &&
Objects.equals(storeFile, that.storeFile);
@ -52,13 +60,15 @@ public class ImageCacheObject implements EventObject {
@Override
public int hashCode() {
return Objects.hash(imageCache, illustId, downloadLink, storeFile);
return Objects.hash(imageCache, illustId, pageIndex, downloadLink, storeFile);
}
@Override
public String toString() {
return "ImageCacheObject@" + Integer.toHexString(hashCode()) + "{" +
"illustId=" + illustId +
return "ImageCacheObject{" +
"imageCache=" + imageCache +
", illustId=" + illustId +
", pageIndex=" + pageIndex +
", downloadLink='" + downloadLink + '\'' +
", storeFile=" + storeFile +
'}';