LamGC/ContentGrabbingJi
1
0
Fork 0
mirror of https://github.com/LamGC/ContentGrabbingJi.git synced 2025-04-30 06:37:36 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[Change] PixivDownload 因日志系统升级, 重新调整日志等级;

Browse Source
This commit is contained in:
LamGC 2020-04-11 12:28:36 +08:00
parent bc6aaa49a0
commit 841a368767
1 changed files with 25 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
src/main/java/net/lamgc/cgj/pixiv/PixivDownload.java
View File

@ -85,13 +85,13 @@ public class PixivDownload {
do { do {
request = new HttpGet(PixivURL.PIXIV_USER_COLLECTION_PAGE.replace("{pageIndex}", Integer.toString(++pageIndex))); request = new HttpGet(PixivURL.PIXIV_USER_COLLECTION_PAGE.replace("{pageIndex}", Integer.toString(++pageIndex)));
setCookieInRequest(request, cookieStore); setCookieInRequest(request, cookieStore);
log.info("Request Link: " + request.getURI().toString()); log.debug("Request Link: " + request.getURI().toString());
HttpResponse response = httpClient.execute(request); HttpResponse response = httpClient.execute(request);
// 解析网页内容，获得所有的收藏信息 // 解析网页内容，获得所有的收藏信息
document = Jsoup.parse(EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8)); document = Jsoup.parse(EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8));
Elements items = document.select(".display_editable_works .image-item a.work"); Elements items = document.select(".display_editable_works .image-item a.work");
List<String> hrefList = items.eachAttr("href"); List<String> hrefList = items.eachAttr("href");
log.info("第 {} 页获取到的图片项数量: {}", pageIndex, hrefList.size()); log.debug("第 {} 页获取到的图片项数量: {}", pageIndex, hrefList.size());
if(hrefList.size() == 0) { if(hrefList.size() == 0) {
break; break;
} }
@ -114,10 +114,10 @@ public class PixivDownload {
} }
} }
} while(!document.select(".pager-container>.next").isEmpty()); } while(!document.select(".pager-container>.next").isEmpty());
log.info("获取完成."); log.debug("获取完成.");
AtomicInteger count = new AtomicInteger(1); AtomicInteger count = new AtomicInteger(1);
linkList.forEach(link -> { linkList.forEach(link -> {
log.info("Next Link [{}]: {}", count.getAndIncrement(), link); log.debug("Next Link [{}]: {}", count.getAndIncrement(), link);
InputStream imageInputStream = null; InputStream imageInputStream = null;
int tryCount = 0; int tryCount = 0;
do { do {
@ -156,12 +156,12 @@ public class PixivDownload {
for(int illustIndex = 0; illustIndex < elements.size(); illustIndex++){ for(int illustIndex = 0; illustIndex < elements.size(); illustIndex++){
String href = elements.get(illustIndex).attr("href"); String href = elements.get(illustIndex).attr("href");
int illustId = Integer.parseInt(href.substring(href.lastIndexOf("/") + 1)); int illustId = Integer.parseInt(href.substring(href.lastIndexOf("/") + 1));
log.info("({}/{}) Href: {}, IllustID: {}", illustIndex + 1, elements.size(), href, illustId); log.debug("({}/{}) Href: {}, IllustID: {}", illustIndex + 1, elements.size(), href, illustId);
List<String> pageLinkList = getIllustAllPageDownload(httpClient, this.cookieStore, illustId, quality); List<String> pageLinkList = getIllustAllPageDownload(httpClient, this.cookieStore, illustId, quality);
for (int linkIndex = 0; linkIndex < pageLinkList.size(); linkIndex++) { for (int linkIndex = 0; linkIndex < pageLinkList.size(); linkIndex++) {
String link = pageLinkList.get(linkIndex); String link = pageLinkList.get(linkIndex);
String fileName = link.substring(link.lastIndexOf("/") + 1); String fileName = link.substring(link.lastIndexOf("/") + 1);
log.info("({}/{})正在处理 {}", linkIndex, pageLinkList.size(), fileName); log.debug("({}/{}) 正在处理 {}", linkIndex, pageLinkList.size(), fileName);
InputStream imageInputStream = null; InputStream imageInputStream = null;
int tryCount = 0; int tryCount = 0;
do { do {
@ -178,10 +178,10 @@ public class PixivDownload {
try(InputStream pageInputStream = new BufferedInputStream(imageInputStream, 256 * 1024)) { try(InputStream pageInputStream = new BufferedInputStream(imageInputStream, 256 * 1024)) {
fn.accept(fileName, pageInputStream); fn.accept(fileName, pageInputStream);
} }
log.info("Done!"); log.debug("Done!");
} }
log.info("IllustId {} 处理完成.", illustId); log.debug("IllustId {} 处理完成.", illustId);
} }
} }
@ -219,8 +219,8 @@ public class PixivDownload {
int authorId = rankInfo.get("user_id").getAsInt(); int authorId = rankInfo.get("user_id").getAsInt();
String authorName = rankInfo.get("user_name").getAsString(); String authorName = rankInfo.get("user_name").getAsString();
String title = rankInfo.get("title").getAsString(); String title = rankInfo.get("title").getAsString();
log.info("当前到第 {}/{} 名(总共 {} 名), IllustID: {}, Author: ({}) {}, Title: {}", rank, rankStart + range - 1, range, illustId, authorId, authorName, title); log.debug("当前到第 {}/{} 名(总共 {} 名), IllustID: {}, Author: ({}) {}, Title: {}", rank, rankStart + range - 1, range, illustId, authorId, authorName, title);
log.info("正在获取PagesLink..."); log.debug("正在获取PagesLink...");
List<String> linkList; List<String> linkList;
try { try {
linkList = getIllustAllPageDownload(httpClient, this.cookieStore, illustId, quality); linkList = getIllustAllPageDownload(httpClient, this.cookieStore, illustId, quality);
@ -232,17 +232,17 @@ public class PixivDownload {
} }
return; return;
} }
log.info("PagesLink 获取完成, 总数: {}", linkList.size()); log.debug("PagesLink 获取完成, 总数: {}", linkList.size());
for (int pageIndex = 0; pageIndex < linkList.size(); pageIndex++) { for (int pageIndex = 0; pageIndex < linkList.size(); pageIndex++) {
String downloadLink = linkList.get(pageIndex); String downloadLink = linkList.get(pageIndex);
log.info("当前Page: {}/{}", pageIndex + 1, linkList.size()); log.debug("当前Page: {}/{}", pageIndex + 1, linkList.size());
try(InputStream imageInputStream = new BufferedInputStream(getImageAsInputStream(HttpClientBuilder.create().build(), downloadLink), 256 * 1024)) { try(InputStream imageInputStream = new BufferedInputStream(getImageAsInputStream(HttpClientBuilder.create().build(), downloadLink), 256 * 1024)) {
fn.download(rank, downloadLink, rankInfo.deepCopy(), imageInputStream); fn.download(rank, downloadLink, rankInfo.deepCopy(), imageInputStream);
} catch(IOException e) { } catch(IOException e) {
log.error("下载插画时发生异常", e); log.error("下载插画时发生异常", e);
return; return;
} }
log.info("完成."); log.debug("完成.");
} }
}); });
} }
@ -257,7 +257,7 @@ public class PixivDownload {
public static List<JsonObject> getRanking(JsonArray rankingArray, int rankStart, int range) { public static List<JsonObject> getRanking(JsonArray rankingArray, int rankStart, int range) {
//需要添加一个总量, 否则会完整跑完一次. //需要添加一个总量, 否则会完整跑完一次.
//检查是否为最后一次请求，和剩余量有多少 //检查是否为最后一次请求，和剩余量有多少
log.info("正在读取JsonArray...(rankStart: {}, range: {})", rankStart, range); log.debug("正在读取JsonArray...(rankStart: {}, range: {})", rankStart, range);
ArrayList<JsonObject> results = new ArrayList<>(rankingArray.size()); ArrayList<JsonObject> results = new ArrayList<>(rankingArray.size());
for (int rankIndex = rankStart; rankIndex < rankingArray.size() && rankIndex < range; rankIndex++) { for (int rankIndex = rankStart; rankIndex < rankingArray.size() && rankIndex < range; rankIndex++) {
JsonElement jsonElement = rankingArray.get(rankIndex); JsonElement jsonElement = rankingArray.get(rankIndex);
@ -267,10 +267,10 @@ public class PixivDownload {
int authorId = rankInfo.get("user_id").getAsInt(); int authorId = rankInfo.get("user_id").getAsInt();
String authorName = rankInfo.get("user_name").getAsString(); String authorName = rankInfo.get("user_name").getAsString();
String title = rankInfo.get("title").getAsString(); String title = rankInfo.get("title").getAsString();
log.info("Array-当前到第 {}/{} 名(总共 {} 名), IllustID: {}, Author: ({}) {}, Title: {}", rank, rankStart + range - 1, range, illustId, authorId, authorName, title); log.debug("Array-当前到第 {}/{} 名(总共 {} 名), IllustID: {}, Author: ({}) {}, Title: {}", rank, rankStart + range, range, illustId, authorId, authorName, title);
results.add(rankInfo); results.add(rankInfo);
} }
log.info("JsonArray读取完成."); log.debug("JsonArray读取完成.");
return results; return results;
} }
@ -305,9 +305,9 @@ public class PixivDownload {
ArrayList<JsonObject> results = new ArrayList<>(); ArrayList<JsonObject> results = new ArrayList<>();
for (int requestCount = startPage; requestCount <= requestFrequency && requestCount <= 10; requestCount++) { for (int requestCount = startPage; requestCount <= requestFrequency && requestCount <= 10; requestCount++) {
int rangeStart = (requestCount - 1) * 50 + 1; int rangeStart = (requestCount - 1) * 50 + 1;
log.info("正在请求第 {} 到 {} 位排名榜 (第{}次请求, 共 {} 次)", rangeStart, rangeStart + 49, requestCount - startPage + 1, requestFrequency - startPage); log.debug("正在请求第 {} 到 {} 位排名榜 (第{}次请求, 共 {} 次)", rangeStart, rangeStart + 49, requestCount - startPage + 1, requestFrequency - startPage);
HttpGet request = createHttpGetRequest(PixivURL.getRankingLink(contentType, mode, time, requestCount, true)); HttpGet request = createHttpGetRequest(PixivURL.getRankingLink(contentType, mode, time, requestCount, true));
log.info("Request URL: {}", request.getURI()); log.debug("Request URL: {}", request.getURI());
HttpResponse response = httpClient.execute(request); HttpResponse response = httpClient.execute(request);
String content = EntityUtils.toString(response.getEntity()); String content = EntityUtils.toString(response.getEntity());
log.debug("Content: " + content); log.debug("Content: " + content);
@ -317,7 +317,7 @@ public class PixivDownload {
break; break;
} }
JsonArray rankingArray = contentObject.getAsJsonArray("contents"); JsonArray rankingArray = contentObject.getAsJsonArray("contents");
log.info("正在解析数据..."); log.debug("正在解析数据...");
//需要添加一个总量, 否则会完整跑完一次. //需要添加一个总量, 否则会完整跑完一次.
//检查是否为最后一次请求，和剩余量有多少 //检查是否为最后一次请求，和剩余量有多少
@ -330,11 +330,11 @@ public class PixivDownload {
int authorId = rankInfo.get("user_id").getAsInt(); int authorId = rankInfo.get("user_id").getAsInt();
String authorName = rankInfo.get("user_name").getAsString(); String authorName = rankInfo.get("user_name").getAsString();
String title = rankInfo.get("title").getAsString(); String title = rankInfo.get("title").getAsString();
log.info("Download-当前到第 {}/{} 名(总共 {} 名), IllustID: {}, Author: ({}) {}, Title: {}", rank, rankStart + range - 1, range, illustId, authorId, authorName, title); log.debug("Download-当前到第 {}/{} 名(总共 {} 名), IllustID: {}, Author: ({}) {}, Title: {}", rank, rankStart + range - 1, range, illustId, authorId, authorName, title);
results.add(rankInfo); results.add(rankInfo);
} }
firstRequest = false; firstRequest = false;
log.info("第 {} 到 {} 位排名榜完成. (第{}次请求)", rangeStart, rangeStart + 49, requestCount); log.debug("第 {} 到 {} 位排名榜完成. (第{}次请求)", rangeStart, rangeStart + 49, requestCount);
} }
if(requestFrequency > 10) { if(requestFrequency > 10) {
@ -402,7 +402,7 @@ public class PixivDownload {
if(resultObject.get("error").getAsBoolean()) { if(resultObject.get("error").getAsBoolean()) {
String message = resultObject.get("message").getAsString(); String message = resultObject.get("message").getAsString();
log.info("请求错误, 错误信息: {}", message); log.debug("请求错误, 错误信息: {}", message);
throw new IOException(message); throw new IOException(message);
} }
@ -410,7 +410,7 @@ public class PixivDownload {
ArrayList<String> resultList = new ArrayList<>(); ArrayList<String> resultList = new ArrayList<>();
String qualityType = quality == null ? "original" : quality.toString().toLowerCase(); String qualityType = quality == null ? "original" : quality.toString().toLowerCase();
log.info("已选择插画类型: {}", qualityType); log.debug("已选择插画类型: {}", qualityType);
linkArray.forEach(el -> { linkArray.forEach(el -> {
JsonObject urlObj = el.getAsJsonObject().getAsJsonObject("urls"); JsonObject urlObj = el.getAsJsonObject().getAsJsonObject("urls");
resultList.add(urlObj.get(qualityType).getAsString()); resultList.add(urlObj.get(qualityType).getAsString());
@ -476,8 +476,8 @@ public class PixivDownload {
HttpResponse response = httpClient.execute(request); HttpResponse response = httpClient.execute(request);
log.debug("response: {}", response); log.debug("response: {}", response);
log.info("Content Length: {}KB", Float.parseFloat(response.getFirstHeader(HttpHeaderNames.CONTENT_LENGTH.toString()).getValue()) / 1024F); log.debug("Content Length: {}KB", Float.parseFloat(response.getFirstHeader(HttpHeaderNames.CONTENT_LENGTH.toString()).getValue()) / 1024F);
log.info("{}", response.getFirstHeader(HttpHeaderNames.CONTENT_TYPE.toString())); log.debug("{}", response.getFirstHeader(HttpHeaderNames.CONTENT_TYPE.toString()));
return response.getEntity().getContent(); return response.getEntity().getContent();
} }