LamGC/ContentGrabbingJi
1
0
Fork 0
mirror of https://github.com/LamGC/ContentGrabbingJi.git synced 2025-07-03 22:07:26 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[Delete] 删除不再使用的类和方法;

Browse Source 
[Update] 优化Javadoc并设置编译注解, 优化代码;
[Change] 调整部分日志的输出级别;
[Change] RedisPoolCacheStore clear方法将根据返回信息确定是否清空成功;
[Change] Dockerfile.sample 调整镜像构建步骤;
This commit is contained in:
LamGC
2020-04-30 02:23:31 +08:00
parent 0075446412
commit fba6d3532c
18 changed files with 29 additions and 743 deletions
Download Patch File Download Diff File Expand all files Collapse all files

69
src/main/java/net/lamgc/cgj/pixiv/IllustInfo.java
View File

@ -1,69 +0,0 @@
package net.lamgc.cgj.pixiv;
/**
* 插图(集)的信息
*/
public class IllustInfo{
/**
* 插图(集)ID
*/
public final int illustID;
/**
* 第几页
*/
public final int page;
/**
* 插图标题
*/
public final String title;
/**
* 插图说明
*/
public final String description;
/**
* 插图标签
*/
public final String[] tags;
/**
* 插图图片长度
*/
public final int width;
/**
* 插图图片高度
*/
public final int height;
/**
* 作者名
*/
public final String authorName;
/**
* 作者用户ID
*/
public final int authorUserID;
public IllustInfo(int illustID, String title, String description, String[] tags, int width, int height, String authorName, int authorUserID){
this(illustID, 0, title, description, tags, width, height, authorName, authorUserID);
}
public IllustInfo(int illustID, int p, String title, String description, String[] tags, int width, int height, String authorName, int authorUserID){
this.illustID = illustID;
this.page = p;
this.title = title;
this.description = description;
this.tags = tags;
this.width = width;
this.height = height;
this.authorName = authorName;
this.authorUserID = authorUserID;
}
}

267
src/main/java/net/lamgc/cgj/pixiv/Pixiv.java
View File

@ -1,267 +0,0 @@
package net.lamgc.cgj.pixiv;
import com.google.gson.JsonArray;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import org.apache.http.HttpHost;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.conn.ConnectionPoolTimeoutException;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.util.EntityUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.util.*;
public class Pixiv {
/**
* illust Link
*/
public final static String ATTR_LINK = "link";
/**
* illust Id
*/
public final static String ATTR_ILLUST_ID = "illustId";
/**
* illust Title
*/
public final static String ATTR_TITLE = "title";
/**
* illust Author Name
*/
public final static String ATTR_AUTHOR_NAME = "authorName";
/**
* illust Author UserID
*/
public final static String ATTR_AUTHOR_ID = "authorId";
private final HttpClient httpClient;
public Pixiv(HttpClient client){
this.httpClient = client;
}
/**
* 使用帐号密码登录Pixiv
* @param PixivID Pixiv账户登录名
* @param Password Pixiv帐号密码
* @throws IOException 当登录连接出现异常时抛出
* @deprecated {@link PixivSession#Login(String, String)} 已经废除, 故本方法不可用
*/
public Pixiv(String PixivID, String Password) throws IOException {
this(PixivID, Password, null);
}
/**
* 使用帐号密码登录Pixiv
* @param PixivID Pixiv账户登录名
* @param Password Pixiv帐号密码
* @param proxy 代理设置
* @throws IOException 当登录连接出现异常时抛出
* @deprecated {@link PixivSession#Login(String, String)} 已经废除, 故本方法不可用
*/
public Pixiv(String PixivID, String Password, HttpHost proxy) throws IOException {
PixivSession pixivSession = new PixivSession(proxy, null);
if(pixivSession.Login(PixivID, Password)){
System.out.println("P站登录成功!");
}else{
System.out.println("P站登录失败!错误信息: " + pixivSession.getErrMsg());
throw new RuntimeException(pixivSession.getErrMsg());
}
//httpClient = pixivSession.getHttpClient();
httpClient = HttpClientBuilder.create()
.setDefaultCookieStore(pixivSession.getCookieStore())
.build();
}
/**
* 获取首页推荐列表
* @return 首页推荐列表, 一个Map对应一个推荐项, 使用<code>ATTR_</code>开头常量访问即可
* @throws IOException
*/
public List<Map<String, String>> getRecommend() throws IOException {
HttpGet getRequest = new HttpGet(PixivURL.PIXIV_INDEX_URL);
HttpResponse response = httpClient.execute(getRequest);
String pageAsXML = EntityUtils.toString(response.getEntity(),"utf-8");
//获取推荐图列表(li)
//System.out.println(pageAsXML);
Document document = Jsoup.parse(pageAsXML);
List<String> links = document.select("._image-items.gtm-illust-recommend-zone>li>.gtm-illust-recommend-thumbnail-link").eachAttr("href");
List<String> illustId = document.select("._image-items.gtm-illust-recommend-zone>li>.gtm-illust-recommend-thumbnail-link").eachAttr("data-gtm-recommend-illust-id");
List<String> title = document.select("._image-items.gtm-illust-recommend-zone>li>.gtm-illust-recommend-title>h1").eachAttr("title");
List<String> authorName = document.select("._image-items.gtm-illust-recommend-zone>li>.gtm-illust-recommend-user-name").eachText();
List<String> authorId = document.select("._image-items.gtm-illust-recommend-zone>li>.gtm-illust-recommend-user-name").eachAttr("data-user_id");
List<Map<String, String>> recommendList = new ArrayList<>();
for(int i = 0; i < links.size(); i++){
//System.out.println(links.get(i));
Map<String, String> info = new HashMap<>();
info.put(ATTR_LINK, PixivURL.PIXIV_INDEX_URL + links.get(i));
info.put(ATTR_ILLUST_ID, illustId.get(i));
info.put(ATTR_TITLE, title.get(i));
info.put(ATTR_AUTHOR_NAME, authorName.get(i));
info.put(ATTR_AUTHOR_ID, authorId.get(i));
recommendList.add(info);
}
return recommendList;
}
public String[] getAllDownloadLink(int illustID) throws IOException {
HttpGet illustPage = new HttpGet(PixivURL.PIXIV_ILLUST_API_URL.replaceAll("\\{illustId}", String.valueOf(illustID)));
HttpResponse response = httpClient.execute(illustPage);
String pageAsXML = EntityUtils.toString(response.getEntity(),"utf-8");
//System.out.println(pageAsXML);
JsonObject resultObj = (JsonObject) new JsonParser().parse(pageAsXML);
if(!resultObj.get("error").getAsBoolean()){
JsonArray bodyArray = resultObj.get("body").getAsJsonArray();
int length = bodyArray.size();
String[] result = new String[length];
for(int i = 0; i < length; i++){
JsonObject childObj = bodyArray.get(i).getAsJsonObject();
result[i] = childObj.get("urls").getAsJsonObject().get("original").getAsString();
}
return result;
}else{
return null;
}
}
/**
* 下载P站图片
* @param illustID 插图ID
* @return 成功返回图片输入流,失败或为多图则返回null
*/
public InputStream[] downloadIllustImage(int illustID) throws IOException {
String[] links = getAllDownloadLink(illustID);
List<InputStream> inputStreamList = new ArrayList<>();
int count = 1;
boolean retry = false;
for(int i = 0; i < links.length; i++){
try {
long sleepTime = (new Random().nextInt(4) + 2) * 1000;
System.out.println("nextTime: " + (float)(sleepTime / 1000));
Thread.sleep(sleepTime);
} catch (InterruptedException ignored) {}
String link = links[i];
System.out.print("page:" + count++ + "/" + links.length + " ...");
HttpGet imgGet = new HttpGet(link);
//关键!如果不加上Referer的话,会返回403
imgGet.setHeader("Referer", PixivURL.PIXIV_ILLUST_MEDIUM_URL.replaceAll("\\{illustId}", String.valueOf(illustID)));
RequestConfig config = RequestConfig.custom()
.setConnectTimeout(20 * 1000)
.setConnectionRequestTimeout(20 * 1000)
.setSocketTimeout(30 * 1000)
.build();
imgGet.setConfig(config);
HttpResponse response;
try {
response = httpClient.execute(imgGet);
}catch(ConnectionPoolTimeoutException e){
if(retry){
retry = false;
System.out.println("获取失败,跳过...");
continue;
}
System.out.println("连接超时,重新获取...");
retry = true;
i--;
continue;
}
retry = false;
ByteArrayOutputStream cacheOutputStream = new ByteArrayOutputStream((int)response.getEntity().getContentLength());
InputStream content = response.getEntity().getContent();
int readLength;
byte[] cache = new byte[4096];
while((readLength = content.read(cache)) != -1){
cacheOutputStream.write(cache, 0, readLength);
}
byte[] data = cacheOutputStream.toByteArray();
//System.out.println("读到数据: " + data.length);
inputStreamList.add(new ByteArrayInputStream(data));
System.out.println("done!（length: " + response.getEntity().getContentLength() + ")");
}
return inputStreamList.toArray(new InputStream[0]);
}
/**
* 下载P站图片
* @param illustID 插图ID
* @return 成功返回图片输入流,失败或为多图则返回null
*/
public InputStream downloadIllustImages(int illustID){
throw new UnsupportedOperationException();
}
/**
* 通过解析插图详情页获取
* - 插图标题
* - 插图作者(及其UserId)
* - 插图上传时间
* - 插图标签(原始标签)
* ...
* @return 成功返回IllustInfo对象,失败返回null
*/
public IllustInfo[] getIllustInfo(int[] illustIDs) throws IOException {
//获取Api
HttpGet apiRequest = new HttpGet(PixivURL.getPixivIllustInfoAPI(illustIDs));
HttpResponse response = httpClient.execute(apiRequest);
String resultText = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8);
System.out.println(resultText);
JsonObject resultObj = ((JsonObject) new JsonParser().parse(resultText));
if(resultObj.get("error").getAsBoolean()){
System.err.println("获取失败!");
return null;
}
List<IllustInfo> illustInfoList = new ArrayList<>();
JsonArray illustArray = resultObj.get("body").getAsJsonObject().get("illusts").getAsJsonArray();
illustArray.forEach(jsonElement -> {
JsonObject illustInfoObj = jsonElement.getAsJsonObject();
JsonArray tagsArray = illustInfoObj.get("tags").getAsJsonArray();
String[] tags = new String[tagsArray.size()];
for(int i = 0; i < tags.length; i++){
tags[i] = tagsArray.get(i).getAsString();
}
//TODO: 通过不需要作者id就能获取图片信息的api无法获取图片尺寸
IllustInfo illustInfo = new IllustInfo(
illustInfoObj.get("workId").getAsInt(),
illustInfoObj.get("title").getAsString(),
null,
tags,
-1,
-1,
illustInfoObj.get("userName").getAsString(),
illustInfoObj.get("userId").getAsInt()
);
});
return null;
}
/**
* 获取指定用户的所有插画
*/
public void getUserAllIllustTest() {
}
}

7
src/main/java/net/lamgc/cgj/pixiv/PixivSearchBuilder.java
View File

@ -9,6 +9,13 @@ import java.util.Date;
import java.util.HashSet;
import java.util.Objects;
/**
* Pixiv搜索URL构造器
* <p>该构造器通过分析Pixiv搜索链接可用的参数而开发, 对搜索链接的构造有高度自定义能力.</p>
* @author LamGC
* @see PixivURL#PIXIV_SEARCH_CONTENT_URL
*/
@SuppressWarnings("ALL")
public class PixivSearchBuilder {
private final String content;

171
src/main/java/net/lamgc/cgj/pixiv/PixivSession.java
View File

@ -1,171 +0,0 @@
package net.lamgc.cgj.pixiv;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import org.apache.http.Header;
import org.apache.http.HttpHost;
import org.apache.http.HttpResponse;
import org.apache.http.NameValuePair;
import org.apache.http.client.CookieStore;
import org.apache.http.client.HttpClient;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.impl.client.BasicCookieStore;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.message.BasicHeader;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
public class PixivSession {
public static final String USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36";
/**
* 全登陆过程的关键,
* 保存会话用的cookieStore!
*/
private CookieStore cookieStore = new BasicCookieStore();
/**
* 可以直接使用的HttpClient对象
*/
private HttpClient httpClient;
/**
* 最后一次登录的错误信息
*/
private String errMsg;
public PixivSession(){
this(null);
}
public PixivSession(CookieStore cookieStore){
this(null, cookieStore);
}
/**
* 创建一个Pixiv登录会话
*/
public PixivSession(HttpHost proxy, CookieStore cookieStore) {
if(cookieStore != null){
this.cookieStore = cookieStore;
}
List<Header> defaultHeader = new ArrayList<>();
defaultHeader.add(new BasicHeader("accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9"));
defaultHeader.add(new BasicHeader("user-agent", PixivSession.USER_AGENT));
defaultHeader.add(new BasicHeader("accept-encoding", "gzip, deflate, br"));
defaultHeader.add(new BasicHeader("accept-language", "zh-CN,zh;q=0.9"));
/*defaultHeader.add(new BasicHeader("sec-fetch-mode", "navigate"));
defaultHeader.add(new BasicHeader("sec-fetch-site", "same-origin"));
defaultHeader.add(new BasicHeader("upgrade-insecure-requests", "1"));*/
//创建一个Http访问器
httpClient = HttpClients.custom()
.setDefaultCookieStore(cookieStore)
.setDefaultHeaders(defaultHeader)
.setProxy(proxy)
.build();
}
/**
* 程序自行通过帐号密码登录Pixiv.
* @param PixivID Pixiv帐号
* @param Password Pixiv密码
* @return 登录成功返回true
* @throws IOException 当登录抛出异常时返回
* @deprecated Pixiv已经新增Google人机验证, 程序已无法自行登录Pixiv
*/
public boolean Login(String PixivID, String Password) throws IOException {
// 获取登录接口所需的PostKey
String post_key = getPostKey();
HttpPost postRequest = new HttpPost(PixivURL.PIXIV_LOGIN_URL); //https://accounts.pixiv.net/api/login?lang=zh
List<NameValuePair> params = new ArrayList<>();
params.add(new BasicNameValuePair("pixiv_id", PixivID));
params.add(new BasicNameValuePair("password", Password));
params.add(new BasicNameValuePair("post_key", post_key));
//Form编码表单,作为Post的数据
postRequest.setEntity(new UrlEncodedFormEntity(params, StandardCharsets.UTF_8));
//启动访问
HttpResponse response = httpClient.execute(postRequest);
//获取接口返回数据
String httpXML = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8);
System.out.println(httpXML);
JsonObject responseJson = (JsonObject) new JsonParser().parse(httpXML);
if(!responseJson.get("error").getAsBoolean() && !responseJson.get("body").getAsJsonObject().has("validation_errors")){
errMsg = null;
return true;
}else{
errMsg = responseJson.get("body").getAsJsonObject().get("validation_errors").toString();
//System.err.println("登录失败！MSG: " + errMsg);
return false;
}
}
/**
* 登录前准备, 获取PostKey
* @return Post_Key
*/
private String getPostKey() throws IOException {
//创建请求,获取PostKey
HttpGet getRequest = new HttpGet(PixivURL.PIXIV_LOGIN_PAGE_URL);
//设置请求
//getRequest.setConfig(config);
getRequest.setHeader("User-Agent", USER_AGENT);
//启动访问
HttpResponse response = httpClient.execute(getRequest);
//获取网页内容
String pageAsXML = EntityUtils.toString(response.getEntity(),"utf-8");
//创建Http解析器
Document document = Jsoup.parse(pageAsXML);
//获取init-config内容
String init_config = document.getElementById("init-config").val();
//System.out.println(init_config);
//创建Json解析器解析init-config
JsonObject initConfigObj = (JsonObject) new JsonParser().parse(init_config);
//检查是否有postKey
if(!initConfigObj.has("pixivAccount.postKey")){
throw new RuntimeException("postKey获取失败!可能是Pixiv修改了登录过程!");
}
//获取postKey
return initConfigObj.get("pixivAccount.postKey").getAsString();
}
/**
* 获取CookieStore
* @return CookieStore
*/
public CookieStore getCookieStore(){
return cookieStore;
}
/**
* 获取可直接使用的HttpClient对象
* @return 已配置好的HttpClient对象
*/
public HttpClient getHttpClient(){
return this.httpClient;
}
public boolean hasError(){
return errMsg == null;
}
/**
* 获取错误信息
* @return 返回登录错误信息
* @deprecated {@link #Login(String, String)}已经废除, 故本接口废除
*/
public String getErrMsg(){
return errMsg;
}
}

16
src/main/java/net/lamgc/cgj/pixiv/PixivTag.java
View File

@ -1,16 +0,0 @@
package net.lamgc.cgj.pixiv;
public class PixivTag {
public static PixivTag create(String tagName) {
return null;
}
private PixivTag() {
}
}

1
src/main/java/net/lamgc/cgj/pixiv/PixivURL.java
View File

@ -9,6 +9,7 @@ import java.util.GregorianCalendar;
/**
* 目前已整理的一些Pixiv接口列表
*/
@SuppressWarnings("unused")
public class PixivURL {

3
src/main/java/net/lamgc/cgj/pixiv/PixivUgoiraBuilder.java
View File

@ -26,8 +26,7 @@ import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
/**
* Pixiv动图构建器.
* 可便捷的接收并处理动图.
* Pixiv动图构建器
*/
public final class PixivUgoiraBuilder {