LamGC/ContentGrabbingJi
1
0
Fork 0
mirror of https://github.com/LamGC/ContentGrabbingJi.git synced 2025-04-30 06:37:36 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[Delete] PixivAccessProxyServer 移除弃用的类;

Browse Source 
[Change] Locker 删除不必要的代码;
[Change] PixivDownload 删除多余的Javadoc内容;
[Change] TimeLimitThreadPoolExecutorTest 调整日志输出方向, 补充测试细节;
This commit is contained in:
LamGC 2020-06-17 19:49:36 +08:00
parent acbd990181
commit 26fd18917d
4 changed files with 16 additions and 202 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
src/main/java/net/lamgc/cgj/pixiv/PixivDownload.java
View File

@ -505,45 +505,7 @@ public class PixivDownload {
/** /**
* 获取作品信息 * 获取作品信息
* @param illustId 作品ID * @param illustId 作品ID
* @return 成功获取返回JsonObject, 失败返回null, <br/> * @return 成功获取返回JsonObject, 失败返回null.
* Json示例: <br/>
* <pre>
* {
* "illustId": "79584670",
* "illustTitle": "このヤンキーはウブすぎる",
* "id": "79584670",
* "title": "このヤンキーはウブすぎる",
* "illustType": 1,
* "xRestrict": 0,
* "restrict": 0,
* "sl": 2,
* "url": "https://i.pximg.net/c/360x360_70/img-master/img/2020/02/19/00/38/23/79584670_p0_square1200.jpg",
* "description": "",
* "tags": [
* "漫画",
* "オリジナル",
* "創作",
* "創作男女",
* "コロさん、ポリさん此方です!",
* "恋の予感",
* "あまずっぺー",
* "交換日記",
* "続編希望!!",
* "オリジナル10000users入り"
* ],
* "userId": "4778293",
* "userName": "隈浪さえ",
* "width": 3288,
* "height": 4564,
* "pageCount": 4,
* "isBookmarkable": true,
* "bookmarkData": null,
* "alt": "#オリジナル このヤンキーはウブすぎる - 隈浪さえ的漫画",
* "isAdContainer": false,
* "profileImageUrl": "https://i.pximg.net/user-profile/img/2019/12/04/18/56/19/16639046_fea29ce38ea89b0cb2313b40b3a72f9a_50.jpg",
* "type": "illust"
* }
* </pre>
* @throws IOException 当请求发生异常, 或接口返回错误信息时抛出. * @throws IOException 当请求发生异常, 或接口返回错误信息时抛出.
* @throws NoSuchElementException 当该作品不存在时抛出异常 * @throws NoSuchElementException 当该作品不存在时抛出异常
*/ */

149
src/main/java/net/lamgc/cgj/proxy/PixivAccessProxyServer.java
View File

@ -1,149 +0,0 @@
package net.lamgc.cgj.proxy;
import com.github.monkeywie.proxyee.intercept.HttpProxyIntercept;
import com.github.monkeywie.proxyee.intercept.HttpProxyInterceptInitializer;
import com.github.monkeywie.proxyee.intercept.HttpProxyInterceptPipeline;
import com.github.monkeywie.proxyee.intercept.common.CertDownIntercept;
import com.github.monkeywie.proxyee.proxy.ProxyConfig;
import com.github.monkeywie.proxyee.server.HttpProxyServer;
import com.github.monkeywie.proxyee.server.HttpProxyServerConfig;
import io.netty.channel.Channel;
import io.netty.handler.codec.http.HttpHeaderNames;
import io.netty.handler.codec.http.HttpHeaders;
import io.netty.handler.codec.http.HttpRequest;
import io.netty.handler.codec.http.HttpResponse;
import org.apache.http.client.CookieStore;
import org.apache.http.impl.cookie.BasicClientCookie;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.net.HttpCookie;
import java.util.Date;
import java.util.List;
/**
* 登录成功后提供CookieStore, 然后由程序自动登录Pixiv
* @author LamGC
*/
public class PixivAccessProxyServer {
private final Logger log = LoggerFactory.getLogger(PixivAccessProxyServer.class);
private final HttpProxyServer proxyServer;
private final CookieStore cookieStore;
public PixivAccessProxyServer(CookieStore cookieStore){
this(cookieStore, null);
}
public PixivAccessProxyServer(CookieStore cookieStore, ProxyConfig proxyConfig){
HttpProxyServerConfig config = new HttpProxyServerConfig();
this.cookieStore = cookieStore;
config.setHandleSsl(true);
this.proxyServer = new HttpProxyServer();
this.proxyServer
.serverConfig(config)
.proxyConfig(proxyConfig)
.proxyInterceptInitializer(new HttpProxyInterceptInitializer(){
@Override
public void init(HttpProxyInterceptPipeline pipeline) {
pipeline.addLast(new CertDownIntercept());
pipeline.addLast(new HttpProxyIntercept(){
private boolean match(HttpRequest request){
String host = request.headers().get(HttpHeaderNames.HOST);
return host.equalsIgnoreCase("pixiv.net") || host.contains(".pixiv.net");
}
@Override
public void beforeRequest(Channel clientChannel, HttpRequest httpRequest, HttpProxyInterceptPipeline pipeline) throws Exception {
log.info("URL: " + httpRequest.headers().get(HttpHeaderNames.HOST) + httpRequest.uri());
if(!match(httpRequest)){
super.beforeRequest(clientChannel, httpRequest, pipeline);
return;
}
log.info("正在注入Cookies...");
HttpHeaders requestHeaders = httpRequest.headers();
if(requestHeaders.contains(HttpHeaderNames.COOKIE)){
log.info("原请求存在自带Cookies, 正在清除Cookies...");
log.debug("原Cookies: {}", requestHeaders.getAsString(HttpHeaderNames.COOKIE));
requestHeaders.remove(HttpHeaderNames.COOKIE);
}
StringBuilder cookieBuilder = new StringBuilder();
cookieStore.getCookies().forEach(cookie -> {
if(cookie.isExpired(new Date())){
return;
}
cookieBuilder.append(cookie.getName()).append("=").append(cookie.getValue()).append("; ");
});
log.info("Cookies构造完成, 结果: " + cookieBuilder.toString());
requestHeaders.add(HttpHeaderNames.COOKIE, cookieBuilder.toString());
log.info("Cookies注入完成.");
super.beforeRequest(clientChannel, httpRequest, pipeline);
}
@Override
public void afterResponse(Channel clientChannel, Channel proxyChannel, HttpResponse httpResponse, HttpProxyInterceptPipeline pipeline) throws Exception {
if(!match(pipeline.getHttpRequest())){
super.afterResponse(clientChannel, proxyChannel, httpResponse, pipeline);
return;
}
log.info("正在更新Response Cookie...(Header Name: " + HttpHeaderNames.SET_COOKIE + ")");
List<String> responseCookies = httpResponse.headers().getAll(HttpHeaderNames.SET_COOKIE);
responseCookies.forEach(value -> {
/*if(check(value)){
log.info("黑名单Cookie, 已忽略: " + value);
return;
}*/
log.info("Response Cookie: " + value);
BasicClientCookie cookie = parseRawCookie(value);
cookieStore.addCookie(cookie);
});
httpResponse.headers().remove(HttpHeaderNames.SET_COOKIE);
super.afterResponse(clientChannel, proxyChannel, httpResponse, pipeline);
}
protected BasicClientCookie parseRawCookie(String rawCookie) {
List<HttpCookie> cookies = HttpCookie.parse(rawCookie);
if (cookies.size() < 1)
return null;
HttpCookie httpCookie = cookies.get(0);
BasicClientCookie cookie = new BasicClientCookie(httpCookie.getName(), httpCookie.getValue());
if (httpCookie.getMaxAge() >= 0) {
Date expiryDate = new Date(System.currentTimeMillis() + httpCookie.getMaxAge() * 1000);
cookie.setExpiryDate(expiryDate);
}
if (httpCookie.getDomain() != null)
cookie.setDomain(httpCookie.getDomain());
if (httpCookie.getPath() != null)
cookie.setPath(httpCookie.getPath());
if (httpCookie.getComment() != null)
cookie.setComment(httpCookie.getComment());
cookie.setSecure(httpCookie.getSecure());
return cookie;
}
});
}
});
}
public void start(int port){
this.proxyServer.start(port);
}
public void close(){
this.proxyServer.close();
}
/**
* 导出CookieStore.
* 注意!该方法导出的CookieStore不适用于ApacheHttpClient, 如需使用则需要进行转换.
* @return CookieStore对象
*/
public CookieStore getCookieStore(){
return this.cookieStore;
}
}

10
src/main/java/net/lamgc/cgj/util/Locker.java
View File

@ -1,14 +1,9 @@
package net.lamgc.cgj.util; package net.lamgc.cgj.util;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicInteger;
public final class Locker<K> { public final class Locker<K> {
private final static Logger log = LoggerFactory.getLogger(Locker.class);
private final LockerMap<K> fromMap; private final LockerMap<K> fromMap;
private final K key; private final K key;
@ -67,9 +62,4 @@ public final class Locker<K> {
'}'; '}';
} }
@Override
protected void finalize() throws Throwable {
super.finalize();
log.trace("{} 已销毁.", this.toString());
}
} }

19
src/test/java/net/lamgc/cgj/util/TimeLimitThreadPoolExecutorTest.java
View File

@ -2,26 +2,37 @@ package net.lamgc.cgj.util;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Test; import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
public class TimeLimitThreadPoolExecutorTest { public class TimeLimitThreadPoolExecutorTest {
private final static Logger log = LoggerFactory.getLogger(TimeLimitThreadPoolExecutorTest.class);
@Test @Test
public void timeoutTest() throws InterruptedException { public void timeoutTest() throws InterruptedException {
TimeLimitThreadPoolExecutor executor = new TimeLimitThreadPoolExecutor(1000, 1, 1, 30, TimeUnit.SECONDS, new ArrayBlockingQueue<>(50)); TimeLimitThreadPoolExecutor executor = new TimeLimitThreadPoolExecutor(1000, 1, 1, 30, TimeUnit.SECONDS, new ArrayBlockingQueue<>(50));
System.out.println(executor.isTerminated()); log.info("ThreadPoolExecutor.isTerminated: {}", executor.isTerminated());
System.out.println(executor.isShutdown()); log.info("ThreadPoolExecutor.isShutdown: {}", executor.isShutdown());
executor.setTimeoutCheckInterval(150); executor.setTimeoutCheckInterval(150);
System.out.println("当前设定: ETL: " + executor.getExecuteTimeLimit() + "ms, TCI: " + executor.getTimeoutCheckInterval() + "ms"); log.info("当前设定: ExecuteTimeLimit: {}ms, CheckInterval: {}ms", executor.getExecuteTimeLimit(),
executor.getTimeoutCheckInterval());
executor.execute(() -> { executor.execute(() -> {
try { try {
Thread.sleep(5 * 1000); Thread.sleep(5000);
} catch (InterruptedException e) { } catch (InterruptedException e) {
System.out.println("线程 " + Thread.currentThread().getName() + " 被中断"); System.out.println("线程 " + Thread.currentThread().getName() + " 被中断");
} }
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
Assert.fail("Multiple interrupts occurred");
}
}); });
executor.shutdown(); executor.shutdown();
Assert.assertTrue(executor.awaitTermination(5 * 1000, TimeUnit.MILLISECONDS)); Assert.assertTrue(executor.awaitTermination(5 * 1000, TimeUnit.MILLISECONDS));