362 lines
15 KiB
Java
362 lines
15 KiB
Java
package ctbrec.io;
|
|
|
|
import com.fasterxml.jackson.core.type.TypeReference;
|
|
import ctbrec.Config;
|
|
import ctbrec.LoggingInterceptor;
|
|
import ctbrec.Settings.ProxyType;
|
|
import ctbrec.io.json.ObjectMapperFactory;
|
|
import ctbrec.io.json.dto.CookieDto;
|
|
import ctbrec.io.json.mapper.CookieMapper;
|
|
import lombok.Data;
|
|
import lombok.Getter;
|
|
import lombok.extern.slf4j.Slf4j;
|
|
import okhttp3.*;
|
|
import okhttp3.OkHttpClient.Builder;
|
|
import org.mapstruct.factory.Mappers;
|
|
|
|
import javax.net.ssl.*;
|
|
import java.io.ByteArrayOutputStream;
|
|
import java.io.File;
|
|
import java.io.IOException;
|
|
import java.net.Authenticator;
|
|
import java.net.PasswordAuthentication;
|
|
import java.nio.file.Files;
|
|
import java.security.KeyManagementException;
|
|
import java.security.NoSuchAlgorithmException;
|
|
import java.security.SecureRandom;
|
|
import java.security.cert.X509Certificate;
|
|
import java.text.NumberFormat;
|
|
import java.util.*;
|
|
import java.util.concurrent.TimeUnit;
|
|
import java.util.stream.Collectors;
|
|
import java.util.zip.GZIPInputStream;
|
|
|
|
import static ctbrec.io.HttpConstants.ACCEPT_ENCODING_GZIP;
|
|
import static ctbrec.io.HttpConstants.CONTENT_ENCODING;
|
|
import static java.nio.charset.StandardCharsets.UTF_8;
|
|
|
|
@Slf4j
|
|
public abstract class HttpClient {
|
|
@Getter
|
|
private static final ConnectionPool GLOBAL_HTTP_CONN_POOL = new ConnectionPool(256, 2, TimeUnit.MINUTES);
|
|
|
|
@Getter
|
|
protected CookieJarImpl cookieJar;
|
|
protected OkHttpClient client;
|
|
protected Cache cache;
|
|
protected Config config;
|
|
protected boolean loggedIn = false;
|
|
protected long cacheSize;
|
|
protected int cacheLifeTime = 600;
|
|
private final String name;
|
|
|
|
protected HttpClient(String name, Config config) {
|
|
this.name = name;
|
|
this.config = config;
|
|
cookieJar = createCookieJar();
|
|
reconfigure();
|
|
}
|
|
|
|
protected CookieJarImpl createCookieJar() {
|
|
return new CookieJarImpl();
|
|
}
|
|
|
|
private void loadProxySettings() {
|
|
ProxyType proxyType = config.getSettings().proxyType;
|
|
switch (proxyType) {
|
|
case HTTP:
|
|
System.setProperty(ProxyConstants.HTTP_PROXY_HOST, config.getSettings().proxyHost);
|
|
System.setProperty(ProxyConstants.HTTP_PROXY_PORT, config.getSettings().proxyPort);
|
|
System.setProperty(ProxyConstants.HTTPS_PROXY_HOST, config.getSettings().proxyHost);
|
|
System.setProperty(ProxyConstants.HTTPS_PROXY_PORT, config.getSettings().proxyPort);
|
|
if (config.getSettings().proxyUser != null && !config.getSettings().proxyUser.isEmpty()) {
|
|
String username = config.getSettings().proxyUser;
|
|
String password = config.getSettings().proxyPassword;
|
|
System.setProperty(ProxyConstants.HTTP_PROXY_USER, username);
|
|
System.setProperty(ProxyConstants.HTTP_PROXY_PASSWORD, password);
|
|
}
|
|
break;
|
|
case SOCKS4:
|
|
System.setProperty(ProxyConstants.SOCKS_PROXY_VERSION, "4");
|
|
System.setProperty(ProxyConstants.SOCKS_PROXY_HOST, config.getSettings().proxyHost);
|
|
System.setProperty(ProxyConstants.SOCKS_PROXY_PORT, config.getSettings().proxyPort);
|
|
break;
|
|
case SOCKS5:
|
|
System.setProperty(ProxyConstants.SOCKS_PROXY_VERSION, "5");
|
|
System.setProperty(ProxyConstants.SOCKS_PROXY_HOST, config.getSettings().proxyHost);
|
|
System.setProperty(ProxyConstants.SOCKS_PROXY_PORT, config.getSettings().proxyPort);
|
|
if (config.getSettings().proxyUser != null && !config.getSettings().proxyUser.isEmpty()) {
|
|
String username = config.getSettings().proxyUser;
|
|
String password = config.getSettings().proxyPassword;
|
|
Authenticator.setDefault(new SocksProxyAuth(username, password));
|
|
}
|
|
break;
|
|
case DIRECT:
|
|
default:
|
|
System.clearProperty(ProxyConstants.HTTP_PROXY_HOST);
|
|
System.clearProperty(ProxyConstants.HTTP_PROXY_PORT);
|
|
System.clearProperty(ProxyConstants.HTTPS_PROXY_HOST);
|
|
System.clearProperty(ProxyConstants.HTTPS_PROXY_PORT);
|
|
System.clearProperty(ProxyConstants.SOCKS_PROXY_VERSION);
|
|
System.clearProperty(ProxyConstants.SOCKS_PROXY_HOST);
|
|
System.clearProperty(ProxyConstants.SOCKS_PROXY_PORT);
|
|
System.clearProperty(ProxyConstants.JAVA_NET_SOCKS_USERNAME);
|
|
System.clearProperty(ProxyConstants.JAVA_NET_SOCKS_PASSWORD);
|
|
System.clearProperty(ProxyConstants.HTTP_PROXY_USER);
|
|
System.clearProperty(ProxyConstants.HTTP_PROXY_PASSWORD);
|
|
break;
|
|
}
|
|
}
|
|
|
|
public Response execute(Request req) throws IOException {
|
|
Response resp = client.newCall(req).execute();
|
|
return resp;
|
|
}
|
|
|
|
public Response execute(Request request, int timeoutInMillis) throws IOException {
|
|
return client.newBuilder() //
|
|
.connectTimeout(timeoutInMillis, TimeUnit.MILLISECONDS) //
|
|
.readTimeout(timeoutInMillis, TimeUnit.MILLISECONDS).build() //
|
|
.newCall(request).execute();
|
|
}
|
|
|
|
public Response executeWithCache(Request req) throws IOException {
|
|
log.trace("Cached request for {}", req.url());
|
|
if (Objects.nonNull(cache)) {
|
|
log.trace("Cache hit ratio {}/{} = {}", cache.hitCount(), cache.requestCount(), NumberFormat.getPercentInstance().format(cache.hitCount() / (double) cache.requestCount()));
|
|
}
|
|
if (cacheSize > 0 && Objects.nonNull(cache)) {
|
|
Request r = req.newBuilder()
|
|
.cacheControl(new CacheControl.Builder().maxAge(cacheLifeTime, TimeUnit.SECONDS).build())
|
|
.build();
|
|
return execute(r);
|
|
} else {
|
|
return execute(req);
|
|
}
|
|
}
|
|
|
|
public abstract boolean login() throws IOException;
|
|
|
|
public void reconfigure() {
|
|
loadProxySettings();
|
|
loadCookies();
|
|
cacheSize = (long) config.getSettings().thumbCacheSize * 1024 * 1024;
|
|
|
|
Builder builder = new OkHttpClient.Builder()
|
|
.cookieJar(cookieJar)
|
|
.connectionPool(GLOBAL_HTTP_CONN_POOL)
|
|
.connectTimeout(config.getSettings().httpTimeout, TimeUnit.MILLISECONDS)
|
|
.readTimeout(config.getSettings().httpTimeout, TimeUnit.MILLISECONDS)
|
|
.addNetworkInterceptor(new LoggingInterceptor());
|
|
|
|
if (cacheSize > 0) {
|
|
cache = HttpClientCacheProvider.getCache(config);
|
|
if (cache != null) {
|
|
builder.cache(cache);
|
|
}
|
|
}
|
|
|
|
ProxyType proxyType = config.getSettings().proxyType;
|
|
if (proxyType == ProxyType.HTTP) {
|
|
String username = config.getSettings().proxyUser;
|
|
String password = config.getSettings().proxyPassword;
|
|
if (username != null && !username.isEmpty()) {
|
|
builder.proxyAuthenticator(createHttpProxyAuthenticator(username, password));
|
|
}
|
|
}
|
|
|
|
// if transport layer security (TLS) is switched on, accept the self-signed cert from the server
|
|
if (config.getSettings().transportLayerSecurity) {
|
|
acceptAllTlsCerts(builder);
|
|
}
|
|
|
|
client = builder.build();
|
|
}
|
|
|
|
/**
|
|
* This is a very simple and insecure solution to accept the self-signed cert from
|
|
* the server. The side effect is, that certificates from other servers are neither checked!
|
|
* TODO Delegate to the default trustmanager, if it is not the self-signed cert
|
|
*/
|
|
private void acceptAllTlsCerts(Builder builder) {
|
|
X509TrustManager x509TrustManager = new X509TrustManager() {
|
|
@Override
|
|
public X509Certificate[] getAcceptedIssuers() {
|
|
X509Certificate[] x509Certificates = new X509Certificate[0];
|
|
return x509Certificates;
|
|
}
|
|
|
|
@Override
|
|
public void checkServerTrusted(final X509Certificate[] chain, final String authType) { /* noop*/ }
|
|
|
|
@Override
|
|
public void checkClientTrusted(final X509Certificate[] chain, final String authType) { /* noop*/ }
|
|
};
|
|
|
|
try {
|
|
final TrustManager[] trustManagers = new TrustManager[]{x509TrustManager};
|
|
final String PROTOCOL = "TLSv1.2";
|
|
SSLContext sslContext = SSLContext.getInstance(PROTOCOL);
|
|
KeyManager[] keyManagers = null;
|
|
SecureRandom secureRandom = new SecureRandom();
|
|
sslContext.init(keyManagers, trustManagers, secureRandom);
|
|
SSLSocketFactory sslSocketFactory = sslContext.getSocketFactory();
|
|
builder.sslSocketFactory(sslSocketFactory, x509TrustManager);
|
|
builder.hostnameVerifier((hostname, sslSession) -> true);
|
|
} catch (KeyManagementException | NoSuchAlgorithmException e) {
|
|
log.error("Couldn't install trust managers for TLS connections");
|
|
}
|
|
}
|
|
|
|
public void shutdown() {
|
|
persistCookies();
|
|
client.connectionPool().evictAll();
|
|
client.dispatcher().executorService().shutdown();
|
|
}
|
|
|
|
private void persistCookies() {
|
|
try {
|
|
List<CookieContainer> containers = new ArrayList<>();
|
|
cookieJar.getCookies().forEach((domain, cookieList) -> {
|
|
CookieContainer cookies = new CookieContainer();
|
|
cookies.setDomain(domain);
|
|
List<CookieDto> dtos = cookieList.stream().map(Mappers.getMapper(CookieMapper.class)::toDto).toList();
|
|
cookies.setCookies(dtos);
|
|
containers.add(cookies);
|
|
});
|
|
String json = ObjectMapperFactory.getMapper().writeValueAsString(containers);
|
|
File cookieFile = new File(config.getConfigDir(), "cookies-" + name + ".json");
|
|
Files.writeString(cookieFile.toPath(), json);
|
|
} catch (Exception e) {
|
|
log.error("Couldn't persist cookies for {}", name, e);
|
|
}
|
|
}
|
|
|
|
private void loadCookies() {
|
|
try {
|
|
File cookieFile = new File(config.getConfigDir(), "cookies-" + name + ".json");
|
|
if (!cookieFile.exists()) {
|
|
return;
|
|
}
|
|
String json = Files.readString(cookieFile.toPath());
|
|
Map<String, List<Cookie>> cookies = cookieJar.getCookies();
|
|
List<CookieContainer> fromJson = ObjectMapperFactory.getMapper().readValue(json, new TypeReference<>() {
|
|
});
|
|
for (CookieContainer container : fromJson) {
|
|
List<Cookie> filteredCookies = container.getCookies().stream()
|
|
.filter(c -> !Objects.equals("deleted", c.getValue()))
|
|
.map(Mappers.getMapper(CookieMapper.class)::toCookie)
|
|
.collect(Collectors.toList());
|
|
cookies.put(container.getDomain(), filteredCookies);
|
|
}
|
|
} catch (Exception e) {
|
|
log.error("Couldn't load cookies for {}", name, e);
|
|
}
|
|
}
|
|
|
|
@Data
|
|
public static class CookieContainer {
|
|
private String domain;
|
|
private List<CookieDto> cookies;
|
|
}
|
|
|
|
private okhttp3.Authenticator createHttpProxyAuthenticator(String username, String password) {
|
|
return (route, response) -> {
|
|
String credential = Credentials.basic(username, password);
|
|
return response.request().newBuilder().header("Proxy-Authorization", credential).build();
|
|
};
|
|
}
|
|
|
|
public static class SocksProxyAuth extends Authenticator {
|
|
private final PasswordAuthentication auth;
|
|
|
|
private SocksProxyAuth(String user, String password) {
|
|
auth = new PasswordAuthentication(user, password == null ? new char[]{} : password.toCharArray());
|
|
}
|
|
|
|
@Override
|
|
protected PasswordAuthentication getPasswordAuthentication() {
|
|
return auth;
|
|
}
|
|
}
|
|
|
|
public void logout() {
|
|
getCookieJar().clear();
|
|
loggedIn = false;
|
|
}
|
|
|
|
public WebSocket newWebSocket(Request request, WebSocketListener l) {
|
|
return client.newWebSocket(request, l);
|
|
}
|
|
|
|
public List<Cookie> getCookiesByName(String... names) {
|
|
List<Cookie> result = new ArrayList<>();
|
|
Map<String, List<Cookie>> cookies = getCookieJar().getCookies();
|
|
for (List<Cookie> cookieList : cookies.values()) {
|
|
for (Cookie cookie : cookieList) {
|
|
for (String cookieName : names) {
|
|
if (Objects.equals(cookieName, cookie.name())) {
|
|
result.add(cookie);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return result;
|
|
}
|
|
|
|
public static String bodyToJsonObject(Response response) {
|
|
return Optional.ofNullable(response.body()).map(b -> {
|
|
try {
|
|
return b.string();
|
|
} catch (IOException e) {
|
|
return "{}";
|
|
}
|
|
}).orElse("{}");
|
|
}
|
|
|
|
public static String bodyToJsonArray(Response response) {
|
|
return Optional.ofNullable(response.body()).map(b -> {
|
|
try {
|
|
return b.string();
|
|
} catch (IOException e) {
|
|
return "[]";
|
|
}
|
|
}).orElse("[]");
|
|
}
|
|
|
|
|
|
public static String gunzipBody(Response response) throws IOException {
|
|
if (Objects.equals(response.header(CONTENT_ENCODING), ACCEPT_ENCODING_GZIP)) {
|
|
GZIPInputStream gzipIn = new GZIPInputStream(Objects.requireNonNull(response.body()).byteStream());
|
|
ByteArrayOutputStream bos = new ByteArrayOutputStream();
|
|
byte[] b = new byte[1024];
|
|
int len;
|
|
while ((len = gzipIn.read(b)) >= 0) {
|
|
bos.write(b, 0, len);
|
|
}
|
|
return bos.toString(UTF_8);
|
|
} else {
|
|
return Objects.requireNonNull(response.body()).string();
|
|
}
|
|
}
|
|
|
|
public void clearCookies() {
|
|
logout();
|
|
}
|
|
|
|
private static class ProxyConstants {
|
|
public static final String HTTP_PROXY_HOST = "http.proxyHost";
|
|
public static final String HTTP_PROXY_PORT = "http.proxyPort";
|
|
public static final String HTTPS_PROXY_HOST = "https.proxyHost";
|
|
public static final String HTTPS_PROXY_PORT = "https.proxyPort";
|
|
public static final String HTTP_PROXY_USER = "https.proxyUser";
|
|
public static final String HTTP_PROXY_PASSWORD = "https.proxyPassword";
|
|
|
|
public static final String SOCKS_PROXY_HOST = "socksProxyHost";
|
|
public static final String SOCKS_PROXY_PORT = "socksProxyPort";
|
|
public static final String SOCKS_PROXY_VERSION = "socksProxyVersion";
|
|
public static final String JAVA_NET_SOCKS_USERNAME = "java.net.socks.username";
|
|
public static final String JAVA_NET_SOCKS_PASSWORD = "java.net.socks.password";
|
|
}
|
|
}
|