From fda65fdf2ace4653def90e8ed2f5c61561953bae Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Sun, 16 May 2021 15:12:47 +0200 Subject: [PATCH] Add GZIP support for Flirt4Free HTTP requests --- .../Flirt4FreeFavoritesUpdateService.java | 4 +++- .../flirt4free/Flirt4FreeUpdateService.java | 14 ++++++------- .../src/main/java/ctbrec/io/HttpClient.java | 20 +++++++++++++++++++ .../main/java/ctbrec/io/HttpConstants.java | 3 +++ 4 files changed, 33 insertions(+), 8 deletions(-) diff --git a/client/src/main/java/ctbrec/ui/sites/flirt4free/Flirt4FreeFavoritesUpdateService.java b/client/src/main/java/ctbrec/ui/sites/flirt4free/Flirt4FreeFavoritesUpdateService.java index 3b32fd50..d9bb274a 100644 --- a/client/src/main/java/ctbrec/ui/sites/flirt4free/Flirt4FreeFavoritesUpdateService.java +++ b/client/src/main/java/ctbrec/ui/sites/flirt4free/Flirt4FreeFavoritesUpdateService.java @@ -1,5 +1,6 @@ package ctbrec.ui.sites.flirt4free; +import static ctbrec.io.HttpClient.*; import static ctbrec.io.HttpConstants.*; import java.io.IOException; @@ -47,12 +48,13 @@ public class Flirt4FreeFavoritesUpdateService extends PaginatedScheduledService .url(url) .header(ACCEPT, "*/*") .header(ACCEPT_LANGUAGE, Locale.ENGLISH.getLanguage()) + .header(ACCEPT_ENCODING, ACCEPT_ENCODING_GZIP) .header(REFERER, flirt4free.getBaseUrl()) .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) .build(); try (var response = flirt4free.getHttpClient().execute(request)) { if (response.isSuccessful()) { - var body = response.body().string(); + var body = gunzipBody(response); Elements modelContainers = HtmlParser.getTags(body, "div.model-container"); for (Element modelContainer : modelContainers) { String modelHtml = modelContainer.html(); diff --git a/client/src/main/java/ctbrec/ui/sites/flirt4free/Flirt4FreeUpdateService.java b/client/src/main/java/ctbrec/ui/sites/flirt4free/Flirt4FreeUpdateService.java index 88d77258..63ee12cd 100644 --- a/client/src/main/java/ctbrec/ui/sites/flirt4free/Flirt4FreeUpdateService.java +++ b/client/src/main/java/ctbrec/ui/sites/flirt4free/Flirt4FreeUpdateService.java @@ -1,5 +1,6 @@ package ctbrec.ui.sites.flirt4free; +import static ctbrec.io.HttpClient.*; import static ctbrec.io.HttpConstants.*; import java.io.IOException; @@ -21,7 +22,6 @@ import ctbrec.sites.flirt4free.Flirt4FreeModel; import ctbrec.ui.tabs.PaginatedScheduledService; import javafx.concurrent.Task; import okhttp3.Request; -import okhttp3.Response; public class Flirt4FreeUpdateService extends PaginatedScheduledService { @@ -46,10 +46,12 @@ public class Flirt4FreeUpdateService extends PaginatedScheduledService { var request = new Request.Builder() .url(url) .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) + .header(ACCEPT_ENCODING, ACCEPT_ENCODING_GZIP) .build(); try (var response = flirt4Free.getHttpClient().execute(request)) { if (response.isSuccessful()) { - return parseResponse(response); + var body = gunzipBody(response); + return parseResponse(body); } else { throw new HttpException(response.code(), response.message()); } @@ -58,9 +60,8 @@ public class Flirt4FreeUpdateService extends PaginatedScheduledService { }; } - private List parseResponse(Response response) throws IOException { + private List parseResponse(String body) throws IOException { List models = new ArrayList<>(); - var body = response.body().string(); var m = Pattern.compile("window\\.__homePageData__ = (\\{.*\\})", Pattern.DOTALL).matcher(body); if (m.find()) { var data = new JSONObject(m.group(1)); @@ -95,10 +96,9 @@ public class Flirt4FreeUpdateService extends PaginatedScheduledService { var videoHost = modelData.getString("video_host"); var modelId = modelData.getString("model_id"); model.setId(modelId); - String streamUrl = "https://manifest.vscdns.com/manifest.m3u8.m3u8?key=nil&provider=level3&secure=true&host=" + videoHost - + "&model_id=" + modelId; + String streamUrl = "https://manifest.vscdns.com/manifest.m3u8.m3u8?key=nil&provider=level3&secure=true&host=" + videoHost + "&model_id=" + modelId; model.setStreamUrl(streamUrl); - model.setPreview("https://live-screencaps.vscdns.com/"+ modelId +"-desktop.jpg"); + model.setPreview("https://live-screencaps.vscdns.com/" + modelId + "-desktop.jpg"); model.setOnlineState(ctbrec.Model.State.ONLINE); model.setOnline(true); return model; diff --git a/common/src/main/java/ctbrec/io/HttpClient.java b/common/src/main/java/ctbrec/io/HttpClient.java index 4a68c39f..d906bdeb 100644 --- a/common/src/main/java/ctbrec/io/HttpClient.java +++ b/common/src/main/java/ctbrec/io/HttpClient.java @@ -1,12 +1,15 @@ package ctbrec.io; +import static ctbrec.io.HttpConstants.*; import static java.nio.charset.StandardCharsets.*; +import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.net.Authenticator; import java.net.PasswordAuthentication; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.security.KeyManagementException; import java.security.NoSuchAlgorithmException; @@ -22,6 +25,7 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.concurrent.TimeUnit; +import java.util.zip.GZIPInputStream; import javax.net.ssl.KeyManager; import javax.net.ssl.SSLContext; @@ -314,4 +318,20 @@ public abstract class HttpClient { } }).orElse("[]"); } + + + public static String gunzipBody(Response response) throws IOException { + if (Objects.equals(response.header(CONTENT_ENCODING), ACCEPT_ENCODING_GZIP)) { + GZIPInputStream gzipIn = new GZIPInputStream(response.body().byteStream()); + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + byte[] b = new byte[1024]; + int len = -1; + while ((len = gzipIn.read(b)) >= 0) { + bos.write(b, 0, len); + } + return bos.toString(StandardCharsets.UTF_8.toString()); + } else { + return response.body().string(); + } + } } diff --git a/common/src/main/java/ctbrec/io/HttpConstants.java b/common/src/main/java/ctbrec/io/HttpConstants.java index fb706566..4ae7202b 100644 --- a/common/src/main/java/ctbrec/io/HttpConstants.java +++ b/common/src/main/java/ctbrec/io/HttpConstants.java @@ -3,9 +3,12 @@ package ctbrec.io; public class HttpConstants { public static final String ACCEPT = "Accept"; + public static final String ACCEPT_ENCODING = "Accept-Encoding"; + public static final String ACCEPT_ENCODING_GZIP = "gzip"; public static final String ACCEPT_LANGUAGE = "Accept-Language"; public static final String CACHE_CONTROL = "Cache-Control"; public static final String CONNECTION = "Connection"; + public static final String CONTENT_ENCODING = "Content-Encoding"; public static final String CONTENT_TYPE = "Content-Type"; public static final String COOKIE = "Cookie"; public static final String KEEP_ALIVE = "keep-alive";