From c9fd95c247fddd0ceba086fd5d9b6d10870d9969 Mon Sep 17 00:00:00 2001 From: reusedname <155286845+reusedname@users.noreply.github.com> Date: Tue, 1 Apr 2025 10:42:37 +0500 Subject: [PATCH] fix flaresolverr's user agent not applying --- .../chaturbate/ChaturbateApiUpdateService.java | 3 ++- .../ChaturbateElectronLoginDialog.java | 3 ++- .../chaturbate/ChaturbateUpdateService.java | 3 ++- common/src/main/java/ctbrec/io/HttpClient.java | 10 ++++++---- .../java/ctbrec/sites/chaturbate/Chaturbate.java | 5 +++-- .../sites/chaturbate/ChaturbateHttpClient.java | 16 ++++------------ .../ctbrec/sites/chaturbate/ChaturbateModel.java | 15 ++++++++------- 7 files changed, 27 insertions(+), 28 deletions(-) diff --git a/client/src/main/java/ctbrec/ui/sites/chaturbate/ChaturbateApiUpdateService.java b/client/src/main/java/ctbrec/ui/sites/chaturbate/ChaturbateApiUpdateService.java index 3902d8b5..71f574bd 100644 --- a/client/src/main/java/ctbrec/ui/sites/chaturbate/ChaturbateApiUpdateService.java +++ b/client/src/main/java/ctbrec/ui/sites/chaturbate/ChaturbateApiUpdateService.java @@ -15,6 +15,7 @@ import okhttp3.Request; import org.json.JSONArray; import org.json.JSONObject; +import java.net.URI; import java.util.ArrayList; import java.util.List; @@ -35,7 +36,7 @@ public class ChaturbateApiUpdateService extends PaginatedScheduledService { protected List call() throws Exception { var request = new Request.Builder() .url(url) - .header(USER_AGENT, chaturbate.getHttpClient().getEffectiveUserAgent()) + .header(USER_AGENT, chaturbate.getHttpClient().getEffectiveUserAgent(URI.create(url).getHost())) .header(ACCEPT, MIMETYPE_APPLICATION_JSON) .build(); try (var response = chaturbate.getHttpClient().execute(request)) { diff --git a/client/src/main/java/ctbrec/ui/sites/chaturbate/ChaturbateElectronLoginDialog.java b/client/src/main/java/ctbrec/ui/sites/chaturbate/ChaturbateElectronLoginDialog.java index d9b0e673..40ef4536 100644 --- a/client/src/main/java/ctbrec/ui/sites/chaturbate/ChaturbateElectronLoginDialog.java +++ b/client/src/main/java/ctbrec/ui/sites/chaturbate/ChaturbateElectronLoginDialog.java @@ -13,6 +13,7 @@ import org.slf4j.LoggerFactory; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URI; import java.net.URL; import java.util.Collections; import java.util.Objects; @@ -35,7 +36,7 @@ public class ChaturbateElectronLoginDialog { config.put("url", site.getBaseUrl() + "/auth/login/"); config.put("w", 640); config.put("h", 480); - config.put("userAgent", site.getHttpClient().getEffectiveUserAgent()); + config.put("userAgent", site.getHttpClient().getEffectiveUserAgent(URI.create(site.getBaseUrl()).getHost())); var msg = new JSONObject(); msg.put("config", config); browser.run(msg, msgHandler); diff --git a/client/src/main/java/ctbrec/ui/sites/chaturbate/ChaturbateUpdateService.java b/client/src/main/java/ctbrec/ui/sites/chaturbate/ChaturbateUpdateService.java index 90c8e7c0..f1ca2db8 100644 --- a/client/src/main/java/ctbrec/ui/sites/chaturbate/ChaturbateUpdateService.java +++ b/client/src/main/java/ctbrec/ui/sites/chaturbate/ChaturbateUpdateService.java @@ -14,6 +14,7 @@ import org.json.JSONObject; import org.jsoup.Jsoup; import java.io.IOException; +import java.net.URI; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -54,7 +55,7 @@ public class ChaturbateUpdateService extends PaginatedScheduledService { .header(ACCEPT, MIMETYPE_APPLICATION_JSON) .header(ACCEPT_LANGUAGE, Locale.ENGLISH.getLanguage()) .header(X_REQUESTED_WITH, XML_HTTP_REQUEST) - .header(USER_AGENT, chaturbate.getHttpClient().getEffectiveUserAgent()) + .header(USER_AGENT, chaturbate.getHttpClient().getEffectiveUserAgent(URI.create(pageUrl).getHost())) .build(); try (var response = chaturbate.getHttpClient().execute(request)) { if (response.isSuccessful()) { diff --git a/common/src/main/java/ctbrec/io/HttpClient.java b/common/src/main/java/ctbrec/io/HttpClient.java index d65f212d..5e166540 100644 --- a/common/src/main/java/ctbrec/io/HttpClient.java +++ b/common/src/main/java/ctbrec/io/HttpClient.java @@ -26,7 +26,6 @@ import java.security.NoSuchAlgorithmException; import java.security.SecureRandom; import java.security.cert.X509Certificate; import java.text.NumberFormat; -import java.time.Instant; import java.util.*; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; @@ -34,7 +33,6 @@ import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.stream.Collectors; import java.util.zip.GZIPInputStream; import java.time.*; -import java.util.Optional; import static ctbrec.io.HttpConstants.ACCEPT_ENCODING_GZIP; import static ctbrec.io.HttpConstants.CONTENT_ENCODING; @@ -455,7 +453,11 @@ public abstract class HttpClient { } // overridable default user agent (used for Flaresolverr) - public String getEffectiveUserAgent() { - return config.getSettings().httpUserAgent; + public String getEffectiveUserAgent(String host) { + if (config.getSettings().flaresolverr.useForDomains.contains(host)) { + return config.getSettings().flaresolverr.userAgent; + } else { + return config.getSettings().httpUserAgent; + } } } diff --git a/common/src/main/java/ctbrec/sites/chaturbate/Chaturbate.java b/common/src/main/java/ctbrec/sites/chaturbate/Chaturbate.java index 4fc06c9f..132a3a35 100644 --- a/common/src/main/java/ctbrec/sites/chaturbate/Chaturbate.java +++ b/common/src/main/java/ctbrec/sites/chaturbate/Chaturbate.java @@ -11,6 +11,7 @@ import org.json.JSONArray; import org.json.JSONObject; import java.io.IOException; +import java.net.URI; import java.net.URLEncoder; import java.time.Duration; import java.time.Instant; @@ -69,7 +70,7 @@ public class Chaturbate extends AbstractSite { String url = "https://chaturbate.com/p/" + username + "/"; Request req = new Request.Builder() .url(url) - .header(USER_AGENT, getHttpClient().getEffectiveUserAgent()) + .header(USER_AGENT, getHttpClient().getEffectiveUserAgent(URI.create(url).getHost())) .build(); try (Response resp = getHttpClient().execute(req)) { if (resp.isSuccessful()) { @@ -131,7 +132,7 @@ public class Chaturbate extends AbstractSite { // search online models Request req = new Request.Builder() .url(url) - .header(USER_AGENT, getHttpClient().getEffectiveUserAgent()) + .header(USER_AGENT, getHttpClient().getEffectiveUserAgent(URI.create(url).getHost())) .header(ACCEPT, "*/*") .header(X_REQUESTED_WITH, XML_HTTP_REQUEST) .header(REFERER, getBaseUrl()) diff --git a/common/src/main/java/ctbrec/sites/chaturbate/ChaturbateHttpClient.java b/common/src/main/java/ctbrec/sites/chaturbate/ChaturbateHttpClient.java index f031ffed..5060cee2 100644 --- a/common/src/main/java/ctbrec/sites/chaturbate/ChaturbateHttpClient.java +++ b/common/src/main/java/ctbrec/sites/chaturbate/ChaturbateHttpClient.java @@ -8,6 +8,7 @@ import okhttp3.*; import java.io.IOException; import java.io.InterruptedIOException; +import java.net.URI; import java.util.NoSuchElementException; import java.util.concurrent.Semaphore; @@ -28,15 +29,6 @@ public class ChaturbateHttpClient extends HttpClient { super("chaturbate", config); } - @Override - public String getEffectiveUserAgent() { - if (flaresolverr != null) { - return config.getSettings().flaresolverr.userAgent; - } else { - return config.getSettings().httpUserAgent; - } - } - private void extractCsrfToken(Request request) { try { Cookie csrfToken = cookieJar.getCookie(request.url(), "csrftoken"); @@ -65,7 +57,7 @@ public class ChaturbateHttpClient extends HttpClient { } Request login = new Request.Builder() .url(Chaturbate.baseUrl + PATH) - .header(USER_AGENT, getEffectiveUserAgent()) + .header(USER_AGENT, getEffectiveUserAgent(URI.create(Chaturbate.baseUrl).getHost())) .build(); try (var initResponse = client.newCall(login).execute()) { String content = initResponse.body().string(); @@ -81,7 +73,7 @@ public class ChaturbateHttpClient extends HttpClient { login = new Request.Builder() .url(Chaturbate.baseUrl + PATH) .header(REFERER, Chaturbate.baseUrl + PATH) - .header(USER_AGENT, getEffectiveUserAgent()) + .header(USER_AGENT, getEffectiveUserAgent(URI.create(Chaturbate.baseUrl).getHost())) .post(body) .build(); @@ -106,7 +98,7 @@ public class ChaturbateHttpClient extends HttpClient { String url = "https://chaturbate.com/api/ts/chatmessages/pm_users/?offset=0"; Request req = new Request.Builder() .url(url) - .header(USER_AGENT, getEffectiveUserAgent()) + .header(USER_AGENT, getEffectiveUserAgent(URI.create(url).getHost())) .build(); try (Response response = execute(req)) { boolean result = false; diff --git a/common/src/main/java/ctbrec/sites/chaturbate/ChaturbateModel.java b/common/src/main/java/ctbrec/sites/chaturbate/ChaturbateModel.java index 7c2176ef..8f40d8dd 100644 --- a/common/src/main/java/ctbrec/sites/chaturbate/ChaturbateModel.java +++ b/common/src/main/java/ctbrec/sites/chaturbate/ChaturbateModel.java @@ -22,6 +22,7 @@ import java.io.ByteArrayInputStream; import java.io.EOFException; import java.io.IOException; import java.io.InputStream; +import java.net.URI; import java.time.Duration; import java.time.Instant; import java.util.*; @@ -100,7 +101,7 @@ public class ChaturbateModel extends AbstractModel { int imageSize = 0; Request req = new Request.Builder() .url(url) - .header(USER_AGENT, site.getHttpClient().getEffectiveUserAgent()) + .header(USER_AGENT, site.getHttpClient().getEffectiveUserAgent(URI.create(url).getHost())) .head() .build(); try (Response response = getSite().getHttpClient().execute(req)) { @@ -193,7 +194,7 @@ public class ChaturbateModel extends AbstractModel { .post(body) .header(REFERER, "https://chaturbate.com/" + getName() + "/") .header(X_REQUESTED_WITH, XML_HTTP_REQUEST) - .header(USER_AGENT, site.getHttpClient().getEffectiveUserAgent()) + .header(USER_AGENT, site.getHttpClient().getEffectiveUserAgent("chaturbate.com")) .build(); try (Response response = getSite().getHttpClient().execute(req)) { if (!response.isSuccessful()) { @@ -242,7 +243,7 @@ public class ChaturbateModel extends AbstractModel { // do an initial request to get cookies Request req = new Request.Builder() .url(getUrl()) - .header(USER_AGENT, site.getHttpClient().getEffectiveUserAgent()) + .header(USER_AGENT, site.getHttpClient().getEffectiveUserAgent(URI.create(getUrl()).getHost())) .build(); Response resp = site.getHttpClient().execute(req); resp.close(); @@ -261,7 +262,7 @@ public class ChaturbateModel extends AbstractModel { .header(ACCEPT, "*/*") .header(ACCEPT_LANGUAGE, "en-US,en;q=0.5") .header(REFERER, getUrl()) - .header(USER_AGENT, site.getHttpClient().getEffectiveUserAgent()) + .header(USER_AGENT, site.getHttpClient().getEffectiveUserAgent(URI.create(url).getHost())) .header("X-CSRFToken", ((ChaturbateHttpClient) site.getHttpClient()).getToken()) .header(X_REQUESTED_WITH, XML_HTTP_REQUEST) .build(); @@ -305,7 +306,7 @@ public class ChaturbateModel extends AbstractModel { Request req = new Request.Builder() .url(getSite().getBaseUrl() + "/get_edge_hls_url_ajax/") .post(body) - .header(USER_AGENT, site.getHttpClient().getEffectiveUserAgent()) + .header(USER_AGENT, site.getHttpClient().getEffectiveUserAgent(URI.create(getSite().getBaseUrl()).getHost())) .header(X_REQUESTED_WITH, XML_HTTP_REQUEST) .build(); try (Response response = getSite().getHttpClient().execute(req)) { @@ -384,7 +385,7 @@ public class ChaturbateModel extends AbstractModel { log.trace("Loading master playlist {}", streamInfo.url); Request req = new Request.Builder() .url(streamInfo.url) - .header(USER_AGENT, site.getHttpClient().getEffectiveUserAgent()) + .header(USER_AGENT, site.getHttpClient().getEffectiveUserAgent(URI.create(streamInfo.url).getHost())) .build(); try (Response response = getSite().getHttpClient().execute(req)) { if (response.isSuccessful()) { @@ -405,7 +406,7 @@ public class ChaturbateModel extends AbstractModel { public boolean exists() throws IOException { Request req = new Request.Builder() // @formatter:off .url(getUrl()) - .header(USER_AGENT, site.getHttpClient().getEffectiveUserAgent()) + .header(USER_AGENT, site.getHttpClient().getEffectiveUserAgent(URI.create(getUrl()).getHost())) .header(ACCEPT_LANGUAGE, Locale.ENGLISH.getLanguage()) .build(); // @formatter:on try (Response response = getSite().getHttpClient().execute(req)) {