reusedname
/
ctbrec
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix flaresolverr's user agent not applying

This commit is contained in:
reusedname 2025-04-01 10:42:37 +05:00
parent 1f26371283
commit c9fd95c247
7 changed files with 27 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
client/src/main/java/ctbrec/ui/sites/chaturbate/ChaturbateApiUpdateService.java
View File

@ -15,6 +15,7 @@ import okhttp3.Request;
import org.json.JSONArray; import org.json.JSONArray;
import org.json.JSONObject; import org.json.JSONObject;
import java.net.URI;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
@ -35,7 +36,7 @@ public class ChaturbateApiUpdateService extends PaginatedScheduledService {
protected List<Model> call() throws Exception { protected List<Model> call() throws Exception {
var request = new Request.Builder() var request = new Request.Builder()
.url(url) .url(url)
.header(USER_AGENT, chaturbate.getHttpClient().getEffectiveUserAgent()) .header(USER_AGENT, chaturbate.getHttpClient().getEffectiveUserAgent(URI.create(url).getHost()))
.header(ACCEPT, MIMETYPE_APPLICATION_JSON) .header(ACCEPT, MIMETYPE_APPLICATION_JSON)
.build(); .build();
try (var response = chaturbate.getHttpClient().execute(request)) { try (var response = chaturbate.getHttpClient().execute(request)) {

3
client/src/main/java/ctbrec/ui/sites/chaturbate/ChaturbateElectronLoginDialog.java
View File

@ -13,6 +13,7 @@ import org.slf4j.LoggerFactory;
import java.io.IOException; import java.io.IOException;
import java.net.MalformedURLException; import java.net.MalformedURLException;
import java.net.URI;
import java.net.URL; import java.net.URL;
import java.util.Collections; import java.util.Collections;
import java.util.Objects; import java.util.Objects;
@ -35,7 +36,7 @@ public class ChaturbateElectronLoginDialog {
config.put("url", site.getBaseUrl() + "/auth/login/"); config.put("url", site.getBaseUrl() + "/auth/login/");
config.put("w", 640); config.put("w", 640);
config.put("h", 480); config.put("h", 480);
config.put("userAgent", site.getHttpClient().getEffectiveUserAgent()); config.put("userAgent", site.getHttpClient().getEffectiveUserAgent(URI.create(site.getBaseUrl()).getHost()));
var msg = new JSONObject(); var msg = new JSONObject();
msg.put("config", config); msg.put("config", config);
browser.run(msg, msgHandler); browser.run(msg, msgHandler);

3
client/src/main/java/ctbrec/ui/sites/chaturbate/ChaturbateUpdateService.java
View File

@ -14,6 +14,7 @@ import org.json.JSONObject;
import org.jsoup.Jsoup; import org.jsoup.Jsoup;
import java.io.IOException; import java.io.IOException;
import java.net.URI;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collections; import java.util.Collections;
import java.util.List; import java.util.List;
@ -54,7 +55,7 @@ public class ChaturbateUpdateService extends PaginatedScheduledService {
.header(ACCEPT, MIMETYPE_APPLICATION_JSON) .header(ACCEPT, MIMETYPE_APPLICATION_JSON)
.header(ACCEPT_LANGUAGE, Locale.ENGLISH.getLanguage()) .header(ACCEPT_LANGUAGE, Locale.ENGLISH.getLanguage())
.header(X_REQUESTED_WITH, XML_HTTP_REQUEST) .header(X_REQUESTED_WITH, XML_HTTP_REQUEST)
.header(USER_AGENT, chaturbate.getHttpClient().getEffectiveUserAgent()) .header(USER_AGENT, chaturbate.getHttpClient().getEffectiveUserAgent(URI.create(pageUrl).getHost()))
.build(); .build();
try (var response = chaturbate.getHttpClient().execute(request)) { try (var response = chaturbate.getHttpClient().execute(request)) {
if (response.isSuccessful()) { if (response.isSuccessful()) {

8
common/src/main/java/ctbrec/io/HttpClient.java
View File

@ -26,7 +26,6 @@ import java.security.NoSuchAlgorithmException;
import java.security.SecureRandom; import java.security.SecureRandom;
import java.security.cert.X509Certificate; import java.security.cert.X509Certificate;
import java.text.NumberFormat; import java.text.NumberFormat;
import java.time.Instant;
import java.util.*; import java.util.*;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicInteger;
@ -34,7 +33,6 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import java.util.zip.GZIPInputStream; import java.util.zip.GZIPInputStream;
import java.time.*; import java.time.*;
import java.util.Optional;
import static ctbrec.io.HttpConstants.ACCEPT_ENCODING_GZIP; import static ctbrec.io.HttpConstants.ACCEPT_ENCODING_GZIP;
import static ctbrec.io.HttpConstants.CONTENT_ENCODING; import static ctbrec.io.HttpConstants.CONTENT_ENCODING;
@ -455,7 +453,11 @@ public abstract class HttpClient {
} }
// overridable default user agent (used for Flaresolverr) // overridable default user agent (used for Flaresolverr)
public String getEffectiveUserAgent() { public String getEffectiveUserAgent(String host) {
if (config.getSettings().flaresolverr.useForDomains.contains(host)) {
return config.getSettings().flaresolverr.userAgent;
} else {
return config.getSettings().httpUserAgent; return config.getSettings().httpUserAgent;
} }
}
} }

5
common/src/main/java/ctbrec/sites/chaturbate/Chaturbate.java
View File

@ -11,6 +11,7 @@ import org.json.JSONArray;
import org.json.JSONObject; import org.json.JSONObject;
import java.io.IOException; import java.io.IOException;
import java.net.URI;
import java.net.URLEncoder; import java.net.URLEncoder;
import java.time.Duration; import java.time.Duration;
import java.time.Instant; import java.time.Instant;
@ -69,7 +70,7 @@ public class Chaturbate extends AbstractSite {
String url = "https://chaturbate.com/p/" + username + "/"; String url = "https://chaturbate.com/p/" + username + "/";
Request req = new Request.Builder() Request req = new Request.Builder()
.url(url) .url(url)
.header(USER_AGENT, getHttpClient().getEffectiveUserAgent()) .header(USER_AGENT, getHttpClient().getEffectiveUserAgent(URI.create(url).getHost()))
.build(); .build();
try (Response resp = getHttpClient().execute(req)) { try (Response resp = getHttpClient().execute(req)) {
if (resp.isSuccessful()) { if (resp.isSuccessful()) {
@ -131,7 +132,7 @@ public class Chaturbate extends AbstractSite {
// search online models // search online models
Request req = new Request.Builder() Request req = new Request.Builder()
.url(url) .url(url)
.header(USER_AGENT, getHttpClient().getEffectiveUserAgent()) .header(USER_AGENT, getHttpClient().getEffectiveUserAgent(URI.create(url).getHost()))
.header(ACCEPT, "*/*") .header(ACCEPT, "*/*")
.header(X_REQUESTED_WITH, XML_HTTP_REQUEST) .header(X_REQUESTED_WITH, XML_HTTP_REQUEST)
.header(REFERER, getBaseUrl()) .header(REFERER, getBaseUrl())

16
common/src/main/java/ctbrec/sites/chaturbate/ChaturbateHttpClient.java
View File

@ -8,6 +8,7 @@ import okhttp3.*;
import java.io.IOException; import java.io.IOException;
import java.io.InterruptedIOException; import java.io.InterruptedIOException;
import java.net.URI;
import java.util.NoSuchElementException; import java.util.NoSuchElementException;
import java.util.concurrent.Semaphore; import java.util.concurrent.Semaphore;
@ -28,15 +29,6 @@ public class ChaturbateHttpClient extends HttpClient {
super("chaturbate", config); super("chaturbate", config);
} }
@Override
public String getEffectiveUserAgent() {
if (flaresolverr != null) {
return config.getSettings().flaresolverr.userAgent;
} else {
return config.getSettings().httpUserAgent;
}
}
private void extractCsrfToken(Request request) { private void extractCsrfToken(Request request) {
try { try {
Cookie csrfToken = cookieJar.getCookie(request.url(), "csrftoken"); Cookie csrfToken = cookieJar.getCookie(request.url(), "csrftoken");
@ -65,7 +57,7 @@ public class ChaturbateHttpClient extends HttpClient {
} }
Request login = new Request.Builder() Request login = new Request.Builder()
.url(Chaturbate.baseUrl + PATH) .url(Chaturbate.baseUrl + PATH)
.header(USER_AGENT, getEffectiveUserAgent()) .header(USER_AGENT, getEffectiveUserAgent(URI.create(Chaturbate.baseUrl).getHost()))
.build(); .build();
try (var initResponse = client.newCall(login).execute()) { try (var initResponse = client.newCall(login).execute()) {
String content = initResponse.body().string(); String content = initResponse.body().string();
@ -81,7 +73,7 @@ public class ChaturbateHttpClient extends HttpClient {
login = new Request.Builder() login = new Request.Builder()
.url(Chaturbate.baseUrl + PATH) .url(Chaturbate.baseUrl + PATH)
.header(REFERER, Chaturbate.baseUrl + PATH) .header(REFERER, Chaturbate.baseUrl + PATH)
.header(USER_AGENT, getEffectiveUserAgent()) .header(USER_AGENT, getEffectiveUserAgent(URI.create(Chaturbate.baseUrl).getHost()))
.post(body) .post(body)
.build(); .build();
@ -106,7 +98,7 @@ public class ChaturbateHttpClient extends HttpClient {
String url = "https://chaturbate.com/api/ts/chatmessages/pm_users/?offset=0"; String url = "https://chaturbate.com/api/ts/chatmessages/pm_users/?offset=0";
Request req = new Request.Builder() Request req = new Request.Builder()
.url(url) .url(url)
.header(USER_AGENT, getEffectiveUserAgent()) .header(USER_AGENT, getEffectiveUserAgent(URI.create(url).getHost()))
.build(); .build();
try (Response response = execute(req)) { try (Response response = execute(req)) {
boolean result = false; boolean result = false;

15
common/src/main/java/ctbrec/sites/chaturbate/ChaturbateModel.java
View File

@ -22,6 +22,7 @@ import java.io.ByteArrayInputStream;
import java.io.EOFException; import java.io.EOFException;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.net.URI;
import java.time.Duration; import java.time.Duration;
import java.time.Instant; import java.time.Instant;
import java.util.*; import java.util.*;
@ -100,7 +101,7 @@ public class ChaturbateModel extends AbstractModel {
int imageSize = 0; int imageSize = 0;
Request req = new Request.Builder() Request req = new Request.Builder()
.url(url) .url(url)
.header(USER_AGENT, site.getHttpClient().getEffectiveUserAgent()) .header(USER_AGENT, site.getHttpClient().getEffectiveUserAgent(URI.create(url).getHost()))
.head() .head()
.build(); .build();
try (Response response = getSite().getHttpClient().execute(req)) { try (Response response = getSite().getHttpClient().execute(req)) {
@ -193,7 +194,7 @@ public class ChaturbateModel extends AbstractModel {
.post(body) .post(body)
.header(REFERER, "https://chaturbate.com/" + getName() + "/") .header(REFERER, "https://chaturbate.com/" + getName() + "/")
.header(X_REQUESTED_WITH, XML_HTTP_REQUEST) .header(X_REQUESTED_WITH, XML_HTTP_REQUEST)
.header(USER_AGENT, site.getHttpClient().getEffectiveUserAgent()) .header(USER_AGENT, site.getHttpClient().getEffectiveUserAgent("chaturbate.com"))
.build(); .build();
try (Response response = getSite().getHttpClient().execute(req)) { try (Response response = getSite().getHttpClient().execute(req)) {
if (!response.isSuccessful()) { if (!response.isSuccessful()) {
@ -242,7 +243,7 @@ public class ChaturbateModel extends AbstractModel {
// do an initial request to get cookies // do an initial request to get cookies
Request req = new Request.Builder() Request req = new Request.Builder()
.url(getUrl()) .url(getUrl())
.header(USER_AGENT, site.getHttpClient().getEffectiveUserAgent()) .header(USER_AGENT, site.getHttpClient().getEffectiveUserAgent(URI.create(getUrl()).getHost()))
.build(); .build();
Response resp = site.getHttpClient().execute(req); Response resp = site.getHttpClient().execute(req);
resp.close(); resp.close();
@ -261,7 +262,7 @@ public class ChaturbateModel extends AbstractModel {
.header(ACCEPT, "*/*") .header(ACCEPT, "*/*")
.header(ACCEPT_LANGUAGE, "en-US,en;q=0.5") .header(ACCEPT_LANGUAGE, "en-US,en;q=0.5")
.header(REFERER, getUrl()) .header(REFERER, getUrl())
.header(USER_AGENT, site.getHttpClient().getEffectiveUserAgent()) .header(USER_AGENT, site.getHttpClient().getEffectiveUserAgent(URI.create(url).getHost()))
.header("X-CSRFToken", ((ChaturbateHttpClient) site.getHttpClient()).getToken()) .header("X-CSRFToken", ((ChaturbateHttpClient) site.getHttpClient()).getToken())
.header(X_REQUESTED_WITH, XML_HTTP_REQUEST) .header(X_REQUESTED_WITH, XML_HTTP_REQUEST)
.build(); .build();
@ -305,7 +306,7 @@ public class ChaturbateModel extends AbstractModel {
Request req = new Request.Builder() Request req = new Request.Builder()
.url(getSite().getBaseUrl() + "/get_edge_hls_url_ajax/") .url(getSite().getBaseUrl() + "/get_edge_hls_url_ajax/")
.post(body) .post(body)
.header(USER_AGENT, site.getHttpClient().getEffectiveUserAgent()) .header(USER_AGENT, site.getHttpClient().getEffectiveUserAgent(URI.create(getSite().getBaseUrl()).getHost()))
.header(X_REQUESTED_WITH, XML_HTTP_REQUEST) .header(X_REQUESTED_WITH, XML_HTTP_REQUEST)
.build(); .build();
try (Response response = getSite().getHttpClient().execute(req)) { try (Response response = getSite().getHttpClient().execute(req)) {
@ -384,7 +385,7 @@ public class ChaturbateModel extends AbstractModel {
log.trace("Loading master playlist {}", streamInfo.url); log.trace("Loading master playlist {}", streamInfo.url);
Request req = new Request.Builder() Request req = new Request.Builder()
.url(streamInfo.url) .url(streamInfo.url)
.header(USER_AGENT, site.getHttpClient().getEffectiveUserAgent()) .header(USER_AGENT, site.getHttpClient().getEffectiveUserAgent(URI.create(streamInfo.url).getHost()))
.build(); .build();
try (Response response = getSite().getHttpClient().execute(req)) { try (Response response = getSite().getHttpClient().execute(req)) {
if (response.isSuccessful()) { if (response.isSuccessful()) {
@ -405,7 +406,7 @@ public class ChaturbateModel extends AbstractModel {
public boolean exists() throws IOException { public boolean exists() throws IOException {
Request req = new Request.Builder() // @formatter:off Request req = new Request.Builder() // @formatter:off
.url(getUrl()) .url(getUrl())
.header(USER_AGENT, site.getHttpClient().getEffectiveUserAgent()) .header(USER_AGENT, site.getHttpClient().getEffectiveUserAgent(URI.create(getUrl()).getHost()))
.header(ACCEPT_LANGUAGE, Locale.ENGLISH.getLanguage()) .header(ACCEPT_LANGUAGE, Locale.ENGLISH.getLanguage())
.build(); // @formatter:on .build(); // @formatter:on
try (Response response = getSite().getHttpClient().execute(req)) { try (Response response = getSite().getHttpClient().execute(req)) {