From f397ad471d5dbf326e884bf9e19badec6f32885e Mon Sep 17 00:00:00 2001 From: reusedname <155286845+reusedname@users.noreply.github.com> Date: Sun, 11 Feb 2024 12:18:01 +0500 Subject: [PATCH 1/6] (WIP) flaresolverr client --- .../java/ctbrec/io/FlaresolverrClient.java | 67 +++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 common/src/main/java/ctbrec/io/FlaresolverrClient.java diff --git a/common/src/main/java/ctbrec/io/FlaresolverrClient.java b/common/src/main/java/ctbrec/io/FlaresolverrClient.java new file mode 100644 index 00000000..92529dd5 --- /dev/null +++ b/common/src/main/java/ctbrec/io/FlaresolverrClient.java @@ -0,0 +1,67 @@ +package ctbrec.io; + +import org.json.JSONObject; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.json.JsonMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; + +import okhttp3.*; +import okhttp3.OkHttpClient.Builder; +import ctbrec.GlobalThreadPool; + + +public class FlaresolverrClient { + public String api_url = "http://localhost:8191/v1"; + protected OkHttpClient client = new OkHttpClient(); + protected JsonMapper mapper = new JsonMapper(); + + public FlaresolverrClient() { + } + + public FlaresolverrClient(String apiUrl) { + api_url = apiUrl; + } + + public JsonNode createSession(String name) throws Exception { + var body = mapper.createObjectNode() + .put("cmd", "sessions.create") + .put("session", name); + + return makeApiCall(body); + } + + public JsonNode destroySession(String name) throws Exception { + var body = mapper.createObjectNode() + .put("cmd", "sessions.destroy") + .put("session", name); + + return makeApiCall(body); + } + + public JsonNode getCookies(String url) throws Exception { + return getCookies(url, 60000); + } + + public JsonNode getCookies(String url, int timeout_ms) throws Exception { + var body = mapper.createObjectNode() + .put("cmd", "request.get") + .put("url", url) + .put("maxTimeout", timeout_ms) + .put("returnOnlyCookies", true); + + return makeApiCall(body); + } + + protected JsonNode makeApiCall(ObjectNode body) throws Exception { + var requestBody = RequestBody.create(mapper.writeValueAsString(body), MediaType.get("application/json")); + var request = new Request.Builder() + .url(api_url) + .post(requestBody) + .build(); + + var response = client.newCall(request).execute(); + return mapper.readTree(response.body().string()); + } + +} From 257226860034fca649a472946f8a2ddaf242f337 Mon Sep 17 00:00:00 2001 From: reusedname <155286845+reusedname@users.noreply.github.com> Date: Tue, 3 Dec 2024 22:00:42 +0500 Subject: [PATCH 2/6] Flaresolverr support for chaturbate --- .../ChaturbateApiUpdateService.java | 2 +- .../ChaturbateElectronLoginDialog.java | 2 +- .../chaturbate/ChaturbateUpdateService.java | 2 +- common/src/main/java/ctbrec/Settings.java | 7 ++ .../ctbrec/io/CompletableRequestFuture.java | 46 ++++++++++ .../java/ctbrec/io/FlaresolverrClient.java | 86 +++++++++++++----- .../java/ctbrec/io/FlaresolverrResponse.java | 23 +++++ .../io/FlaresolverrSolutionResponse.java | 73 +++++++++++++++ .../src/main/java/ctbrec/io/HttpClient.java | 5 + .../ctbrec/sites/chaturbate/Chaturbate.java | 4 +- .../chaturbate/ChaturbateHttpClient.java | 91 ++++++++++++++++++- .../sites/chaturbate/ChaturbateModel.java | 14 +-- 12 files changed, 316 insertions(+), 39 deletions(-) create mode 100644 common/src/main/java/ctbrec/io/CompletableRequestFuture.java create mode 100644 common/src/main/java/ctbrec/io/FlaresolverrResponse.java create mode 100644 common/src/main/java/ctbrec/io/FlaresolverrSolutionResponse.java diff --git a/client/src/main/java/ctbrec/ui/sites/chaturbate/ChaturbateApiUpdateService.java b/client/src/main/java/ctbrec/ui/sites/chaturbate/ChaturbateApiUpdateService.java index 0c8ac2df..3902d8b5 100644 --- a/client/src/main/java/ctbrec/ui/sites/chaturbate/ChaturbateApiUpdateService.java +++ b/client/src/main/java/ctbrec/ui/sites/chaturbate/ChaturbateApiUpdateService.java @@ -35,7 +35,7 @@ public class ChaturbateApiUpdateService extends PaginatedScheduledService { protected List call() throws Exception { var request = new Request.Builder() .url(url) - .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) + .header(USER_AGENT, chaturbate.getHttpClient().getEffectiveUserAgent()) .header(ACCEPT, MIMETYPE_APPLICATION_JSON) .build(); try (var response = chaturbate.getHttpClient().execute(request)) { diff --git a/client/src/main/java/ctbrec/ui/sites/chaturbate/ChaturbateElectronLoginDialog.java b/client/src/main/java/ctbrec/ui/sites/chaturbate/ChaturbateElectronLoginDialog.java index d376edc6..cf1c0628 100644 --- a/client/src/main/java/ctbrec/ui/sites/chaturbate/ChaturbateElectronLoginDialog.java +++ b/client/src/main/java/ctbrec/ui/sites/chaturbate/ChaturbateElectronLoginDialog.java @@ -35,7 +35,7 @@ public class ChaturbateElectronLoginDialog { config.put("url", site.getBaseUrl() + "/auth/login/"); config.put("w", 640); config.put("h", 480); - config.put("userAgent", Config.getInstance().getSettings().httpUserAgent); + config.put("userAgent", site.getHttpClient().getEffectiveUserAgent()); var msg = new JSONObject(); msg.put("config", config); browser.run(msg, msgHandler); diff --git a/client/src/main/java/ctbrec/ui/sites/chaturbate/ChaturbateUpdateService.java b/client/src/main/java/ctbrec/ui/sites/chaturbate/ChaturbateUpdateService.java index b7f14ada..90c8e7c0 100644 --- a/client/src/main/java/ctbrec/ui/sites/chaturbate/ChaturbateUpdateService.java +++ b/client/src/main/java/ctbrec/ui/sites/chaturbate/ChaturbateUpdateService.java @@ -54,7 +54,7 @@ public class ChaturbateUpdateService extends PaginatedScheduledService { .header(ACCEPT, MIMETYPE_APPLICATION_JSON) .header(ACCEPT_LANGUAGE, Locale.ENGLISH.getLanguage()) .header(X_REQUESTED_WITH, XML_HTTP_REQUEST) - .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) + .header(USER_AGENT, chaturbate.getHttpClient().getEffectiveUserAgent()) .build(); try (var response = chaturbate.getHttpClient().execute(request)) { if (response.isSuccessful()) { diff --git a/common/src/main/java/ctbrec/Settings.java b/common/src/main/java/ctbrec/Settings.java index 55f50c4b..1e8f3eaf 100644 --- a/common/src/main/java/ctbrec/Settings.java +++ b/common/src/main/java/ctbrec/Settings.java @@ -42,7 +42,13 @@ public class Settings { TIME_OR_SIZE } + public class FlaresolverrSettings { + public String apiUrl = "http://localhost:8191/v1"; + public int timeoutInMillis = 60000; + public String userAgent = ""; + }; + public FlaresolverrSettings flaresolverr = new FlaresolverrSettings(); public String amateurTvUsername = ""; public String amateurTvPassword = ""; public String bongacamsBaseUrl = "https://bongacams.com"; @@ -55,6 +61,7 @@ public class Settings { public String chaturbatePassword = ""; public String chaturbateUsername = ""; public String chaturbateBaseUrl = "https://chaturbate.com"; + public boolean chaturbateUseFlaresolverr = false; public int chaturbateMsBetweenRequests = 1000; public String cherryTvPassword = ""; public String cherryTvUsername = ""; diff --git a/common/src/main/java/ctbrec/io/CompletableRequestFuture.java b/common/src/main/java/ctbrec/io/CompletableRequestFuture.java new file mode 100644 index 00000000..faeb5026 --- /dev/null +++ b/common/src/main/java/ctbrec/io/CompletableRequestFuture.java @@ -0,0 +1,46 @@ +package ctbrec.io; + +import java.io.IOException; +import java.util.concurrent.CompletableFuture; + +import lombok.Getter; +import okhttp3.Call; +import okhttp3.Callback; +import okhttp3.Response; +import okhttp3.ResponseBody; + +public class CompletableRequestFuture extends CompletableFuture implements Callback { + @Getter + protected Call call; + + public CompletableRequestFuture(Call call) { + this.call = call; + } + + @Override + public void onResponse(Call c, Response response) throws IOException { + try (var body = response.body()) { + if (response.isSuccessful()) { + processBody(body); + } else { + completeExceptionally(new HttpException(response.code(), response.message())); + } + } catch (Exception e) { + completeExceptionally(e); + } + } + + @Override + public void onFailure(Call c, IOException error) { + completeExceptionally(error); + } + + @Override + public boolean cancel(boolean mayInterruptIfRunning) { + call.cancel(); + return super.cancel(mayInterruptIfRunning); + } + + + protected void processBody(ResponseBody body) throws Exception {} +} \ No newline at end of file diff --git a/common/src/main/java/ctbrec/io/FlaresolverrClient.java b/common/src/main/java/ctbrec/io/FlaresolverrClient.java index 92529dd5..5e759563 100644 --- a/common/src/main/java/ctbrec/io/FlaresolverrClient.java +++ b/common/src/main/java/ctbrec/io/FlaresolverrClient.java @@ -1,67 +1,107 @@ package ctbrec.io; -import org.json.JSONObject; +import ctbrec.io.FlaresolverrResponse; +import ctbrec.io.FlaresolverrSolutionResponse; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.json.JsonMapper; import com.fasterxml.jackson.databind.node.ObjectNode; -import okhttp3.*; -import okhttp3.OkHttpClient.Builder; -import ctbrec.GlobalThreadPool; +import lombok.Getter; +import lombok.Setter; +import okhttp3.Call; +import okhttp3.Callback; +import okhttp3.MediaType; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.RequestBody; +import okhttp3.Response; +import okhttp3.ResponseBody; +import okhttp3.Cookie; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.*; public class FlaresolverrClient { - public String api_url = "http://localhost:8191/v1"; - protected OkHttpClient client = new OkHttpClient(); + public String api_url; + protected int timeout_ms; + protected OkHttpClient client; protected JsonMapper mapper = new JsonMapper(); + @Getter + protected String sessionName = ""; + public FlaresolverrClient() { + this("http://localhost:8191/v1", 60000); } - public FlaresolverrClient(String apiUrl) { + public FlaresolverrClient(String apiUrl, int timeout_ms) { api_url = apiUrl; + this.timeout_ms = timeout_ms; + client = new OkHttpClient.Builder() + .callTimeout(timeout_ms + 10000, TimeUnit.MILLISECONDS) + .readTimeout(timeout_ms + 1000, TimeUnit.MILLISECONDS) + .build(); } - - public JsonNode createSession(String name) throws Exception { + + public CompletableFuture createSession(String name) throws IOException { + if (!sessionName.equals("")) + throw new IOException("Cannot start new session because another one is already started. Finish it before creating a new one"); + + sessionName = name; var body = mapper.createObjectNode() .put("cmd", "sessions.create") .put("session", name); - return makeApiCall(body); + return makeApiCall(body).thenApply(r -> new FlaresolverrResponse(r)); } - public JsonNode destroySession(String name) throws Exception { + public CompletableFuture destroySession(String name) throws IOException { + if (sessionName.equals("")) + throw new IOException("Cannot destroy session because no session is active"); + var body = mapper.createObjectNode() .put("cmd", "sessions.destroy") .put("session", name); - return makeApiCall(body); + sessionName = ""; + return makeApiCall(body).thenApply(r -> new FlaresolverrResponse(r)); } - public JsonNode getCookies(String url) throws Exception { - return getCookies(url, 60000); - } - - public JsonNode getCookies(String url, int timeout_ms) throws Exception { + public CompletableFuture getCookies(String url) throws IOException { var body = mapper.createObjectNode() .put("cmd", "request.get") .put("url", url) .put("maxTimeout", timeout_ms) .put("returnOnlyCookies", true); - return makeApiCall(body); + if (sessionName != "") { + body.put("session", sessionName); + } + + return makeApiCall(body).thenApply(r -> new FlaresolverrSolutionResponse(r)); } - protected JsonNode makeApiCall(ObjectNode body) throws Exception { + protected CompletableRequestFuture makeApiCall(ObjectNode body) throws IOException { var requestBody = RequestBody.create(mapper.writeValueAsString(body), MediaType.get("application/json")); var request = new Request.Builder() .url(api_url) .post(requestBody) .build(); - - var response = client.newCall(request).execute(); - return mapper.readTree(response.body().string()); + + var call = client.newCall(request); + var future = new CompletableRequestFuture(call) { + @Override + public void processBody(ResponseBody body) throws IOException { + complete(mapper.readTree(body.charStream())); + } + }; + + // FIXME?: unfortunate cyclic reference here to allow cancelling through the future, is this bad? + call.enqueue(future); + return future; } - } diff --git a/common/src/main/java/ctbrec/io/FlaresolverrResponse.java b/common/src/main/java/ctbrec/io/FlaresolverrResponse.java new file mode 100644 index 00000000..3151954b --- /dev/null +++ b/common/src/main/java/ctbrec/io/FlaresolverrResponse.java @@ -0,0 +1,23 @@ +package ctbrec.io; + +import com.fasterxml.jackson.databind.JsonNode; + +import lombok.Getter; +import lombok.ToString; + +public class FlaresolverrResponse { + @Getter + protected String status; + @Getter + protected String message; + + public FlaresolverrResponse(JsonNode response) { + status = response.get("status").asText(); + message = response.get("message").asText(); + } + + @Override + public String toString() { + return String.format("{status: %s, message: %s}", status, message); + } +} diff --git a/common/src/main/java/ctbrec/io/FlaresolverrSolutionResponse.java b/common/src/main/java/ctbrec/io/FlaresolverrSolutionResponse.java new file mode 100644 index 00000000..505b5e15 --- /dev/null +++ b/common/src/main/java/ctbrec/io/FlaresolverrSolutionResponse.java @@ -0,0 +1,73 @@ +package ctbrec.io; + +import com.fasterxml.jackson.databind.JsonNode; + +import ctbrec.StringUtil; +import lombok.Getter; +import okhttp3.Cookie; + +import java.time.Instant; +import java.util.*; + +public class FlaresolverrSolutionResponse extends FlaresolverrResponse { + @Getter + protected String userAgent; + @Getter + protected Instant startTimestamp; + @Getter + protected Instant endTimestamp; + @Getter + protected String version; + @Getter + protected List cookies; + + FlaresolverrSolutionResponse(JsonNode response) { + super(response); + + startTimestamp = Instant.ofEpochMilli(response.get("startTimestamp").asLong()); + endTimestamp = Instant.ofEpochMilli(response.get("endTimestamp").asLong()); + version = response.get("version").asText(); + + var solution = response.get("solution"); + userAgent = solution.get("userAgent").asText(); + + cookies = new ArrayList(); + + for (var c : solution.get("cookies")) { + // "domain": c["domain"].lstrip('.'), + // "expiresAt": c["expiry"], + // "hostOnly": c["sameSite"].lower() == "strict", + // "httpOnly": c["httpOnly"], + // "name": c["name"], + // "path": c["path"], + // "persistent": False, + // "secure": c["secure"], + // "value": c["value"] + + var cb = new Cookie.Builder() + .expiresAt(Optional.ofNullable(c.get("expires")).orElse(c.get("expiry")).asLong() * 1000) // seconds -> millis + .name(c.get("name").asText()) + .path(c.get("path").asText()) + .value(c.get("value").asText()) + ; + + var domain = c.get("domain").asText().replaceFirst("\\.", ""); + + // FIXME: is this correct? + if (c.path("sameSite").asText("").equalsIgnoreCase("strict")) { + cb.hostOnlyDomain(domain); + } else { + cb.domain(domain); + } + + if (c.path("httpOnly").asBoolean(false)) + cb.httpOnly(); + + if (c.path("secure").asBoolean(false)) + cb.secure(); + + cookies.add(cb.build()); + } + } + +} diff --git a/common/src/main/java/ctbrec/io/HttpClient.java b/common/src/main/java/ctbrec/io/HttpClient.java index e90a6b79..a198bb45 100644 --- a/common/src/main/java/ctbrec/io/HttpClient.java +++ b/common/src/main/java/ctbrec/io/HttpClient.java @@ -357,4 +357,9 @@ public abstract class HttpClient { public static final String JAVA_NET_SOCKS_USERNAME = "java.net.socks.username"; public static final String JAVA_NET_SOCKS_PASSWORD = "java.net.socks.password"; } + + // overridable default user agent (used for Flaresolverr) + public String getEffectiveUserAgent() { + return config.getSettings().httpUserAgent; + } } diff --git a/common/src/main/java/ctbrec/sites/chaturbate/Chaturbate.java b/common/src/main/java/ctbrec/sites/chaturbate/Chaturbate.java index a6bfaf62..1d19592d 100644 --- a/common/src/main/java/ctbrec/sites/chaturbate/Chaturbate.java +++ b/common/src/main/java/ctbrec/sites/chaturbate/Chaturbate.java @@ -69,7 +69,7 @@ public class Chaturbate extends AbstractSite { String url = "https://chaturbate.com/p/" + username + "/"; Request req = new Request.Builder() .url(url) - .header(USER_AGENT, getConfig().getSettings().httpUserAgent) + .header(USER_AGENT, getHttpClient().getEffectiveUserAgent()) .build(); try (Response resp = getHttpClient().execute(req)) { if (resp.isSuccessful()) { @@ -131,7 +131,7 @@ public class Chaturbate extends AbstractSite { // search online models Request req = new Request.Builder() .url(url) - .header(USER_AGENT, getConfig().getSettings().httpUserAgent) + .header(USER_AGENT, getHttpClient().getEffectiveUserAgent()) .header(ACCEPT, "*/*") .header(X_REQUESTED_WITH, XML_HTTP_REQUEST) .header(REFERER, getBaseUrl()) diff --git a/common/src/main/java/ctbrec/sites/chaturbate/ChaturbateHttpClient.java b/common/src/main/java/ctbrec/sites/chaturbate/ChaturbateHttpClient.java index 923fc50b..d987628d 100644 --- a/common/src/main/java/ctbrec/sites/chaturbate/ChaturbateHttpClient.java +++ b/common/src/main/java/ctbrec/sites/chaturbate/ChaturbateHttpClient.java @@ -1,15 +1,18 @@ package ctbrec.sites.chaturbate; import ctbrec.Config; +import ctbrec.io.FlaresolverrClient; import ctbrec.io.HtmlParser; import ctbrec.io.HttpClient; import lombok.extern.slf4j.Slf4j; import okhttp3.*; +import java.time.*; import java.io.IOException; import java.io.InterruptedIOException; import java.util.NoSuchElementException; import java.util.concurrent.Semaphore; +import java.util.concurrent.locks.ReentrantReadWriteLock; import static ctbrec.io.HttpConstants.REFERER; import static ctbrec.io.HttpConstants.USER_AGENT; @@ -19,12 +22,39 @@ public class ChaturbateHttpClient extends HttpClient { private static final String PATH = "/auth/login/"; // NOSONAR protected String token; + protected final FlaresolverrClient flaresolverr; private static final Semaphore requestThrottle = new Semaphore(2, true); private static long lastRequest = 0; + + // a lock to prevent multiple requests from + ReentrantReadWriteLock cookieRefreshLock = new ReentrantReadWriteLock(); public ChaturbateHttpClient(Config config) { super("chaturbate", config); + + if (config.getSettings().chaturbateUseFlaresolverr) { + flaresolverr = new FlaresolverrClient(config.getSettings().flaresolverr.apiUrl, config.getSettings().flaresolverr.timeoutInMillis); + + // try { + // flaresolverr.createSession("ctbrec").get(); + // } catch (InterruptedException e) { + // Thread.currentThread().interrupt(); + // } catch (Exception e) { + // log.error("Error starting Flaresolverr session", e); + // } + } else { + flaresolverr = null; + } + } + + @Override + public String getEffectiveUserAgent() { + if (flaresolverr != null) { + return config.getSettings().flaresolverr.userAgent; + } else { + return config.getSettings().httpUserAgent; + } } private void extractCsrfToken(Request request) { @@ -55,7 +85,7 @@ public class ChaturbateHttpClient extends HttpClient { } Request login = new Request.Builder() .url(Chaturbate.baseUrl + PATH) - .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) + .header(USER_AGENT, getEffectiveUserAgent()) .build(); try (var initResponse = client.newCall(login).execute()) { String content = initResponse.body().string(); @@ -71,7 +101,7 @@ public class ChaturbateHttpClient extends HttpClient { login = new Request.Builder() .url(Chaturbate.baseUrl + PATH) .header(REFERER, Chaturbate.baseUrl + PATH) - .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) + .header(USER_AGENT, getEffectiveUserAgent()) .post(body) .build(); @@ -96,7 +126,7 @@ public class ChaturbateHttpClient extends HttpClient { String url = "https://chaturbate.com/api/ts/chatmessages/pm_users/?offset=0"; Request req = new Request.Builder() .url(url) - .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) + .header(USER_AGENT, getEffectiveUserAgent()) .build(); try (Response response = execute(req)) { boolean result = false; @@ -124,7 +154,21 @@ public class ChaturbateHttpClient extends HttpClient { if (throttle) { acquireSlot(); } - Response resp = super.execute(req); + + Response resp; + + try { + cookieRefreshLock.readLock().lock(); + resp = super.execute(req); + } finally { + cookieRefreshLock.readLock().unlock(); + } + + // try to solve the cloudflare challenge if we got one (clearance cookie expired, update it) + if (resp.code() == 403 && flaresolverr != null) { + resp = refreshCookiesAndRetry(req, resp); + } + extractCsrfToken(req); return resp; } catch (InterruptedException e) { @@ -136,6 +180,45 @@ public class ChaturbateHttpClient extends HttpClient { } } } + + private Response refreshCookiesAndRetry(Request req, Response origResp) throws IOException { + log.debug("403 received from {}. Trying to refresh cookies with Flaresolverr", req.url().host()); + + try { + cookieRefreshLock.writeLock().lock(); + + // we need to prevent repeated challenge requests from multiple threads, so we check if the clearance cookie needs updating + // maybe this can be done with some syncronization primitive, or maybe an expiresAt() check is enough + var cookie = cookieJar.getCookieFromCollection(cookieJar.getCookies().get(req.url().topPrivateDomain()), "cf_clearance"); + if (cookie.map(c -> Instant.ofEpochMilli(c.expiresAt()).isBefore(Instant.now())).orElse(true)) { + var apiResponse = flaresolverr.getCookies(req.url().toString()).get(); + if (apiResponse.getStatus().equals("ok")) { + // update user agent. It should be the same for all sites, assuming we use the same api address every time + if (!config.getSettings().flaresolverr.userAgent.equals(apiResponse.getUserAgent())) { + config.getSettings().flaresolverr.userAgent = apiResponse.getUserAgent(); + config.save(); + } + + cookieJar.saveFromResponse(req.url(), apiResponse.getCookies()); + log.debug("Cookies successfully refreshed with Flaresolverr in {}", Duration.between(apiResponse.getStartTimestamp(), apiResponse.getEndTimestamp())); + } else { + log.debug("Unsuccessful attempt to refresh cookies. Response from Flaresolverr: {}", apiResponse); + return origResp; + } + } else { + log.debug("Looks like the cookies were refreshed already, skipping refreshing"); + } + } catch (Exception e) { + log.warn("Error refreshing cookies with Flaresolverr", e); + return origResp; + } finally { + cookieRefreshLock.writeLock().unlock(); + } + + origResp.close(); + + return super.execute(req); + } private static void acquireSlot() throws InterruptedException { long pauseBetweenRequests = Config.getInstance().getSettings().chaturbateMsBetweenRequests; diff --git a/common/src/main/java/ctbrec/sites/chaturbate/ChaturbateModel.java b/common/src/main/java/ctbrec/sites/chaturbate/ChaturbateModel.java index 972cb4c1..d1dad529 100644 --- a/common/src/main/java/ctbrec/sites/chaturbate/ChaturbateModel.java +++ b/common/src/main/java/ctbrec/sites/chaturbate/ChaturbateModel.java @@ -97,7 +97,7 @@ public class ChaturbateModel extends AbstractModel { int imageSize = 0; Request req = new Request.Builder() .url(url) - .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) + .header(USER_AGENT, site.getHttpClient().getEffectiveUserAgent()) .head() .build(); try (Response response = getSite().getHttpClient().execute(req)) { @@ -190,7 +190,7 @@ public class ChaturbateModel extends AbstractModel { .post(body) .header(REFERER, "https://chaturbate.com/" + getName() + "/") .header(X_REQUESTED_WITH, XML_HTTP_REQUEST) - .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) + .header(USER_AGENT, site.getHttpClient().getEffectiveUserAgent()) .build(); try (Response response = getSite().getHttpClient().execute(req)) { if (!response.isSuccessful()) { @@ -239,7 +239,7 @@ public class ChaturbateModel extends AbstractModel { // do an initial request to get cookies Request req = new Request.Builder() .url(getUrl()) - .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) + .header(USER_AGENT, site.getHttpClient().getEffectiveUserAgent()) .build(); Response resp = site.getHttpClient().execute(req); resp.close(); @@ -258,7 +258,7 @@ public class ChaturbateModel extends AbstractModel { .header(ACCEPT, "*/*") .header(ACCEPT_LANGUAGE, "en-US,en;q=0.5") .header(REFERER, getUrl()) - .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) + .header(USER_AGENT, site.getHttpClient().getEffectiveUserAgent()) .header("X-CSRFToken", ((ChaturbateHttpClient) site.getHttpClient()).getToken()) .header(X_REQUESTED_WITH, XML_HTTP_REQUEST) .build(); @@ -302,7 +302,7 @@ public class ChaturbateModel extends AbstractModel { Request req = new Request.Builder() .url(getSite().getBaseUrl() + "/get_edge_hls_url_ajax/") .post(body) - .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) + .header(USER_AGENT, site.getHttpClient().getEffectiveUserAgent()) .header(X_REQUESTED_WITH, XML_HTTP_REQUEST) .build(); try (Response response = getSite().getHttpClient().execute(req)) { @@ -364,7 +364,7 @@ public class ChaturbateModel extends AbstractModel { log.trace("Loading master playlist {}", streamInfo.url); Request req = new Request.Builder() .url(streamInfo.url) - .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) + .header(USER_AGENT, site.getHttpClient().getEffectiveUserAgent()) .build(); try (Response response = getSite().getHttpClient().execute(req)) { if (response.isSuccessful()) { @@ -385,7 +385,7 @@ public class ChaturbateModel extends AbstractModel { public boolean exists() throws IOException { Request req = new Request.Builder() // @formatter:off .url(getUrl()) - .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) + .header(USER_AGENT, site.getHttpClient().getEffectiveUserAgent()) .header(ACCEPT_LANGUAGE, Locale.ENGLISH.getLanguage()) .build(); // @formatter:on try (Response response = getSite().getHttpClient().execute(req)) { From ae1d5c7af8eee67c3078ad42456781f846aa444e Mon Sep 17 00:00:00 2001 From: reusedname <155286845+reusedname@users.noreply.github.com> Date: Wed, 4 Dec 2024 16:56:46 +0500 Subject: [PATCH 3/6] Settings for Flaresolverr --- .../ui/settings/CtbrecPreferencesStorage.java | 110 +++++++++--------- .../java/ctbrec/ui/settings/SettingsTab.java | 12 +- .../sites/chaturbate/ChaturbateConfigUi.java | 18 ++- .../src/main/java/ctbrec/io/BoundField.java | 32 +++++ .../ctbrec/recorder/server/ConfigServlet.java | 9 +- 5 files changed, 115 insertions(+), 66 deletions(-) create mode 100644 common/src/main/java/ctbrec/io/BoundField.java diff --git a/client/src/main/java/ctbrec/ui/settings/CtbrecPreferencesStorage.java b/client/src/main/java/ctbrec/ui/settings/CtbrecPreferencesStorage.java index 72117c9c..3900c29f 100644 --- a/client/src/main/java/ctbrec/ui/settings/CtbrecPreferencesStorage.java +++ b/client/src/main/java/ctbrec/ui/settings/CtbrecPreferencesStorage.java @@ -1,6 +1,7 @@ package ctbrec.ui.settings; import java.io.IOException; +import java.lang.reflect.InvocationTargetException; import java.time.LocalTime; import java.util.List; import java.util.Objects; @@ -25,6 +26,7 @@ import ctbrec.ui.settings.api.Setting; import ctbrec.ui.settings.api.SimpleDirectoryProperty; import ctbrec.ui.settings.api.SimpleFileProperty; import ctbrec.ui.settings.api.SimpleRangeProperty; +import ctbrec.io.BoundField; import javafx.beans.property.BooleanProperty; import javafx.beans.property.IntegerProperty; import javafx.beans.property.ListProperty; @@ -115,13 +117,13 @@ public class CtbrecPreferencesStorage implements PreferencesStorage { optionB.setSelected(!optionA.isSelected()); optionB.setToggleGroup(toggleGroup); optionA.selectedProperty().bindBidirectional(prop); - prop.addListener((obs, oldV, newV) -> saveValue(() -> { - var field = Settings.class.getField(setting.getKey()); - field.set(settings, newV); // NOSONAR - if (setting.doesNeedRestart() && !Objects.equals(oldV, newV)) { - runRestartRequiredCallback(); + prop.addListener((obs, oldV, newV) -> saveValue(() -> { + if (setIfChanged(setting.getKey(), newV)) { + if (setting.doesNeedRestart()) { + runRestartRequiredCallback(); + } + config.save(); } - config.save(); })); var row = new HBox(); row.getChildren().addAll(optionA, optionB); @@ -155,15 +157,15 @@ public class CtbrecPreferencesStorage implements PreferencesStorage { resolutionRange.setHigh(highValue >= 0 ? highValue : values.get(values.size() - 1)); resolutionRange.getLow().addListener((obs, o, n) -> saveValue(() -> { int newV = labels.get(n.intValue()); - var field = Settings.class.getField(rangeProperty.getLowKey()); - field.set(settings, newV); // NOSONAR - config.save(); + if (setIfChanged(rangeProperty.getLowKey(), newV)) { + config.save(); + } })); resolutionRange.getHigh().addListener((obs, o, n) -> saveValue(() -> { int newV = labels.get(n.intValue()); - var field = Settings.class.getField(rangeProperty.getHighKey()); - field.set(settings, newV); // NOSONAR - config.save(); + if (setIfChanged(rangeProperty.getHighKey(), newV)) { + config.save(); + } })); return resolutionRange; } @@ -181,11 +183,7 @@ public class CtbrecPreferencesStorage implements PreferencesStorage { private Node createFileSelector(Setting setting) { var programSelector = new ProgramSelectionBox(""); programSelector.fileProperty().addListener((obs, o, n) -> saveValue(() -> { - String path = n; - var field = Settings.class.getField(setting.getKey()); - String oldValue = (String) field.get(settings); - if (!Objects.equals(path, oldValue)) { - field.set(settings, path); // NOSONAR + if (setIfChanged(setting.getKey(), n)) { if (setting.doesNeedRestart()) { runRestartRequiredCallback(); } @@ -201,11 +199,7 @@ public class CtbrecPreferencesStorage implements PreferencesStorage { var directorySelector = new DirectorySelectionBox(""); directorySelector.prefWidth(400); directorySelector.fileProperty().addListener((obs, o, n) -> saveValue(() -> { - String path = n; - var field = Settings.class.getField(setting.getKey()); - String oldValue = (String) field.get(settings); - if (!Objects.equals(path, oldValue)) { - field.set(settings, path); // NOSONAR + if (setIfChanged(setting.getKey(), n)) { if (setting.doesNeedRestart()) { runRestartRequiredCallback(); } @@ -221,10 +215,7 @@ public class CtbrecPreferencesStorage implements PreferencesStorage { LocalTime time = (LocalTime) setting.getProperty().getValue(); var timePicker = new TimePicker(time); timePicker.valueProperty().addListener((obs, o, n) -> saveValue(() -> { - var field = Settings.class.getField(setting.getKey()); - LocalTime oldValue = (LocalTime) field.get(settings); - if (!Objects.equals(n, oldValue)) { - field.set(settings, n); // NOSONAR + if (setIfChanged(setting.getKey(), n)) { if (setting.doesNeedRestart()) { runRestartRequiredCallback(); } @@ -237,12 +228,12 @@ public class CtbrecPreferencesStorage implements PreferencesStorage { private Node createStringProperty(Setting setting) { var ctrl = new TextField(); ctrl.textProperty().addListener((obs, oldV, newV) -> saveValue(() -> { - var field = Settings.class.getField(setting.getKey()); - field.set(settings, newV); // NOSONAR - if (setting.doesNeedRestart() && !Objects.equals(oldV, newV)) { - runRestartRequiredCallback(); + if (setIfChanged(setting.getKey(), newV)) { + if (setting.doesNeedRestart()) { + runRestartRequiredCallback(); + } + config.save(); } - config.save(); })); StringProperty prop = (StringProperty) setting.getProperty(); ctrl.textProperty().bindBidirectional(prop); @@ -256,10 +247,8 @@ public class CtbrecPreferencesStorage implements PreferencesStorage { if (!newV.matches("\\d*")) { ctrl.setText(newV.replaceAll(PATTERN_NOT_A_DIGIT, "")); } - if (!ctrl.getText().isEmpty()) { - var field = Settings.class.getField(setting.getKey()); - field.set(settings, Integer.parseInt(ctrl.getText())); // NOSONAR - if (setting.doesNeedRestart() && !Objects.equals(oldV, newV) && prefs != null) { + if (!ctrl.getText().isEmpty() && setIfChanged(setting.getKey(), Integer.parseInt(ctrl.getText()))) { + if (setting.doesNeedRestart() && prefs != null) { runRestartRequiredCallback(); } config.save(); @@ -282,12 +271,12 @@ public class CtbrecPreferencesStorage implements PreferencesStorage { if (setting.getConverter() != null) { value = (long) setting.getConverter().convertFrom(value); } - var field = Settings.class.getField(setting.getKey()); - field.set(settings, value); // NOSONAR - if (setting.doesNeedRestart() && !Objects.equals(oldV, newV)) { - runRestartRequiredCallback(); + if (setIfChanged(setting.getKey(), value)) { + if (setting.doesNeedRestart() && !Objects.equals(oldV, newV)) { + runRestartRequiredCallback(); + } + config.save(); } - config.save(); } })); Property prop = setting.getProperty(); @@ -298,12 +287,12 @@ public class CtbrecPreferencesStorage implements PreferencesStorage { private Node createBooleanProperty(Setting setting) { var ctrl = new CheckBox(); ctrl.selectedProperty().addListener((obs, oldV, newV) -> saveValue(() -> { - var field = Settings.class.getField(setting.getKey()); - field.set(settings, newV); // NOSONAR - if (setting.doesNeedRestart() && !Objects.equals(oldV, newV)) { - runRestartRequiredCallback(); + if (setIfChanged(setting.getKey(), newV)) { + if (setting.doesNeedRestart()) { + runRestartRequiredCallback(); + } + config.save(); } - config.save(); })); BooleanProperty prop = (BooleanProperty) setting.getProperty(); ctrl.selectedProperty().bindBidirectional(prop); @@ -311,11 +300,10 @@ public class CtbrecPreferencesStorage implements PreferencesStorage { } @SuppressWarnings({ "rawtypes", "unchecked" }) - private Node createComboBox(Setting setting) throws NoSuchFieldException, IllegalAccessException { + private Node createComboBox(Setting setting) throws IllegalAccessException, NoSuchFieldException { ListProperty listProp = (ListProperty) setting.getProperty(); ComboBox comboBox = new ComboBox(listProp); - var field = Settings.class.getField(setting.getKey()); - Object value = field.get(settings); + Object value = BoundField.of(settings, setting.getKey()).get(); if (StringUtil.isNotBlank(value.toString())) { if (setting.getConverter() != null) { comboBox.getSelectionModel().select(setting.getConverter().convertTo(value)); @@ -325,21 +313,29 @@ public class CtbrecPreferencesStorage implements PreferencesStorage { } comboBox.valueProperty().addListener((obs, oldV, newV) -> saveValue(() -> { LOG.debug("Saving setting {}", setting.getKey()); - if (setting.getConverter() != null) { - field.set(settings, setting.getConverter().convertFrom(newV)); // NOSONAR - } else { - field.set(settings, newV); // NOSONAR + if (setIfChanged(setting.getKey(), setting.getConverter() != null ? setting.getConverter().convertFrom(newV) : newV)) { + if (setting.doesNeedRestart()) { + runRestartRequiredCallback(); + } + config.save(); } - if (setting.doesNeedRestart() && !Objects.equals(oldV, newV)) { - runRestartRequiredCallback(); - } - config.save(); })); if (setting.getChangeListener() != null) { comboBox.valueProperty().addListener((ChangeListener) setting.getChangeListener()); } return comboBox; } + + + private boolean setIfChanged(String key, Object n) throws IllegalAccessException, NoSuchFieldException, InvocationTargetException { + var field = BoundField.of(settings, key); + var o = field.get(); + if (!Objects.equals(n, o)) { + field.set(n); // NOSONAR + return true; + } + return false; + } private void saveValue(Exec exe) { try { @@ -351,6 +347,6 @@ public class CtbrecPreferencesStorage implements PreferencesStorage { @FunctionalInterface private interface Exec { - public void run() throws IllegalAccessException, IOException, NoSuchFieldException; + public void run() throws IllegalAccessException, IOException, NoSuchFieldException, NoSuchMethodException, InvocationTargetException; } } diff --git a/client/src/main/java/ctbrec/ui/settings/SettingsTab.java b/client/src/main/java/ctbrec/ui/settings/SettingsTab.java index 0e548f80..d18d9492 100644 --- a/client/src/main/java/ctbrec/ui/settings/SettingsTab.java +++ b/client/src/main/java/ctbrec/ui/settings/SettingsTab.java @@ -60,6 +60,8 @@ public class SettingsTab extends Tab implements TabSelectionListener { private final Settings settings; private boolean initialized = false; + private SimpleStringProperty flaresolverrApiUrl; + private SimpleIntegerProperty flaresolverrTimeoutInMillis; private SimpleStringProperty httpUserAgent; private SimpleStringProperty httpUserAgentMobile; private SimpleIntegerProperty overviewUpdateIntervalInSecs; @@ -144,6 +146,8 @@ public class SettingsTab extends Tab implements TabSelectionListener { } private void initializeProperties() { + flaresolverrApiUrl = new SimpleStringProperty(null, "flaresolverr.apiUrl", settings.flaresolverr.apiUrl); + flaresolverrTimeoutInMillis = new SimpleIntegerProperty(null, "flaresolverr.timeoutInMillis", settings.flaresolverr.timeoutInMillis); httpUserAgent = new SimpleStringProperty(null, "httpUserAgent", settings.httpUserAgent); httpUserAgentMobile = new SimpleStringProperty(null, "httpUserAgentMobile", settings.httpUserAgentMobile); overviewUpdateIntervalInSecs = new SimpleIntegerProperty(null, "overviewUpdateIntervalInSecs", settings.overviewUpdateIntervalInSecs); @@ -258,7 +262,11 @@ public class SettingsTab extends Tab implements TabSelectionListener { Group.of("Browser", Setting.of("Browser", browserOverride), Setting.of("Start parameters", browserParams), - Setting.of("Force use (ignore default browser)", forceBrowserOverride, "Default behaviour will fallback to OS default if the above browser fails"))), + Setting.of("Force use (ignore default browser)", forceBrowserOverride, "Default behaviour will fallback to OS default if the above browser fails")), + + Group.of("Flaresolverr", + Setting.of("API URL", flaresolverrApiUrl), + Setting.of("Request timeout", flaresolverrTimeoutInMillis))), Category.of("Look & Feel", Group.of("Look & Feel", Setting.of("Colors (Base / Accent)", new ColorSettingsPane(Config.getInstance())).needsRestart(), @@ -354,6 +362,8 @@ public class SettingsTab extends Tab implements TabSelectionListener { setContent(stackPane); prefs.expandTree(); + prefs.getSetting("flaresolverr.apiUrl").ifPresent(s -> bindEnabledProperty(s, recordLocal.not())); + prefs.getSetting("flaresolverr.timeoutInMillis").ifPresent(s -> bindEnabledProperty(s, recordLocal.not())); prefs.getSetting("httpServer").ifPresent(s -> bindEnabledProperty(s, recordLocal)); prefs.getSetting("httpPort").ifPresent(s -> bindEnabledProperty(s, recordLocal)); prefs.getSetting("servletContext").ifPresent(s -> bindEnabledProperty(s, recordLocal)); diff --git a/client/src/main/java/ctbrec/ui/sites/chaturbate/ChaturbateConfigUi.java b/client/src/main/java/ctbrec/ui/sites/chaturbate/ChaturbateConfigUi.java index 4ceee14a..30fba0cd 100644 --- a/client/src/main/java/ctbrec/ui/sites/chaturbate/ChaturbateConfigUi.java +++ b/client/src/main/java/ctbrec/ui/sites/chaturbate/ChaturbateConfigUi.java @@ -7,11 +7,7 @@ import ctbrec.ui.settings.SettingsTab; import ctbrec.ui.sites.AbstractConfigUI; import javafx.geometry.Insets; import javafx.scene.Parent; -import javafx.scene.control.Button; -import javafx.scene.control.CheckBox; -import javafx.scene.control.Label; -import javafx.scene.control.PasswordField; -import javafx.scene.control.TextField; +import javafx.scene.control.*; import javafx.scene.layout.GridPane; import javafx.scene.layout.Priority; @@ -103,6 +99,18 @@ public class ChaturbateConfigUi extends AbstractConfigUI { GridPane.setHgrow(requestThrottle, Priority.ALWAYS); GridPane.setColumnSpan(requestThrottle, 2); layout.add(requestThrottle, 1, row++); + + var label = new Label("Use Flaresolverr"); + label.setTooltip(new Tooltip("Use Flaresolverr for solving the Cloudflare challenge. This also overrides the User Agent used for HTTP requests (only for the site)")); + layout.add(label, 0, row); + var flaresolverrToggle = new CheckBox(); + flaresolverrToggle.setSelected(settings.chaturbateUseFlaresolverr); + flaresolverrToggle.setOnAction(e -> { + settings.chaturbateUseFlaresolverr = flaresolverrToggle.isSelected(); + save(); + }); + GridPane.setMargin(flaresolverrToggle, new Insets(0, 0, SettingsTab.CHECKBOX_MARGIN, SettingsTab.CHECKBOX_MARGIN)); + layout.add(flaresolverrToggle, 1, row++); var createAccount = new Button("Create new Account"); createAccount.setOnAction(e -> DesktopIntegration.open(Chaturbate.REGISTRATION_LINK)); diff --git a/common/src/main/java/ctbrec/io/BoundField.java b/common/src/main/java/ctbrec/io/BoundField.java new file mode 100644 index 00000000..9ca891aa --- /dev/null +++ b/common/src/main/java/ctbrec/io/BoundField.java @@ -0,0 +1,32 @@ +package ctbrec.io; + +import java.lang.reflect.*; + +public class BoundField { + public Object object = null; + public Field field = null; + + public BoundField(Object o, Field f) { + object = o; + field = f; + } + + public Object get() throws IllegalAccessException { + return field.get(object); + } + + public void set(Object value) throws IllegalAccessException { + field.set(object, value); + } + + // by-path field resolver (i.e: "a.b.c") + public static BoundField of(Object root, String path) throws NoSuchFieldException, IllegalAccessException { + var keys = path.split("\\."); + var result = new BoundField(root, root.getClass().getField(keys[0])); + for (int i = 1; i Date: Wed, 4 Dec 2024 19:45:28 +0500 Subject: [PATCH 4/6] fix null pointer error --- .../ctbrec/sites/chaturbate/ChaturbateHttpClient.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/common/src/main/java/ctbrec/sites/chaturbate/ChaturbateHttpClient.java b/common/src/main/java/ctbrec/sites/chaturbate/ChaturbateHttpClient.java index d987628d..463459ef 100644 --- a/common/src/main/java/ctbrec/sites/chaturbate/ChaturbateHttpClient.java +++ b/common/src/main/java/ctbrec/sites/chaturbate/ChaturbateHttpClient.java @@ -11,6 +11,7 @@ import java.time.*; import java.io.IOException; import java.io.InterruptedIOException; import java.util.NoSuchElementException; +import java.util.Optional; import java.util.concurrent.Semaphore; import java.util.concurrent.locks.ReentrantReadWriteLock; @@ -189,8 +190,11 @@ public class ChaturbateHttpClient extends HttpClient { // we need to prevent repeated challenge requests from multiple threads, so we check if the clearance cookie needs updating // maybe this can be done with some syncronization primitive, or maybe an expiresAt() check is enough - var cookie = cookieJar.getCookieFromCollection(cookieJar.getCookies().get(req.url().topPrivateDomain()), "cf_clearance"); - if (cookie.map(c -> Instant.ofEpochMilli(c.expiresAt()).isBefore(Instant.now())).orElse(true)) { + var needRefresh = Optional + .ofNullable(cookieJar.getCookies().get(req.url().topPrivateDomain())) + .flatMap(x -> cookieJar.getCookieFromCollection(x, "cf_clearance")) + .map(c -> Instant.ofEpochMilli(c.expiresAt()).isBefore(Instant.now())).orElse(true); + if (needRefresh) { var apiResponse = flaresolverr.getCookies(req.url().toString()).get(); if (apiResponse.getStatus().equals("ok")) { // update user agent. It should be the same for all sites, assuming we use the same api address every time From 4fcf61b9210cb42cff7f1d7e33c0e4eefa00ac9f Mon Sep 17 00:00:00 2001 From: reusedname <155286845+reusedname@users.noreply.github.com> Date: Wed, 4 Dec 2024 22:02:48 +0500 Subject: [PATCH 5/6] more cookie expiring conditions --- .../sites/chaturbate/ChaturbateHttpClient.java | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/common/src/main/java/ctbrec/sites/chaturbate/ChaturbateHttpClient.java b/common/src/main/java/ctbrec/sites/chaturbate/ChaturbateHttpClient.java index 463459ef..038c22b3 100644 --- a/common/src/main/java/ctbrec/sites/chaturbate/ChaturbateHttpClient.java +++ b/common/src/main/java/ctbrec/sites/chaturbate/ChaturbateHttpClient.java @@ -188,13 +188,19 @@ public class ChaturbateHttpClient extends HttpClient { try { cookieRefreshLock.writeLock().lock(); + // we need to prevent repeated challenge requests from multiple threads, so we check if the clearance cookie needs updating // maybe this can be done with some syncronization primitive, or maybe an expiresAt() check is enough - var needRefresh = Optional + var cookie = Optional .ofNullable(cookieJar.getCookies().get(req.url().topPrivateDomain())) - .flatMap(x -> cookieJar.getCookieFromCollection(x, "cf_clearance")) - .map(c -> Instant.ofEpochMilli(c.expiresAt()).isBefore(Instant.now())).orElse(true); - if (needRefresh) { + .flatMap(x -> cookieJar.getCookieFromCollection(x, "cf_clearance")); + + var cookieExpired = cookie.map(c -> + Instant.ofEpochMilli(c.expiresAt()).isBefore(Instant.now()) // by time + || req.headers("Set-Cookie").stream().anyMatch(headerCookie -> headerCookie.contains(c.value())) // we got 403 with current cookie present + ).orElse(true); + + if (cookieExpired) { var apiResponse = flaresolverr.getCookies(req.url().toString()).get(); if (apiResponse.getStatus().equals("ok")) { // update user agent. It should be the same for all sites, assuming we use the same api address every time From 6fd3f97e4b6b9b09bf56e152a96f5d2762ce5837 Mon Sep 17 00:00:00 2001 From: reusedname <155286845+reusedname@users.noreply.github.com> Date: Tue, 10 Dec 2024 21:12:16 +0500 Subject: [PATCH 6/6] refresh cookie on at error threshold --- common/src/main/java/ctbrec/io/HttpClient.java | 4 ++-- .../sites/chaturbate/ChaturbateHttpClient.java | 14 +++++++++----- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/common/src/main/java/ctbrec/io/HttpClient.java b/common/src/main/java/ctbrec/io/HttpClient.java index a198bb45..da70cc89 100644 --- a/common/src/main/java/ctbrec/io/HttpClient.java +++ b/common/src/main/java/ctbrec/io/HttpClient.java @@ -213,7 +213,7 @@ public abstract class HttpClient { client.dispatcher().executorService().shutdown(); } - private void persistCookies() { + protected void persistCookies() { try { List containers = new ArrayList<>(); cookieJar.getCookies().forEach((domain, cookieList) -> { @@ -231,7 +231,7 @@ public abstract class HttpClient { } } - private void loadCookies() { + protected void loadCookies() { try { File cookieFile = new File(config.getConfigDir(), "cookies-" + name + ".json"); if (!cookieFile.exists()) { diff --git a/common/src/main/java/ctbrec/sites/chaturbate/ChaturbateHttpClient.java b/common/src/main/java/ctbrec/sites/chaturbate/ChaturbateHttpClient.java index 038c22b3..87d02625 100644 --- a/common/src/main/java/ctbrec/sites/chaturbate/ChaturbateHttpClient.java +++ b/common/src/main/java/ctbrec/sites/chaturbate/ChaturbateHttpClient.java @@ -13,6 +13,7 @@ import java.io.InterruptedIOException; import java.util.NoSuchElementException; import java.util.Optional; import java.util.concurrent.Semaphore; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.locks.ReentrantReadWriteLock; import static ctbrec.io.HttpConstants.REFERER; @@ -30,6 +31,7 @@ public class ChaturbateHttpClient extends HttpClient { // a lock to prevent multiple requests from ReentrantReadWriteLock cookieRefreshLock = new ReentrantReadWriteLock(); + AtomicInteger cookieErrorCounter = new AtomicInteger(0); public ChaturbateHttpClient(Config config) { super("chaturbate", config); @@ -186,8 +188,7 @@ public class ChaturbateHttpClient extends HttpClient { log.debug("403 received from {}. Trying to refresh cookies with Flaresolverr", req.url().host()); try { - cookieRefreshLock.writeLock().lock(); - + cookieRefreshLock.writeLock().lock(); // we need to prevent repeated challenge requests from multiple threads, so we check if the clearance cookie needs updating // maybe this can be done with some syncronization primitive, or maybe an expiresAt() check is enough @@ -197,10 +198,12 @@ public class ChaturbateHttpClient extends HttpClient { var cookieExpired = cookie.map(c -> Instant.ofEpochMilli(c.expiresAt()).isBefore(Instant.now()) // by time - || req.headers("Set-Cookie").stream().anyMatch(headerCookie -> headerCookie.contains(c.value())) // we got 403 with current cookie present + || req.headers("Cookie").stream().anyMatch(headerCookie -> headerCookie.contains(c.value())) // we got 403 with current cookie present ).orElse(true); - if (cookieExpired) { + if (cookieExpired || cookieErrorCounter.incrementAndGet() >= 5) { + cookieErrorCounter.set(0); + var apiResponse = flaresolverr.getCookies(req.url().toString()).get(); if (apiResponse.getStatus().equals("ok")) { // update user agent. It should be the same for all sites, assuming we use the same api address every time @@ -209,7 +212,8 @@ public class ChaturbateHttpClient extends HttpClient { config.save(); } - cookieJar.saveFromResponse(req.url(), apiResponse.getCookies()); + cookieJar.saveFromResponse(req.url(), apiResponse.getCookies()); + persistCookies(); log.debug("Cookies successfully refreshed with Flaresolverr in {}", Duration.between(apiResponse.getStartTimestamp(), apiResponse.getEndTimestamp())); } else { log.debug("Unsuccessful attempt to refresh cookies. Response from Flaresolverr: {}", apiResponse);