j62
/
ctbrec
2
2
Fork 2
Code Issues 3 Pull Requests Packages Projects Releases 3 Wiki Activity

Flaresolverr support for chaturbate

This commit is contained in:
reusedname 2024-12-03 22:00:42 +05:00
parent f397ad471d
commit 2572268600
12 changed files with 316 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
client/src/main/java/ctbrec/ui/sites/chaturbate/ChaturbateApiUpdateService.java
View File

@ -35,7 +35,7 @@ public class ChaturbateApiUpdateService extends PaginatedScheduledService {
protected List<Model> call() throws Exception {
var request = new Request.Builder()
.url(url)
.header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
.header(USER_AGENT, chaturbate.getHttpClient().getEffectiveUserAgent())
.header(ACCEPT, MIMETYPE_APPLICATION_JSON)
.build();
try (var response = chaturbate.getHttpClient().execute(request)) {

2
client/src/main/java/ctbrec/ui/sites/chaturbate/ChaturbateElectronLoginDialog.java
View File

@ -35,7 +35,7 @@ public class ChaturbateElectronLoginDialog {
config.put("url", site.getBaseUrl() + "/auth/login/");
config.put("w", 640);
config.put("h", 480);
config.put("userAgent", Config.getInstance().getSettings().httpUserAgent);
config.put("userAgent", site.getHttpClient().getEffectiveUserAgent());
var msg = new JSONObject();
msg.put("config", config);
browser.run(msg, msgHandler);

2
client/src/main/java/ctbrec/ui/sites/chaturbate/ChaturbateUpdateService.java
View File

@ -54,7 +54,7 @@ public class ChaturbateUpdateService extends PaginatedScheduledService {
.header(ACCEPT, MIMETYPE_APPLICATION_JSON)
.header(ACCEPT_LANGUAGE, Locale.ENGLISH.getLanguage())
.header(X_REQUESTED_WITH, XML_HTTP_REQUEST)
.header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
.header(USER_AGENT, chaturbate.getHttpClient().getEffectiveUserAgent())
.build();
try (var response = chaturbate.getHttpClient().execute(request)) {
if (response.isSuccessful()) {

7
common/src/main/java/ctbrec/Settings.java
View File

@ -42,7 +42,13 @@ public class Settings {
TIME_OR_SIZE
}
public class FlaresolverrSettings {
public String apiUrl = "http://localhost:8191/v1";
public int timeoutInMillis = 60000;
public String userAgent = "";
};
public FlaresolverrSettings flaresolverr = new FlaresolverrSettings();
public String amateurTvUsername = "";
public String amateurTvPassword = "";
public String bongacamsBaseUrl = "https://bongacams.com";
@ -55,6 +61,7 @@ public class Settings {
public String chaturbatePassword = "";
public String chaturbateUsername = "";
public String chaturbateBaseUrl = "https://chaturbate.com";
public boolean chaturbateUseFlaresolverr = false;
public int chaturbateMsBetweenRequests = 1000;
public String cherryTvPassword = "";
public String cherryTvUsername = "";

46
common/src/main/java/ctbrec/io/CompletableRequestFuture.java Normal file
View File

@ -0,0 +1,46 @@
package ctbrec.io;
import java.io.IOException;
import java.util.concurrent.CompletableFuture;
import lombok.Getter;
import okhttp3.Call;
import okhttp3.Callback;
import okhttp3.Response;
import okhttp3.ResponseBody;
public class CompletableRequestFuture<T> extends CompletableFuture<T> implements Callback {
@Getter
protected Call call;
public CompletableRequestFuture(Call call) {
this.call = call;
}
@Override
public void onResponse(Call c, Response response) throws IOException {
try (var body = response.body()) {
if (response.isSuccessful()) {
processBody(body);
} else {
completeExceptionally(new HttpException(response.code(), response.message()));
}
} catch (Exception e) {
completeExceptionally(e);
}
}
@Override
public void onFailure(Call c, IOException error) {
completeExceptionally(error);
}
@Override
public boolean cancel(boolean mayInterruptIfRunning) {
call.cancel();
return super.cancel(mayInterruptIfRunning);
}
protected void processBody(ResponseBody body) throws Exception {}
}

86
common/src/main/java/ctbrec/io/FlaresolverrClient.java
View File

@ -1,67 +1,107 @@
package ctbrec.io;
import org.json.JSONObject;
import ctbrec.io.FlaresolverrResponse;
import ctbrec.io.FlaresolverrSolutionResponse;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.json.JsonMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
import okhttp3.*;
import okhttp3.OkHttpClient.Builder;
import ctbrec.GlobalThreadPool;
import lombok.Getter;
import lombok.Setter;
import okhttp3.Call;
import okhttp3.Callback;
import okhttp3.MediaType;
import okhttp3.OkHttpClient;
import okhttp3.Request;
import okhttp3.RequestBody;
import okhttp3.Response;
import okhttp3.ResponseBody;
import okhttp3.Cookie;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.*;
public class FlaresolverrClient {
public String api_url = "http://localhost:8191/v1";
protected OkHttpClient client = new OkHttpClient();
public String api_url;
protected int timeout_ms;
protected OkHttpClient client;
protected JsonMapper mapper = new JsonMapper();
@Getter
protected String sessionName = "";
public FlaresolverrClient() {
this("http://localhost:8191/v1", 60000);
}
public FlaresolverrClient(String apiUrl) {
public FlaresolverrClient(String apiUrl, int timeout_ms) {
api_url = apiUrl;
this.timeout_ms = timeout_ms;
client = new OkHttpClient.Builder()
.callTimeout(timeout_ms + 10000, TimeUnit.MILLISECONDS)
.readTimeout(timeout_ms + 1000, TimeUnit.MILLISECONDS)
.build();
}
public JsonNode createSession(String name) throws Exception {
public CompletableFuture<FlaresolverrResponse> createSession(String name) throws IOException {
if (!sessionName.equals(""))
throw new IOException("Cannot start new session because another one is already started. Finish it before creating a new one");
sessionName = name;
var body = mapper.createObjectNode()
.put("cmd", "sessions.create")
.put("session", name);
return makeApiCall(body);
return makeApiCall(body).thenApply(r -> new FlaresolverrResponse(r));
}
public JsonNode destroySession(String name) throws Exception {
public CompletableFuture<FlaresolverrResponse> destroySession(String name) throws IOException {
if (sessionName.equals(""))
throw new IOException("Cannot destroy session because no session is active");
var body = mapper.createObjectNode()
.put("cmd", "sessions.destroy")
.put("session", name);
return makeApiCall(body);
sessionName = "";
return makeApiCall(body).thenApply(r -> new FlaresolverrResponse(r));
}
public JsonNode getCookies(String url) throws Exception {
return getCookies(url, 60000);
}
public JsonNode getCookies(String url, int timeout_ms) throws Exception {
public CompletableFuture<FlaresolverrSolutionResponse> getCookies(String url) throws IOException {
var body = mapper.createObjectNode()
.put("cmd", "request.get")
.put("url", url)
.put("maxTimeout", timeout_ms)
.put("returnOnlyCookies", true);
return makeApiCall(body);
if (sessionName != "") {
body.put("session", sessionName);
}
return makeApiCall(body).thenApply(r -> new FlaresolverrSolutionResponse(r));
}
protected JsonNode makeApiCall(ObjectNode body) throws Exception {
protected CompletableRequestFuture<JsonNode> makeApiCall(ObjectNode body) throws IOException {
var requestBody = RequestBody.create(mapper.writeValueAsString(body), MediaType.get("application/json"));
var request = new Request.Builder()
.url(api_url)
.post(requestBody)
.build();
var response = client.newCall(request).execute();
return mapper.readTree(response.body().string());
var call = client.newCall(request);
var future = new CompletableRequestFuture<JsonNode>(call) {
@Override
public void processBody(ResponseBody body) throws IOException {
complete(mapper.readTree(body.charStream()));
}
};
// FIXME?: unfortunate cyclic reference here to allow cancelling through the future, is this bad?
call.enqueue(future);
return future;
}
}

23
common/src/main/java/ctbrec/io/FlaresolverrResponse.java Normal file
View File

@ -0,0 +1,23 @@
package ctbrec.io;
import com.fasterxml.jackson.databind.JsonNode;
import lombok.Getter;
import lombok.ToString;
public class FlaresolverrResponse {
@Getter
protected String status;
@Getter
protected String message;
public FlaresolverrResponse(JsonNode response) {
status = response.get("status").asText();
message = response.get("message").asText();
}
@Override
public String toString() {
return String.format("{status: %s, message: %s}", status, message);
}
}

73
common/src/main/java/ctbrec/io/FlaresolverrSolutionResponse.java Normal file
View File

@ -0,0 +1,73 @@
package ctbrec.io;
import com.fasterxml.jackson.databind.JsonNode;
import ctbrec.StringUtil;
import lombok.Getter;
import okhttp3.Cookie;
import java.time.Instant;
import java.util.*;
public class FlaresolverrSolutionResponse extends FlaresolverrResponse {
@Getter
protected String userAgent;
@Getter
protected Instant startTimestamp;
@Getter
protected Instant endTimestamp;
@Getter
protected String version;
@Getter
protected List<Cookie> cookies;
FlaresolverrSolutionResponse(JsonNode response) {
super(response);
startTimestamp = Instant.ofEpochMilli(response.get("startTimestamp").asLong());
endTimestamp = Instant.ofEpochMilli(response.get("endTimestamp").asLong());
version = response.get("version").asText();
var solution = response.get("solution");
userAgent = solution.get("userAgent").asText();
cookies = new ArrayList<Cookie>();
for (var c : solution.get("cookies")) {
// "domain": c["domain"].lstrip('.'),
// "expiresAt": c["expiry"],
// "hostOnly": c["sameSite"].lower() == "strict",
// "httpOnly": c["httpOnly"],
// "name": c["name"],
// "path": c["path"],
// "persistent": False,
// "secure": c["secure"],
// "value": c["value"]
var cb = new Cookie.Builder()
.expiresAt(Optional.ofNullable(c.get("expires")).orElse(c.get("expiry")).asLong() * 1000) // seconds -> millis
.name(c.get("name").asText())
.path(c.get("path").asText())
.value(c.get("value").asText())
;
var domain = c.get("domain").asText().replaceFirst("\\.", "");
// FIXME: is this correct?
if (c.path("sameSite").asText("").equalsIgnoreCase("strict")) {
cb.hostOnlyDomain(domain);
} else {
cb.domain(domain);
}
if (c.path("httpOnly").asBoolean(false))
cb.httpOnly();
if (c.path("secure").asBoolean(false))
cb.secure();
cookies.add(cb.build());
}
}
}

5
common/src/main/java/ctbrec/io/HttpClient.java
View File

@ -357,4 +357,9 @@ public abstract class HttpClient {
public static final String JAVA_NET_SOCKS_USERNAME = "java.net.socks.username";
public static final String JAVA_NET_SOCKS_PASSWORD = "java.net.socks.password";
}
// overridable default user agent (used for Flaresolverr)
public String getEffectiveUserAgent() {
return config.getSettings().httpUserAgent;
}
}

4
common/src/main/java/ctbrec/sites/chaturbate/Chaturbate.java
View File

@ -69,7 +69,7 @@ public class Chaturbate extends AbstractSite {
String url = "https://chaturbate.com/p/" + username + "/";
Request req = new Request.Builder()
.url(url)
.header(USER_AGENT, getConfig().getSettings().httpUserAgent)
.header(USER_AGENT, getHttpClient().getEffectiveUserAgent())
.build();
try (Response resp = getHttpClient().execute(req)) {
if (resp.isSuccessful()) {
@ -131,7 +131,7 @@ public class Chaturbate extends AbstractSite {
// search online models
Request req = new Request.Builder()
.url(url)
.header(USER_AGENT, getConfig().getSettings().httpUserAgent)
.header(USER_AGENT, getHttpClient().getEffectiveUserAgent())
.header(ACCEPT, "*/*")
.header(X_REQUESTED_WITH, XML_HTTP_REQUEST)
.header(REFERER, getBaseUrl())

91
common/src/main/java/ctbrec/sites/chaturbate/ChaturbateHttpClient.java
View File

@ -1,15 +1,18 @@
package ctbrec.sites.chaturbate;
import ctbrec.Config;
import ctbrec.io.FlaresolverrClient;
import ctbrec.io.HtmlParser;
import ctbrec.io.HttpClient;
import lombok.extern.slf4j.Slf4j;
import okhttp3.*;
import java.time.*;
import java.io.IOException;
import java.io.InterruptedIOException;
import java.util.NoSuchElementException;
import java.util.concurrent.Semaphore;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import static ctbrec.io.HttpConstants.REFERER;
import static ctbrec.io.HttpConstants.USER_AGENT;
@ -19,12 +22,39 @@ public class ChaturbateHttpClient extends HttpClient {
private static final String PATH = "/auth/login/"; // NOSONAR
protected String token;
protected final FlaresolverrClient flaresolverr;
private static final Semaphore requestThrottle = new Semaphore(2, true);
private static long lastRequest = 0;
// a lock to prevent multiple requests from
ReentrantReadWriteLock cookieRefreshLock = new ReentrantReadWriteLock();
public ChaturbateHttpClient(Config config) {
super("chaturbate", config);
if (config.getSettings().chaturbateUseFlaresolverr) {
flaresolverr = new FlaresolverrClient(config.getSettings().flaresolverr.apiUrl, config.getSettings().flaresolverr.timeoutInMillis);
// try {
// flaresolverr.createSession("ctbrec").get();
// } catch (InterruptedException e) {
// Thread.currentThread().interrupt();
// } catch (Exception e) {
// log.error("Error starting Flaresolverr session", e);
// }
} else {
flaresolverr = null;
}
}
@Override
public String getEffectiveUserAgent() {
if (flaresolverr != null) {
return config.getSettings().flaresolverr.userAgent;
} else {
return config.getSettings().httpUserAgent;
}
}
private void extractCsrfToken(Request request) {
@ -55,7 +85,7 @@ public class ChaturbateHttpClient extends HttpClient {
}
Request login = new Request.Builder()
.url(Chaturbate.baseUrl + PATH)
.header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
.header(USER_AGENT, getEffectiveUserAgent())
.build();
try (var initResponse = client.newCall(login).execute()) {
String content = initResponse.body().string();
@ -71,7 +101,7 @@ public class ChaturbateHttpClient extends HttpClient {
login = new Request.Builder()
.url(Chaturbate.baseUrl + PATH)
.header(REFERER, Chaturbate.baseUrl + PATH)
.header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
.header(USER_AGENT, getEffectiveUserAgent())
.post(body)
.build();
@ -96,7 +126,7 @@ public class ChaturbateHttpClient extends HttpClient {
String url = "https://chaturbate.com/api/ts/chatmessages/pm_users/?offset=0";
Request req = new Request.Builder()
.url(url)
.header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
.header(USER_AGENT, getEffectiveUserAgent())
.build();
try (Response response = execute(req)) {
boolean result = false;
@ -124,7 +154,21 @@ public class ChaturbateHttpClient extends HttpClient {
if (throttle) {
acquireSlot();
}
Response resp = super.execute(req);
Response resp;
try {
cookieRefreshLock.readLock().lock();
resp = super.execute(req);
} finally {
cookieRefreshLock.readLock().unlock();
}
// try to solve the cloudflare challenge if we got one (clearance cookie expired, update it)
if (resp.code() == 403 && flaresolverr != null) {
resp = refreshCookiesAndRetry(req, resp);
}
extractCsrfToken(req);
return resp;
} catch (InterruptedException e) {
@ -136,6 +180,45 @@ public class ChaturbateHttpClient extends HttpClient {
}
}
}
private Response refreshCookiesAndRetry(Request req, Response origResp) throws IOException {
log.debug("403 received from {}. Trying to refresh cookies with Flaresolverr", req.url().host());
try {
cookieRefreshLock.writeLock().lock();
// we need to prevent repeated challenge requests from multiple threads, so we check if the clearance cookie needs updating
// maybe this can be done with some syncronization primitive, or maybe an expiresAt() check is enough
var cookie = cookieJar.getCookieFromCollection(cookieJar.getCookies().get(req.url().topPrivateDomain()), "cf_clearance");
if (cookie.map(c -> Instant.ofEpochMilli(c.expiresAt()).isBefore(Instant.now())).orElse(true)) {
var apiResponse = flaresolverr.getCookies(req.url().toString()).get();
if (apiResponse.getStatus().equals("ok")) {
// update user agent. It should be the same for all sites, assuming we use the same api address every time
if (!config.getSettings().flaresolverr.userAgent.equals(apiResponse.getUserAgent())) {
config.getSettings().flaresolverr.userAgent = apiResponse.getUserAgent();
config.save();
}
cookieJar.saveFromResponse(req.url(), apiResponse.getCookies());
log.debug("Cookies successfully refreshed with Flaresolverr in {}", Duration.between(apiResponse.getStartTimestamp(), apiResponse.getEndTimestamp()));
} else {
log.debug("Unsuccessful attempt to refresh cookies. Response from Flaresolverr: {}", apiResponse);
return origResp;
}
} else {
log.debug("Looks like the cookies were refreshed already, skipping refreshing");
}
} catch (Exception e) {
log.warn("Error refreshing cookies with Flaresolverr", e);
return origResp;
} finally {
cookieRefreshLock.writeLock().unlock();
}
origResp.close();
return super.execute(req);
}
private static void acquireSlot() throws InterruptedException {
long pauseBetweenRequests = Config.getInstance().getSettings().chaturbateMsBetweenRequests;

14
common/src/main/java/ctbrec/sites/chaturbate/ChaturbateModel.java
View File

@ -97,7 +97,7 @@ public class ChaturbateModel extends AbstractModel {
int imageSize = 0;
Request req = new Request.Builder()
.url(url)
.header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
.header(USER_AGENT, site.getHttpClient().getEffectiveUserAgent())
.head()
.build();
try (Response response = getSite().getHttpClient().execute(req)) {
@ -190,7 +190,7 @@ public class ChaturbateModel extends AbstractModel {
.post(body)
.header(REFERER, "https://chaturbate.com/" + getName() + "/")
.header(X_REQUESTED_WITH, XML_HTTP_REQUEST)
.header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
.header(USER_AGENT, site.getHttpClient().getEffectiveUserAgent())
.build();
try (Response response = getSite().getHttpClient().execute(req)) {
if (!response.isSuccessful()) {
@ -239,7 +239,7 @@ public class ChaturbateModel extends AbstractModel {
// do an initial request to get cookies
Request req = new Request.Builder()
.url(getUrl())
.header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
.header(USER_AGENT, site.getHttpClient().getEffectiveUserAgent())
.build();
Response resp = site.getHttpClient().execute(req);
resp.close();
@ -258,7 +258,7 @@ public class ChaturbateModel extends AbstractModel {
.header(ACCEPT, "*/*")
.header(ACCEPT_LANGUAGE, "en-US,en;q=0.5")
.header(REFERER, getUrl())
.header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
.header(USER_AGENT, site.getHttpClient().getEffectiveUserAgent())
.header("X-CSRFToken", ((ChaturbateHttpClient) site.getHttpClient()).getToken())
.header(X_REQUESTED_WITH, XML_HTTP_REQUEST)
.build();
@ -302,7 +302,7 @@ public class ChaturbateModel extends AbstractModel {
Request req = new Request.Builder()
.url(getSite().getBaseUrl() + "/get_edge_hls_url_ajax/")
.post(body)
.header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
.header(USER_AGENT, site.getHttpClient().getEffectiveUserAgent())
.header(X_REQUESTED_WITH, XML_HTTP_REQUEST)
.build();
try (Response response = getSite().getHttpClient().execute(req)) {
@ -364,7 +364,7 @@ public class ChaturbateModel extends AbstractModel {
log.trace("Loading master playlist {}", streamInfo.url);
Request req = new Request.Builder()
.url(streamInfo.url)
.header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
.header(USER_AGENT, site.getHttpClient().getEffectiveUserAgent())
.build();
try (Response response = getSite().getHttpClient().execute(req)) {
if (response.isSuccessful()) {
@ -385,7 +385,7 @@ public class ChaturbateModel extends AbstractModel {
public boolean exists() throws IOException {
Request req = new Request.Builder() // @formatter:off
.url(getUrl())
.header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
.header(USER_AGENT, site.getHttpClient().getEffectiveUserAgent())
.header(ACCEPT_LANGUAGE, Locale.ENGLISH.getLanguage())
.build(); // @formatter:on
try (Response response = getSite().getHttpClient().execute(req)) {