j62
/
ctbrec
2
2
Fork 2
Code Issues 3 Pull Requests Packages Projects Releases 3 Wiki Activity

Merge branch 'flaresolverr' into reusedname-dev

This commit is contained in:
reusedname 2025-02-15 13:50:08 +05:00
parent 9c2566e79c 6fd3f97e4b
commit 8ac6cf8c07
17 changed files with 491 additions and 84 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

110
client/src/main/java/ctbrec/ui/settings/CtbrecPreferencesStorage.java
View File

@ -1,6 +1,7 @@
package ctbrec.ui.settings;
import java.io.IOException;
import java.lang.reflect.InvocationTargetException;
import java.time.LocalTime;
import java.util.List;
import java.util.Objects;
@ -25,6 +26,7 @@ import ctbrec.ui.settings.api.Setting;
import ctbrec.ui.settings.api.SimpleDirectoryProperty;
import ctbrec.ui.settings.api.SimpleFileProperty;
import ctbrec.ui.settings.api.SimpleRangeProperty;
import ctbrec.io.BoundField;
import javafx.beans.property.BooleanProperty;
import javafx.beans.property.IntegerProperty;
import javafx.beans.property.ListProperty;
@ -115,13 +117,13 @@ public class CtbrecPreferencesStorage implements PreferencesStorage {
optionB.setSelected(!optionA.isSelected());
optionB.setToggleGroup(toggleGroup);
optionA.selectedProperty().bindBidirectional(prop);
prop.addListener((obs, oldV, newV) -> saveValue(() -> {
var field = Settings.class.getField(setting.getKey());
field.set(settings, newV); // NOSONAR
if (setting.doesNeedRestart() && !Objects.equals(oldV, newV)) {
runRestartRequiredCallback();
prop.addListener((obs, oldV, newV) -> saveValue(() -> {
if (setIfChanged(setting.getKey(), newV)) {
if (setting.doesNeedRestart()) {
runRestartRequiredCallback();
}
config.save();
}
config.save();
}));
var row = new HBox();
row.getChildren().addAll(optionA, optionB);
@ -155,15 +157,15 @@ public class CtbrecPreferencesStorage implements PreferencesStorage {
resolutionRange.setHigh(highValue >= 0 ? highValue : values.get(values.size() - 1));
resolutionRange.getLow().addListener((obs, o, n) -> saveValue(() -> {
int newV = labels.get(n.intValue());
var field = Settings.class.getField(rangeProperty.getLowKey());
field.set(settings, newV); // NOSONAR
config.save();
if (setIfChanged(rangeProperty.getLowKey(), newV)) {
config.save();
}
}));
resolutionRange.getHigh().addListener((obs, o, n) -> saveValue(() -> {
int newV = labels.get(n.intValue());
var field = Settings.class.getField(rangeProperty.getHighKey());
field.set(settings, newV); // NOSONAR
config.save();
if (setIfChanged(rangeProperty.getHighKey(), newV)) {
config.save();
}
}));
return resolutionRange;
}
@ -181,11 +183,7 @@ public class CtbrecPreferencesStorage implements PreferencesStorage {
private Node createFileSelector(Setting setting) {
var programSelector = new ProgramSelectionBox("");
programSelector.fileProperty().addListener((obs, o, n) -> saveValue(() -> {
String path = n;
var field = Settings.class.getField(setting.getKey());
String oldValue = (String) field.get(settings);
if (!Objects.equals(path, oldValue)) {
field.set(settings, path); // NOSONAR
if (setIfChanged(setting.getKey(), n)) {
if (setting.doesNeedRestart()) {
runRestartRequiredCallback();
}
@ -201,11 +199,7 @@ public class CtbrecPreferencesStorage implements PreferencesStorage {
var directorySelector = new DirectorySelectionBox("");
directorySelector.prefWidth(400);
directorySelector.fileProperty().addListener((obs, o, n) -> saveValue(() -> {
String path = n;
var field = Settings.class.getField(setting.getKey());
String oldValue = (String) field.get(settings);
if (!Objects.equals(path, oldValue)) {
field.set(settings, path); // NOSONAR
if (setIfChanged(setting.getKey(), n)) {
if (setting.doesNeedRestart()) {
runRestartRequiredCallback();
}
@ -221,10 +215,7 @@ public class CtbrecPreferencesStorage implements PreferencesStorage {
LocalTime time = (LocalTime) setting.getProperty().getValue();
var timePicker = new TimePicker(time);
timePicker.valueProperty().addListener((obs, o, n) -> saveValue(() -> {
var field = Settings.class.getField(setting.getKey());
LocalTime oldValue = (LocalTime) field.get(settings);
if (!Objects.equals(n, oldValue)) {
field.set(settings, n); // NOSONAR
if (setIfChanged(setting.getKey(), n)) {
if (setting.doesNeedRestart()) {
runRestartRequiredCallback();
}
@ -237,12 +228,12 @@ public class CtbrecPreferencesStorage implements PreferencesStorage {
private Node createStringProperty(Setting setting) {
var ctrl = new TextField();
ctrl.textProperty().addListener((obs, oldV, newV) -> saveValue(() -> {
var field = Settings.class.getField(setting.getKey());
field.set(settings, newV); // NOSONAR
if (setting.doesNeedRestart() && !Objects.equals(oldV, newV)) {
runRestartRequiredCallback();
if (setIfChanged(setting.getKey(), newV)) {
if (setting.doesNeedRestart()) {
runRestartRequiredCallback();
}
config.save();
}
config.save();
}));
StringProperty prop = (StringProperty) setting.getProperty();
ctrl.textProperty().bindBidirectional(prop);
@ -256,10 +247,8 @@ public class CtbrecPreferencesStorage implements PreferencesStorage {
if (!newV.matches("\\d*")) {
ctrl.setText(newV.replaceAll(PATTERN_NOT_A_DIGIT, ""));
}
if (!ctrl.getText().isEmpty()) {
var field = Settings.class.getField(setting.getKey());
field.set(settings, Integer.parseInt(ctrl.getText())); // NOSONAR
if (setting.doesNeedRestart() && !Objects.equals(oldV, newV) && prefs != null) {
if (!ctrl.getText().isEmpty() && setIfChanged(setting.getKey(), Integer.parseInt(ctrl.getText()))) {
if (setting.doesNeedRestart() && prefs != null) {
runRestartRequiredCallback();
}
config.save();
@ -282,12 +271,12 @@ public class CtbrecPreferencesStorage implements PreferencesStorage {
if (setting.getConverter() != null) {
value = (long) setting.getConverter().convertFrom(value);
}
var field = Settings.class.getField(setting.getKey());
field.set(settings, value); // NOSONAR
if (setting.doesNeedRestart() && !Objects.equals(oldV, newV)) {
runRestartRequiredCallback();
if (setIfChanged(setting.getKey(), value)) {
if (setting.doesNeedRestart() && !Objects.equals(oldV, newV)) {
runRestartRequiredCallback();
}
config.save();
}
config.save();
}
}));
Property<Number> prop = setting.getProperty();
@ -298,12 +287,12 @@ public class CtbrecPreferencesStorage implements PreferencesStorage {
private Node createBooleanProperty(Setting setting) {
var ctrl = new CheckBox();
ctrl.selectedProperty().addListener((obs, oldV, newV) -> saveValue(() -> {
var field = Settings.class.getField(setting.getKey());
field.set(settings, newV); // NOSONAR
if (setting.doesNeedRestart() && !Objects.equals(oldV, newV)) {
runRestartRequiredCallback();
if (setIfChanged(setting.getKey(), newV)) {
if (setting.doesNeedRestart()) {
runRestartRequiredCallback();
}
config.save();
}
config.save();
}));
BooleanProperty prop = (BooleanProperty) setting.getProperty();
ctrl.selectedProperty().bindBidirectional(prop);
@ -311,11 +300,10 @@ public class CtbrecPreferencesStorage implements PreferencesStorage {
}
@SuppressWarnings({ "rawtypes", "unchecked" })
private Node createComboBox(Setting setting) throws NoSuchFieldException, IllegalAccessException {
private Node createComboBox(Setting setting) throws IllegalAccessException, NoSuchFieldException {
ListProperty<?> listProp = (ListProperty<?>) setting.getProperty();
ComboBox<Object> comboBox = new ComboBox(listProp);
var field = Settings.class.getField(setting.getKey());
Object value = field.get(settings);
Object value = BoundField.of(settings, setting.getKey()).get();
if (StringUtil.isNotBlank(value.toString())) {
if (setting.getConverter() != null) {
comboBox.getSelectionModel().select(setting.getConverter().convertTo(value));
@ -325,21 +313,29 @@ public class CtbrecPreferencesStorage implements PreferencesStorage {
}
comboBox.valueProperty().addListener((obs, oldV, newV) -> saveValue(() -> {
LOG.debug("Saving setting {}", setting.getKey());
if (setting.getConverter() != null) {
field.set(settings, setting.getConverter().convertFrom(newV)); // NOSONAR
} else {
field.set(settings, newV); // NOSONAR
if (setIfChanged(setting.getKey(), setting.getConverter() != null ? setting.getConverter().convertFrom(newV) : newV)) {
if (setting.doesNeedRestart()) {
runRestartRequiredCallback();
}
config.save();
}
if (setting.doesNeedRestart() && !Objects.equals(oldV, newV)) {
runRestartRequiredCallback();
}
config.save();
}));
if (setting.getChangeListener() != null) {
comboBox.valueProperty().addListener((ChangeListener<? super Object>) setting.getChangeListener());
}
return comboBox;
}
private boolean setIfChanged(String key, Object n) throws IllegalAccessException, NoSuchFieldException, InvocationTargetException {
var field = BoundField.of(settings, key);
var o = field.get();
if (!Objects.equals(n, o)) {
field.set(n); // NOSONAR
return true;
}
return false;
}
private void saveValue(Exec exe) {
try {
@ -351,6 +347,6 @@ public class CtbrecPreferencesStorage implements PreferencesStorage {
@FunctionalInterface
private interface Exec {
public void run() throws IllegalAccessException, IOException, NoSuchFieldException;
public void run() throws IllegalAccessException, IOException, NoSuchFieldException, NoSuchMethodException, InvocationTargetException;
}
}

12
client/src/main/java/ctbrec/ui/settings/SettingsTab.java
View File

@ -60,6 +60,8 @@ public class SettingsTab extends Tab implements TabSelectionListener {
private final Settings settings;
private boolean initialized = false;
private SimpleStringProperty flaresolverrApiUrl;
private SimpleIntegerProperty flaresolverrTimeoutInMillis;
private SimpleStringProperty httpUserAgent;
private SimpleStringProperty httpUserAgentMobile;
private SimpleIntegerProperty overviewUpdateIntervalInSecs;
@ -144,6 +146,8 @@ public class SettingsTab extends Tab implements TabSelectionListener {
}
private void initializeProperties() {
flaresolverrApiUrl = new SimpleStringProperty(null, "flaresolverr.apiUrl", settings.flaresolverr.apiUrl);
flaresolverrTimeoutInMillis = new SimpleIntegerProperty(null, "flaresolverr.timeoutInMillis", settings.flaresolverr.timeoutInMillis);
httpUserAgent = new SimpleStringProperty(null, "httpUserAgent", settings.httpUserAgent);
httpUserAgentMobile = new SimpleStringProperty(null, "httpUserAgentMobile", settings.httpUserAgentMobile);
overviewUpdateIntervalInSecs = new SimpleIntegerProperty(null, "overviewUpdateIntervalInSecs", settings.overviewUpdateIntervalInSecs);
@ -258,7 +262,11 @@ public class SettingsTab extends Tab implements TabSelectionListener {
Group.of("Browser",
Setting.of("Browser", browserOverride),
Setting.of("Start parameters", browserParams),
Setting.of("Force use (ignore default browser)", forceBrowserOverride, "Default behaviour will fallback to OS default if the above browser fails"))),
Setting.of("Force use (ignore default browser)", forceBrowserOverride, "Default behaviour will fallback to OS default if the above browser fails")),
Group.of("Flaresolverr",
Setting.of("API URL", flaresolverrApiUrl),
Setting.of("Request timeout", flaresolverrTimeoutInMillis))),
Category.of("Look & Feel",
Group.of("Look & Feel",
Setting.of("Colors (Base / Accent)", new ColorSettingsPane(Config.getInstance())).needsRestart(),
@ -354,6 +362,8 @@ public class SettingsTab extends Tab implements TabSelectionListener {
setContent(stackPane);
prefs.expandTree();
prefs.getSetting("flaresolverr.apiUrl").ifPresent(s -> bindEnabledProperty(s, recordLocal.not()));
prefs.getSetting("flaresolverr.timeoutInMillis").ifPresent(s -> bindEnabledProperty(s, recordLocal.not()));
prefs.getSetting("httpServer").ifPresent(s -> bindEnabledProperty(s, recordLocal));
prefs.getSetting("httpPort").ifPresent(s -> bindEnabledProperty(s, recordLocal));
prefs.getSetting("servletContext").ifPresent(s -> bindEnabledProperty(s, recordLocal));

2
client/src/main/java/ctbrec/ui/sites/chaturbate/ChaturbateApiUpdateService.java
View File

@ -35,7 +35,7 @@ public class ChaturbateApiUpdateService extends PaginatedScheduledService {
protected List<Model> call() throws Exception {
var request = new Request.Builder()
.url(url)
.header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
.header(USER_AGENT, chaturbate.getHttpClient().getEffectiveUserAgent())
.header(ACCEPT, MIMETYPE_APPLICATION_JSON)
.build();
try (var response = chaturbate.getHttpClient().execute(request)) {

18
client/src/main/java/ctbrec/ui/sites/chaturbate/ChaturbateConfigUi.java
View File

@ -7,11 +7,7 @@ import ctbrec.ui.settings.SettingsTab;
import ctbrec.ui.sites.AbstractConfigUI;
import javafx.geometry.Insets;
import javafx.scene.Parent;
import javafx.scene.control.Button;
import javafx.scene.control.CheckBox;
import javafx.scene.control.Label;
import javafx.scene.control.PasswordField;
import javafx.scene.control.TextField;
import javafx.scene.control.*;
import javafx.scene.layout.GridPane;
import javafx.scene.layout.Priority;
@ -103,6 +99,18 @@ public class ChaturbateConfigUi extends AbstractConfigUI {
GridPane.setHgrow(requestThrottle, Priority.ALWAYS);
GridPane.setColumnSpan(requestThrottle, 2);
layout.add(requestThrottle, 1, row++);
var label = new Label("Use Flaresolverr");
label.setTooltip(new Tooltip("Use Flaresolverr for solving the Cloudflare challenge. This also overrides the User Agent used for HTTP requests (only for the site)"));
layout.add(label, 0, row);
var flaresolverrToggle = new CheckBox();
flaresolverrToggle.setSelected(settings.chaturbateUseFlaresolverr);
flaresolverrToggle.setOnAction(e -> {
settings.chaturbateUseFlaresolverr = flaresolverrToggle.isSelected();
save();
});
GridPane.setMargin(flaresolverrToggle, new Insets(0, 0, SettingsTab.CHECKBOX_MARGIN, SettingsTab.CHECKBOX_MARGIN));
layout.add(flaresolverrToggle, 1, row++);
var createAccount = new Button("Create new Account");
createAccount.setOnAction(e -> DesktopIntegration.open(Chaturbate.REGISTRATION_LINK));

2
client/src/main/java/ctbrec/ui/sites/chaturbate/ChaturbateElectronLoginDialog.java
View File

@ -35,7 +35,7 @@ public class ChaturbateElectronLoginDialog {
config.put("url", site.getBaseUrl() + "/auth/login/");
config.put("w", 640);
config.put("h", 480);
config.put("userAgent", Config.getInstance().getSettings().httpUserAgent);
config.put("userAgent", site.getHttpClient().getEffectiveUserAgent());
var msg = new JSONObject();
msg.put("config", config);
browser.run(msg, msgHandler);

2
client/src/main/java/ctbrec/ui/sites/chaturbate/ChaturbateUpdateService.java
View File

@ -54,7 +54,7 @@ public class ChaturbateUpdateService extends PaginatedScheduledService {
.header(ACCEPT, MIMETYPE_APPLICATION_JSON)
.header(ACCEPT_LANGUAGE, Locale.ENGLISH.getLanguage())
.header(X_REQUESTED_WITH, XML_HTTP_REQUEST)
.header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
.header(USER_AGENT, chaturbate.getHttpClient().getEffectiveUserAgent())
.build();
try (var response = chaturbate.getHttpClient().execute(request)) {
if (response.isSuccessful()) {

7
common/src/main/java/ctbrec/Settings.java
View File

@ -42,7 +42,13 @@ public class Settings {
TIME_OR_SIZE
}
public class FlaresolverrSettings {
public String apiUrl = "http://localhost:8191/v1";
public int timeoutInMillis = 60000;
public String userAgent = "";
};
public FlaresolverrSettings flaresolverr = new FlaresolverrSettings();
public String amateurTvUsername = "";
public String amateurTvPassword = "";
public String bongacamsBaseUrl = "https://bongacams.com";
@ -55,6 +61,7 @@ public class Settings {
public String chaturbatePassword = "";
public String chaturbateUsername = "";
public String chaturbateBaseUrl = "https://chaturbate.com";
public boolean chaturbateUseFlaresolverr = false;
public int chaturbateMsBetweenRequests = 1000;
public String cherryTvPassword = "";
public String cherryTvUsername = "";

32
common/src/main/java/ctbrec/io/BoundField.java Normal file
View File

@ -0,0 +1,32 @@
package ctbrec.io;
import java.lang.reflect.*;
public class BoundField {
public Object object = null;
public Field field = null;
public BoundField(Object o, Field f) {
object = o;
field = f;
}
public Object get() throws IllegalAccessException {
return field.get(object);
}
public void set(Object value) throws IllegalAccessException {
field.set(object, value);
}
// by-path field resolver (i.e: "a.b.c")
public static BoundField of(Object root, String path) throws NoSuchFieldException, IllegalAccessException {
var keys = path.split("\\.");
var result = new BoundField(root, root.getClass().getField(keys[0]));
for (int i = 1; i<keys.length; i++) {
result.object = result.field.get(result.object);
result.field = result.object.getClass().getField(keys[i]);
}
return result;
}
}

46
common/src/main/java/ctbrec/io/CompletableRequestFuture.java Normal file
View File

@ -0,0 +1,46 @@
package ctbrec.io;
import java.io.IOException;
import java.util.concurrent.CompletableFuture;
import lombok.Getter;
import okhttp3.Call;
import okhttp3.Callback;
import okhttp3.Response;
import okhttp3.ResponseBody;
public class CompletableRequestFuture<T> extends CompletableFuture<T> implements Callback {
@Getter
protected Call call;
public CompletableRequestFuture(Call call) {
this.call = call;
}
@Override
public void onResponse(Call c, Response response) throws IOException {
try (var body = response.body()) {
if (response.isSuccessful()) {
processBody(body);
} else {
completeExceptionally(new HttpException(response.code(), response.message()));
}
} catch (Exception e) {
completeExceptionally(e);
}
}
@Override
public void onFailure(Call c, IOException error) {
completeExceptionally(error);
}
@Override
public boolean cancel(boolean mayInterruptIfRunning) {
call.cancel();
return super.cancel(mayInterruptIfRunning);
}
protected void processBody(ResponseBody body) throws Exception {}
}

107
common/src/main/java/ctbrec/io/FlaresolverrClient.java Normal file
View File

@ -0,0 +1,107 @@
package ctbrec.io;
import ctbrec.io.FlaresolverrResponse;
import ctbrec.io.FlaresolverrSolutionResponse;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.json.JsonMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
import lombok.Getter;
import lombok.Setter;
import okhttp3.Call;
import okhttp3.Callback;
import okhttp3.MediaType;
import okhttp3.OkHttpClient;
import okhttp3.Request;
import okhttp3.RequestBody;
import okhttp3.Response;
import okhttp3.ResponseBody;
import okhttp3.Cookie;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.*;
public class FlaresolverrClient {
public String api_url;
protected int timeout_ms;
protected OkHttpClient client;
protected JsonMapper mapper = new JsonMapper();
@Getter
protected String sessionName = "";
public FlaresolverrClient() {
this("http://localhost:8191/v1", 60000);
}
public FlaresolverrClient(String apiUrl, int timeout_ms) {
api_url = apiUrl;
this.timeout_ms = timeout_ms;
client = new OkHttpClient.Builder()
.callTimeout(timeout_ms + 10000, TimeUnit.MILLISECONDS)
.readTimeout(timeout_ms + 1000, TimeUnit.MILLISECONDS)
.build();
}
public CompletableFuture<FlaresolverrResponse> createSession(String name) throws IOException {
if (!sessionName.equals(""))
throw new IOException("Cannot start new session because another one is already started. Finish it before creating a new one");
sessionName = name;
var body = mapper.createObjectNode()
.put("cmd", "sessions.create")
.put("session", name);
return makeApiCall(body).thenApply(r -> new FlaresolverrResponse(r));
}
public CompletableFuture<FlaresolverrResponse> destroySession(String name) throws IOException {
if (sessionName.equals(""))
throw new IOException("Cannot destroy session because no session is active");
var body = mapper.createObjectNode()
.put("cmd", "sessions.destroy")
.put("session", name);
sessionName = "";
return makeApiCall(body).thenApply(r -> new FlaresolverrResponse(r));
}
public CompletableFuture<FlaresolverrSolutionResponse> getCookies(String url) throws IOException {
var body = mapper.createObjectNode()
.put("cmd", "request.get")
.put("url", url)
.put("maxTimeout", timeout_ms)
.put("returnOnlyCookies", true);
if (sessionName != "") {
body.put("session", sessionName);
}
return makeApiCall(body).thenApply(r -> new FlaresolverrSolutionResponse(r));
}
protected CompletableRequestFuture<JsonNode> makeApiCall(ObjectNode body) throws IOException {
var requestBody = RequestBody.create(mapper.writeValueAsString(body), MediaType.get("application/json"));
var request = new Request.Builder()
.url(api_url)
.post(requestBody)
.build();
var call = client.newCall(request);
var future = new CompletableRequestFuture<JsonNode>(call) {
@Override
public void processBody(ResponseBody body) throws IOException {
complete(mapper.readTree(body.charStream()));
}
};
// FIXME?: unfortunate cyclic reference here to allow cancelling through the future, is this bad?
call.enqueue(future);
return future;
}
}

23
common/src/main/java/ctbrec/io/FlaresolverrResponse.java Normal file
View File

@ -0,0 +1,23 @@
package ctbrec.io;
import com.fasterxml.jackson.databind.JsonNode;
import lombok.Getter;
import lombok.ToString;
public class FlaresolverrResponse {
@Getter
protected String status;
@Getter
protected String message;
public FlaresolverrResponse(JsonNode response) {
status = response.get("status").asText();
message = response.get("message").asText();
}
@Override
public String toString() {
return String.format("{status: %s, message: %s}", status, message);
}
}

73
common/src/main/java/ctbrec/io/FlaresolverrSolutionResponse.java Normal file
View File

@ -0,0 +1,73 @@
package ctbrec.io;
import com.fasterxml.jackson.databind.JsonNode;
import ctbrec.StringUtil;
import lombok.Getter;
import okhttp3.Cookie;
import java.time.Instant;
import java.util.*;
public class FlaresolverrSolutionResponse extends FlaresolverrResponse {
@Getter
protected String userAgent;
@Getter
protected Instant startTimestamp;
@Getter
protected Instant endTimestamp;
@Getter
protected String version;
@Getter
protected List<Cookie> cookies;
FlaresolverrSolutionResponse(JsonNode response) {
super(response);
startTimestamp = Instant.ofEpochMilli(response.get("startTimestamp").asLong());
endTimestamp = Instant.ofEpochMilli(response.get("endTimestamp").asLong());
version = response.get("version").asText();
var solution = response.get("solution");
userAgent = solution.get("userAgent").asText();
cookies = new ArrayList<Cookie>();
for (var c : solution.get("cookies")) {
// "domain": c["domain"].lstrip('.'),
// "expiresAt": c["expiry"],
// "hostOnly": c["sameSite"].lower() == "strict",
// "httpOnly": c["httpOnly"],
// "name": c["name"],
// "path": c["path"],
// "persistent": False,
// "secure": c["secure"],
// "value": c["value"]
var cb = new Cookie.Builder()
.expiresAt(Optional.ofNullable(c.get("expires")).orElse(c.get("expiry")).asLong() * 1000) // seconds -> millis
.name(c.get("name").asText())
.path(c.get("path").asText())
.value(c.get("value").asText())
;
var domain = c.get("domain").asText().replaceFirst("\\.", "");
// FIXME: is this correct?
if (c.path("sameSite").asText("").equalsIgnoreCase("strict")) {
cb.hostOnlyDomain(domain);
} else {
cb.domain(domain);
}
if (c.path("httpOnly").asBoolean(false))
cb.httpOnly();
if (c.path("secure").asBoolean(false))
cb.secure();
cookies.add(cb.build());
}
}
}

9
common/src/main/java/ctbrec/io/HttpClient.java
View File

@ -213,7 +213,7 @@ public abstract class HttpClient {
client.dispatcher().executorService().shutdown();
}
private void persistCookies() {
protected void persistCookies() {
try {
List<CookieContainer> containers = new ArrayList<>();
cookieJar.getCookies().forEach((domain, cookieList) -> {
@ -231,7 +231,7 @@ public abstract class HttpClient {
}
}
private void loadCookies() {
protected void loadCookies() {
try {
File cookieFile = new File(config.getConfigDir(), "cookies-" + name + ".json");
if (!cookieFile.exists()) {
@ -357,4 +357,9 @@ public abstract class HttpClient {
public static final String JAVA_NET_SOCKS_USERNAME = "java.net.socks.username";
public static final String JAVA_NET_SOCKS_PASSWORD = "java.net.socks.password";
}
// overridable default user agent (used for Flaresolverr)
public String getEffectiveUserAgent() {
return config.getSettings().httpUserAgent;
}
}

4
common/src/main/java/ctbrec/sites/chaturbate/Chaturbate.java
View File

@ -69,7 +69,7 @@ public class Chaturbate extends AbstractSite {
String url = "https://chaturbate.com/p/" + username + "/";
Request req = new Request.Builder()
.url(url)
.header(USER_AGENT, getConfig().getSettings().httpUserAgent)
.header(USER_AGENT, getHttpClient().getEffectiveUserAgent())
.build();
try (Response resp = getHttpClient().execute(req)) {
if (resp.isSuccessful()) {
@ -131,7 +131,7 @@ public class Chaturbate extends AbstractSite {
// search online models
Request req = new Request.Builder()
.url(url)
.header(USER_AGENT, getConfig().getSettings().httpUserAgent)
.header(USER_AGENT, getHttpClient().getEffectiveUserAgent())
.header(ACCEPT, "*/*")
.header(X_REQUESTED_WITH, XML_HTTP_REQUEST)
.header(REFERER, getBaseUrl())

105
common/src/main/java/ctbrec/sites/chaturbate/ChaturbateHttpClient.java
View File

@ -1,15 +1,20 @@
package ctbrec.sites.chaturbate;
import ctbrec.Config;
import ctbrec.io.FlaresolverrClient;
import ctbrec.io.HtmlParser;
import ctbrec.io.HttpClient;
import lombok.extern.slf4j.Slf4j;
import okhttp3.*;
import java.time.*;
import java.io.IOException;
import java.io.InterruptedIOException;
import java.util.NoSuchElementException;
import java.util.Optional;
import java.util.concurrent.Semaphore;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import static ctbrec.io.HttpConstants.REFERER;
import static ctbrec.io.HttpConstants.USER_AGENT;
@ -19,12 +24,40 @@ public class ChaturbateHttpClient extends HttpClient {
private static final String PATH = "/auth/login/"; // NOSONAR
protected String token;
protected final FlaresolverrClient flaresolverr;
private static final Semaphore requestThrottle = new Semaphore(2, true);
private static long lastRequest = 0;
// a lock to prevent multiple requests from
ReentrantReadWriteLock cookieRefreshLock = new ReentrantReadWriteLock();
AtomicInteger cookieErrorCounter = new AtomicInteger(0);
public ChaturbateHttpClient(Config config) {
super("chaturbate", config);
if (config.getSettings().chaturbateUseFlaresolverr) {
flaresolverr = new FlaresolverrClient(config.getSettings().flaresolverr.apiUrl, config.getSettings().flaresolverr.timeoutInMillis);
// try {
// flaresolverr.createSession("ctbrec").get();
// } catch (InterruptedException e) {
// Thread.currentThread().interrupt();
// } catch (Exception e) {
// log.error("Error starting Flaresolverr session", e);
// }
} else {
flaresolverr = null;
}
}
@Override
public String getEffectiveUserAgent() {
if (flaresolverr != null) {
return config.getSettings().flaresolverr.userAgent;
} else {
return config.getSettings().httpUserAgent;
}
}
private void extractCsrfToken(Request request) {
@ -55,7 +88,7 @@ public class ChaturbateHttpClient extends HttpClient {
}
Request login = new Request.Builder()
.url(Chaturbate.baseUrl + PATH)
.header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
.header(USER_AGENT, getEffectiveUserAgent())
.build();
try (var initResponse = client.newCall(login).execute()) {
String content = initResponse.body().string();
@ -71,7 +104,7 @@ public class ChaturbateHttpClient extends HttpClient {
login = new Request.Builder()
.url(Chaturbate.baseUrl + PATH)
.header(REFERER, Chaturbate.baseUrl + PATH)
.header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
.header(USER_AGENT, getEffectiveUserAgent())
.post(body)
.build();
@ -96,7 +129,7 @@ public class ChaturbateHttpClient extends HttpClient {
String url = "https://chaturbate.com/api/ts/chatmessages/pm_users/?offset=0";
Request req = new Request.Builder()
.url(url)
.header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
.header(USER_AGENT, getEffectiveUserAgent())
.build();
try (Response response = execute(req)) {
boolean result = false;
@ -124,7 +157,21 @@ public class ChaturbateHttpClient extends HttpClient {
if (throttle) {
acquireSlot();
}
Response resp = super.execute(req);
Response resp;
try {
cookieRefreshLock.readLock().lock();
resp = super.execute(req);
} finally {
cookieRefreshLock.readLock().unlock();
}
// try to solve the cloudflare challenge if we got one (clearance cookie expired, update it)
if (resp.code() == 403 && flaresolverr != null) {
resp = refreshCookiesAndRetry(req, resp);
}
extractCsrfToken(req);
return resp;
} catch (InterruptedException e) {
@ -136,6 +183,56 @@ public class ChaturbateHttpClient extends HttpClient {
}
}
}
private Response refreshCookiesAndRetry(Request req, Response origResp) throws IOException {
log.debug("403 received from {}. Trying to refresh cookies with Flaresolverr", req.url().host());
try {
cookieRefreshLock.writeLock().lock();
// we need to prevent repeated challenge requests from multiple threads, so we check if the clearance cookie needs updating
// maybe this can be done with some syncronization primitive, or maybe an expiresAt() check is enough
var cookie = Optional
.ofNullable(cookieJar.getCookies().get(req.url().topPrivateDomain()))
.flatMap(x -> cookieJar.getCookieFromCollection(x, "cf_clearance"));
var cookieExpired = cookie.map(c ->
Instant.ofEpochMilli(c.expiresAt()).isBefore(Instant.now()) // by time
|| req.headers("Cookie").stream().anyMatch(headerCookie -> headerCookie.contains(c.value())) // we got 403 with current cookie present
).orElse(true);
if (cookieExpired || cookieErrorCounter.incrementAndGet() >= 5) {
cookieErrorCounter.set(0);
var apiResponse = flaresolverr.getCookies(req.url().toString()).get();
if (apiResponse.getStatus().equals("ok")) {
// update user agent. It should be the same for all sites, assuming we use the same api address every time
if (!config.getSettings().flaresolverr.userAgent.equals(apiResponse.getUserAgent())) {
config.getSettings().flaresolverr.userAgent = apiResponse.getUserAgent();
config.save();
}
cookieJar.saveFromResponse(req.url(), apiResponse.getCookies());
persistCookies();
log.debug("Cookies successfully refreshed with Flaresolverr in {}", Duration.between(apiResponse.getStartTimestamp(), apiResponse.getEndTimestamp()));
} else {
log.debug("Unsuccessful attempt to refresh cookies. Response from Flaresolverr: {}", apiResponse);
return origResp;
}
} else {
log.debug("Looks like the cookies were refreshed already, skipping refreshing");
}
} catch (Exception e) {
log.warn("Error refreshing cookies with Flaresolverr", e);
return origResp;
} finally {
cookieRefreshLock.writeLock().unlock();
}
origResp.close();
return super.execute(req);
}
private static void acquireSlot() throws InterruptedException {
long pauseBetweenRequests = Config.getInstance().getSettings().chaturbateMsBetweenRequests;

14
common/src/main/java/ctbrec/sites/chaturbate/ChaturbateModel.java
View File

@ -97,7 +97,7 @@ public class ChaturbateModel extends AbstractModel {
int imageSize = 0;
Request req = new Request.Builder()
.url(url)
.header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
.header(USER_AGENT, site.getHttpClient().getEffectiveUserAgent())
.head()
.build();
try (Response response = getSite().getHttpClient().execute(req)) {
@ -190,7 +190,7 @@ public class ChaturbateModel extends AbstractModel {
.post(body)
.header(REFERER, "https://chaturbate.com/" + getName() + "/")
.header(X_REQUESTED_WITH, XML_HTTP_REQUEST)
.header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
.header(USER_AGENT, site.getHttpClient().getEffectiveUserAgent())
.build();
try (Response response = getSite().getHttpClient().execute(req)) {
if (!response.isSuccessful()) {
@ -239,7 +239,7 @@ public class ChaturbateModel extends AbstractModel {
// do an initial request to get cookies
Request req = new Request.Builder()
.url(getUrl())
.header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
.header(USER_AGENT, site.getHttpClient().getEffectiveUserAgent())
.build();
Response resp = site.getHttpClient().execute(req);
resp.close();
@ -258,7 +258,7 @@ public class ChaturbateModel extends AbstractModel {
.header(ACCEPT, "*/*")
.header(ACCEPT_LANGUAGE, "en-US,en;q=0.5")
.header(REFERER, getUrl())
.header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
.header(USER_AGENT, site.getHttpClient().getEffectiveUserAgent())
.header("X-CSRFToken", ((ChaturbateHttpClient) site.getHttpClient()).getToken())
.header(X_REQUESTED_WITH, XML_HTTP_REQUEST)
.build();
@ -302,7 +302,7 @@ public class ChaturbateModel extends AbstractModel {
Request req = new Request.Builder()
.url(getSite().getBaseUrl() + "/get_edge_hls_url_ajax/")
.post(body)
.header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
.header(USER_AGENT, site.getHttpClient().getEffectiveUserAgent())
.header(X_REQUESTED_WITH, XML_HTTP_REQUEST)
.build();
try (Response response = getSite().getHttpClient().execute(req)) {
@ -364,7 +364,7 @@ public class ChaturbateModel extends AbstractModel {
log.trace("Loading master playlist {}", streamInfo.url);
Request req = new Request.Builder()
.url(streamInfo.url)
.header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
.header(USER_AGENT, site.getHttpClient().getEffectiveUserAgent())
.build();
try (Response response = getSite().getHttpClient().execute(req)) {
if (response.isSuccessful()) {
@ -385,7 +385,7 @@ public class ChaturbateModel extends AbstractModel {
public boolean exists() throws IOException {
Request req = new Request.Builder() // @formatter:off
.url(getUrl())
.header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
.header(USER_AGENT, site.getHttpClient().getEffectiveUserAgent())
.header(ACCEPT_LANGUAGE, Locale.ENGLISH.getLanguage())
.build(); // @formatter:on
try (Response response = getSite().getHttpClient().execute(req)) {

9
server/src/main/java/ctbrec/recorder/server/ConfigServlet.java
View File

@ -3,7 +3,6 @@ package ctbrec.recorder.server;
import static javax.servlet.http.HttpServletResponse.*;
import java.io.IOException;
import java.lang.reflect.Field;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import java.time.LocalTime;
@ -21,6 +20,7 @@ import org.slf4j.LoggerFactory;
import ctbrec.Config;
import ctbrec.Settings;
import ctbrec.Settings.SplitStrategy;
import ctbrec.io.BoundField;
public class ConfigServlet extends AbstractCtbrecServlet {
@ -76,6 +76,9 @@ public class ConfigServlet extends AbstractCtbrecServlet {
addParameter("webinterfacePassword", "Web-Interface Password", DataType.STRING, settings.webinterfacePassword, json);
addParameter("servletContext", "Servlet Context", DataType.STRING, settings.servletContext, json);
addParameter("logFFmpegOutput", "Log FFmpeg Output", DataType.BOOLEAN, settings.logFFmpegOutput, json);
addParameter("flaresolverr.apiUrl", "Flaresolverr API URL", DataType.STRING, settings.flaresolverr.apiUrl, json);
addParameter("flaresolverr.timeoutInMillis", "Flaresolverr request timeout (ms)", DataType.INTEGER, settings.flaresolverr.timeoutInMillis, json);
addParameter("chaturbateUseFlaresolverr", "Chaturbate: use Flaresolverr", DataType.BOOLEAN, settings.chaturbateUseFlaresolverr, json);
resp.setStatus(SC_OK);
resp.setContentType("application/json");
@ -135,8 +138,8 @@ public class ConfigServlet extends AbstractCtbrecServlet {
Object typeCorrectedValue = correctType(type, value);
LOG.debug("{}: {}", key, value);
Field field = Settings.class.getField(key);
field.set(settings, typeCorrectedValue);
var field = BoundField.of(settings, key);
field.set(typeCorrectedValue);
}
config.save();
} catch (Exception e) {