Merge branch 'flaresolverr' into reusedname-dev

client/src/main/java/ctbrec/ui/settings/CtbrecPreferencesStorage.java

@@ -1,6 +1,7 @@
 package ctbrec.ui.settings;
 
 import java.io.IOException;
+import java.lang.reflect.InvocationTargetException;
 import java.time.LocalTime;
 import java.util.List;
 import java.util.Objects;
@@ -25,6 +26,7 @@ import ctbrec.ui.settings.api.Setting;
 import ctbrec.ui.settings.api.SimpleDirectoryProperty;
 import ctbrec.ui.settings.api.SimpleFileProperty;
 import ctbrec.ui.settings.api.SimpleRangeProperty;
+import ctbrec.io.BoundField;
 import javafx.beans.property.BooleanProperty;
 import javafx.beans.property.IntegerProperty;
 import javafx.beans.property.ListProperty;
@@ -116,12 +118,12 @@ public class CtbrecPreferencesStorage implements PreferencesStorage {
         optionB.setToggleGroup(toggleGroup);
         optionA.selectedProperty().bindBidirectional(prop);
         prop.addListener((obs, oldV, newV) -> saveValue(() -> {            
-            var field = Settings.class.getField(setting.getKey());
+            if (setIfChanged(setting.getKey(), newV)) {
-            field.set(settings, newV); // NOSONAR
+                if (setting.doesNeedRestart()) {
-            if (setting.doesNeedRestart() && !Objects.equals(oldV, newV)) {
                     runRestartRequiredCallback();
                 }
                 config.save();
+            }
         }));
         var row = new HBox();
         row.getChildren().addAll(optionA, optionB);
@@ -155,15 +157,15 @@ public class CtbrecPreferencesStorage implements PreferencesStorage {
         resolutionRange.setHigh(highValue >= 0 ? highValue : values.get(values.size() - 1));
         resolutionRange.getLow().addListener((obs, o, n) -> saveValue(() -> {
             int newV = labels.get(n.intValue());
-            var field = Settings.class.getField(rangeProperty.getLowKey());
+            if (setIfChanged(rangeProperty.getLowKey(), newV)) {
-            field.set(settings, newV); // NOSONAR
                 config.save();
+            }
         }));
         resolutionRange.getHigh().addListener((obs, o, n) -> saveValue(() -> {
             int newV = labels.get(n.intValue());
-            var field = Settings.class.getField(rangeProperty.getHighKey());
+            if (setIfChanged(rangeProperty.getHighKey(), newV)) {
-            field.set(settings, newV); // NOSONAR
                 config.save();
+            }
         }));
         return resolutionRange;
     }
@@ -181,11 +183,7 @@ public class CtbrecPreferencesStorage implements PreferencesStorage {
     private Node createFileSelector(Setting setting) {
         var programSelector = new ProgramSelectionBox("");
         programSelector.fileProperty().addListener((obs, o, n) -> saveValue(() -> {
-            String path = n;
+            if (setIfChanged(setting.getKey(), n)) {
-            var field = Settings.class.getField(setting.getKey());
-            String oldValue = (String) field.get(settings);
-            if (!Objects.equals(path, oldValue)) {
-                field.set(settings, path); // NOSONAR
                 if (setting.doesNeedRestart()) {
                     runRestartRequiredCallback();
                 }
@@ -201,11 +199,7 @@ public class CtbrecPreferencesStorage implements PreferencesStorage {
         var directorySelector = new DirectorySelectionBox("");
         directorySelector.prefWidth(400);
         directorySelector.fileProperty().addListener((obs, o, n) -> saveValue(() -> {
-            String path = n;
+            if (setIfChanged(setting.getKey(), n)) {
-            var field = Settings.class.getField(setting.getKey());
-            String oldValue = (String) field.get(settings);
-            if (!Objects.equals(path, oldValue)) {
-                field.set(settings, path); // NOSONAR
                 if (setting.doesNeedRestart()) {
                     runRestartRequiredCallback();
                 }
@@ -221,10 +215,7 @@ public class CtbrecPreferencesStorage implements PreferencesStorage {
         LocalTime time = (LocalTime) setting.getProperty().getValue();
         var timePicker = new TimePicker(time);
         timePicker.valueProperty().addListener((obs, o, n) -> saveValue(() -> {
-            var field = Settings.class.getField(setting.getKey());
+            if (setIfChanged(setting.getKey(), n)) {
-            LocalTime oldValue = (LocalTime) field.get(settings);
-            if (!Objects.equals(n, oldValue)) {
-                field.set(settings, n); // NOSONAR
                 if (setting.doesNeedRestart()) {
                     runRestartRequiredCallback();
                 }
@@ -237,12 +228,12 @@ public class CtbrecPreferencesStorage implements PreferencesStorage {
     private Node createStringProperty(Setting setting) {
         var ctrl = new TextField();
         ctrl.textProperty().addListener((obs, oldV, newV) -> saveValue(() -> {
-            var field = Settings.class.getField(setting.getKey());
+            if (setIfChanged(setting.getKey(), newV)) {
-            field.set(settings, newV); // NOSONAR
+                if (setting.doesNeedRestart()) {
-            if (setting.doesNeedRestart() && !Objects.equals(oldV, newV)) {
                     runRestartRequiredCallback();
                 }
                 config.save();
+            }
         }));
         StringProperty prop = (StringProperty) setting.getProperty();
         ctrl.textProperty().bindBidirectional(prop);
@@ -256,10 +247,8 @@ public class CtbrecPreferencesStorage implements PreferencesStorage {
             if (!newV.matches("\\d*")) {
                 ctrl.setText(newV.replaceAll(PATTERN_NOT_A_DIGIT, ""));
             }
-            if (!ctrl.getText().isEmpty()) {
+            if (!ctrl.getText().isEmpty() && setIfChanged(setting.getKey(), Integer.parseInt(ctrl.getText()))) {
-                var field = Settings.class.getField(setting.getKey());
+                if (setting.doesNeedRestart() && prefs != null) {
-                field.set(settings, Integer.parseInt(ctrl.getText())); // NOSONAR
-                if (setting.doesNeedRestart() && !Objects.equals(oldV, newV) && prefs != null) {
                     runRestartRequiredCallback();
                 }
                 config.save();
@@ -282,13 +271,13 @@ public class CtbrecPreferencesStorage implements PreferencesStorage {
                 if (setting.getConverter() != null) {
                     value = (long) setting.getConverter().convertFrom(value);
                 }
-                var field = Settings.class.getField(setting.getKey());
+                if (setIfChanged(setting.getKey(), value)) {
-                field.set(settings, value); // NOSONAR
                     if (setting.doesNeedRestart() && !Objects.equals(oldV, newV)) {
                         runRestartRequiredCallback();
                     }
                     config.save();
                 }
+            }
         }));
         Property<Number> prop = setting.getProperty();
         ctrl.textProperty().bindBidirectional(prop, new NumberStringConverter());
@@ -298,12 +287,12 @@ public class CtbrecPreferencesStorage implements PreferencesStorage {
     private Node createBooleanProperty(Setting setting) {
         var ctrl = new CheckBox();
         ctrl.selectedProperty().addListener((obs, oldV, newV) -> saveValue(() -> {
-            var field = Settings.class.getField(setting.getKey());
+            if (setIfChanged(setting.getKey(), newV)) {
-            field.set(settings, newV); // NOSONAR
+                if (setting.doesNeedRestart()) {
-            if (setting.doesNeedRestart() && !Objects.equals(oldV, newV)) {
                     runRestartRequiredCallback();
                 }
                 config.save();
+            }
         }));
         BooleanProperty prop = (BooleanProperty) setting.getProperty();
         ctrl.selectedProperty().bindBidirectional(prop);
@@ -311,11 +300,10 @@ public class CtbrecPreferencesStorage implements PreferencesStorage {
     }
 
     @SuppressWarnings({ "rawtypes", "unchecked" })
-    private Node createComboBox(Setting setting) throws NoSuchFieldException, IllegalAccessException {
+    private Node createComboBox(Setting setting) throws IllegalAccessException, NoSuchFieldException  {
         ListProperty<?> listProp = (ListProperty<?>) setting.getProperty();
         ComboBox<Object> comboBox = new ComboBox(listProp);
-        var field = Settings.class.getField(setting.getKey());
+        Object value = BoundField.of(settings, setting.getKey()).get();
-        Object value = field.get(settings);
         if (StringUtil.isNotBlank(value.toString())) {
             if (setting.getConverter() != null) {
                 comboBox.getSelectionModel().select(setting.getConverter().convertTo(value));
@@ -325,15 +313,12 @@ public class CtbrecPreferencesStorage implements PreferencesStorage {
         }
         comboBox.valueProperty().addListener((obs, oldV, newV) -> saveValue(() -> {
             LOG.debug("Saving setting {}", setting.getKey());
-            if (setting.getConverter() != null) {
+            if (setIfChanged(setting.getKey(), setting.getConverter() != null ? setting.getConverter().convertFrom(newV) : newV)) {
-                field.set(settings, setting.getConverter().convertFrom(newV)); // NOSONAR
+                if (setting.doesNeedRestart()) {
-            } else {
-                field.set(settings, newV); // NOSONAR
-            }
-            if (setting.doesNeedRestart() && !Objects.equals(oldV, newV)) {
                     runRestartRequiredCallback();
                 }
                 config.save();
+            }
         }));
         if (setting.getChangeListener() != null) {
             comboBox.valueProperty().addListener((ChangeListener<? super Object>) setting.getChangeListener());
@@ -341,6 +326,17 @@ public class CtbrecPreferencesStorage implements PreferencesStorage {
         return comboBox;
     }
     
+    
+    private boolean setIfChanged(String key, Object n) throws IllegalAccessException, NoSuchFieldException, InvocationTargetException {        
+        var field = BoundField.of(settings, key);
+        var o = field.get();
+        if (!Objects.equals(n, o)) {
+            field.set(n); // NOSONAR
+            return true;
+        }
+        return false;
+    }
+
     private void saveValue(Exec exe) {
         try {
             exe.run();
@@ -351,6 +347,6 @@ public class CtbrecPreferencesStorage implements PreferencesStorage {
 
     @FunctionalInterface
     private interface Exec {
-        public void run() throws IllegalAccessException, IOException, NoSuchFieldException;
+        public void run() throws IllegalAccessException, IOException, NoSuchFieldException, NoSuchMethodException, InvocationTargetException;
     }
 }

client/src/main/java/ctbrec/ui/settings/SettingsTab.java

@@ -60,6 +60,8 @@ public class SettingsTab extends Tab implements TabSelectionListener {
     private final Settings settings;
     private boolean initialized = false;
 
+    private SimpleStringProperty flaresolverrApiUrl;
+    private SimpleIntegerProperty flaresolverrTimeoutInMillis;
     private SimpleStringProperty httpUserAgent;
     private SimpleStringProperty httpUserAgentMobile;
     private SimpleIntegerProperty overviewUpdateIntervalInSecs;
@@ -144,6 +146,8 @@ public class SettingsTab extends Tab implements TabSelectionListener {
     }
 
     private void initializeProperties() {
+        flaresolverrApiUrl = new SimpleStringProperty(null, "flaresolverr.apiUrl", settings.flaresolverr.apiUrl);
+        flaresolverrTimeoutInMillis = new SimpleIntegerProperty(null, "flaresolverr.timeoutInMillis", settings.flaresolverr.timeoutInMillis);
         httpUserAgent = new SimpleStringProperty(null, "httpUserAgent", settings.httpUserAgent);
         httpUserAgentMobile = new SimpleStringProperty(null, "httpUserAgentMobile", settings.httpUserAgentMobile);
         overviewUpdateIntervalInSecs = new SimpleIntegerProperty(null, "overviewUpdateIntervalInSecs", settings.overviewUpdateIntervalInSecs);
@@ -258,7 +262,11 @@ public class SettingsTab extends Tab implements TabSelectionListener {
                         Group.of("Browser",
                                 Setting.of("Browser", browserOverride),
                                 Setting.of("Start parameters", browserParams),
-                                Setting.of("Force use (ignore default browser)", forceBrowserOverride, "Default behaviour will fallback to OS default if the above browser fails"))),
+                                Setting.of("Force use (ignore default browser)", forceBrowserOverride, "Default behaviour will fallback to OS default if the above browser fails")),
+                                
+                        Group.of("Flaresolverr",
+                                Setting.of("API URL", flaresolverrApiUrl),
+                                Setting.of("Request timeout", flaresolverrTimeoutInMillis))),
                 Category.of("Look & Feel",
                         Group.of("Look & Feel",
                                 Setting.of("Colors (Base / Accent)", new ColorSettingsPane(Config.getInstance())).needsRestart(),
@@ -354,6 +362,8 @@ public class SettingsTab extends Tab implements TabSelectionListener {
         setContent(stackPane);
         prefs.expandTree();
 
+        prefs.getSetting("flaresolverr.apiUrl").ifPresent(s -> bindEnabledProperty(s, recordLocal.not()));
+        prefs.getSetting("flaresolverr.timeoutInMillis").ifPresent(s -> bindEnabledProperty(s, recordLocal.not()));
         prefs.getSetting("httpServer").ifPresent(s -> bindEnabledProperty(s, recordLocal));
         prefs.getSetting("httpPort").ifPresent(s -> bindEnabledProperty(s, recordLocal));
         prefs.getSetting("servletContext").ifPresent(s -> bindEnabledProperty(s, recordLocal));

client/src/main/java/ctbrec/ui/sites/chaturbate/ChaturbateApiUpdateService.java

@@ -35,7 +35,7 @@ public class ChaturbateApiUpdateService extends PaginatedScheduledService {
             protected List<Model> call() throws Exception {
                 var request = new Request.Builder()
                         .url(url)
-                        .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
+                        .header(USER_AGENT, chaturbate.getHttpClient().getEffectiveUserAgent())
                         .header(ACCEPT, MIMETYPE_APPLICATION_JSON)
                         .build();
                 try (var response = chaturbate.getHttpClient().execute(request)) {

client/src/main/java/ctbrec/ui/sites/chaturbate/ChaturbateConfigUi.java

@@ -7,11 +7,7 @@ import ctbrec.ui.settings.SettingsTab;
 import ctbrec.ui.sites.AbstractConfigUI;
 import javafx.geometry.Insets;
 import javafx.scene.Parent;
-import javafx.scene.control.Button;
+import javafx.scene.control.*;
-import javafx.scene.control.CheckBox;
-import javafx.scene.control.Label;
-import javafx.scene.control.PasswordField;
-import javafx.scene.control.TextField;
 import javafx.scene.layout.GridPane;
 import javafx.scene.layout.Priority;
 
@@ -104,6 +100,18 @@ public class ChaturbateConfigUi extends AbstractConfigUI {
         GridPane.setColumnSpan(requestThrottle, 2);
         layout.add(requestThrottle, 1, row++);
         
+        var label = new Label("Use Flaresolverr");
+        label.setTooltip(new Tooltip("Use Flaresolverr for solving the Cloudflare challenge. This also overrides the User Agent used for HTTP requests (only for the site)"));
+        layout.add(label, 0, row);
+        var flaresolverrToggle = new CheckBox();
+        flaresolverrToggle.setSelected(settings.chaturbateUseFlaresolverr);
+        flaresolverrToggle.setOnAction(e -> {
+            settings.chaturbateUseFlaresolverr = flaresolverrToggle.isSelected();
+            save();
+        });
+        GridPane.setMargin(flaresolverrToggle, new Insets(0, 0, SettingsTab.CHECKBOX_MARGIN, SettingsTab.CHECKBOX_MARGIN));
+        layout.add(flaresolverrToggle, 1, row++);
+
         var createAccount = new Button("Create new Account");
         createAccount.setOnAction(e -> DesktopIntegration.open(Chaturbate.REGISTRATION_LINK));
         layout.add(createAccount, 1, row++);

client/src/main/java/ctbrec/ui/sites/chaturbate/ChaturbateElectronLoginDialog.java

@@ -35,7 +35,7 @@ public class ChaturbateElectronLoginDialog {
             config.put("url", site.getBaseUrl() + "/auth/login/");
             config.put("w", 640);
             config.put("h", 480);
-            config.put("userAgent", Config.getInstance().getSettings().httpUserAgent);
+            config.put("userAgent", site.getHttpClient().getEffectiveUserAgent());
             var msg = new JSONObject();
             msg.put("config", config);
             browser.run(msg, msgHandler);

client/src/main/java/ctbrec/ui/sites/chaturbate/ChaturbateUpdateService.java

@@ -54,7 +54,7 @@ public class ChaturbateUpdateService extends PaginatedScheduledService {
                             .header(ACCEPT, MIMETYPE_APPLICATION_JSON)
                             .header(ACCEPT_LANGUAGE, Locale.ENGLISH.getLanguage())
                             .header(X_REQUESTED_WITH, XML_HTTP_REQUEST)
-                            .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
+                            .header(USER_AGENT, chaturbate.getHttpClient().getEffectiveUserAgent())
                             .build();
                     try (var response = chaturbate.getHttpClient().execute(request)) {
                         if (response.isSuccessful()) {

common/src/main/java/ctbrec/Settings.java

@@ -42,7 +42,13 @@ public class Settings {
         TIME_OR_SIZE
     }
 
+    public class FlaresolverrSettings {
+        public String apiUrl = "http://localhost:8191/v1";
+        public int timeoutInMillis = 60000;
+        public String userAgent = "";
+    };
 
+    public FlaresolverrSettings flaresolverr = new FlaresolverrSettings();
     public String amateurTvUsername = "";
     public String amateurTvPassword = "";
     public String bongacamsBaseUrl = "https://bongacams.com";
@@ -55,6 +61,7 @@ public class Settings {
     public String chaturbatePassword = "";
     public String chaturbateUsername = "";
     public String chaturbateBaseUrl = "https://chaturbate.com";
+    public boolean chaturbateUseFlaresolverr = false;
     public int chaturbateMsBetweenRequests = 1000;
     public String cherryTvPassword = "";
     public String cherryTvUsername = "";

common/src/main/java/ctbrec/io/BoundField.java

@@ -0,0 +1,32 @@
+package ctbrec.io;
+
+import java.lang.reflect.*;
+
+public class BoundField {
+    public Object object = null;
+    public Field field = null;
+    
+    public BoundField(Object o, Field f) {
+        object = o;
+        field = f;
+    }
+    
+    public Object get() throws IllegalAccessException {
+        return field.get(object);
+    }
+    
+    public void set(Object value) throws IllegalAccessException {
+        field.set(object, value);
+    }
+    
+    // by-path field resolver (i.e: "a.b.c")
+    public static BoundField of(Object root, String path) throws NoSuchFieldException, IllegalAccessException {
+        var keys = path.split("\\.");
+        var result = new BoundField(root, root.getClass().getField(keys[0]));
+        for (int i = 1; i<keys.length; i++) {
+            result.object = result.field.get(result.object);
+            result.field = result.object.getClass().getField(keys[i]);
+        }        
+        return result;
+    }
+}

common/src/main/java/ctbrec/io/CompletableRequestFuture.java

@@ -0,0 +1,46 @@
+package ctbrec.io;
+
+import java.io.IOException;
+import java.util.concurrent.CompletableFuture;
+
+import lombok.Getter;
+import okhttp3.Call;
+import okhttp3.Callback;
+import okhttp3.Response;
+import okhttp3.ResponseBody;
+
+public class CompletableRequestFuture<T> extends CompletableFuture<T> implements Callback {
+    @Getter
+    protected Call call;
+    
+    public CompletableRequestFuture(Call call) {
+        this.call = call;
+    }
+    
+    @Override
+    public void onResponse(Call c, Response response) throws IOException {
+        try (var body = response.body()) {
+            if (response.isSuccessful()) {
+                processBody(body);
+            } else {
+                completeExceptionally(new HttpException(response.code(), response.message()));
+            }
+        } catch (Exception e) {
+            completeExceptionally(e);
+        }
+    }
+
+    @Override
+    public void onFailure(Call c, IOException error) {            
+        completeExceptionally(error);
+    }
+    
+    @Override
+    public boolean cancel(boolean mayInterruptIfRunning) {
+        call.cancel();
+        return super.cancel(mayInterruptIfRunning);
+    }
+    
+    
+    protected void processBody(ResponseBody body) throws Exception {}
+}

common/src/main/java/ctbrec/io/FlaresolverrClient.java

@@ -0,0 +1,107 @@
+package ctbrec.io;
+
+import ctbrec.io.FlaresolverrResponse;
+import ctbrec.io.FlaresolverrSolutionResponse;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.json.JsonMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+
+import lombok.Getter;
+import lombok.Setter;
+import okhttp3.Call;
+import okhttp3.Callback;
+import okhttp3.MediaType;
+import okhttp3.OkHttpClient;
+import okhttp3.Request;
+import okhttp3.RequestBody;
+import okhttp3.Response;
+import okhttp3.ResponseBody;
+import okhttp3.Cookie;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.*;
+
+
+public class FlaresolverrClient {
+    public String api_url;
+    protected int timeout_ms;
+    protected OkHttpClient client;
+    protected JsonMapper mapper = new JsonMapper();
+
+    @Getter
+    protected String sessionName = "";
+
+    public FlaresolverrClient() {
+        this("http://localhost:8191/v1", 60000);
+    }
+
+    public FlaresolverrClient(String apiUrl, int timeout_ms) {
+        api_url = apiUrl;
+        this.timeout_ms = timeout_ms;
+        client = new OkHttpClient.Builder()
+            .callTimeout(timeout_ms + 10000, TimeUnit.MILLISECONDS)
+            .readTimeout(timeout_ms + 1000, TimeUnit.MILLISECONDS)
+            .build();
+    }
+    
+    public CompletableFuture<FlaresolverrResponse> createSession(String name) throws IOException {
+        if (!sessionName.equals("")) 
+            throw new IOException("Cannot start new session because another one is already started. Finish it before creating a new one");
+        
+        sessionName = name;
+        var body = mapper.createObjectNode()
+            .put("cmd", "sessions.create")
+            .put("session", name);
+
+        return makeApiCall(body).thenApply(r -> new FlaresolverrResponse(r));
+    }
+
+    public CompletableFuture<FlaresolverrResponse> destroySession(String name) throws IOException {
+        if (sessionName.equals(""))
+            throw new IOException("Cannot destroy session because no session is active");
+        
+        var body = mapper.createObjectNode()
+            .put("cmd", "sessions.destroy")
+            .put("session", name);
+
+        sessionName = "";
+        return makeApiCall(body).thenApply(r -> new FlaresolverrResponse(r));
+    }
+
+    public CompletableFuture<FlaresolverrSolutionResponse> getCookies(String url) throws IOException {
+        var body = mapper.createObjectNode()
+            .put("cmd", "request.get")
+            .put("url", url)
+            .put("maxTimeout", timeout_ms)
+            .put("returnOnlyCookies", true);
+
+        if (sessionName != "") {
+            body.put("session", sessionName);
+        }
+
+        return makeApiCall(body).thenApply(r -> new FlaresolverrSolutionResponse(r));
+    }
+
+    protected CompletableRequestFuture<JsonNode> makeApiCall(ObjectNode body) throws IOException {
+        var requestBody = RequestBody.create(mapper.writeValueAsString(body), MediaType.get("application/json"));
+        var request = new Request.Builder()
+            .url(api_url)
+            .post(requestBody)
+            .build();
+        
+        var call = client.newCall(request);
+        var future = new CompletableRequestFuture<JsonNode>(call) {
+            @Override
+            public void processBody(ResponseBody body) throws IOException {
+                complete(mapper.readTree(body.charStream()));                
+            }
+        };
+        
+        // FIXME?: unfortunate cyclic reference here to allow cancelling through the future, is this bad?
+        call.enqueue(future);
+        return future;
+    }
+}

common/src/main/java/ctbrec/io/FlaresolverrResponse.java

@@ -0,0 +1,23 @@
+package ctbrec.io;
+
+import com.fasterxml.jackson.databind.JsonNode;
+
+import lombok.Getter;
+import lombok.ToString;
+
+public class FlaresolverrResponse {
+    @Getter
+    protected String status;
+    @Getter
+    protected String message;
+    
+    public FlaresolverrResponse(JsonNode response) {
+        status = response.get("status").asText();
+        message = response.get("message").asText();
+    }
+    
+    @Override
+    public String toString() {
+        return String.format("{status: %s, message: %s}", status, message);
+    }
+}

common/src/main/java/ctbrec/io/FlaresolverrSolutionResponse.java

@@ -0,0 +1,73 @@
+package ctbrec.io;
+
+import com.fasterxml.jackson.databind.JsonNode;
+
+import ctbrec.StringUtil;
+import lombok.Getter;
+import okhttp3.Cookie;
+
+import java.time.Instant;
+import java.util.*;
+
+public class FlaresolverrSolutionResponse extends FlaresolverrResponse {
+    @Getter
+    protected String userAgent;
+    @Getter
+    protected Instant startTimestamp;
+    @Getter
+    protected Instant endTimestamp;
+    @Getter
+    protected String version;
+    @Getter
+    protected List<Cookie> cookies;
+
+    FlaresolverrSolutionResponse(JsonNode response) {
+        super(response);
+
+        startTimestamp = Instant.ofEpochMilli(response.get("startTimestamp").asLong());
+        endTimestamp = Instant.ofEpochMilli(response.get("endTimestamp").asLong());
+        version = response.get("version").asText();
+        
+        var solution = response.get("solution");
+        userAgent = solution.get("userAgent").asText();
+        
+        cookies = new ArrayList<Cookie>();
+
+        for (var c : solution.get("cookies")) {
+            //     "domain": c["domain"].lstrip('.'),
+            //     "expiresAt": c["expiry"],
+            //     "hostOnly": c["sameSite"].lower() == "strict",
+            //     "httpOnly": c["httpOnly"],
+            //     "name": c["name"],
+            //     "path": c["path"],
+            //     "persistent": False,
+            //     "secure": c["secure"],
+            //     "value": c["value"]
+
+            var cb = new Cookie.Builder()
+                .expiresAt(Optional.ofNullable(c.get("expires")).orElse(c.get("expiry")).asLong() * 1000) // seconds -> millis
+                .name(c.get("name").asText())
+                .path(c.get("path").asText())
+                .value(c.get("value").asText())
+                ;
+
+            var domain = c.get("domain").asText().replaceFirst("\\.", "");
+
+            // FIXME: is this correct?
+            if (c.path("sameSite").asText("").equalsIgnoreCase("strict")) {
+                cb.hostOnlyDomain(domain);
+            } else {
+                cb.domain(domain);
+            }
+
+            if (c.path("httpOnly").asBoolean(false))
+                cb.httpOnly();
+
+            if (c.path("secure").asBoolean(false))
+                cb.secure();
+
+            cookies.add(cb.build());
+        }
+    }
+
+}

common/src/main/java/ctbrec/io/HttpClient.java

@@ -213,7 +213,7 @@ public abstract class HttpClient {
         client.dispatcher().executorService().shutdown();
     }
 
-    private void persistCookies() {
+    protected void persistCookies() {
         try {
             List<CookieContainer> containers = new ArrayList<>();
             cookieJar.getCookies().forEach((domain, cookieList) -> {
@@ -231,7 +231,7 @@ public abstract class HttpClient {
         }
     }
 
-    private void loadCookies() {
+    protected void loadCookies() {
         try {
             File cookieFile = new File(config.getConfigDir(), "cookies-" + name + ".json");
             if (!cookieFile.exists()) {
@@ -357,4 +357,9 @@ public abstract class HttpClient {
         public static final String JAVA_NET_SOCKS_USERNAME = "java.net.socks.username";
         public static final String JAVA_NET_SOCKS_PASSWORD = "java.net.socks.password";
     }
+    
+    // overridable default user agent (used for Flaresolverr)
+    public String getEffectiveUserAgent() {
+        return config.getSettings().httpUserAgent;
+    }
 }

common/src/main/java/ctbrec/sites/chaturbate/Chaturbate.java

@@ -69,7 +69,7 @@ public class Chaturbate extends AbstractSite {
         String url = "https://chaturbate.com/p/" + username + "/";
         Request req = new Request.Builder()
                 .url(url)
-                .header(USER_AGENT, getConfig().getSettings().httpUserAgent)
+                .header(USER_AGENT, getHttpClient().getEffectiveUserAgent())
                 .build();
         try (Response resp = getHttpClient().execute(req)) {
             if (resp.isSuccessful()) {
@@ -131,7 +131,7 @@ public class Chaturbate extends AbstractSite {
         // search online models
         Request req = new Request.Builder()
                 .url(url)
-                .header(USER_AGENT, getConfig().getSettings().httpUserAgent)
+                .header(USER_AGENT, getHttpClient().getEffectiveUserAgent())
                 .header(ACCEPT, "*/*")
                 .header(X_REQUESTED_WITH, XML_HTTP_REQUEST)
                 .header(REFERER, getBaseUrl())

common/src/main/java/ctbrec/sites/chaturbate/ChaturbateHttpClient.java

@@ -1,15 +1,20 @@
 package ctbrec.sites.chaturbate;
 
 import ctbrec.Config;
+import ctbrec.io.FlaresolverrClient;
 import ctbrec.io.HtmlParser;
 import ctbrec.io.HttpClient;
 import lombok.extern.slf4j.Slf4j;
 import okhttp3.*;
 
+import java.time.*;
 import java.io.IOException;
 import java.io.InterruptedIOException;
 import java.util.NoSuchElementException;
+import java.util.Optional;
 import java.util.concurrent.Semaphore;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
 
 import static ctbrec.io.HttpConstants.REFERER;
 import static ctbrec.io.HttpConstants.USER_AGENT;
@@ -19,12 +24,40 @@ public class ChaturbateHttpClient extends HttpClient {
 
     private static final String PATH = "/auth/login/"; // NOSONAR
     protected String token;
+    protected final FlaresolverrClient flaresolverr;
 
     private static final Semaphore requestThrottle = new Semaphore(2, true);
     private static long lastRequest = 0;
     
+    // a lock to prevent multiple requests from 
+    ReentrantReadWriteLock cookieRefreshLock = new ReentrantReadWriteLock();
+    AtomicInteger cookieErrorCounter = new AtomicInteger(0);
+
     public ChaturbateHttpClient(Config config) {
         super("chaturbate", config);
+        
+        if (config.getSettings().chaturbateUseFlaresolverr) {
+            flaresolverr = new FlaresolverrClient(config.getSettings().flaresolverr.apiUrl, config.getSettings().flaresolverr.timeoutInMillis);
+            
+            // try {
+            //     flaresolverr.createSession("ctbrec").get();
+            // } catch (InterruptedException e) {
+            //     Thread.currentThread().interrupt();
+            // } catch (Exception e) {
+            //     log.error("Error starting Flaresolverr session", e);
+            // }
+        } else {
+            flaresolverr = null;
+        }
+    }
+    
+    @Override
+    public String getEffectiveUserAgent() {
+        if (flaresolverr != null) {
+            return config.getSettings().flaresolverr.userAgent;
+        } else {
+            return config.getSettings().httpUserAgent;
+        }
     }
 
     private void extractCsrfToken(Request request) {
@@ -55,7 +88,7 @@ public class ChaturbateHttpClient extends HttpClient {
         }
         Request login = new Request.Builder()
                 .url(Chaturbate.baseUrl + PATH)
-                .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
+                .header(USER_AGENT, getEffectiveUserAgent())
                 .build();
         try (var initResponse = client.newCall(login).execute()) {
             String content = initResponse.body().string();
@@ -71,7 +104,7 @@ public class ChaturbateHttpClient extends HttpClient {
             login = new Request.Builder()
                     .url(Chaturbate.baseUrl + PATH)
                     .header(REFERER, Chaturbate.baseUrl + PATH)
-                    .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
+                    .header(USER_AGENT, getEffectiveUserAgent())
                     .post(body)
                     .build();
 
@@ -96,7 +129,7 @@ public class ChaturbateHttpClient extends HttpClient {
         String url = "https://chaturbate.com/api/ts/chatmessages/pm_users/?offset=0";
         Request req = new Request.Builder()
                 .url(url)
-                .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
+                .header(USER_AGENT, getEffectiveUserAgent())
                 .build();
         try (Response response = execute(req)) {
             boolean result = false;
@@ -124,7 +157,21 @@ public class ChaturbateHttpClient extends HttpClient {
             if (throttle) {
                 acquireSlot();
             }
-            Response resp = super.execute(req);
+            
+            Response resp;
+            
+            try {
+                cookieRefreshLock.readLock().lock();
+                resp = super.execute(req);
+            } finally {
+                cookieRefreshLock.readLock().unlock();
+            }
+            
+            // try to solve the cloudflare challenge if we got one (clearance cookie expired, update it)
+            if (resp.code() == 403 && flaresolverr != null) {
+                resp = refreshCookiesAndRetry(req, resp);
+            }
+            
             extractCsrfToken(req);
             return resp;
         } catch (InterruptedException e) {
@@ -137,6 +184,56 @@ public class ChaturbateHttpClient extends HttpClient {
         }
     }
     
+    private Response refreshCookiesAndRetry(Request req, Response origResp) throws IOException {
+        log.debug("403 received from {}. Trying to refresh cookies with Flaresolverr", req.url().host());
+        
+        try {
+            cookieRefreshLock.writeLock().lock();            
+            
+            // we need to prevent repeated challenge requests from multiple threads, so we check if the clearance cookie needs updating
+            // maybe this can be done with some syncronization primitive, or maybe an expiresAt() check is enough
+            var cookie = Optional
+                .ofNullable(cookieJar.getCookies().get(req.url().topPrivateDomain()))
+                .flatMap(x -> cookieJar.getCookieFromCollection(x, "cf_clearance"));
+                
+            var cookieExpired = cookie.map(c -> 
+                Instant.ofEpochMilli(c.expiresAt()).isBefore(Instant.now())  // by time
+                || req.headers("Cookie").stream().anyMatch(headerCookie -> headerCookie.contains(c.value()))  // we got 403 with current cookie present
+            ).orElse(true);
+                
+            if (cookieExpired || cookieErrorCounter.incrementAndGet() >= 5) {
+                cookieErrorCounter.set(0);
+                
+                var apiResponse = flaresolverr.getCookies(req.url().toString()).get();
+                if (apiResponse.getStatus().equals("ok")) {
+                    // update user agent. It should be the same for all sites, assuming we use the same api address every time
+                    if (!config.getSettings().flaresolverr.userAgent.equals(apiResponse.getUserAgent())) {
+                        config.getSettings().flaresolverr.userAgent = apiResponse.getUserAgent();
+                        config.save();
+                    }
+                    
+                    cookieJar.saveFromResponse(req.url(), apiResponse.getCookies());    
+                    persistCookies();  
+                    log.debug("Cookies successfully refreshed with Flaresolverr in {}", Duration.between(apiResponse.getStartTimestamp(), apiResponse.getEndTimestamp()));
+                } else {
+                    log.debug("Unsuccessful attempt to refresh cookies. Response from Flaresolverr: {}", apiResponse);
+                    return origResp;
+                }
+            } else {
+                log.debug("Looks like the cookies were refreshed already, skipping refreshing");
+            }
+        } catch (Exception e) {
+            log.warn("Error refreshing cookies with Flaresolverr", e);
+            return origResp;
+        } finally {
+            cookieRefreshLock.writeLock().unlock();
+        }
+        
+        origResp.close();
+        
+        return super.execute(req);
+    }
+
     private static void acquireSlot() throws InterruptedException {
         long pauseBetweenRequests = Config.getInstance().getSettings().chaturbateMsBetweenRequests;
         requestThrottle.acquire();

common/src/main/java/ctbrec/sites/chaturbate/ChaturbateModel.java

@@ -97,7 +97,7 @@ public class ChaturbateModel extends AbstractModel {
         int imageSize = 0;
         Request req = new Request.Builder()
                 .url(url)
-                .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
+                .header(USER_AGENT, site.getHttpClient().getEffectiveUserAgent())
                 .head()
                 .build();
         try (Response response = getSite().getHttpClient().execute(req)) {
@@ -190,7 +190,7 @@ public class ChaturbateModel extends AbstractModel {
                     .post(body)
                     .header(REFERER, "https://chaturbate.com/" + getName() + "/")
                     .header(X_REQUESTED_WITH, XML_HTTP_REQUEST)
-                    .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
+                    .header(USER_AGENT, site.getHttpClient().getEffectiveUserAgent())
                     .build();
             try (Response response = getSite().getHttpClient().execute(req)) {
                 if (!response.isSuccessful()) {
@@ -239,7 +239,7 @@ public class ChaturbateModel extends AbstractModel {
         // do an initial request to get cookies
         Request req = new Request.Builder()
                 .url(getUrl())
-                .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
+                .header(USER_AGENT, site.getHttpClient().getEffectiveUserAgent())
                 .build();
         Response resp = site.getHttpClient().execute(req);
         resp.close();
@@ -258,7 +258,7 @@ public class ChaturbateModel extends AbstractModel {
                 .header(ACCEPT, "*/*")
                 .header(ACCEPT_LANGUAGE, "en-US,en;q=0.5")
                 .header(REFERER, getUrl())
-                .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
+                .header(USER_AGENT, site.getHttpClient().getEffectiveUserAgent())
                 .header("X-CSRFToken", ((ChaturbateHttpClient) site.getHttpClient()).getToken())
                 .header(X_REQUESTED_WITH, XML_HTTP_REQUEST)
                 .build();
@@ -302,7 +302,7 @@ public class ChaturbateModel extends AbstractModel {
         Request req = new Request.Builder()
                 .url(getSite().getBaseUrl() + "/get_edge_hls_url_ajax/")
                 .post(body)
-                .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
+                .header(USER_AGENT, site.getHttpClient().getEffectiveUserAgent())
                 .header(X_REQUESTED_WITH, XML_HTTP_REQUEST)
                 .build();
         try (Response response = getSite().getHttpClient().execute(req)) {
@@ -364,7 +364,7 @@ public class ChaturbateModel extends AbstractModel {
         log.trace("Loading master playlist {}", streamInfo.url);
         Request req = new Request.Builder()
                 .url(streamInfo.url)
-                .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
+                .header(USER_AGENT, site.getHttpClient().getEffectiveUserAgent())
                 .build();
         try (Response response = getSite().getHttpClient().execute(req)) {
             if (response.isSuccessful()) {
@@ -385,7 +385,7 @@ public class ChaturbateModel extends AbstractModel {
     public boolean exists() throws IOException {
         Request req = new Request.Builder() // @formatter:off
                 .url(getUrl())
-                .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
+                .header(USER_AGENT, site.getHttpClient().getEffectiveUserAgent())
                 .header(ACCEPT_LANGUAGE, Locale.ENGLISH.getLanguage())
                 .build(); // @formatter:on
         try (Response response = getSite().getHttpClient().execute(req)) {

server/src/main/java/ctbrec/recorder/server/ConfigServlet.java

@@ -3,7 +3,6 @@ package ctbrec.recorder.server;
 import static javax.servlet.http.HttpServletResponse.*;
 
 import java.io.IOException;
-import java.lang.reflect.Field;
 import java.security.InvalidKeyException;
 import java.security.NoSuchAlgorithmException;
 import java.time.LocalTime;
@@ -21,6 +20,7 @@ import org.slf4j.LoggerFactory;
 import ctbrec.Config;
 import ctbrec.Settings;
 import ctbrec.Settings.SplitStrategy;
+import ctbrec.io.BoundField;
 
 public class ConfigServlet extends AbstractCtbrecServlet {
 
@@ -76,6 +76,9 @@ public class ConfigServlet extends AbstractCtbrecServlet {
             addParameter("webinterfacePassword", "Web-Interface Password", DataType.STRING, settings.webinterfacePassword, json);
             addParameter("servletContext", "Servlet Context", DataType.STRING, settings.servletContext, json);
             addParameter("logFFmpegOutput", "Log FFmpeg Output", DataType.BOOLEAN, settings.logFFmpegOutput, json);
+            addParameter("flaresolverr.apiUrl", "Flaresolverr API URL", DataType.STRING, settings.flaresolverr.apiUrl, json);
+            addParameter("flaresolverr.timeoutInMillis", "Flaresolverr request timeout (ms)", DataType.INTEGER, settings.flaresolverr.timeoutInMillis, json);
+            addParameter("chaturbateUseFlaresolverr", "Chaturbate: use Flaresolverr", DataType.BOOLEAN, settings.chaturbateUseFlaresolverr, json);
 
             resp.setStatus(SC_OK);
             resp.setContentType("application/json");
@@ -135,8 +138,8 @@ public class ConfigServlet extends AbstractCtbrecServlet {
                 Object typeCorrectedValue = correctType(type, value);
                 LOG.debug("{}: {}", key, value);
 
-                Field field = Settings.class.getField(key);
+                var field = BoundField.of(settings, key);
-                field.set(settings, typeCorrectedValue);
+                field.set(typeCorrectedValue);
             }
             config.save();
         } catch (Exception e) {