build-all.sh

@@ -1,6 +0,0 @@
-#!/bin/sh
-mvn clean -f ./master
-mvn verify -am -f ./master -pl :client -Djavafx.platform=win
-mvn verify -am -f ./master -pl :client -Djavafx.platform=linux
-mvn verify -am -f ./master -pl :client -Djavafx.platform=mac
-mvn verify -am -f ./master -pl :server

client/pom.xml

@@ -8,7 +8,7 @@
     <parent>
         <groupId>ctbrec</groupId>
         <artifactId>master</artifactId>
-        <version>5.3.2</version>
+        <version>5.3.1</version>
         <relativePath>../master</relativePath>
     </parent>
 

client/src/main/java/ctbrec/ui/controls/table/StatePersistingTableView.java

@@ -54,7 +54,7 @@ public class StatePersistingTableView<T> extends TableView<T> {
     protected void restoreColumnVisibility() {
         Map<String, Boolean> visibility = stateStore.loadColumnVisibility();
         for (TableColumn<T, ?> tc : getColumns()) {
-            tc.setVisible(visibility.getOrDefault(tc.getId(), tc.isVisible()));
+            tc.setVisible(visibility.getOrDefault(tc.getId(), true));
         }
     }
 

client/src/main/java/ctbrec/ui/settings/CtbrecPreferencesStorage.java

@@ -3,7 +3,6 @@ package ctbrec.ui.settings;
 import java.io.IOException;
 import java.lang.reflect.InvocationTargetException;
 import java.time.LocalTime;
-import java.util.Arrays;
 import java.util.List;
 import java.util.Objects;
 import java.util.Optional;
@@ -26,7 +25,6 @@ import ctbrec.ui.settings.api.PreferencesStorage;
 import ctbrec.ui.settings.api.Setting;
 import ctbrec.ui.settings.api.SimpleDirectoryProperty;
 import ctbrec.ui.settings.api.SimpleFileProperty;
-import ctbrec.ui.settings.api.SimpleJoinedStringListProperty;
 import ctbrec.ui.settings.api.SimpleRangeProperty;
 import ctbrec.io.BoundField;
 import javafx.beans.property.BooleanProperty;
@@ -42,7 +40,6 @@ import javafx.scene.control.CheckBox;
 import javafx.scene.control.ComboBox;
 import javafx.scene.control.Label;
 import javafx.scene.control.RadioButton;
-import javafx.scene.control.TextArea;
 import javafx.scene.control.TextField;
 import javafx.scene.control.ToggleGroup;
 import javafx.scene.layout.HBox;
@@ -100,8 +97,6 @@ public class CtbrecPreferencesStorage implements PreferencesStorage {
                 return createBooleanProperty(setting);
             } else if (prop instanceof ListProperty) {
                 return createComboBox(setting);
-            } else if (prop instanceof SimpleJoinedStringListProperty) {
-                return createStringListProperty(setting);
             } else if (prop instanceof StringProperty) {
                 return createStringProperty(setting);
             } else {
@@ -245,20 +240,6 @@ public class CtbrecPreferencesStorage implements PreferencesStorage {
         return ctrl;
     }
 
-    private Node createStringListProperty(Setting setting) {
-        var ctrl = new TextArea();
-        StringProperty prop = (StringProperty) setting.getProperty();
-        ctrl.textProperty().bindBidirectional(prop);
-        prop.addListener((obs, oldV, newV) -> saveValue(() -> {
-            //setUnchecked(setting.getKey(), Arrays.asList(newV.split("\n")));
-            if (setting.doesNeedRestart()) {
-                runRestartRequiredCallback();
-            }
-            config.save();
-        }));
-        return ctrl;
-    }
-
     @SuppressWarnings("unchecked")
     private Node createIntegerProperty(Setting setting) {
         var ctrl = new TextField();
@@ -350,30 +331,11 @@ public class CtbrecPreferencesStorage implements PreferencesStorage {
         var field = BoundField.of(settings, key);
         var o = field.get();
         if (!Objects.equals(n, o)) {
-            if (n instanceof List && o instanceof List) {
+            field.set(n); // NOSONAR
-                var list = (List<String>)o;
-                list.clear();
-                list.addAll((List<String>)n);
-            } else {
-                field.set(n); // NOSONAR
-            }
             return true;
         }
         return false;
     }
-    
-    private boolean setUnchecked(String key, Object n) throws IllegalAccessException, NoSuchFieldException, InvocationTargetException {        
-        var field = BoundField.of(settings, key);
-        var o = field.get();
-        if (n instanceof List && o instanceof List) {
-            var list = (List<String>)o;
-            list.clear();
-            list.addAll((List<String>)n);
-        } else {
-            field.set(n); // NOSONAR
-        }
-        return true;
-    }
 
     private void saveValue(Exec exe) {
         try {

client/src/main/java/ctbrec/ui/settings/SettingsTab.java

@@ -62,7 +62,6 @@ public class SettingsTab extends Tab implements TabSelectionListener {
 
     private SimpleStringProperty flaresolverrApiUrl;
     private SimpleIntegerProperty flaresolverrTimeoutInMillis;
-    private SimpleJoinedStringListProperty flaresolverrUseForDomains;
     private SimpleStringProperty httpUserAgent;
     private SimpleStringProperty httpUserAgentMobile;
     private SimpleIntegerProperty overviewUpdateIntervalInSecs;
@@ -136,9 +135,6 @@ public class SettingsTab extends Tab implements TabSelectionListener {
     private SimpleStringProperty filterWhitelist;
     private SimpleBooleanProperty deleteOrphanedRecordingMetadata;
     private SimpleIntegerProperty restrictBitrate;
-    private SimpleIntegerProperty configSavingDelayMs;
-    private SimpleIntegerProperty httpClientMaxRequests;
-    private SimpleIntegerProperty httpClientMaxRequestsPerHost;
 
     public SettingsTab(List<Site> sites, Recorder recorder) {
         this.sites = sites;
@@ -152,8 +148,6 @@ public class SettingsTab extends Tab implements TabSelectionListener {
     private void initializeProperties() {
         flaresolverrApiUrl = new SimpleStringProperty(null, "flaresolverr.apiUrl", settings.flaresolverr.apiUrl);
         flaresolverrTimeoutInMillis = new SimpleIntegerProperty(null, "flaresolverr.timeoutInMillis", settings.flaresolverr.timeoutInMillis);
-        flaresolverrUseForDomains = new SimpleJoinedStringListProperty(null, "flaresolverr.useForDomains", "\n", 
-                FXCollections.observableList(settings.flaresolverr.useForDomains));
         httpUserAgent = new SimpleStringProperty(null, "httpUserAgent", settings.httpUserAgent);
         httpUserAgentMobile = new SimpleStringProperty(null, "httpUserAgentMobile", settings.httpUserAgentMobile);
         overviewUpdateIntervalInSecs = new SimpleIntegerProperty(null, "overviewUpdateIntervalInSecs", settings.overviewUpdateIntervalInSecs);
@@ -225,9 +219,6 @@ public class SettingsTab extends Tab implements TabSelectionListener {
         filterWhitelist = new SimpleStringProperty(null, "filterWhitelist", settings.filterWhitelist);
         deleteOrphanedRecordingMetadata = new SimpleBooleanProperty(null, "deleteOrphanedRecordingMetadata", settings.deleteOrphanedRecordingMetadata);
         restrictBitrate = new SimpleIntegerProperty(null, "restrictBitrate", settings.restrictBitrate);
-        configSavingDelayMs = new SimpleIntegerProperty(null, "configSavingDelayMs", settings.configSavingDelayMs);
-        httpClientMaxRequests = new SimpleIntegerProperty(null, "httpClientMaxRequests", settings.httpClientMaxRequests);
-        httpClientMaxRequestsPerHost = new SimpleIntegerProperty(null, "httpClientMaxRequestsPerHost", settings.httpClientMaxRequestsPerHost);
     }
 
     private void createGui() {
@@ -275,8 +266,7 @@ public class SettingsTab extends Tab implements TabSelectionListener {
                                 
                         Group.of("Flaresolverr",
                                 Setting.of("API URL", flaresolverrApiUrl),
-                                Setting.of("Request timeout", flaresolverrTimeoutInMillis),
+                                Setting.of("Request timeout", flaresolverrTimeoutInMillis))),
-                                Setting.of("Use for domains (one per line)", flaresolverrUseForDomains))),
                 Category.of("Look & Feel",
                         Group.of("Look & Feel",
                                 Setting.of("Colors (Base / Accent)", new ColorSettingsPane(Config.getInstance())).needsRestart(),
@@ -342,17 +332,7 @@ public class SettingsTab extends Tab implements TabSelectionListener {
                                 Setting.of("Password", proxyPassword).needsRestart())),
                 Category.of("Advanced / Devtools",
                         Group.of("Networking",
-                                Setting.of("Playlist request timeout (ms)", playlistRequestTimeout, "Timeout in ms for playlist requests"),
+                                Setting.of("Playlist request timeout (ms)", playlistRequestTimeout, "Timeout in ms for playlist requests")),
-                                Setting.of("Max requests", httpClientMaxRequests, 
-                                                "The maximum number of requests to execute concurrently. Above this requests queue in memory,\n" + //
-                                                "waiting for the running calls to complete.\n\n" + //
-                                                "If more than [maxRequests] requests are in flight when this is invoked, those requests will remain in flight."),
-                                Setting.of("Max requests per host", httpClientMaxRequestsPerHost, 
-                                                "The maximum number of requests for each host to execute concurrently. This limits requests by\n" + //
-                                                "the URL's host name. Note that concurrent requests to a single IP address may still exceed this\n" + //
-                                                "limit: multiple hostnames may share an IP address or be routed through the same HTTP proxy.\n\n" + //
-                                                "If more than [maxRequestsPerHost] requests are in flight when this is invoked, those requests will remain in flight.\n\n" + //
-                                                "WebSocket connections to hosts **do not** count against this limit.")),
                         Group.of("Logging",
                                 Setting.of("Log FFmpeg output", logFFmpegOutput, "Log FFmpeg output to files in the system's temp directory"),
                                 Setting.of("Log missed segments", logMissedSegments,
@@ -361,10 +341,7 @@ public class SettingsTab extends Tab implements TabSelectionListener {
                                 Setting.of("Use hlsdl (if possible)", useHlsdl,
                                         "Use hlsdl to record the live streams. Some features might not work correctly."),
                                 Setting.of("hlsdl executable", hlsdlExecutable, "Path to the hlsdl executable"),
-                                Setting.of("Log hlsdl output", loghlsdlOutput, "Log hlsdl output to files in the system's temp directory")),
+                                Setting.of("Log hlsdl output", loghlsdlOutput, "Log hlsdl output to files in the system's temp directory"))));
-                        Group.of("Miscelaneous",
-                                Setting.of("Config file saving delay (ms)", configSavingDelayMs, 
-                                        "Wait specified number of milliseconds before actually writing config to disk"))));
         Region preferencesView = prefs.getView();
         prefs.onRestartRequired(this::showRestartRequired);
         storage.setPreferences(prefs);

client/src/main/java/ctbrec/ui/settings/api/SimpleJoinedStringListProperty.java

@@ -1,48 +0,0 @@
-package ctbrec.ui.settings.api;
-
-import javafx.beans.binding.Bindings;
-import javafx.beans.property.SimpleListProperty;
-import javafx.beans.property.SimpleStringProperty;
-import javafx.collections.ListChangeListener;
-import javafx.collections.ObservableList;
-import lombok.Getter;
-
-public class SimpleJoinedStringListProperty extends SimpleStringProperty implements ListChangeListener<String> {    
-    private ObservableList<String> list;
-    @Getter
-    private String delimiter;
-    private boolean updating = false;
-
-    public SimpleJoinedStringListProperty(Object bean, String name, String delimiter, ObservableList<String> initialValue) {
-        super(bean, name, String.join(delimiter, initialValue));
-        this.delimiter = delimiter;   
-        this.list = initialValue;
-        initialValue.addListener(this);
-    }
-
-
-    @Override
-    public void setValue(String newValue) {
-        if (!updating) {
-            try {
-                updating = true;
-                list.setAll(newValue.split(delimiter));
-                super.setValue(newValue);
-            } finally {
-                updating = false;
-            }
-        }
-    }
-
-    @Override
-    public void onChanged(Change<? extends String> c) {
-        if (!updating) {
-            try {
-                updating = true;
-                super.setValue(String.join(delimiter, list));
-            } finally {
-                updating = false;
-            }
-        }
-    }
-}

client/src/main/java/ctbrec/ui/sites/chaturbate/ChaturbateConfigUi.java

@@ -99,6 +99,18 @@ public class ChaturbateConfigUi extends AbstractConfigUI {
         GridPane.setHgrow(requestThrottle, Priority.ALWAYS);
         GridPane.setColumnSpan(requestThrottle, 2);
         layout.add(requestThrottle, 1, row++);
+        
+        var label = new Label("Use Flaresolverr");
+        label.setTooltip(new Tooltip("Use Flaresolverr for solving the Cloudflare challenge. This also overrides the User Agent used for HTTP requests (only for the site)"));
+        layout.add(label, 0, row);
+        var flaresolverrToggle = new CheckBox();
+        flaresolverrToggle.setSelected(settings.chaturbateUseFlaresolverr);
+        flaresolverrToggle.setOnAction(e -> {
+            settings.chaturbateUseFlaresolverr = flaresolverrToggle.isSelected();
+            save();
+        });
+        GridPane.setMargin(flaresolverrToggle, new Insets(0, 0, SettingsTab.CHECKBOX_MARGIN, SettingsTab.CHECKBOX_MARGIN));
+        layout.add(flaresolverrToggle, 1, row++);
 
         var createAccount = new Button("Create new Account");
         createAccount.setOnAction(e -> DesktopIntegration.open(Chaturbate.REGISTRATION_LINK));

common/pom.xml

@@ -8,7 +8,7 @@
     <parent>
         <groupId>ctbrec</groupId>
         <artifactId>master</artifactId>
-        <version>5.3.2</version>
+        <version>5.3.1</version>
         <relativePath>../master</relativePath>
     </parent>
 

common/src/main/java/ctbrec/Config.java

@@ -184,15 +184,12 @@ public class Config {
         migrateOldSettings();
     }
 
-    private String migrateJson(String jsonStr) {
+    private String migrateJson(String json) {
-        var json = new JSONObject(jsonStr);
+        return migrateTo5_1_2(json);
-        migrateTo5_1_2(json);
-        migrateTo5_3_2(json);
-        return json.toString();
     }
 
-    private void migrateTo5_1_2(JSONObject json) {
+    private String migrateTo5_1_2(String json) {
-        JSONObject s = json;
+        JSONObject s = new JSONObject(json);
 
         if (s.has("models")) {
             JSONArray models = s.getJSONArray("models");
@@ -220,16 +217,7 @@ public class Config {
                 }
             }
         }
-    }
+        return s.toString();
-    
-    private void migrateTo5_3_2(JSONObject json) {
-        if (json.has("chaturbateUseFlaresolverr") && json.has("flaresolverr")) {
-            var fsr = json.getJSONObject("flaresolverr");
-            
-            if (!fsr.has("useForDomains") && json.getBoolean("chaturbateUseFlaresolverr")) {
-                fsr.put("useForDomains", new JSONArray().put("chaturbate.com"));
-            }
-        }
     }
 
     private void migrateOldSettings() {

common/src/main/java/ctbrec/Settings.java

@@ -46,7 +46,6 @@ public class Settings {
         public String apiUrl = "http://localhost:8191/v1";
         public int timeoutInMillis = 60000;
         public String userAgent = "";
-        public List<String> useForDomains = new ArrayList<>(); //(List.of("chaturbate.com", "bongacams.com"));
     };
 
     public FlaresolverrSettings flaresolverr = new FlaresolverrSettings();
@@ -62,7 +61,6 @@ public class Settings {
     public String chaturbatePassword = "";
     public String chaturbateUsername = "";
     public String chaturbateBaseUrl = "https://chaturbate.com";
-    @Deprecated
     public boolean chaturbateUseFlaresolverr = false;
     public int chaturbateMsBetweenRequests = 1000;
     public String cherryTvPassword = "";
@@ -145,8 +143,6 @@ public class Settings {
     @Deprecated
     public String postProcessing = "";
     public int playlistRequestTimeout = 2000;
-    public int httpClientMaxRequests = 64;
-    public int httpClientMaxRequestsPerHost = 16;
     public int postProcessingThreads = 2;
     public List<PostProcessorDto> postProcessors = new ArrayList<>();
     public String proxyHost;
@@ -232,5 +228,4 @@ public class Settings {
     public String filterWhitelist = "";
     public boolean checkResolutionByMinSide = false;
     public int restrictBitrate = 0;
-    public int configSavingDelayMs = 400;
 }

common/src/main/java/ctbrec/io/HttpClient.java

@@ -26,21 +26,15 @@ import java.security.NoSuchAlgorithmException;
 import java.security.SecureRandom;
 import java.security.cert.X509Certificate;
 import java.text.NumberFormat;
-import java.time.Instant;
 import java.util.*;
 import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
 import java.util.stream.Collectors;
 import java.util.zip.GZIPInputStream;
-import java.time.*;
-import java.util.Optional;
 
 import static ctbrec.io.HttpConstants.ACCEPT_ENCODING_GZIP;
 import static ctbrec.io.HttpConstants.CONTENT_ENCODING;
 import static java.nio.charset.StandardCharsets.UTF_8;
 
-
 @Slf4j
 public abstract class HttpClient {
     @Getter
@@ -55,31 +49,12 @@ public abstract class HttpClient {
     protected long cacheSize;
     protected int cacheLifeTime = 600;
     private final String name;
-    
-    protected final FlaresolverrClient flaresolverr;
-    // a lock to prevent multiple requests from 
-    ReentrantReadWriteLock cookieRefreshLock = new ReentrantReadWriteLock();
-    AtomicInteger cookieErrorCounter = new AtomicInteger(0);
 
     protected HttpClient(String name, Config config) {
         this.name = name;
         this.config = config;
         cookieJar = createCookieJar();
         reconfigure();
-                
-        if (!config.getSettings().flaresolverr.useForDomains.isEmpty()) {
-            flaresolverr = new FlaresolverrClient(config.getSettings().flaresolverr.apiUrl, config.getSettings().flaresolverr.timeoutInMillis);
-            
-            // try {
-            //     flaresolverr.createSession("ctbrec").get();
-            // } catch (InterruptedException e) {
-            //     Thread.currentThread().interrupt();
-            // } catch (Exception e) {
-            //     log.error("Error starting Flaresolverr session", e);
-            // }
-        } else {
-            flaresolverr = null;
-        }
     }
 
     protected CookieJarImpl createCookieJar() {
@@ -133,33 +108,16 @@ public abstract class HttpClient {
         }
     }
 
-    private Response execute(Call call) throws IOException {   
+    public Response execute(Request req) throws IOException {
-        Response resp;
+        Response resp = client.newCall(req).execute();
-        
-        try {
-            cookieRefreshLock.readLock().lock();
-            resp = call.execute();
-        } finally {
-            cookieRefreshLock.readLock().unlock();
-        }
-        
-        // try to solve the cloudflare challenge if we got one (clearance cookie expired, update it)
-        if (resp.code() == 403 && config.getSettings().flaresolverr.useForDomains.contains(call.request().url().host())) {
-            resp = refreshCookiesAndRetry(call.request(), resp);
-        }
-        
         return resp;
     }
 
-    public Response execute(Request req) throws IOException {   
-        return execute(client.newCall(req));
-    }
-
     public Response execute(Request request, int timeoutInMillis) throws IOException {
-            return execute(client.newBuilder() //
+        return client.newBuilder() //
-                            .connectTimeout(timeoutInMillis, TimeUnit.MILLISECONDS) //
+                .connectTimeout(timeoutInMillis, TimeUnit.MILLISECONDS) //
-                            .readTimeout(timeoutInMillis, TimeUnit.MILLISECONDS).build() //
+                .readTimeout(timeoutInMillis, TimeUnit.MILLISECONDS).build() //
-                            .newCall(request));
+                .newCall(request).execute();
     }
 
     public Response executeWithCache(Request req) throws IOException {
@@ -176,56 +134,6 @@ public abstract class HttpClient {
             return execute(req);
         }
     }
-    
-    private Response refreshCookiesAndRetry(Request req, Response origResp) throws IOException {
-        log.debug("403 received from {}. Trying to refresh cookies with Flaresolverr", req.url().host());
-        
-        try {
-            cookieRefreshLock.writeLock().lock();            
-            
-            // we need to prevent repeated challenge requests from multiple threads, so we check if the clearance cookie needs updating
-            // maybe this can be done with some syncronization primitive, or maybe an expiresAt() check is enough
-            var cookie = Optional
-                .ofNullable(cookieJar.getCookies().get(req.url().topPrivateDomain()))
-                .flatMap(x -> cookieJar.getCookieFromCollection(x, "cf_clearance"));
-                
-            var cookieExpired = cookie.map(c -> 
-                Instant.ofEpochMilli(c.expiresAt()).isBefore(Instant.now())  // by time
-                || req.headers("Cookie").stream().anyMatch(headerCookie -> headerCookie.contains(c.value()))  // we got 403 with current cookie present
-            ).orElse(true);
-                
-            if (cookieExpired || cookieErrorCounter.incrementAndGet() >= 5) {
-                cookieErrorCounter.set(0);
-                
-                var apiResponse = flaresolverr.getCookies(req.url().toString()).get();
-                if (apiResponse.getStatus().equals("ok")) {
-                    // update user agent. It should be the same for all sites, assuming we use the same api address every time
-                    if (!config.getSettings().flaresolverr.userAgent.equals(apiResponse.getUserAgent())) {
-                        config.getSettings().flaresolverr.userAgent = apiResponse.getUserAgent();
-                        config.save();
-                    }
-                    
-                    cookieJar.saveFromResponse(req.url(), apiResponse.getCookies());    
-                    persistCookies();  
-                    log.debug("Cookies successfully refreshed with Flaresolverr in {}", Duration.between(apiResponse.getStartTimestamp(), apiResponse.getEndTimestamp()));
-                } else {
-                    log.debug("Unsuccessful attempt to refresh cookies. Response from Flaresolverr: {}", apiResponse);
-                    return origResp;
-                }
-            } else {
-                log.debug("Looks like the cookies were refreshed already, skipping refreshing");
-            }
-        } catch (Exception e) {
-            log.warn("Error refreshing cookies with Flaresolverr", e);
-            return origResp;
-        } finally {
-            cookieRefreshLock.writeLock().unlock();
-        }
-        
-        origResp.close();
-        
-        return execute(req);
-    }
 
     public abstract boolean login() throws IOException;
 
@@ -263,8 +171,6 @@ public abstract class HttpClient {
         }
 
         client = builder.build();
-        client.dispatcher().setMaxRequests(config.getSettings().httpClientMaxRequests);
-        client.dispatcher().setMaxRequestsPerHost(config.getSettings().httpClientMaxRequestsPerHost);
     }
 
     /**

common/src/main/java/ctbrec/recorder/SimplifiedLocalRecorder.java

@@ -580,7 +580,7 @@ public class SimplifiedLocalRecorder implements Recorder {
                 }
             }
         };
-        saveConfigTimer.schedule(saveConfigTask, config.getSettings().configSavingDelayMs);
+        saveConfigTimer.schedule(saveConfigTask, 400);
     }
 
     @Override

common/src/main/java/ctbrec/recorder/download/hls/AbstractHlsDownload.java

@@ -122,11 +122,7 @@ public abstract class AbstractHlsDownload extends AbstractDownload {
             handleMissedSegments(segmentPlaylist, nextSegmentNumber);
             enqueueNewSegments(segmentPlaylist, nextSegmentNumber);
             splitRecordingIfNecessary();
-            // by the spec we must wait `targetDuration` before next playlist request if there are changes
+            calculateRescheduleTime();
-            // if there are none - half that amount
-            calculateRescheduleTime(playlistChanged(segmentPlaylist, nextSegmentNumber) 
-                                    ? segmentPlaylist.targetDuration*1000 
-                                    : segmentPlaylist.targetDuration*500);
             processFinishedSegments();
 
             // this if-check makes sure, that we don't decrease nextSegment. for some reason
@@ -179,11 +175,6 @@ public abstract class AbstractHlsDownload extends AbstractDownload {
         
         return this;
     }
-    
-    private boolean playlistChanged(SegmentPlaylist segmentPlaylist, int nextSegmentNumber) {
-        return segmentPlaylist.seq + segmentPlaylist.segments.size() > nextSegmentNumber;
-    }
-    
 
     protected void processFinishedSegments() {
         downloadExecutor.submit(() -> {
@@ -436,8 +427,8 @@ public abstract class AbstractHlsDownload extends AbstractDownload {
         return new SegmentDownload(model, playlist, segment, client, targetStream);
     }
 
-    private void calculateRescheduleTime(float duration_ms) {
+    private void calculateRescheduleTime() {
-        rescheduleTime = beforeLastPlaylistRequest.plusMillis(Math.max((int)Math.ceil(duration_ms), 250));
+        rescheduleTime = beforeLastPlaylistRequest.plusMillis(1000);
         if (Instant.now().isAfter(rescheduleTime))
             rescheduleTime = Instant.now();
         recordingEvents.add(RecordingEvent.of("next playlist download scheduled for " + rescheduleTime.toString()));

common/src/main/java/ctbrec/sites/bonga/BongaCamsModel.java

@@ -38,6 +38,8 @@ public class BongaCamsModel extends AbstractModel {
     private static final String SUCCESS = "success";
     private static final String STATUS = "status";
 
+    private static final Pattern ONLINE_BADGE_REGEX = Pattern.compile("class=\"badge_online\s*\"");
+
     @Setter
     private boolean online = false;
     private final transient List<StreamSource> streamSources = new ArrayList<>();
@@ -49,6 +51,13 @@ public class BongaCamsModel extends AbstractModel {
     @Override
     public boolean isOnline(boolean ignoreCache) throws IOException, ExecutionException, InterruptedException {
         if (ignoreCache) {
+            boolean modelIsConnected = basicOnlineCheck();
+            if (!modelIsConnected) {
+                onlineState = OFFLINE;
+                online = false;
+                return false;
+            }
+
             return completeOnlineCheck();
         }
         return online;
@@ -64,34 +73,45 @@ public class BongaCamsModel extends AbstractModel {
         setDisplayName(performerData.optString("displayName"));
         String chatType = performerData.optString("showType");
         boolean isAway = performerData.optBoolean("isAway");
-        
-        // looks like isOnline key is new. Treat it's absence as true (old behavior)
-        boolean jsonIsOnline = performerData.optBoolean("isOnline", true);
 
-        if (!jsonIsOnline) {
+        onlineState = mapState(chatType);
-            onlineState = OFFLINE;
+        if (onlineState == ONLINE) {
-            online = false;
+            if (isStreamAvailable()) {
-        } else {
+                if (isAway) {
-            onlineState = mapState(chatType);
-            if (onlineState == ONLINE) {
-                if (isStreamAvailable()) {
-                    if (isAway) {
-                        onlineState = AWAY;
-                        online = false;
-                    } else {
-                        online = true;
-                    }
-                } else {
-                    online = false;
                     onlineState = AWAY;
+                    online = false;
+                } else {
+                    online = true;
                 }
             } else {
                 online = false;
+                onlineState = AWAY;
             }
+        } else {
+            online = false;
         }
         return online;
     }
 
+    private boolean basicOnlineCheck() {
+        try {
+            String url = site.getBaseUrl() + "/profile/" + getName().toLowerCase();
+            Request req = newRequestBuilder().url(url).build();
+            try (Response resp = site.getHttpClient().execute(req)) {
+                if (resp.isSuccessful()) {
+                    String body = Objects.requireNonNull(resp.body(), HTTP_RESPONSE_BODY_IS_NULL).string();
+                    Matcher m = ONLINE_BADGE_REGEX.matcher(body);
+                    return m.find();
+                } else {
+                    return false;
+                }
+            }
+        } catch (Exception e) {
+            log.warn("Couldn't check if model is connected: {}", e.getLocalizedMessage());
+            return false;
+        }
+    }
+
     public State mapState(String roomState) {
         return switch (roomState) {
             case "private", "fullprivate" -> PRIVATE;

common/src/main/java/ctbrec/sites/chaturbate/ChaturbateHttpClient.java

@@ -1,15 +1,20 @@
 package ctbrec.sites.chaturbate;
 
 import ctbrec.Config;
+import ctbrec.io.FlaresolverrClient;
 import ctbrec.io.HtmlParser;
 import ctbrec.io.HttpClient;
 import lombok.extern.slf4j.Slf4j;
 import okhttp3.*;
 
+import java.time.*;
 import java.io.IOException;
 import java.io.InterruptedIOException;
 import java.util.NoSuchElementException;
+import java.util.Optional;
 import java.util.concurrent.Semaphore;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
 
 import static ctbrec.io.HttpConstants.REFERER;
 import static ctbrec.io.HttpConstants.USER_AGENT;
@@ -19,13 +24,31 @@ public class ChaturbateHttpClient extends HttpClient {
 
     private static final String PATH = "/auth/login/"; // NOSONAR
     protected String token;
+    protected final FlaresolverrClient flaresolverr;
 
     private static final Semaphore requestThrottle = new Semaphore(2, true);
     private static long lastRequest = 0;
     
+    // a lock to prevent multiple requests from 
+    ReentrantReadWriteLock cookieRefreshLock = new ReentrantReadWriteLock();
+    AtomicInteger cookieErrorCounter = new AtomicInteger(0);
 
     public ChaturbateHttpClient(Config config) {
         super("chaturbate", config);
+        
+        if (config.getSettings().chaturbateUseFlaresolverr) {
+            flaresolverr = new FlaresolverrClient(config.getSettings().flaresolverr.apiUrl, config.getSettings().flaresolverr.timeoutInMillis);
+            
+            // try {
+            //     flaresolverr.createSession("ctbrec").get();
+            // } catch (InterruptedException e) {
+            //     Thread.currentThread().interrupt();
+            // } catch (Exception e) {
+            //     log.error("Error starting Flaresolverr session", e);
+            // }
+        } else {
+            flaresolverr = null;
+        }
     }
     
     @Override
@@ -135,7 +158,19 @@ public class ChaturbateHttpClient extends HttpClient {
                 acquireSlot();
             }
             
-            Response resp = super.execute(req);
+            Response resp;
+            
+            try {
+                cookieRefreshLock.readLock().lock();
+                resp = super.execute(req);
+            } finally {
+                cookieRefreshLock.readLock().unlock();
+            }
+            
+            // try to solve the cloudflare challenge if we got one (clearance cookie expired, update it)
+            if (resp.code() == 403 && flaresolverr != null) {
+                resp = refreshCookiesAndRetry(req, resp);
+            }
             
             extractCsrfToken(req);
             return resp;
@@ -149,6 +184,55 @@ public class ChaturbateHttpClient extends HttpClient {
         }
     }
     
+    private Response refreshCookiesAndRetry(Request req, Response origResp) throws IOException {
+        log.debug("403 received from {}. Trying to refresh cookies with Flaresolverr", req.url().host());
+        
+        try {
+            cookieRefreshLock.writeLock().lock();            
+            
+            // we need to prevent repeated challenge requests from multiple threads, so we check if the clearance cookie needs updating
+            // maybe this can be done with some syncronization primitive, or maybe an expiresAt() check is enough
+            var cookie = Optional
+                .ofNullable(cookieJar.getCookies().get(req.url().topPrivateDomain()))
+                .flatMap(x -> cookieJar.getCookieFromCollection(x, "cf_clearance"));
+                
+            var cookieExpired = cookie.map(c -> 
+                Instant.ofEpochMilli(c.expiresAt()).isBefore(Instant.now())  // by time
+                || req.headers("Cookie").stream().anyMatch(headerCookie -> headerCookie.contains(c.value()))  // we got 403 with current cookie present
+            ).orElse(true);
+                
+            if (cookieExpired || cookieErrorCounter.incrementAndGet() >= 5) {
+                cookieErrorCounter.set(0);
+                
+                var apiResponse = flaresolverr.getCookies(req.url().toString()).get();
+                if (apiResponse.getStatus().equals("ok")) {
+                    // update user agent. It should be the same for all sites, assuming we use the same api address every time
+                    if (!config.getSettings().flaresolverr.userAgent.equals(apiResponse.getUserAgent())) {
+                        config.getSettings().flaresolverr.userAgent = apiResponse.getUserAgent();
+                        config.save();
+                    }
+                    
+                    cookieJar.saveFromResponse(req.url(), apiResponse.getCookies());    
+                    persistCookies();  
+                    log.debug("Cookies successfully refreshed with Flaresolverr in {}", Duration.between(apiResponse.getStartTimestamp(), apiResponse.getEndTimestamp()));
+                } else {
+                    log.debug("Unsuccessful attempt to refresh cookies. Response from Flaresolverr: {}", apiResponse);
+                    return origResp;
+                }
+            } else {
+                log.debug("Looks like the cookies were refreshed already, skipping refreshing");
+            }
+        } catch (Exception e) {
+            log.warn("Error refreshing cookies with Flaresolverr", e);
+            return origResp;
+        } finally {
+            cookieRefreshLock.writeLock().unlock();
+        }
+        
+        origResp.close();
+        
+        return super.execute(req);
+    }
 
     private static void acquireSlot() throws InterruptedException {
         long pauseBetweenRequests = Config.getInstance().getSettings().chaturbateMsBetweenRequests;

common/src/test/java/ctbrec/recorder/RecordingPreconditionsTest.java

@@ -159,14 +159,12 @@ class RecordingPreconditionsTest {
         when(mockita.toString()).thenReturn("Mockita Boobilicious");
         when(mockita.isOnline(true)).thenReturn(true);
         when(mockita.getUrl()).thenReturn("http://localhost/mockita");
-        when(mockita.getPriority()).thenReturn(0);
 
         Model theOtherOne = mock(Model.class);
         when(theOtherOne.getRecordUntil()).thenReturn(Instant.MAX);
         when(theOtherOne.toString()).thenReturn("The Other One");
         when(theOtherOne.isOnline(true)).thenReturn(true);
         when(theOtherOne.getUrl()).thenReturn("http://localhost/theOtherOne");
-        when(theOtherOne.getPriority()).thenReturn(10);
 
         ModelGroup group = new ModelGroup();
         group.add(theOtherOne);

master/pom.xml

@@ -6,7 +6,7 @@
     <groupId>ctbrec</groupId>
     <artifactId>master</artifactId>
     <packaging>pom</packaging>
-    <version>5.3.2</version>
+    <version>5.3.1</version>
 
     <modules>
         <module>../common</module>

server/pom.xml

@@ -8,7 +8,7 @@
     <parent>
         <groupId>ctbrec</groupId>
         <artifactId>master</artifactId>
-        <version>5.3.2</version>
+        <version>5.3.1</version>
         <relativePath>../master</relativePath>
     </parent>
 

server/src/main/java/ctbrec/recorder/server/ConfigServlet.java

@@ -6,10 +6,6 @@ import java.io.IOException;
 import java.security.InvalidKeyException;
 import java.security.NoSuchAlgorithmException;
 import java.time.LocalTime;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Enumeration;
-import java.util.*;
 
 import javax.servlet.ServletException;
 import javax.servlet.http.HttpServletRequest;
@@ -33,7 +29,7 @@ public class ConfigServlet extends AbstractCtbrecServlet {
     private Settings settings;
 
     public enum DataType {
-        STRING, BOOLEAN, INTEGER, LONG, DOUBLE, SPLIT_STRATEGY, TIME, STRING_LIST
+        STRING, BOOLEAN, INTEGER, LONG, DOUBLE, SPLIT_STRATEGY, TIME
     }
 
     public ConfigServlet(Config config) {
@@ -55,8 +51,6 @@ public class ConfigServlet extends AbstractCtbrecServlet {
             JSONArray json = new JSONArray();
             addParameter("concurrentRecordings", "Concurrent Recordings", DataType.INTEGER, settings.concurrentRecordings, json);
             addParameter("chaturbateMsBetweenRequests", "Chaturbate time between requests (ms)", DataType.INTEGER, settings.chaturbateMsBetweenRequests, json);
-            addParameter("defaultPriority", "Default Priority", DataType.INTEGER, settings.defaultPriority, json);
-            addParameter("deleteOrphanedRecordingMetadata", "Delete orphaned recording metadata", DataType.BOOLEAN, settings.deleteOrphanedRecordingMetadata, json);
             addParameter("ffmpegFileSuffix", "File Suffix", DataType.STRING, settings.ffmpegFileSuffix, json);
             addParameter("ffmpegMergedDownloadArgs", "FFmpeg Parameters", DataType.STRING, settings.ffmpegMergedDownloadArgs, json);
             addParameter("httpPort", "HTTP port", DataType.INTEGER, settings.httpPort, json);
@@ -84,7 +78,7 @@ public class ConfigServlet extends AbstractCtbrecServlet {
             addParameter("logFFmpegOutput", "Log FFmpeg Output", DataType.BOOLEAN, settings.logFFmpegOutput, json);
             addParameter("flaresolverr.apiUrl", "Flaresolverr API URL", DataType.STRING, settings.flaresolverr.apiUrl, json);
             addParameter("flaresolverr.timeoutInMillis", "Flaresolverr request timeout (ms)", DataType.INTEGER, settings.flaresolverr.timeoutInMillis, json);
-            addParameter("flaresolverr.useForDomains", "Use Flaresolverr for domains (one per line)", DataType.STRING_LIST, settings.flaresolverr.useForDomains, json);
+            addParameter("chaturbateUseFlaresolverr", "Chaturbate: use Flaresolverr", DataType.BOOLEAN, settings.chaturbateUseFlaresolverr, json);
 
             resp.setStatus(SC_OK);
             resp.setContentType("application/json");
@@ -177,9 +171,6 @@ public class ConfigServlet extends AbstractCtbrecServlet {
         case TIME:
             corrected = LocalTime.parse(value.toString());
             break;
-        case STRING_LIST:
-            corrected = ((JSONArray)value).toList();
-            break;
         default:
             break;
         }

server/src/main/resources/html/static/config.js

@@ -16,29 +16,7 @@ function loadConfig() {
                 }
                 for (let i = 0; i < data.length; i++) {
                     let param = data[i];
-                    if (param.type !== 'STRING_LIST') {
+                    param.ko_value = ko.observable(param.value);
-                      // could not get ko.observable() to write to param.value
-                      // TODO: either fix that, or we need to take ko_value in saveConfig() instead
-                      param.ko_value = ko.pureComputed({
-                          read: function () {
-                              return this.value;
-                          },
-                          write: function (value) {
-                              this.value = value
-                          },
-                          owner: param
-                      });
-                    } else {
-                      param.ko_value = ko.pureComputed({
-                          read: function () {
-                              return this.value.join('\n');
-                          },
-                          write: function (value) {
-                              this.value = value.split('\n')
-                          },
-                          owner: param
-                      });
-                    }
                     observableSettingsArray.push(param);
                 }
             } else {

server/src/main/resources/html/static/index.html

@@ -210,14 +210,7 @@
                                 <tbody data-bind="foreach: settings">
                                 <tr>
                                     <td data-bind="text: name"></td>
-                                    <td>
+                                    <td><input class="form-control" data-bind="value: value" style="width: 100%"/></td>
-                                      <div data-bind="ifnot: type === 'STRING_LIST'">
-                                        <input class="form-control" data-bind="value: ko_value" style="width: 100%;" />
-                                      </div>
-                                      <div data-bind="if: type === 'STRING_LIST'">
-                                        <textarea rows="3" class="form-control" data-bind="value: ko_value" style="width: 100%;"></textarea>
-                                      </div>
-                                    </td>
                                 </tr>
                                 </tbody>
                             </table>