Respect column default visibility for new columns

- check new (?) field isOnline before attempting to get stream sources (was failing due to missing json field when offline)

build-all.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+mvn clean -f ./master
+mvn verify -am -f ./master -pl :client -Djavafx.platform=win
+mvn verify -am -f ./master -pl :client -Djavafx.platform=linux
+mvn verify -am -f ./master -pl :client -Djavafx.platform=mac
+mvn verify -am -f ./master -pl :server

client/pom.xml

@@ -8,7 +8,7 @@
     <parent>
         <groupId>ctbrec</groupId>
         <artifactId>master</artifactId>
-        <version>5.3.1</version>
+        <version>5.3.2</version>
         <relativePath>../master</relativePath>
     </parent>
 

client/src/main/java/ctbrec/ui/controls/table/StatePersistingTableView.java

@@ -54,7 +54,7 @@ public class StatePersistingTableView<T> extends TableView<T> {
     protected void restoreColumnVisibility() {
         Map<String, Boolean> visibility = stateStore.loadColumnVisibility();
         for (TableColumn<T, ?> tc : getColumns()) {
-            tc.setVisible(visibility.getOrDefault(tc.getId(), true));
+            tc.setVisible(visibility.getOrDefault(tc.getId(), tc.isVisible()));
         }
     }
 

client/src/main/java/ctbrec/ui/settings/CtbrecPreferencesStorage.java

@@ -3,6 +3,7 @@ package ctbrec.ui.settings;
 import java.io.IOException;
 import java.lang.reflect.InvocationTargetException;
 import java.time.LocalTime;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Objects;
 import java.util.Optional;
@@ -25,6 +26,7 @@ import ctbrec.ui.settings.api.PreferencesStorage;
 import ctbrec.ui.settings.api.Setting;
 import ctbrec.ui.settings.api.SimpleDirectoryProperty;
 import ctbrec.ui.settings.api.SimpleFileProperty;
+import ctbrec.ui.settings.api.SimpleJoinedStringListProperty;
 import ctbrec.ui.settings.api.SimpleRangeProperty;
 import ctbrec.io.BoundField;
 import javafx.beans.property.BooleanProperty;
@@ -40,6 +42,7 @@ import javafx.scene.control.CheckBox;
 import javafx.scene.control.ComboBox;
 import javafx.scene.control.Label;
 import javafx.scene.control.RadioButton;
+import javafx.scene.control.TextArea;
 import javafx.scene.control.TextField;
 import javafx.scene.control.ToggleGroup;
 import javafx.scene.layout.HBox;
@@ -97,6 +100,8 @@ public class CtbrecPreferencesStorage implements PreferencesStorage {
                 return createBooleanProperty(setting);
             } else if (prop instanceof ListProperty) {
                 return createComboBox(setting);
+            } else if (prop instanceof SimpleJoinedStringListProperty) {
+                return createStringListProperty(setting);
             } else if (prop instanceof StringProperty) {
                 return createStringProperty(setting);
             } else {
@@ -240,6 +245,20 @@ public class CtbrecPreferencesStorage implements PreferencesStorage {
         return ctrl;
     }
 
+    private Node createStringListProperty(Setting setting) {
+        var ctrl = new TextArea();
+        StringProperty prop = (StringProperty) setting.getProperty();
+        ctrl.textProperty().bindBidirectional(prop);
+        prop.addListener((obs, oldV, newV) -> saveValue(() -> {
+            //setUnchecked(setting.getKey(), Arrays.asList(newV.split("\n")));
+            if (setting.doesNeedRestart()) {
+                runRestartRequiredCallback();
+            }
+            config.save();
+        }));
+        return ctrl;
+    }
+
     @SuppressWarnings("unchecked")
     private Node createIntegerProperty(Setting setting) {
         var ctrl = new TextField();
@@ -331,12 +350,31 @@ public class CtbrecPreferencesStorage implements PreferencesStorage {
         var field = BoundField.of(settings, key);
         var o = field.get();
         if (!Objects.equals(n, o)) {
+            if (n instanceof List && o instanceof List) {
+                var list = (List<String>)o;
+                list.clear();
+                list.addAll((List<String>)n);
+            } else {
                 field.set(n); // NOSONAR
+            }
             return true;
         }
         return false;
     }
     
+    private boolean setUnchecked(String key, Object n) throws IllegalAccessException, NoSuchFieldException, InvocationTargetException {        
+        var field = BoundField.of(settings, key);
+        var o = field.get();
+        if (n instanceof List && o instanceof List) {
+            var list = (List<String>)o;
+            list.clear();
+            list.addAll((List<String>)n);
+        } else {
+            field.set(n); // NOSONAR
+        }
+        return true;
+    }
+
     private void saveValue(Exec exe) {
         try {
             exe.run();

client/src/main/java/ctbrec/ui/settings/SettingsTab.java

@@ -62,6 +62,7 @@ public class SettingsTab extends Tab implements TabSelectionListener {
 
     private SimpleStringProperty flaresolverrApiUrl;
     private SimpleIntegerProperty flaresolverrTimeoutInMillis;
+    private SimpleJoinedStringListProperty flaresolverrUseForDomains;
     private SimpleStringProperty httpUserAgent;
     private SimpleStringProperty httpUserAgentMobile;
     private SimpleIntegerProperty overviewUpdateIntervalInSecs;
@@ -135,6 +136,9 @@ public class SettingsTab extends Tab implements TabSelectionListener {
     private SimpleStringProperty filterWhitelist;
     private SimpleBooleanProperty deleteOrphanedRecordingMetadata;
     private SimpleIntegerProperty restrictBitrate;
+    private SimpleIntegerProperty configSavingDelayMs;
+    private SimpleIntegerProperty httpClientMaxRequests;
+    private SimpleIntegerProperty httpClientMaxRequestsPerHost;
 
     public SettingsTab(List<Site> sites, Recorder recorder) {
         this.sites = sites;
@@ -148,6 +152,8 @@ public class SettingsTab extends Tab implements TabSelectionListener {
     private void initializeProperties() {
         flaresolverrApiUrl = new SimpleStringProperty(null, "flaresolverr.apiUrl", settings.flaresolverr.apiUrl);
         flaresolverrTimeoutInMillis = new SimpleIntegerProperty(null, "flaresolverr.timeoutInMillis", settings.flaresolverr.timeoutInMillis);
+        flaresolverrUseForDomains = new SimpleJoinedStringListProperty(null, "flaresolverr.useForDomains", "\n", 
+                FXCollections.observableList(settings.flaresolverr.useForDomains));
         httpUserAgent = new SimpleStringProperty(null, "httpUserAgent", settings.httpUserAgent);
         httpUserAgentMobile = new SimpleStringProperty(null, "httpUserAgentMobile", settings.httpUserAgentMobile);
         overviewUpdateIntervalInSecs = new SimpleIntegerProperty(null, "overviewUpdateIntervalInSecs", settings.overviewUpdateIntervalInSecs);
@@ -219,6 +225,9 @@ public class SettingsTab extends Tab implements TabSelectionListener {
         filterWhitelist = new SimpleStringProperty(null, "filterWhitelist", settings.filterWhitelist);
         deleteOrphanedRecordingMetadata = new SimpleBooleanProperty(null, "deleteOrphanedRecordingMetadata", settings.deleteOrphanedRecordingMetadata);
         restrictBitrate = new SimpleIntegerProperty(null, "restrictBitrate", settings.restrictBitrate);
+        configSavingDelayMs = new SimpleIntegerProperty(null, "configSavingDelayMs", settings.configSavingDelayMs);
+        httpClientMaxRequests = new SimpleIntegerProperty(null, "httpClientMaxRequests", settings.httpClientMaxRequests);
+        httpClientMaxRequestsPerHost = new SimpleIntegerProperty(null, "httpClientMaxRequestsPerHost", settings.httpClientMaxRequestsPerHost);
     }
 
     private void createGui() {
@@ -266,7 +275,8 @@ public class SettingsTab extends Tab implements TabSelectionListener {
                                 
                         Group.of("Flaresolverr",
                                 Setting.of("API URL", flaresolverrApiUrl),
-                                Setting.of("Request timeout", flaresolverrTimeoutInMillis))),
+                                Setting.of("Request timeout", flaresolverrTimeoutInMillis),
+                                Setting.of("Use for domains (one per line)", flaresolverrUseForDomains))),
                 Category.of("Look & Feel",
                         Group.of("Look & Feel",
                                 Setting.of("Colors (Base / Accent)", new ColorSettingsPane(Config.getInstance())).needsRestart(),
@@ -332,7 +342,17 @@ public class SettingsTab extends Tab implements TabSelectionListener {
                                 Setting.of("Password", proxyPassword).needsRestart())),
                 Category.of("Advanced / Devtools",
                         Group.of("Networking",
-                                Setting.of("Playlist request timeout (ms)", playlistRequestTimeout, "Timeout in ms for playlist requests")),
+                                Setting.of("Playlist request timeout (ms)", playlistRequestTimeout, "Timeout in ms for playlist requests"),
+                                Setting.of("Max requests", httpClientMaxRequests, 
+                                                "The maximum number of requests to execute concurrently. Above this requests queue in memory,\n" + //
+                                                "waiting for the running calls to complete.\n\n" + //
+                                                "If more than [maxRequests] requests are in flight when this is invoked, those requests will remain in flight."),
+                                Setting.of("Max requests per host", httpClientMaxRequestsPerHost, 
+                                                "The maximum number of requests for each host to execute concurrently. This limits requests by\n" + //
+                                                "the URL's host name. Note that concurrent requests to a single IP address may still exceed this\n" + //
+                                                "limit: multiple hostnames may share an IP address or be routed through the same HTTP proxy.\n\n" + //
+                                                "If more than [maxRequestsPerHost] requests are in flight when this is invoked, those requests will remain in flight.\n\n" + //
+                                                "WebSocket connections to hosts **do not** count against this limit.")),
                         Group.of("Logging",
                                 Setting.of("Log FFmpeg output", logFFmpegOutput, "Log FFmpeg output to files in the system's temp directory"),
                                 Setting.of("Log missed segments", logMissedSegments,
@@ -341,7 +361,10 @@ public class SettingsTab extends Tab implements TabSelectionListener {
                                 Setting.of("Use hlsdl (if possible)", useHlsdl,
                                         "Use hlsdl to record the live streams. Some features might not work correctly."),
                                 Setting.of("hlsdl executable", hlsdlExecutable, "Path to the hlsdl executable"),
-                                Setting.of("Log hlsdl output", loghlsdlOutput, "Log hlsdl output to files in the system's temp directory"))));
+                                Setting.of("Log hlsdl output", loghlsdlOutput, "Log hlsdl output to files in the system's temp directory")),
+                        Group.of("Miscelaneous",
+                                Setting.of("Config file saving delay (ms)", configSavingDelayMs, 
+                                        "Wait specified number of milliseconds before actually writing config to disk"))));
         Region preferencesView = prefs.getView();
         prefs.onRestartRequired(this::showRestartRequired);
         storage.setPreferences(prefs);

client/src/main/java/ctbrec/ui/settings/api/SimpleJoinedStringListProperty.java

@@ -0,0 +1,48 @@
+package ctbrec.ui.settings.api;
+
+import javafx.beans.binding.Bindings;
+import javafx.beans.property.SimpleListProperty;
+import javafx.beans.property.SimpleStringProperty;
+import javafx.collections.ListChangeListener;
+import javafx.collections.ObservableList;
+import lombok.Getter;
+
+public class SimpleJoinedStringListProperty extends SimpleStringProperty implements ListChangeListener<String> {    
+    private ObservableList<String> list;
+    @Getter
+    private String delimiter;
+    private boolean updating = false;
+
+    public SimpleJoinedStringListProperty(Object bean, String name, String delimiter, ObservableList<String> initialValue) {
+        super(bean, name, String.join(delimiter, initialValue));
+        this.delimiter = delimiter;   
+        this.list = initialValue;
+        initialValue.addListener(this);
+    }
+
+
+    @Override
+    public void setValue(String newValue) {
+        if (!updating) {
+            try {
+                updating = true;
+                list.setAll(newValue.split(delimiter));
+                super.setValue(newValue);
+            } finally {
+                updating = false;
+            }
+        }
+    }
+
+    @Override
+    public void onChanged(Change<? extends String> c) {
+        if (!updating) {
+            try {
+                updating = true;
+                super.setValue(String.join(delimiter, list));
+            } finally {
+                updating = false;
+            }
+        }
+    }
+}

client/src/main/java/ctbrec/ui/sites/chaturbate/ChaturbateConfigUi.java

@@ -100,18 +100,6 @@ public class ChaturbateConfigUi extends AbstractConfigUI {
         GridPane.setColumnSpan(requestThrottle, 2);
         layout.add(requestThrottle, 1, row++);
 
-        var label = new Label("Use Flaresolverr");
-        label.setTooltip(new Tooltip("Use Flaresolverr for solving the Cloudflare challenge. This also overrides the User Agent used for HTTP requests (only for the site)"));
-        layout.add(label, 0, row);
-        var flaresolverrToggle = new CheckBox();
-        flaresolverrToggle.setSelected(settings.chaturbateUseFlaresolverr);
-        flaresolverrToggle.setOnAction(e -> {
-            settings.chaturbateUseFlaresolverr = flaresolverrToggle.isSelected();
-            save();
-        });
-        GridPane.setMargin(flaresolverrToggle, new Insets(0, 0, SettingsTab.CHECKBOX_MARGIN, SettingsTab.CHECKBOX_MARGIN));
-        layout.add(flaresolverrToggle, 1, row++);
-
         var createAccount = new Button("Create new Account");
         createAccount.setOnAction(e -> DesktopIntegration.open(Chaturbate.REGISTRATION_LINK));
         layout.add(createAccount, 1, row++);

common/pom.xml

@@ -8,7 +8,7 @@
     <parent>
         <groupId>ctbrec</groupId>
         <artifactId>master</artifactId>
-        <version>5.3.1</version>
+        <version>5.3.2</version>
         <relativePath>../master</relativePath>
     </parent>
 

common/src/main/java/ctbrec/Config.java

@@ -184,12 +184,15 @@ public class Config {
         migrateOldSettings();
     }
 
-    private String migrateJson(String json) {
-        return migrateTo5_1_2(json);
+    private String migrateJson(String jsonStr) {
+        var json = new JSONObject(jsonStr);
+        migrateTo5_1_2(json);
+        migrateTo5_3_2(json);
+        return json.toString();
     }
 
-    private String migrateTo5_1_2(String json) {
-        JSONObject s = new JSONObject(json);
+    private void migrateTo5_1_2(JSONObject json) {
+        JSONObject s = json;
 
         if (s.has("models")) {
             JSONArray models = s.getJSONArray("models");
@@ -217,7 +220,16 @@ public class Config {
                 }
             }
         }
-        return s.toString();
+    }
+    
+    private void migrateTo5_3_2(JSONObject json) {
+        if (json.has("chaturbateUseFlaresolverr") && json.has("flaresolverr")) {
+            var fsr = json.getJSONObject("flaresolverr");
+            
+            if (!fsr.has("useForDomains") && json.getBoolean("chaturbateUseFlaresolverr")) {
+                fsr.put("useForDomains", new JSONArray().put("chaturbate.com"));
+            }
+        }
     }
 
     private void migrateOldSettings() {

common/src/main/java/ctbrec/Settings.java

@@ -46,6 +46,7 @@ public class Settings {
         public String apiUrl = "http://localhost:8191/v1";
         public int timeoutInMillis = 60000;
         public String userAgent = "";
+        public List<String> useForDomains = new ArrayList<>(); //(List.of("chaturbate.com", "bongacams.com"));
     };
 
     public FlaresolverrSettings flaresolverr = new FlaresolverrSettings();
@@ -61,6 +62,7 @@ public class Settings {
     public String chaturbatePassword = "";
     public String chaturbateUsername = "";
     public String chaturbateBaseUrl = "https://chaturbate.com";
+    @Deprecated
     public boolean chaturbateUseFlaresolverr = false;
     public int chaturbateMsBetweenRequests = 1000;
     public String cherryTvPassword = "";
@@ -143,6 +145,8 @@ public class Settings {
     @Deprecated
     public String postProcessing = "";
     public int playlistRequestTimeout = 2000;
+    public int httpClientMaxRequests = 64;
+    public int httpClientMaxRequestsPerHost = 16;
     public int postProcessingThreads = 2;
     public List<PostProcessorDto> postProcessors = new ArrayList<>();
     public String proxyHost;
@@ -228,4 +232,5 @@ public class Settings {
     public String filterWhitelist = "";
     public boolean checkResolutionByMinSide = false;
     public int restrictBitrate = 0;
+    public int configSavingDelayMs = 400;
 }

common/src/main/java/ctbrec/io/HttpClient.java

@@ -26,15 +26,21 @@ import java.security.NoSuchAlgorithmException;
 import java.security.SecureRandom;
 import java.security.cert.X509Certificate;
 import java.text.NumberFormat;
+import java.time.Instant;
 import java.util.*;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
 import java.util.stream.Collectors;
 import java.util.zip.GZIPInputStream;
+import java.time.*;
+import java.util.Optional;
 
 import static ctbrec.io.HttpConstants.ACCEPT_ENCODING_GZIP;
 import static ctbrec.io.HttpConstants.CONTENT_ENCODING;
 import static java.nio.charset.StandardCharsets.UTF_8;
 
+
 @Slf4j
 public abstract class HttpClient {
     @Getter
@@ -50,11 +56,30 @@ public abstract class HttpClient {
     protected int cacheLifeTime = 600;
     private final String name;
     
+    protected final FlaresolverrClient flaresolverr;
+    // a lock to prevent multiple requests from 
+    ReentrantReadWriteLock cookieRefreshLock = new ReentrantReadWriteLock();
+    AtomicInteger cookieErrorCounter = new AtomicInteger(0);
+
     protected HttpClient(String name, Config config) {
         this.name = name;
         this.config = config;
         cookieJar = createCookieJar();
         reconfigure();
+                
+        if (!config.getSettings().flaresolverr.useForDomains.isEmpty()) {
+            flaresolverr = new FlaresolverrClient(config.getSettings().flaresolverr.apiUrl, config.getSettings().flaresolverr.timeoutInMillis);
+            
+            // try {
+            //     flaresolverr.createSession("ctbrec").get();
+            // } catch (InterruptedException e) {
+            //     Thread.currentThread().interrupt();
+            // } catch (Exception e) {
+            //     log.error("Error starting Flaresolverr session", e);
+            // }
+        } else {
+            flaresolverr = null;
+        }
     }
 
     protected CookieJarImpl createCookieJar() {
@@ -108,16 +133,33 @@ public abstract class HttpClient {
         }
     }
 
-    public Response execute(Request req) throws IOException {
-        Response resp = client.newCall(req).execute();
+    private Response execute(Call call) throws IOException {   
+        Response resp;
+        
+        try {
+            cookieRefreshLock.readLock().lock();
+            resp = call.execute();
+        } finally {
+            cookieRefreshLock.readLock().unlock();
+        }
+        
+        // try to solve the cloudflare challenge if we got one (clearance cookie expired, update it)
+        if (resp.code() == 403 && config.getSettings().flaresolverr.useForDomains.contains(call.request().url().host())) {
+            resp = refreshCookiesAndRetry(call.request(), resp);
+        }
+        
         return resp;
     }
 
+    public Response execute(Request req) throws IOException {   
+        return execute(client.newCall(req));
+    }
+
     public Response execute(Request request, int timeoutInMillis) throws IOException {
-        return client.newBuilder() //
+            return execute(client.newBuilder() //
                             .connectTimeout(timeoutInMillis, TimeUnit.MILLISECONDS) //
                             .readTimeout(timeoutInMillis, TimeUnit.MILLISECONDS).build() //
-                .newCall(request).execute();
+                            .newCall(request));
     }
 
     public Response executeWithCache(Request req) throws IOException {
@@ -135,6 +177,56 @@ public abstract class HttpClient {
         }
     }
     
+    private Response refreshCookiesAndRetry(Request req, Response origResp) throws IOException {
+        log.debug("403 received from {}. Trying to refresh cookies with Flaresolverr", req.url().host());
+        
+        try {
+            cookieRefreshLock.writeLock().lock();            
+            
+            // we need to prevent repeated challenge requests from multiple threads, so we check if the clearance cookie needs updating
+            // maybe this can be done with some syncronization primitive, or maybe an expiresAt() check is enough
+            var cookie = Optional
+                .ofNullable(cookieJar.getCookies().get(req.url().topPrivateDomain()))
+                .flatMap(x -> cookieJar.getCookieFromCollection(x, "cf_clearance"));
+                
+            var cookieExpired = cookie.map(c -> 
+                Instant.ofEpochMilli(c.expiresAt()).isBefore(Instant.now())  // by time
+                || req.headers("Cookie").stream().anyMatch(headerCookie -> headerCookie.contains(c.value()))  // we got 403 with current cookie present
+            ).orElse(true);
+                
+            if (cookieExpired || cookieErrorCounter.incrementAndGet() >= 5) {
+                cookieErrorCounter.set(0);
+                
+                var apiResponse = flaresolverr.getCookies(req.url().toString()).get();
+                if (apiResponse.getStatus().equals("ok")) {
+                    // update user agent. It should be the same for all sites, assuming we use the same api address every time
+                    if (!config.getSettings().flaresolverr.userAgent.equals(apiResponse.getUserAgent())) {
+                        config.getSettings().flaresolverr.userAgent = apiResponse.getUserAgent();
+                        config.save();
+                    }
+                    
+                    cookieJar.saveFromResponse(req.url(), apiResponse.getCookies());    
+                    persistCookies();  
+                    log.debug("Cookies successfully refreshed with Flaresolverr in {}", Duration.between(apiResponse.getStartTimestamp(), apiResponse.getEndTimestamp()));
+                } else {
+                    log.debug("Unsuccessful attempt to refresh cookies. Response from Flaresolverr: {}", apiResponse);
+                    return origResp;
+                }
+            } else {
+                log.debug("Looks like the cookies were refreshed already, skipping refreshing");
+            }
+        } catch (Exception e) {
+            log.warn("Error refreshing cookies with Flaresolverr", e);
+            return origResp;
+        } finally {
+            cookieRefreshLock.writeLock().unlock();
+        }
+        
+        origResp.close();
+        
+        return execute(req);
+    }
+
     public abstract boolean login() throws IOException;
 
     public void reconfigure() {
@@ -171,6 +263,8 @@ public abstract class HttpClient {
         }
 
         client = builder.build();
+        client.dispatcher().setMaxRequests(config.getSettings().httpClientMaxRequests);
+        client.dispatcher().setMaxRequestsPerHost(config.getSettings().httpClientMaxRequestsPerHost);
     }
 
     /**

common/src/main/java/ctbrec/recorder/SimplifiedLocalRecorder.java

@@ -580,7 +580,7 @@ public class SimplifiedLocalRecorder implements Recorder {
                 }
             }
         };
-        saveConfigTimer.schedule(saveConfigTask, 400);
+        saveConfigTimer.schedule(saveConfigTask, config.getSettings().configSavingDelayMs);
     }
 
     @Override

common/src/main/java/ctbrec/recorder/download/hls/AbstractHlsDownload.java

@@ -122,7 +122,11 @@ public abstract class AbstractHlsDownload extends AbstractDownload {
             handleMissedSegments(segmentPlaylist, nextSegmentNumber);
             enqueueNewSegments(segmentPlaylist, nextSegmentNumber);
             splitRecordingIfNecessary();
-            calculateRescheduleTime();
+            // by the spec we must wait `targetDuration` before next playlist request if there are changes
+            // if there are none - half that amount
+            calculateRescheduleTime(playlistChanged(segmentPlaylist, nextSegmentNumber) 
+                                    ? segmentPlaylist.targetDuration*1000 
+                                    : segmentPlaylist.targetDuration*500);
             processFinishedSegments();
 
             // this if-check makes sure, that we don't decrease nextSegment. for some reason
@@ -176,6 +180,11 @@ public abstract class AbstractHlsDownload extends AbstractDownload {
         return this;
     }
     
+    private boolean playlistChanged(SegmentPlaylist segmentPlaylist, int nextSegmentNumber) {
+        return segmentPlaylist.seq + segmentPlaylist.segments.size() > nextSegmentNumber;
+    }
+    
+
     protected void processFinishedSegments() {
         downloadExecutor.submit(() -> {
             Future<SegmentDownload> future;
@@ -427,8 +436,8 @@ public abstract class AbstractHlsDownload extends AbstractDownload {
         return new SegmentDownload(model, playlist, segment, client, targetStream);
     }
 
-    private void calculateRescheduleTime() {
-        rescheduleTime = beforeLastPlaylistRequest.plusMillis(1000);
+    private void calculateRescheduleTime(float duration_ms) {
+        rescheduleTime = beforeLastPlaylistRequest.plusMillis(Math.max((int)Math.ceil(duration_ms), 250));
         if (Instant.now().isAfter(rescheduleTime))
             rescheduleTime = Instant.now();
         recordingEvents.add(RecordingEvent.of("next playlist download scheduled for " + rescheduleTime.toString()));

common/src/main/java/ctbrec/sites/bonga/BongaCamsModel.java

@@ -38,8 +38,6 @@ public class BongaCamsModel extends AbstractModel {
     private static final String SUCCESS = "success";
     private static final String STATUS = "status";
 
-    private static final Pattern ONLINE_BADGE_REGEX = Pattern.compile("class=\"badge_online\s*\"");
-
     @Setter
     private boolean online = false;
     private final transient List<StreamSource> streamSources = new ArrayList<>();
@@ -51,13 +49,6 @@ public class BongaCamsModel extends AbstractModel {
     @Override
     public boolean isOnline(boolean ignoreCache) throws IOException, ExecutionException, InterruptedException {
         if (ignoreCache) {
-            boolean modelIsConnected = basicOnlineCheck();
-            if (!modelIsConnected) {
-                onlineState = OFFLINE;
-                online = false;
-                return false;
-            }
-
             return completeOnlineCheck();
         }
         return online;
@@ -74,6 +65,13 @@ public class BongaCamsModel extends AbstractModel {
         String chatType = performerData.optString("showType");
         boolean isAway = performerData.optBoolean("isAway");
         
+        // looks like isOnline key is new. Treat it's absence as true (old behavior)
+        boolean jsonIsOnline = performerData.optBoolean("isOnline", true);
+
+        if (!jsonIsOnline) {
+            onlineState = OFFLINE;
+            online = false;
+        } else {
             onlineState = mapState(chatType);
             if (onlineState == ONLINE) {
                 if (isStreamAvailable()) {
@@ -90,28 +88,10 @@ public class BongaCamsModel extends AbstractModel {
             } else {
                 online = false;
             }
+        }
         return online;
     }
 
-    private boolean basicOnlineCheck() {
-        try {
-            String url = site.getBaseUrl() + "/profile/" + getName().toLowerCase();
-            Request req = newRequestBuilder().url(url).build();
-            try (Response resp = site.getHttpClient().execute(req)) {
-                if (resp.isSuccessful()) {
-                    String body = Objects.requireNonNull(resp.body(), HTTP_RESPONSE_BODY_IS_NULL).string();
-                    Matcher m = ONLINE_BADGE_REGEX.matcher(body);
-                    return m.find();
-                } else {
-                    return false;
-                }
-            }
-        } catch (Exception e) {
-            log.warn("Couldn't check if model is connected: {}", e.getLocalizedMessage());
-            return false;
-        }
-    }
-
     public State mapState(String roomState) {
         return switch (roomState) {
             case "private", "fullprivate" -> PRIVATE;

common/src/main/java/ctbrec/sites/chaturbate/ChaturbateHttpClient.java

@@ -1,20 +1,15 @@
 package ctbrec.sites.chaturbate;
 
 import ctbrec.Config;
-import ctbrec.io.FlaresolverrClient;
 import ctbrec.io.HtmlParser;
 import ctbrec.io.HttpClient;
 import lombok.extern.slf4j.Slf4j;
 import okhttp3.*;
 
-import java.time.*;
 import java.io.IOException;
 import java.io.InterruptedIOException;
 import java.util.NoSuchElementException;
-import java.util.Optional;
 import java.util.concurrent.Semaphore;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
 
 import static ctbrec.io.HttpConstants.REFERER;
 import static ctbrec.io.HttpConstants.USER_AGENT;
@@ -24,31 +19,13 @@ public class ChaturbateHttpClient extends HttpClient {
 
     private static final String PATH = "/auth/login/"; // NOSONAR
     protected String token;
-    protected final FlaresolverrClient flaresolverr;
 
     private static final Semaphore requestThrottle = new Semaphore(2, true);
     private static long lastRequest = 0;
     
-    // a lock to prevent multiple requests from 
-    ReentrantReadWriteLock cookieRefreshLock = new ReentrantReadWriteLock();
-    AtomicInteger cookieErrorCounter = new AtomicInteger(0);
 
     public ChaturbateHttpClient(Config config) {
         super("chaturbate", config);
-        
-        if (config.getSettings().chaturbateUseFlaresolverr) {
-            flaresolverr = new FlaresolverrClient(config.getSettings().flaresolverr.apiUrl, config.getSettings().flaresolverr.timeoutInMillis);
-            
-            // try {
-            //     flaresolverr.createSession("ctbrec").get();
-            // } catch (InterruptedException e) {
-            //     Thread.currentThread().interrupt();
-            // } catch (Exception e) {
-            //     log.error("Error starting Flaresolverr session", e);
-            // }
-        } else {
-            flaresolverr = null;
-        }
     }
     
     @Override
@@ -158,19 +135,7 @@ public class ChaturbateHttpClient extends HttpClient {
                 acquireSlot();
             }
             
-            Response resp;
-            
-            try {
-                cookieRefreshLock.readLock().lock();
-                resp = super.execute(req);
-            } finally {
-                cookieRefreshLock.readLock().unlock();
-            }
-            
-            // try to solve the cloudflare challenge if we got one (clearance cookie expired, update it)
-            if (resp.code() == 403 && flaresolverr != null) {
-                resp = refreshCookiesAndRetry(req, resp);
-            }
+            Response resp = super.execute(req);
             
             extractCsrfToken(req);
             return resp;
@@ -184,55 +149,6 @@ public class ChaturbateHttpClient extends HttpClient {
         }
     }
     
-    private Response refreshCookiesAndRetry(Request req, Response origResp) throws IOException {
-        log.debug("403 received from {}. Trying to refresh cookies with Flaresolverr", req.url().host());
-        
-        try {
-            cookieRefreshLock.writeLock().lock();            
-            
-            // we need to prevent repeated challenge requests from multiple threads, so we check if the clearance cookie needs updating
-            // maybe this can be done with some syncronization primitive, or maybe an expiresAt() check is enough
-            var cookie = Optional
-                .ofNullable(cookieJar.getCookies().get(req.url().topPrivateDomain()))
-                .flatMap(x -> cookieJar.getCookieFromCollection(x, "cf_clearance"));
-                
-            var cookieExpired = cookie.map(c -> 
-                Instant.ofEpochMilli(c.expiresAt()).isBefore(Instant.now())  // by time
-                || req.headers("Cookie").stream().anyMatch(headerCookie -> headerCookie.contains(c.value()))  // we got 403 with current cookie present
-            ).orElse(true);
-                
-            if (cookieExpired || cookieErrorCounter.incrementAndGet() >= 5) {
-                cookieErrorCounter.set(0);
-                
-                var apiResponse = flaresolverr.getCookies(req.url().toString()).get();
-                if (apiResponse.getStatus().equals("ok")) {
-                    // update user agent. It should be the same for all sites, assuming we use the same api address every time
-                    if (!config.getSettings().flaresolverr.userAgent.equals(apiResponse.getUserAgent())) {
-                        config.getSettings().flaresolverr.userAgent = apiResponse.getUserAgent();
-                        config.save();
-                    }
-                    
-                    cookieJar.saveFromResponse(req.url(), apiResponse.getCookies());    
-                    persistCookies();  
-                    log.debug("Cookies successfully refreshed with Flaresolverr in {}", Duration.between(apiResponse.getStartTimestamp(), apiResponse.getEndTimestamp()));
-                } else {
-                    log.debug("Unsuccessful attempt to refresh cookies. Response from Flaresolverr: {}", apiResponse);
-                    return origResp;
-                }
-            } else {
-                log.debug("Looks like the cookies were refreshed already, skipping refreshing");
-            }
-        } catch (Exception e) {
-            log.warn("Error refreshing cookies with Flaresolverr", e);
-            return origResp;
-        } finally {
-            cookieRefreshLock.writeLock().unlock();
-        }
-        
-        origResp.close();
-        
-        return super.execute(req);
-    }
 
     private static void acquireSlot() throws InterruptedException {
         long pauseBetweenRequests = Config.getInstance().getSettings().chaturbateMsBetweenRequests;

common/src/test/java/ctbrec/recorder/RecordingPreconditionsTest.java

@@ -159,12 +159,14 @@ class RecordingPreconditionsTest {
         when(mockita.toString()).thenReturn("Mockita Boobilicious");
         when(mockita.isOnline(true)).thenReturn(true);
         when(mockita.getUrl()).thenReturn("http://localhost/mockita");
+        when(mockita.getPriority()).thenReturn(0);
 
         Model theOtherOne = mock(Model.class);
         when(theOtherOne.getRecordUntil()).thenReturn(Instant.MAX);
         when(theOtherOne.toString()).thenReturn("The Other One");
         when(theOtherOne.isOnline(true)).thenReturn(true);
         when(theOtherOne.getUrl()).thenReturn("http://localhost/theOtherOne");
+        when(theOtherOne.getPriority()).thenReturn(10);
 
         ModelGroup group = new ModelGroup();
         group.add(theOtherOne);

master/pom.xml

@@ -6,7 +6,7 @@
     <groupId>ctbrec</groupId>
     <artifactId>master</artifactId>
     <packaging>pom</packaging>
-    <version>5.3.1</version>
+    <version>5.3.2</version>
 
     <modules>
         <module>../common</module>

server/pom.xml

@@ -8,7 +8,7 @@
     <parent>
         <groupId>ctbrec</groupId>
         <artifactId>master</artifactId>
-        <version>5.3.1</version>
+        <version>5.3.2</version>
         <relativePath>../master</relativePath>
     </parent>
 

server/src/main/java/ctbrec/recorder/server/ConfigServlet.java

@@ -6,6 +6,10 @@ import java.io.IOException;
 import java.security.InvalidKeyException;
 import java.security.NoSuchAlgorithmException;
 import java.time.LocalTime;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Enumeration;
+import java.util.*;
 
 import javax.servlet.ServletException;
 import javax.servlet.http.HttpServletRequest;
@@ -29,7 +33,7 @@ public class ConfigServlet extends AbstractCtbrecServlet {
     private Settings settings;
 
     public enum DataType {
-        STRING, BOOLEAN, INTEGER, LONG, DOUBLE, SPLIT_STRATEGY, TIME
+        STRING, BOOLEAN, INTEGER, LONG, DOUBLE, SPLIT_STRATEGY, TIME, STRING_LIST
     }
 
     public ConfigServlet(Config config) {
@@ -51,6 +55,8 @@ public class ConfigServlet extends AbstractCtbrecServlet {
             JSONArray json = new JSONArray();
             addParameter("concurrentRecordings", "Concurrent Recordings", DataType.INTEGER, settings.concurrentRecordings, json);
             addParameter("chaturbateMsBetweenRequests", "Chaturbate time between requests (ms)", DataType.INTEGER, settings.chaturbateMsBetweenRequests, json);
+            addParameter("defaultPriority", "Default Priority", DataType.INTEGER, settings.defaultPriority, json);
+            addParameter("deleteOrphanedRecordingMetadata", "Delete orphaned recording metadata", DataType.BOOLEAN, settings.deleteOrphanedRecordingMetadata, json);
             addParameter("ffmpegFileSuffix", "File Suffix", DataType.STRING, settings.ffmpegFileSuffix, json);
             addParameter("ffmpegMergedDownloadArgs", "FFmpeg Parameters", DataType.STRING, settings.ffmpegMergedDownloadArgs, json);
             addParameter("httpPort", "HTTP port", DataType.INTEGER, settings.httpPort, json);
@@ -78,7 +84,7 @@ public class ConfigServlet extends AbstractCtbrecServlet {
             addParameter("logFFmpegOutput", "Log FFmpeg Output", DataType.BOOLEAN, settings.logFFmpegOutput, json);
             addParameter("flaresolverr.apiUrl", "Flaresolverr API URL", DataType.STRING, settings.flaresolverr.apiUrl, json);
             addParameter("flaresolverr.timeoutInMillis", "Flaresolverr request timeout (ms)", DataType.INTEGER, settings.flaresolverr.timeoutInMillis, json);
-            addParameter("chaturbateUseFlaresolverr", "Chaturbate: use Flaresolverr", DataType.BOOLEAN, settings.chaturbateUseFlaresolverr, json);
+            addParameter("flaresolverr.useForDomains", "Use Flaresolverr for domains (one per line)", DataType.STRING_LIST, settings.flaresolverr.useForDomains, json);
 
             resp.setStatus(SC_OK);
             resp.setContentType("application/json");
@@ -171,6 +177,9 @@ public class ConfigServlet extends AbstractCtbrecServlet {
         case TIME:
             corrected = LocalTime.parse(value.toString());
             break;
+        case STRING_LIST:
+            corrected = ((JSONArray)value).toList();
+            break;
         default:
             break;
         }

server/src/main/resources/html/static/config.js

@@ -16,7 +16,29 @@ function loadConfig() {
                 }
                 for (let i = 0; i < data.length; i++) {
                     let param = data[i];
-                    param.ko_value = ko.observable(param.value);
+                    if (param.type !== 'STRING_LIST') {
+                      // could not get ko.observable() to write to param.value
+                      // TODO: either fix that, or we need to take ko_value in saveConfig() instead
+                      param.ko_value = ko.pureComputed({
+                          read: function () {
+                              return this.value;
+                          },
+                          write: function (value) {
+                              this.value = value
+                          },
+                          owner: param
+                      });
+                    } else {
+                      param.ko_value = ko.pureComputed({
+                          read: function () {
+                              return this.value.join('\n');
+                          },
+                          write: function (value) {
+                              this.value = value.split('\n')
+                          },
+                          owner: param
+                      });
+                    }
                     observableSettingsArray.push(param);
                 }
             } else {

server/src/main/resources/html/static/index.html

@@ -210,7 +210,14 @@
                                 <tbody data-bind="foreach: settings">
                                 <tr>
                                     <td data-bind="text: name"></td>
-                                    <td><input class="form-control" data-bind="value: value" style="width: 100%"/></td>
+                                    <td>
+                                      <div data-bind="ifnot: type === 'STRING_LIST'">
+                                        <input class="form-control" data-bind="value: ko_value" style="width: 100%;" />
+                                      </div>
+                                      <div data-bind="if: type === 'STRING_LIST'">
+                                        <textarea rows="3" class="form-control" data-bind="value: ko_value" style="width: 100%;"></textarea>
+                                      </div>
+                                    </td>
                                 </tr>
                                 </tbody>
                             </table>