j62
/
ctbrec
2
2
Fork 2
Code Issues 3 Pull Requests Packages Projects Releases 3 Wiki Activity

Generalize Flaresolverr for any domain 

- replace per-site setting with list of hosts
- add new setting type for simple lists of strings
This commit is contained in:
reusedname 2025-03-02 13:07:42 +05:00
parent 131a9d54c9
commit 7ab0c1e237
11 changed files with 250 additions and 115 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
client/src/main/java/ctbrec/ui/settings/CtbrecPreferencesStorage.java
View File

@ -3,6 +3,7 @@ package ctbrec.ui.settings;
import java.io.IOException;
import java.lang.reflect.InvocationTargetException;
import java.time.LocalTime;
import java.util.Arrays;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
@ -25,6 +26,7 @@ import ctbrec.ui.settings.api.PreferencesStorage;
import ctbrec.ui.settings.api.Setting;
import ctbrec.ui.settings.api.SimpleDirectoryProperty;
import ctbrec.ui.settings.api.SimpleFileProperty;
import ctbrec.ui.settings.api.SimpleJoinedStringListProperty;
import ctbrec.ui.settings.api.SimpleRangeProperty;
import ctbrec.io.BoundField;
import javafx.beans.property.BooleanProperty;
@ -40,6 +42,7 @@ import javafx.scene.control.CheckBox;
import javafx.scene.control.ComboBox;
import javafx.scene.control.Label;
import javafx.scene.control.RadioButton;
import javafx.scene.control.TextArea;
import javafx.scene.control.TextField;
import javafx.scene.control.ToggleGroup;
import javafx.scene.layout.HBox;
@ -97,6 +100,8 @@ public class CtbrecPreferencesStorage implements PreferencesStorage {
return createBooleanProperty(setting);
} else if (prop instanceof ListProperty) {
return createComboBox(setting);
} else if (prop instanceof SimpleJoinedStringListProperty) {
return createStringListProperty(setting);
} else if (prop instanceof StringProperty) {
return createStringProperty(setting);
} else {
@ -240,6 +245,20 @@ public class CtbrecPreferencesStorage implements PreferencesStorage {
return ctrl;
}
private Node createStringListProperty(Setting setting) {
var ctrl = new TextArea();
StringProperty prop = (StringProperty) setting.getProperty();
ctrl.textProperty().bindBidirectional(prop);
prop.addListener((obs, oldV, newV) -> saveValue(() -> {
//setUnchecked(setting.getKey(), Arrays.asList(newV.split("\n")));
if (setting.doesNeedRestart()) {
runRestartRequiredCallback();
}
config.save();
}));
return ctrl;
}
@SuppressWarnings("unchecked")
private Node createIntegerProperty(Setting setting) {
var ctrl = new TextField();
@ -331,11 +350,30 @@ public class CtbrecPreferencesStorage implements PreferencesStorage {
var field = BoundField.of(settings, key);
var o = field.get();
if (!Objects.equals(n, o)) {
field.set(n); // NOSONAR
if (n instanceof List && o instanceof List) {
var list = (List<String>)o;
list.clear();
list.addAll((List<String>)n);
} else {
field.set(n); // NOSONAR
}
return true;
}
return false;
}
private boolean setUnchecked(String key, Object n) throws IllegalAccessException, NoSuchFieldException, InvocationTargetException {
var field = BoundField.of(settings, key);
var o = field.get();
if (n instanceof List && o instanceof List) {
var list = (List<String>)o;
list.clear();
list.addAll((List<String>)n);
} else {
field.set(n); // NOSONAR
}
return true;
}
private void saveValue(Exec exe) {
try {

6
client/src/main/java/ctbrec/ui/settings/SettingsTab.java
View File

@ -62,6 +62,7 @@ public class SettingsTab extends Tab implements TabSelectionListener {
private SimpleStringProperty flaresolverrApiUrl;
private SimpleIntegerProperty flaresolverrTimeoutInMillis;
private SimpleJoinedStringListProperty flaresolverrUseForDomains;
private SimpleStringProperty httpUserAgent;
private SimpleStringProperty httpUserAgentMobile;
private SimpleIntegerProperty overviewUpdateIntervalInSecs;
@ -151,6 +152,8 @@ public class SettingsTab extends Tab implements TabSelectionListener {
private void initializeProperties() {
flaresolverrApiUrl = new SimpleStringProperty(null, "flaresolverr.apiUrl", settings.flaresolverr.apiUrl);
flaresolverrTimeoutInMillis = new SimpleIntegerProperty(null, "flaresolverr.timeoutInMillis", settings.flaresolverr.timeoutInMillis);
flaresolverrUseForDomains = new SimpleJoinedStringListProperty(null, "flaresolverr.useForDomains", "\n",
FXCollections.observableList(settings.flaresolverr.useForDomains));
httpUserAgent = new SimpleStringProperty(null, "httpUserAgent", settings.httpUserAgent);
httpUserAgentMobile = new SimpleStringProperty(null, "httpUserAgentMobile", settings.httpUserAgentMobile);
overviewUpdateIntervalInSecs = new SimpleIntegerProperty(null, "overviewUpdateIntervalInSecs", settings.overviewUpdateIntervalInSecs);
@ -272,7 +275,8 @@ public class SettingsTab extends Tab implements TabSelectionListener {
Group.of("Flaresolverr",
Setting.of("API URL", flaresolverrApiUrl),
Setting.of("Request timeout", flaresolverrTimeoutInMillis))),
Setting.of("Request timeout", flaresolverrTimeoutInMillis),
Setting.of("Use for domains (one per line)", flaresolverrUseForDomains))),
Category.of("Look & Feel",
Group.of("Look & Feel",
Setting.of("Colors (Base / Accent)", new ColorSettingsPane(Config.getInstance())).needsRestart(),

48
client/src/main/java/ctbrec/ui/settings/api/SimpleJoinedStringListProperty.java Normal file
View File

@ -0,0 +1,48 @@
package ctbrec.ui.settings.api;
import javafx.beans.binding.Bindings;
import javafx.beans.property.SimpleListProperty;
import javafx.beans.property.SimpleStringProperty;
import javafx.collections.ListChangeListener;
import javafx.collections.ObservableList;
import lombok.Getter;
public class SimpleJoinedStringListProperty extends SimpleStringProperty implements ListChangeListener<String> {
private ObservableList<String> list;
@Getter
private String delimiter;
private boolean updating = false;
public SimpleJoinedStringListProperty(Object bean, String name, String delimiter, ObservableList<String> initialValue) {
super(bean, name, String.join(delimiter, initialValue));
this.delimiter = delimiter;
this.list = initialValue;
initialValue.addListener(this);
}
@Override
public void setValue(String newValue) {
if (!updating) {
try {
updating = true;
list.setAll(newValue.split(delimiter));
super.setValue(newValue);
} finally {
updating = false;
}
}
}
@Override
public void onChanged(Change<? extends String> c) {
if (!updating) {
try {
updating = true;
super.setValue(String.join(delimiter, list));
} finally {
updating = false;
}
}
}
}

12
client/src/main/java/ctbrec/ui/sites/chaturbate/ChaturbateConfigUi.java
View File

@ -99,18 +99,6 @@ public class ChaturbateConfigUi extends AbstractConfigUI {
GridPane.setHgrow(requestThrottle, Priority.ALWAYS);
GridPane.setColumnSpan(requestThrottle, 2);
layout.add(requestThrottle, 1, row++);
var label = new Label("Use Flaresolverr");
label.setTooltip(new Tooltip("Use Flaresolverr for solving the Cloudflare challenge. This also overrides the User Agent used for HTTP requests (only for the site)"));
layout.add(label, 0, row);
var flaresolverrToggle = new CheckBox();
flaresolverrToggle.setSelected(settings.chaturbateUseFlaresolverr);
flaresolverrToggle.setOnAction(e -> {
settings.chaturbateUseFlaresolverr = flaresolverrToggle.isSelected();
save();
});
GridPane.setMargin(flaresolverrToggle, new Insets(0, 0, SettingsTab.CHECKBOX_MARGIN, SettingsTab.CHECKBOX_MARGIN));
layout.add(flaresolverrToggle, 1, row++);
var createAccount = new Button("Create new Account");
createAccount.setOnAction(e -> DesktopIntegration.open(Chaturbate.REGISTRATION_LINK));

22
common/src/main/java/ctbrec/Config.java
View File

@ -184,12 +184,15 @@ public class Config {
migrateOldSettings();
}
private String migrateJson(String json) {
return migrateTo5_1_2(json);
private String migrateJson(String jsonStr) {
var json = new JSONObject(jsonStr);
migrateTo5_1_2(json);
migrateTo5_3_2(json);
return json.toString();
}
private String migrateTo5_1_2(String json) {
JSONObject s = new JSONObject(json);
private void migrateTo5_1_2(JSONObject json) {
JSONObject s = json;
if (s.has("models")) {
JSONArray models = s.getJSONArray("models");
@ -217,7 +220,16 @@ public class Config {
}
}
}
return s.toString();
}
private void migrateTo5_3_2(JSONObject json) {
if (json.has("chaturbateUseFlaresolverr") && json.has("flaresolverr")) {
var fsr = json.getJSONObject("flaresolverr");
if (!fsr.has("useForDomains") && json.getBoolean("chaturbateUseFlaresolverr")) {
fsr.put("useForDomains", new JSONArray().put("chaturbate.com"));
}
}
}
private void migrateOldSettings() {

2
common/src/main/java/ctbrec/Settings.java
View File

@ -46,6 +46,7 @@ public class Settings {
public String apiUrl = "http://localhost:8191/v1";
public int timeoutInMillis = 60000;
public String userAgent = "";
public List<String> useForDomains = new ArrayList<>(); //(List.of("chaturbate.com", "bongacams.com"));
};
public FlaresolverrSettings flaresolverr = new FlaresolverrSettings();
@ -61,6 +62,7 @@ public class Settings {
public String chaturbatePassword = "";
public String chaturbateUsername = "";
public String chaturbateBaseUrl = "https://chaturbate.com";
@Deprecated
public boolean chaturbateUseFlaresolverr = false;
public int chaturbateMsBetweenRequests = 1000;
public String cherryTvPassword = "";

105
common/src/main/java/ctbrec/io/HttpClient.java
View File

@ -20,22 +20,27 @@ import java.io.File;
import java.io.IOException;
import java.net.Authenticator;
import java.net.PasswordAuthentication;
import java.net.SocketTimeoutException;
import java.nio.file.Files;
import java.security.KeyManagementException;
import java.security.NoSuchAlgorithmException;
import java.security.SecureRandom;
import java.security.cert.X509Certificate;
import java.text.NumberFormat;
import java.time.Instant;
import java.util.*;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.stream.Collectors;
import java.util.zip.GZIPInputStream;
import java.time.*;
import java.util.Optional;
import static ctbrec.io.HttpConstants.ACCEPT_ENCODING_GZIP;
import static ctbrec.io.HttpConstants.CONTENT_ENCODING;
import static java.nio.charset.StandardCharsets.UTF_8;
@Slf4j
public abstract class HttpClient {
@Getter
@ -50,12 +55,31 @@ public abstract class HttpClient {
protected long cacheSize;
protected int cacheLifeTime = 600;
private final String name;
protected final FlaresolverrClient flaresolverr;
// a lock to prevent multiple requests from
ReentrantReadWriteLock cookieRefreshLock = new ReentrantReadWriteLock();
AtomicInteger cookieErrorCounter = new AtomicInteger(0);
protected HttpClient(String name, Config config) {
this.name = name;
this.config = config;
cookieJar = createCookieJar();
reconfigure();
if (!config.getSettings().flaresolverr.useForDomains.isEmpty()) {
flaresolverr = new FlaresolverrClient(config.getSettings().flaresolverr.apiUrl, config.getSettings().flaresolverr.timeoutInMillis);
// try {
// flaresolverr.createSession("ctbrec").get();
// } catch (InterruptedException e) {
// Thread.currentThread().interrupt();
// } catch (Exception e) {
// log.error("Error starting Flaresolverr session", e);
// }
} else {
flaresolverr = null;
}
}
protected CookieJarImpl createCookieJar() {
@ -109,16 +133,33 @@ public abstract class HttpClient {
}
}
public Response execute(Request req) throws IOException {
Response resp = client.newCall(req).execute();
private Response execute(Call call) throws IOException {
Response resp;
try {
cookieRefreshLock.readLock().lock();
resp = call.execute();
} finally {
cookieRefreshLock.readLock().unlock();
}
// try to solve the cloudflare challenge if we got one (clearance cookie expired, update it)
if (resp.code() == 403 && config.getSettings().flaresolverr.useForDomains.contains(call.request().url().host())) {
resp = refreshCookiesAndRetry(call.request(), resp);
}
return resp;
}
public Response execute(Request req) throws IOException {
return execute(client.newCall(req));
}
public Response execute(Request request, int timeoutInMillis) throws IOException {
return client.newBuilder() //
.connectTimeout(timeoutInMillis, TimeUnit.MILLISECONDS) //
.readTimeout(timeoutInMillis, TimeUnit.MILLISECONDS).build() //
.newCall(request).execute();
return execute(client.newBuilder() //
.connectTimeout(timeoutInMillis, TimeUnit.MILLISECONDS) //
.readTimeout(timeoutInMillis, TimeUnit.MILLISECONDS).build() //
.newCall(request));
}
public Response executeWithCache(Request req) throws IOException {
@ -135,6 +176,56 @@ public abstract class HttpClient {
return execute(req);
}
}
private Response refreshCookiesAndRetry(Request req, Response origResp) throws IOException {
log.debug("403 received from {}. Trying to refresh cookies with Flaresolverr", req.url().host());
try {
cookieRefreshLock.writeLock().lock();
// we need to prevent repeated challenge requests from multiple threads, so we check if the clearance cookie needs updating
// maybe this can be done with some syncronization primitive, or maybe an expiresAt() check is enough
var cookie = Optional
.ofNullable(cookieJar.getCookies().get(req.url().topPrivateDomain()))
.flatMap(x -> cookieJar.getCookieFromCollection(x, "cf_clearance"));
var cookieExpired = cookie.map(c ->
Instant.ofEpochMilli(c.expiresAt()).isBefore(Instant.now()) // by time
|| req.headers("Cookie").stream().anyMatch(headerCookie -> headerCookie.contains(c.value())) // we got 403 with current cookie present
).orElse(true);
if (cookieExpired || cookieErrorCounter.incrementAndGet() >= 5) {
cookieErrorCounter.set(0);
var apiResponse = flaresolverr.getCookies(req.url().toString()).get();
if (apiResponse.getStatus().equals("ok")) {
// update user agent. It should be the same for all sites, assuming we use the same api address every time
if (!config.getSettings().flaresolverr.userAgent.equals(apiResponse.getUserAgent())) {
config.getSettings().flaresolverr.userAgent = apiResponse.getUserAgent();
config.save();
}
cookieJar.saveFromResponse(req.url(), apiResponse.getCookies());
persistCookies();
log.debug("Cookies successfully refreshed with Flaresolverr in {}", Duration.between(apiResponse.getStartTimestamp(), apiResponse.getEndTimestamp()));
} else {
log.debug("Unsuccessful attempt to refresh cookies. Response from Flaresolverr: {}", apiResponse);
return origResp;
}
} else {
log.debug("Looks like the cookies were refreshed already, skipping refreshing");
}
} catch (Exception e) {
log.warn("Error refreshing cookies with Flaresolverr", e);
return origResp;
} finally {
cookieRefreshLock.writeLock().unlock();
}
origResp.close();
return execute(req);
}
public abstract boolean login() throws IOException;

86
common/src/main/java/ctbrec/sites/chaturbate/ChaturbateHttpClient.java
View File

@ -1,20 +1,15 @@
package ctbrec.sites.chaturbate;
import ctbrec.Config;
import ctbrec.io.FlaresolverrClient;
import ctbrec.io.HtmlParser;
import ctbrec.io.HttpClient;
import lombok.extern.slf4j.Slf4j;
import okhttp3.*;
import java.time.*;
import java.io.IOException;
import java.io.InterruptedIOException;
import java.util.NoSuchElementException;
import java.util.Optional;
import java.util.concurrent.Semaphore;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import static ctbrec.io.HttpConstants.REFERER;
import static ctbrec.io.HttpConstants.USER_AGENT;
@ -24,31 +19,13 @@ public class ChaturbateHttpClient extends HttpClient {
private static final String PATH = "/auth/login/"; // NOSONAR
protected String token;
protected final FlaresolverrClient flaresolverr;
private static final Semaphore requestThrottle = new Semaphore(2, true);
private static long lastRequest = 0;
// a lock to prevent multiple requests from
ReentrantReadWriteLock cookieRefreshLock = new ReentrantReadWriteLock();
AtomicInteger cookieErrorCounter = new AtomicInteger(0);
public ChaturbateHttpClient(Config config) {
super("chaturbate", config);
if (config.getSettings().chaturbateUseFlaresolverr) {
flaresolverr = new FlaresolverrClient(config.getSettings().flaresolverr.apiUrl, config.getSettings().flaresolverr.timeoutInMillis);
// try {
// flaresolverr.createSession("ctbrec").get();
// } catch (InterruptedException e) {
// Thread.currentThread().interrupt();
// } catch (Exception e) {
// log.error("Error starting Flaresolverr session", e);
// }
} else {
flaresolverr = null;
}
}
@Override
@ -158,19 +135,7 @@ public class ChaturbateHttpClient extends HttpClient {
acquireSlot();
}
Response resp;
try {
cookieRefreshLock.readLock().lock();
resp = super.execute(req);
} finally {
cookieRefreshLock.readLock().unlock();
}
// try to solve the cloudflare challenge if we got one (clearance cookie expired, update it)
if (resp.code() == 403 && flaresolverr != null) {
resp = refreshCookiesAndRetry(req, resp);
}
Response resp = super.execute(req);
extractCsrfToken(req);
return resp;
@ -184,55 +149,6 @@ public class ChaturbateHttpClient extends HttpClient {
}
}
private Response refreshCookiesAndRetry(Request req, Response origResp) throws IOException {
log.debug("403 received from {}. Trying to refresh cookies with Flaresolverr", req.url().host());
try {
cookieRefreshLock.writeLock().lock();
// we need to prevent repeated challenge requests from multiple threads, so we check if the clearance cookie needs updating
// maybe this can be done with some syncronization primitive, or maybe an expiresAt() check is enough
var cookie = Optional
.ofNullable(cookieJar.getCookies().get(req.url().topPrivateDomain()))
.flatMap(x -> cookieJar.getCookieFromCollection(x, "cf_clearance"));
var cookieExpired = cookie.map(c ->
Instant.ofEpochMilli(c.expiresAt()).isBefore(Instant.now()) // by time
|| req.headers("Cookie").stream().anyMatch(headerCookie -> headerCookie.contains(c.value())) // we got 403 with current cookie present
).orElse(true);
if (cookieExpired || cookieErrorCounter.incrementAndGet() >= 5) {
cookieErrorCounter.set(0);
var apiResponse = flaresolverr.getCookies(req.url().toString()).get();
if (apiResponse.getStatus().equals("ok")) {
// update user agent. It should be the same for all sites, assuming we use the same api address every time
if (!config.getSettings().flaresolverr.userAgent.equals(apiResponse.getUserAgent())) {
config.getSettings().flaresolverr.userAgent = apiResponse.getUserAgent();
config.save();
}
cookieJar.saveFromResponse(req.url(), apiResponse.getCookies());
persistCookies();
log.debug("Cookies successfully refreshed with Flaresolverr in {}", Duration.between(apiResponse.getStartTimestamp(), apiResponse.getEndTimestamp()));
} else {
log.debug("Unsuccessful attempt to refresh cookies. Response from Flaresolverr: {}", apiResponse);
return origResp;
}
} else {
log.debug("Looks like the cookies were refreshed already, skipping refreshing");
}
} catch (Exception e) {
log.warn("Error refreshing cookies with Flaresolverr", e);
return origResp;
} finally {
cookieRefreshLock.writeLock().unlock();
}
origResp.close();
return super.execute(req);
}
private static void acquireSlot() throws InterruptedException {
long pauseBetweenRequests = Config.getInstance().getSettings().chaturbateMsBetweenRequests;

11
server/src/main/java/ctbrec/recorder/server/ConfigServlet.java
View File

@ -6,6 +6,10 @@ import java.io.IOException;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import java.time.LocalTime;
import java.util.Arrays;
import java.util.Collections;
import java.util.Enumeration;
import java.util.*;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
@ -29,7 +33,7 @@ public class ConfigServlet extends AbstractCtbrecServlet {
private Settings settings;
public enum DataType {
STRING, BOOLEAN, INTEGER, LONG, DOUBLE, SPLIT_STRATEGY, TIME
STRING, BOOLEAN, INTEGER, LONG, DOUBLE, SPLIT_STRATEGY, TIME, STRING_LIST
}
public ConfigServlet(Config config) {
@ -80,7 +84,7 @@ public class ConfigServlet extends AbstractCtbrecServlet {
addParameter("logFFmpegOutput", "Log FFmpeg Output", DataType.BOOLEAN, settings.logFFmpegOutput, json);
addParameter("flaresolverr.apiUrl", "Flaresolverr API URL", DataType.STRING, settings.flaresolverr.apiUrl, json);
addParameter("flaresolverr.timeoutInMillis", "Flaresolverr request timeout (ms)", DataType.INTEGER, settings.flaresolverr.timeoutInMillis, json);
addParameter("chaturbateUseFlaresolverr", "Chaturbate: use Flaresolverr", DataType.BOOLEAN, settings.chaturbateUseFlaresolverr, json);
addParameter("flaresolverr.useForDomains", "Use Flaresolverr for domains (one per line)", DataType.STRING_LIST, settings.flaresolverr.useForDomains, json);
resp.setStatus(SC_OK);
resp.setContentType("application/json");
@ -173,6 +177,9 @@ public class ConfigServlet extends AbstractCtbrecServlet {
case TIME:
corrected = LocalTime.parse(value.toString());
break;
case STRING_LIST:
corrected = ((JSONArray)value).toList();
break;
default:
break;
}

24
server/src/main/resources/html/static/config.js
View File

@ -16,7 +16,29 @@ function loadConfig() {
}
for (let i = 0; i < data.length; i++) {
let param = data[i];
param.ko_value = ko.observable(param.value);
if (param.type !== 'STRING_LIST') {
// could not get ko.observable() to write to param.value
// TODO: either fix that, or we need to take ko_value in saveConfig() instead
param.ko_value = ko.pureComputed({
read: function () {
return this.value;
},
write: function (value) {
this.value = value
},
owner: param
});
} else {
param.ko_value = ko.pureComputed({
read: function () {
return this.value.join('\n');
},
write: function (value) {
this.value = value.split('\n')
},
owner: param
});
}
observableSettingsArray.push(param);
}
} else {

9
server/src/main/resources/html/static/index.html
View File

@ -210,7 +210,14 @@
<tbody data-bind="foreach: settings">
<tr>
<td data-bind="text: name"></td>
<td><input class="form-control" data-bind="value: value" style="width: 100%"/></td>
<td>
<div data-bind="ifnot: type === 'STRING_LIST'">
<input class="form-control" data-bind="value: ko_value" style="width: 100%;" />
</div>
<div data-bind="if: type === 'STRING_LIST'">
<textarea rows="3" class="form-control" data-bind="value: ko_value" style="width: 100%;"></textarea>
</div>
</td>
</tr>
</tbody>
</table>