davebluke
/
ctbrec
forked from j62/ctbrec
1
0
Fork 0
Code Pull Requests Activity

Merge branch 'dev' into manyvids 

# Conflicts:
#	common/src/main/java/ctbrec/io/HttpClient.java
This commit is contained in:
0xb00bface 2020-08-16 17:00:43 +02:00
parent 6600b0da94 acec91ee69
commit db186e65f4
73 changed files with 1468 additions and 524 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

55
CHANGELOG.md
View File

@ -1,3 +1,54 @@
3.8.6
========================
* Added setting to disable the online check for paused models
* Speed up shutdown process by stopping all recordings simultaneously
* Fixed Streamate followed tab once again
* Fixed: Flirt4Free models loose their name after some time
* Made loading of config file more robust for Flirt4Free models
* Added tab which shows the log output
3.8.5
========================
* Fixed Stripchat followed tab. It didn't work, if you have many favorited
models
* Fixed: Some Stripchat models didn't get recorded
* Fixed: Some LiveJasmin models didn't get recorded
* Added support for temporary recordings. On the recording tab you can now set
a date, when to stop recording a model and what to do afterwards
(pause or remove the model)
* Changed the look of the model table in the web interface a bit
3.8.4
========================
* Added support for xHamsterLive (go to Settings -> Sites -> Stripchat,
switch to xHamsterLive, enter your credentials and restart)
* Fixed follow / unfollow for Stripchat
* Enable rerun PP for multiple recordings
* Fixed bug, which prevented recordings to finish properly on app
shutdown. Recordings now shouldn't end up in state waiting anymore
3.8.3
========================
* Fixed Streamate
* Fixed favorites tab for Cam4; kind of, because only the online tab works.
I currently don't see a way to retrieve the offline favorites
* Fixed favorites tab for CamSoda
* Fixed CamSoda recordings
* Added external login dialog for Stripchat to support the captcha
3.8.2
========================
* Fixed misconfiguration in global connection pool, which caused a lot of
threads to spawn while browsing in the thumbnail overviews
* Improved memory handling for the thumbnail overviews. Thumbnail images were
not released, when a tab was switched. This caused a huge memory consumption,
if you opened a lot of different tabs.
* Fixed a bug in MFC websocket client, which caused to spawn a bunch of
"keep-alive" threads, if there was a problem with the connection
* Reworked the settings tab
* Fire recording finished event, if a download from the server is finished
* Ignore min/max resolution, if the resolution is unknown
3.8.1
========================
* Fixed recent MFC error
@ -10,7 +61,7 @@
* Models can be added by name in the web-interface
* Added a bandwidth monitor
* Added possibility to add notes to recordings
* Added range slider to restrict the recording resolution in a range; e.g. 480p - 1080p
* Added range slider to restrict the recording resolution in a range; e.g. 480p - 1080p
* Improved MFC SD downloads (much less blocking, I think)
3.7.3
@ -26,7 +77,7 @@
3.7.1
========================
* Server now logs in on startup, if credentials are set
* Show confirmation dialog on shutdown, if the are active downloads from the
* Show confirmation dialog on shutdown, if the are active downloads from the
server
* Added setting to remove recordings after post-processing
* Added max resolution setting for the player (click on the gear!)

1
client/build.sh
View File

@ -1,5 +1,4 @@
#!/bin/bash
export JAVA_HOME=/opt/jdk-11.0.1
mvn clean
mvn -Djavafx.platform=win package verify
mvn -Djavafx.platform=linux package verify

10
client/pom.xml
View File

@ -8,7 +8,7 @@
<parent>
<groupId>ctbrec</groupId>
<artifactId>master</artifactId>
<version>3.8.1</version>
<version>3.8.6</version>
<relativePath>../master</relativePath>
</parent>
@ -71,7 +71,7 @@
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
<scope>runtime</scope>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.openjfx</groupId>
@ -91,7 +91,7 @@
</dependency>
<dependency>
<groupId>com.vladsch.flexmark</groupId>
<artifactId>flexmark-all</artifactId>
<artifactId>flexmark</artifactId>
<version>0.40.34</version>
</dependency>
</dependencies>
@ -140,8 +140,8 @@
<versionInfo>
<fileVersion>${project.version}.0</fileVersion>
<txtFileVersion>${project.version}</txtFileVersion>
<fileDescription>Recorder for Charturbate streams</fileDescription>
<copyright>2018 0xboobface</copyright>
<fileDescription>Software to record live streams</fileDescription>
<copyright>2020 0xboobface</copyright>
<productVersion>${project.version}.0</productVersion>
<txtProductVersion>${project.version}</txtProductVersion>
<productName>CTB Recorder</productName>

6
client/src/main/java/ctbrec/ui/CamrecApplication.java
View File

@ -62,6 +62,7 @@ import ctbrec.ui.tabs.RecordingsTab;
import ctbrec.ui.tabs.SiteTab;
import ctbrec.ui.tabs.TabSelectionListener;
import ctbrec.ui.tabs.UpdateTab;
import ctbrec.ui.tabs.logging.LoggingTab;
import javafx.application.Application;
import javafx.application.HostServices;
import javafx.application.Platform;
@ -143,8 +144,6 @@ public class CamrecApplication extends Application {
}
private void startOnlineMonitor() {
onlineMonitor = new OnlineMonitor(recorder);
onlineMonitor.start();
for (Site site : sites) {
if(site.isEnabled()) {
try {
@ -155,6 +154,8 @@ public class CamrecApplication extends Application {
}
}
}
onlineMonitor = new OnlineMonitor(recorder, config);
onlineMonitor.start();
}
private void logEnvironment() {
@ -193,6 +194,7 @@ public class CamrecApplication extends Application {
tabPane.getTabs().add(new NewsTab());
tabPane.getTabs().add(new DonateTabFx());
tabPane.getTabs().add(new HelpTab());
tabPane.getTabs().add(new LoggingTab());
switchToStartTab();
writeColorSchemeStyleSheet();

21
client/src/main/java/ctbrec/ui/JavaFxModel.java
View File

@ -13,6 +13,7 @@ import com.squareup.moshi.JsonReader;
import com.squareup.moshi.JsonWriter;
import ctbrec.Model;
import ctbrec.SubsequentAction;
import ctbrec.recorder.download.Download;
import ctbrec.recorder.download.HttpHeaderFactory;
import ctbrec.recorder.download.StreamSource;
@ -286,4 +287,24 @@ public class JavaFxModel implements Model {
public HttpHeaderFactory getHttpHeaderFactory() {
return delegate.getHttpHeaderFactory();
}
@Override
public Instant getRecordUntil() {
return delegate.getRecordUntil();
}
@Override
public void setRecordUntil(Instant instant) {
delegate.setRecordUntil(instant);
}
@Override
public SubsequentAction getRecordUntilSubsequentAction() {
return delegate.getRecordUntilSubsequentAction();
}
@Override
public void setRecordUntilSubsequentAction(SubsequentAction action) {
delegate.setRecordUntilSubsequentAction(action);
}
}

18
client/src/main/java/ctbrec/ui/controls/Dialogs.java
View File

@ -16,6 +16,7 @@ import javafx.scene.control.Dialog;
import javafx.scene.control.TextArea;
import javafx.scene.image.Image;
import javafx.scene.layout.GridPane;
import javafx.scene.layout.Region;
import javafx.stage.Modality;
import javafx.stage.Stage;
@ -87,6 +88,23 @@ public class Dialogs {
return dialog.showAndWait();
}
public static Boolean showCustomInput(Scene parent, String title, Region region) {
Dialog<?> dialog = new Dialog<>();
dialog.setTitle(title);
dialog.getDialogPane().getButtonTypes().addAll(ButtonType.OK, ButtonType.CANCEL);
dialog.initModality(Modality.APPLICATION_MODAL);
dialog.setResizable(true);
InputStream icon = Dialogs.class.getResourceAsStream("/icon.png");
Stage stage = (Stage) dialog.getDialogPane().getScene().getWindow();
stage.getIcons().add(new Image(icon));
if (parent != null) {
stage.getScene().getStylesheets().addAll(parent.getStylesheets());
}
dialog.getDialogPane().setContent(region);
dialog.showAndWait();
return dialog.getResult() == ButtonType.OK;
}
public static boolean showConfirmDialog(String title, String message, String header, Scene parent) {
AutosizeAlert confirm = new AutosizeAlert(AlertType.CONFIRMATION, message, parent, YES, NO);
confirm.setTitle(title);

8
client/src/main/java/ctbrec/ui/settings/SettingsTab.java
View File

@ -86,6 +86,7 @@ public class SettingsTab extends Tab implements TabSelectionListener {
private DiscreteRange<Integer> rangeValues = new DiscreteRange<>(values, labels);
private SimpleIntegerProperty concurrentRecordings;
private SimpleIntegerProperty onlineCheckIntervalInSecs;
private SimpleBooleanProperty onlineCheckSkipsPausedModels;
private SimpleLongProperty leaveSpaceOnDevice;
private SimpleIntegerProperty minimumLengthInSecs;
private SimpleStringProperty ffmpegParameters;
@ -149,6 +150,7 @@ public class SettingsTab extends Tab implements TabSelectionListener {
postProcessing = new SimpleFileProperty(null, "postProcessing", settings.postProcessing);
postProcessingThreads = new SimpleIntegerProperty(null, "postProcessingThreads", settings.postProcessingThreads);
removeRecordingAfterPp = new SimpleBooleanProperty(null, "removeRecordingAfterPostProcessing", settings.removeRecordingAfterPostProcessing);
onlineCheckSkipsPausedModels = new SimpleBooleanProperty(null, "onlineCheckSkipsPausedModels", settings.onlineCheckSkipsPausedModels);
}
private void createGui() {
@ -187,10 +189,11 @@ public class SettingsTab extends Tab implements TabSelectionListener {
Setting.of("Split recordings after (minutes)", splitAfter).converter(SplitAfterOption.converter()),
Setting.of("Restrict Resolution", resolutionRange, "Only record streams with resolution within the given range"),
Setting.of("Concurrent Recordings (0 = unlimited)", concurrentRecordings),
Setting.of("Check online state every (seconds)", onlineCheckIntervalInSecs, "Check every x seconds, if a model came online"),
Setting.of("Leave space on device (GiB)", leaveSpaceOnDevice, "Stop recording, if the free space on the device gets below this threshold").converter(new GigabytesConverter()),
Setting.of("FFmpeg parameters", ffmpegParameters, "FFmpeg parameters to use when merging stream segments"),
Setting.of("File Extension", fileExtension, "File extension to use for recordings")
Setting.of("File Extension", fileExtension, "File extension to use for recordings"),
Setting.of("Check online state every (seconds)", onlineCheckIntervalInSecs, "Check every x seconds, if a model came online"),
Setting.of("Skip online check for paused models", onlineCheckSkipsPausedModels, "Skip online check for paused models")
),
Group.of("Location",
Setting.of("Record Location", recordLocal),
@ -236,6 +239,7 @@ public class SettingsTab extends Tab implements TabSelectionListener {
prefs.getSetting("minimumResolution").ifPresent(s -> bindEnabledProperty(s, recordLocal.not()));
prefs.getSetting("recordingsDirStructure").ifPresent(s -> bindEnabledProperty(s, recordLocal.not()));
prefs.getSetting("onlineCheckIntervalInSecs").ifPresent(s -> bindEnabledProperty(s, recordLocal.not()));
prefs.getSetting("onlineCheckSkipsPausedModels").ifPresent(s -> bindEnabledProperty(s, recordLocal.not()));
prefs.getSetting("minimumSpaceLeftInBytes").ifPresent(s -> bindEnabledProperty(s, recordLocal.not()));
prefs.getSetting("postProcessing").ifPresent(s -> bindEnabledProperty(s, recordLocal.not()));
prefs.getSetting("postProcessingThreads").ifPresent(s -> bindEnabledProperty(s, recordLocal.not()));

1
client/src/main/java/ctbrec/ui/sites/bonga/BongaCamsElectronLoginDialog.java
View File

@ -63,6 +63,7 @@ public class BongaCamsElectronLoginDialog {
}
String password = Config.getInstance().getSettings().bongaPassword;
if (password != null && !password.trim().isEmpty()) {
password = password.replace("'", "\\'");
browser.executeJavaScript("$('input[name=\"log_in[password]\"]').attr('value','" + password + "')");
}
String[] simplify = new String[] {

1
client/src/main/java/ctbrec/ui/sites/cam4/Cam4ElectronLoginDialog.java
View File

@ -62,6 +62,7 @@ public class Cam4ElectronLoginDialog {
}
String password = Config.getInstance().getSettings().cam4Password;
if (password != null && !password.trim().isEmpty()) {
password = password.replace("'", "\\'");
browser.executeJavaScript("document.querySelector('#loginPageForm input[name=\"password\"]').value = '" + password + "';");
}
browser.executeJavaScript("document.getElementById('footer').setAttribute('style', 'display:none');");

55
client/src/main/java/ctbrec/ui/sites/cam4/Cam4FollowedUpdateService.java
View File

@ -3,19 +3,13 @@ package ctbrec.ui.sites.cam4;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.stream.Collectors;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.json.JSONArray;
import org.json.JSONObject;
import ctbrec.Config;
import ctbrec.Model;
import ctbrec.io.HtmlParser;
import ctbrec.io.HttpException;
import ctbrec.sites.cam4.Cam4;
import ctbrec.sites.cam4.Cam4Model;
@ -27,7 +21,6 @@ import okhttp3.Response;
public class Cam4FollowedUpdateService extends PaginatedScheduledService {
private static final Logger LOG = LoggerFactory.getLogger(Cam4FollowedUpdateService.class);
private Cam4 site;
private boolean showOnline = true;
@ -50,46 +43,28 @@ public class Cam4FollowedUpdateService extends PaginatedScheduledService {
// login first
SiteUiFactory.getUi(site).login();
List<Model> models = new ArrayList<>();
String username = Config.getInstance().getSettings().cam4Username;
String url = site.getBaseUrl() + '/' + username + "/edit/friends_favorites";
String url = site.getBaseUrl() + "/directoryCams?directoryJson=true&online=" + showOnline + "&url=true&friends=true&favorites=true&resultsPerPage=90";
Request req = new Request.Builder().url(url).build();
try(Response response = site.getHttpClient().execute(req)) {
if(response.isSuccessful()) {
try (Response response = site.getHttpClient().execute(req)) {
if (response.isSuccessful()) {
String content = response.body().string();
Elements cells = HtmlParser.getTags(content, "div#favorites div.ff_thumb");
for (Element cell : cells) {
String cellHtml = cell.html();
Element link = HtmlParser.getTag(cellHtml, "div.ff_img a");
String path = link.attr("href");
String modelName = path.substring(1);
Cam4Model model = (Cam4Model) site.createModel(modelName);
model.setPreview("https://snapshots.xcdnpro.com/thumbnails/"+model.getName()+"?s=" + System.currentTimeMillis());
model.setOnlineStateByShowType(parseOnlineState(cellHtml));
JSONObject json = new JSONObject(content);
JSONArray users = json.getJSONArray("users");
for (int i = 0; i < users.length(); i++) {
JSONObject modelJson = users.getJSONObject(i);
String username = modelJson.optString("username");
Cam4Model model = site.createModel(username);
model.setPreview(modelJson.optString("snapshotImageLink"));
model.setOnlineStateByShowType(modelJson.optString("showType"));
model.setDescription(modelJson.optString("statusMessage"));
models.add(model);
}
return models.stream()
.filter(m -> {
try {
return m.isOnline() == showOnline;
} catch (IOException | ExecutionException e) {
LOG.error("Couldn't determine online state", e);
return false;
} catch(InterruptedException e) {
Thread.currentThread().interrupt();
LOG.error("Couldn't determine online state", e);
return false;
}
}).collect(Collectors.toList());
return models;
} else {
throw new HttpException(response.code(), response.message());
}
}
}
private String parseOnlineState(String cellHtml) {
Element state = HtmlParser.getTag(cellHtml, "div.ff_name div");
return state.attr("class").equals("online") ? "NORMAL" : "OFFLINE";
}
};
}

24
client/src/main/java/ctbrec/ui/sites/camsoda/CamsodaFollowedTab.java
View File

@ -1,6 +1,9 @@
package ctbrec.ui.sites.camsoda;
import java.util.function.Predicate;
import ctbrec.sites.camsoda.Camsoda;
import ctbrec.sites.camsoda.CamsodaModel;
import ctbrec.ui.tabs.FollowedTab;
import ctbrec.ui.tabs.ThumbOverviewTab;
import javafx.concurrent.WorkerStateEvent;
@ -18,9 +21,10 @@ public class CamsodaFollowedTab extends ThumbOverviewTab implements FollowedTab
boolean showOnline = true;
public CamsodaFollowedTab(String title, Camsoda camsoda) {
super(title, new CamsodaFollowedUpdateService(camsoda), camsoda);
super(title, new CamsodaUpdateService(camsoda.getBaseUrl() + "/api/v1/browse/following", true, camsoda, m -> true), camsoda);
status = new Label("Logging in...");
grid.getChildren().add(status);
((CamsodaUpdateService)updateService).setFilter(createFilter(this));
}
@Override
@ -40,9 +44,9 @@ public class CamsodaFollowedTab extends ThumbOverviewTab implements FollowedTab
HBox.setMargin(online, new Insets(5, 5, 5, 40));
HBox.setMargin(offline, new Insets(5, 5, 5, 5));
online.setSelected(true);
group.selectedToggleProperty().addListener((e) -> {
group.selectedToggleProperty().addListener(e -> {
showOnline = online.isSelected();
queue.clear();
((CamsodaFollowedUpdateService)updateService).showOnline(online.isSelected());
updateService.restart();
});
}
@ -78,4 +82,18 @@ public class CamsodaFollowedTab extends ThumbOverviewTab implements FollowedTab
}
});
}
private static Predicate<CamsodaModel> createFilter(CamsodaFollowedTab tab) {
return m -> {
try {
return m.isOnline() == tab.showOnline;
} catch(InterruptedException e) {
Thread.currentThread().interrupt();
return false;
} catch (Exception e) {
return false;
}
};
}
}

79
client/src/main/java/ctbrec/ui/sites/camsoda/CamsodaFollowedUpdateService.java
View File

@ -1,79 +0,0 @@
package ctbrec.ui.sites.camsoda;
import static ctbrec.Model.State.*;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.stream.Collectors;
import org.json.JSONArray;
import org.json.JSONObject;
import ctbrec.Model;
import ctbrec.io.HttpException;
import ctbrec.sites.camsoda.Camsoda;
import ctbrec.sites.camsoda.CamsodaModel;
import ctbrec.ui.SiteUiFactory;
import ctbrec.ui.tabs.PaginatedScheduledService;
import javafx.concurrent.Task;
import okhttp3.Request;
import okhttp3.Response;
public class CamsodaFollowedUpdateService extends PaginatedScheduledService {
private Camsoda camsoda;
private boolean showOnline = true;
public CamsodaFollowedUpdateService(Camsoda camsoda) {
this.camsoda = camsoda;
}
@Override
protected Task<List<Model>> createTask() {
return new Task<List<Model>>() {
@Override
public List<Model> call() throws IOException {
List<Model> models = new ArrayList<>();
String url = camsoda.getBaseUrl() + "/api/v1/user/current";
SiteUiFactory.getUi(camsoda).login();
Request request = new Request.Builder().url(url).build();
try(Response response = camsoda.getHttpClient().execute(request)) {
if (response.isSuccessful()) {
JSONObject json = new JSONObject(response.body().string());
if(json.has("status") && json.getBoolean("status")) {
JSONObject user = json.getJSONObject("user");
JSONArray following = user.getJSONArray("following");
for (int i = 0; i < following.length(); i++) {
JSONObject m = following.getJSONObject(i);
CamsodaModel model = (CamsodaModel) camsoda.createModel(m.getString("followname"));
boolean online = m.getInt("online") == 1;
model.setOnlineState(online ? ONLINE : OFFLINE);
model.setPreview("https://md.camsoda.com/thumbs/" + model.getName() + ".jpg");
models.add(model);
}
return models.stream()
.filter((m) -> {
try {
return m.isOnline() == showOnline;
} catch (IOException | ExecutionException | InterruptedException e) {
return false;
}
}).collect(Collectors.toList());
} else {
response.close();
return Collections.emptyList();
}
} else {
throw new HttpException(response.code(), response.message());
}
}
}
};
}
void showOnline(boolean online) {
this.showOnline = online;
}
}

34
client/src/main/java/ctbrec/ui/sites/camsoda/CamsodaUpdateService.java
View File

@ -1,5 +1,7 @@
package ctbrec.ui.sites.camsoda;
import static ctbrec.Model.State.*;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
@ -60,53 +62,55 @@ public class CamsodaUpdateService extends PaginatedScheduledService {
protected List<CamsodaModel> loadOnlineModels() throws IOException {
List<CamsodaModel> models = new ArrayList<>();
if(loginRequired && StringUtil.isBlank(ctbrec.Config.getInstance().getSettings().username)) {
if(loginRequired && StringUtil.isBlank(ctbrec.Config.getInstance().getSettings().camsodaUsername)) {
return Collections.emptyList();
} else {
String url = CamsodaUpdateService.this.url;
LOG.debug("Fetching page {}", url);
if(loginRequired) {
SiteUiFactory.getUi(camsoda).login();
}
Request request = new Request.Builder().url(url).build();
try(Response response = camsoda.getHttpClient().execute(request)) {
try (Response response = camsoda.getHttpClient().execute(request)) {
if (response.isSuccessful()) {
JSONObject json = new JSONObject(response.body().string());
if(json.has("status") && json.getBoolean("status")) {
String body = response.body().string();
JSONObject json = new JSONObject(body);
if (json.optBoolean("status")) {
JSONArray template = json.getJSONArray("template");
JSONArray results = json.getJSONArray("results");
for (int i = 0; i < results.length(); i++) {
JSONObject result = results.getJSONObject(i);
try {
if(result.has("tpl")) {
CamsodaModel model;
if (result.has("tpl")) {
JSONArray tpl = result.getJSONArray("tpl");
String name = tpl.getString(getTemplateIndex(template, "username"));
CamsodaModel model = (CamsodaModel) camsoda.createModel(name);
model = (CamsodaModel) camsoda.createModel(name);
model.setDescription(tpl.getString(getTemplateIndex(template, "subject_html")));
model.setSortOrder(tpl.getFloat(getTemplateIndex(template, "sort_value")));
String preview = "https:" + tpl.getString(getTemplateIndex(template, "thumb"));
model.setPreview(preview);
String displayName = tpl.getString(getTemplateIndex(template, "display_name"));
model.setDisplayName(displayName.replaceAll("[^a-zA-Z0-9]", ""));
if(model.getDisplayName().isBlank()) {
if (model.getDisplayName().isBlank()) {
model.setDisplayName(name);
}
model.setNew(result.optBoolean("new"));
model.setOnlineState(tpl.getString(getTemplateIndex(template, "stream_name")).isEmpty() ? OFFLINE : ONLINE);
models.add(model);
} else {
String name = result.getString("username");
CamsodaModel model = (CamsodaModel) camsoda.createModel(name);
model = (CamsodaModel) camsoda.createModel(name);
model.setSortOrder(result.getFloat("sort_value"));
if(result.has("status")) {
if (result.has("status")) {
model.setOnlineStateByStatus(result.getString("status"));
}
if(result.has("display_name")) {
if (result.has("display_name")) {
model.setDisplayName(result.getString("display_name").replaceAll("[^a-zA-Z0-9]", ""));
if(model.getDisplayName().isBlank()) {
if (model.getDisplayName().isBlank()) {
model.setDisplayName(name);
}
}
if(result.has("thumb")) {
if (result.has("thumb")) {
String previewUrl = "https:" + result.getString("thumb");
model.setPreview(previewUrl);
}
@ -138,4 +142,8 @@ public class CamsodaUpdateService extends PaginatedScheduledService {
}
throw new NoSuchElementException(string + " not found in template: " + template.toString());
}
public void setFilter(Predicate<CamsodaModel> filter) {
this.filter = filter;
}
}

1
client/src/main/java/ctbrec/ui/sites/jasmin/LiveJasminElectronLoginDialog.java
View File

@ -60,6 +60,7 @@ public class LiveJasminElectronLoginDialog {
}
String password = Config.getInstance().getSettings().livejasminPassword;
if (password != null && !password.trim().isEmpty()) {
password = password.replace("'", "\\'");
browser.executeJavaScript("document.querySelector('#login_form input[name=\"password\"]').value = '" + password + "';");
}
browser.executeJavaScript("document.getElementById('header_container').setAttribute('style', 'display:none');");

1
client/src/main/java/ctbrec/ui/sites/myfreecams/MyFreeCamsTableTab.java
View File

@ -526,6 +526,7 @@ public class MyFreeCamsTableTab extends Tab implements TabSelectionListener {
updateService.cancel();
}
saveData();
observableModels.clear();
}
private void saveData() {

1
client/src/main/java/ctbrec/ui/sites/showup/ShowupElectronLoginDialog.java
View File

@ -84,6 +84,7 @@ public class ShowupElectronLoginDialog {
}
String password = Config.getInstance().getSettings().showupPassword;
if (password != null && !password.trim().isEmpty()) {
password = password.replace("'", "\\'");
browser.executeJavaScript("$('input[name=\"password\"]').attr('value','" + password + "')");
}
browser.executeJavaScript("$('input[name=\"remember\"]').attr('value','true')");

2
client/src/main/java/ctbrec/ui/sites/streamate/StreamateFollowedService.java
View File

@ -41,7 +41,7 @@ public class StreamateFollowedService extends PaginatedScheduledService {
public StreamateFollowedService(Streamate streamate) {
this.streamate = streamate;
this.httpClient = (StreamateHttpClient) streamate.getHttpClient();
this.url = "https://member.naiadsystems.com/search/favorites?domain=streamate.com&skipXmentSelection=true";
this.url = "https://member.naiadsystems.com/search/v3/favorites?skipXmentSelection=true&skinConfig=%7B%7D&filters=";
}
@Override

23
client/src/main/java/ctbrec/ui/sites/stripchat/StripchatConfigUI.java
View File

@ -12,8 +12,11 @@ import javafx.scene.control.Button;
import javafx.scene.control.CheckBox;
import javafx.scene.control.Label;
import javafx.scene.control.PasswordField;
import javafx.scene.control.RadioButton;
import javafx.scene.control.TextField;
import javafx.scene.control.ToggleGroup;
import javafx.scene.layout.GridPane;
import javafx.scene.layout.HBox;
import javafx.scene.layout.Priority;
public class StripchatConfigUI extends AbstractConfigUI {
@ -44,6 +47,26 @@ public class StripchatConfigUI extends AbstractConfigUI {
GridPane.setMargin(enabled, new Insets(0, 0, 0, SettingsTab.CHECKBOX_MARGIN));
layout.add(enabled, 1, row++);
l = new Label("Site");
layout.add(l, 0, row);
ToggleGroup toggleGroup = new ToggleGroup();
RadioButton optionA = new RadioButton("Stripchat");
optionA.setSelected(!Config.getInstance().getSettings().stripchatUseXhamster);
optionA.setToggleGroup(toggleGroup);
RadioButton optionB = new RadioButton("xHamsterLive");
optionB.setSelected(!optionA.isSelected());
optionB.setToggleGroup(toggleGroup);
optionA.selectedProperty().addListener((obs, oldV, newV) -> {
Config.getInstance().getSettings().stripchatUseXhamster = !newV;
save();
});
HBox hbox = new HBox();
hbox.getChildren().addAll(optionA, optionB);
HBox.setMargin(optionA, new Insets(5));
HBox.setMargin(optionB, new Insets(5));
GridPane.setMargin(hbox, new Insets(0, 0, 0, SettingsTab.CHECKBOX_MARGIN));
layout.add(hbox, 1, row++);
layout.add(new Label("Stripchat User"), 0, row);
TextField username = new TextField(Config.getInstance().getSettings().stripchatUsername);
username.textProperty().addListener((ob, o, n) -> {

125
client/src/main/java/ctbrec/ui/sites/stripchat/StripchatElectronLoginDialog.java Normal file
View File

@ -0,0 +1,125 @@
package ctbrec.ui.sites.stripchat;
import java.io.IOException;
import java.util.Collections;
import java.util.function.Consumer;
import org.json.JSONArray;
import org.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import ctbrec.Config;
import ctbrec.sites.stripchat.Stripchat;
import ctbrec.ui.ExternalBrowser;
import okhttp3.Cookie;
import okhttp3.Cookie.Builder;
import okhttp3.CookieJar;
import okhttp3.HttpUrl;
public class StripchatElectronLoginDialog {
private static final transient Logger LOG = LoggerFactory.getLogger(StripchatElectronLoginDialog.class);
public String DOMAIN = Stripchat.domain;
public String URL = Stripchat.baseUri;
private CookieJar cookieJar;
private ExternalBrowser browser;
public StripchatElectronLoginDialog(CookieJar cookieJar) throws IOException {
this.cookieJar = cookieJar;
browser = ExternalBrowser.getInstance();
try {
JSONObject config = new JSONObject();
config.put("url", URL);
config.put("w", 640);
config.put("h", 640);
JSONObject msg = new JSONObject();
msg.put("config", config);
browser.run(msg, msgHandler);
} catch (InterruptedException e) {
throw new IOException("Couldn't wait for login dialog", e);
} finally {
browser.close();
}
}
private Consumer<String> msgHandler = (line) -> {
if(!line.startsWith("{")) {
System.err.println(line);
} else {
JSONObject json = new JSONObject(line);
if(json.has("url")) {
String url = json.getString("url");
if(url.endsWith(DOMAIN) || url.endsWith(DOMAIN + '/')) {
try {
browser.executeJavaScript("document.querySelector('button[class~=\"btn-visitors-agreement-accept\"]').click();");
browser.executeJavaScript("document.querySelector('div[class~=\"header-dropdown\"] a[class~=\"dropdown-link\"]').click();");
browser.executeJavaScript("document.querySelector('a[class~=\"btn\"][href*=\"login\"]').click();");
String username = Config.getInstance().getSettings().stripchatUsername;
if (username != null && !username.trim().isEmpty()) {
browser.executeJavaScript("document.querySelector('#login_login_or_email').value = '" + username + "';");
}
String password = Config.getInstance().getSettings().stripchatPassword;
if (password != null && !password.trim().isEmpty()) {
password = password.replace("'", "\\'");
browser.executeJavaScript("document.querySelector('#login_password').value = '" + password + "';");
}
browser.executeJavaScript("document.querySelector('#recaptcha-checkbox-border').click();");
browser.executeJavaScript("document.querySelector('*[class~=btn-login]').addEventListener('click', function() {window.setTimeout(function() {location.reload()}, 2000)});");
} catch(Exception e) {
LOG.warn("Couldn't auto fill username and password for Stripchat", e);
}
}
if (json.has("cookies")) {
JSONArray _cookies = json.getJSONArray("cookies");
boolean sessionCookieFound = false;
for (int i = 0; i < _cookies.length(); i++) {
JSONObject cookie = _cookies.getJSONObject(i);
if (cookie.getString("domain").contains(DOMAIN)) {
String domain = cookie.getString("domain");
if (domain.startsWith(".")) {
domain = domain.substring(1);
}
Cookie c = createCookie(domain, cookie);
cookieJar.saveFromResponse(HttpUrl.parse(url), Collections.singletonList(c));
c = createCookie(DOMAIN, cookie);
cookieJar.saveFromResponse(HttpUrl.parse(Stripchat.baseUri), Collections.singletonList(c));
if (c.name().contains("_com_sessionId")) {
sessionCookieFound = true;
}
}
}
if(sessionCookieFound) {
try {
browser.close();
} catch (IOException e) {
LOG.error("Couldn't send close request to browser", e);
}
}
}
}
}
};
private Cookie createCookie(String domain, JSONObject cookie) {
Builder b = new Cookie.Builder()
.path(cookie.getString("path"))
.domain(domain)
.name(cookie.getString("name"))
.value(cookie.getString("value"))
.expiresAt(Double.valueOf(cookie.optDouble("expirationDate")).longValue());
if(cookie.optBoolean("hostOnly")) {
b.hostOnlyDomain(domain);
}
if(cookie.optBoolean("httpOnly")) {
b.httpOnly();
}
if(cookie.optBoolean("secure")) {
b.secure();
}
return b.build();
}
}

23
client/src/main/java/ctbrec/ui/sites/stripchat/StripchatFollowedTab.java
View File

@ -4,14 +4,10 @@ import ctbrec.sites.stripchat.Stripchat;
import ctbrec.ui.tabs.FollowedTab;
import ctbrec.ui.tabs.ThumbOverviewTab;
import javafx.concurrent.WorkerStateEvent;
import javafx.geometry.Insets;
import javafx.scene.Scene;
import javafx.scene.control.Label;
import javafx.scene.control.RadioButton;
import javafx.scene.control.ToggleGroup;
import javafx.scene.input.KeyCode;
import javafx.scene.input.KeyEvent;
import javafx.scene.layout.HBox;
public class StripchatFollowedTab extends ThumbOverviewTab implements FollowedTab {
private Label status;
@ -26,25 +22,6 @@ public class StripchatFollowedTab extends ThumbOverviewTab implements FollowedTa
@Override
protected void createGui() {
super.createGui();
addOnlineOfflineSelector();
}
private void addOnlineOfflineSelector() {
ToggleGroup group = new ToggleGroup();
RadioButton online = new RadioButton("online");
online.setToggleGroup(group);
RadioButton offline = new RadioButton("offline");
offline.setToggleGroup(group);
pagination.getChildren().add(online);
pagination.getChildren().add(offline);
HBox.setMargin(online, new Insets(5, 5, 5, 40));
HBox.setMargin(offline, new Insets(5, 5, 5, 5));
online.setSelected(true);
group.selectedToggleProperty().addListener(e -> {
queue.clear();
((StripchatFollowedUpdateService)updateService).showOnline(online.isSelected());
updateService.restart();
});
}
@Override

43
client/src/main/java/ctbrec/ui/sites/stripchat/StripchatFollowedUpdateService.java
View File

@ -4,8 +4,8 @@ import static ctbrec.io.HttpConstants.*;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import org.json.JSONArray;
import org.json.JSONObject;
@ -24,8 +24,8 @@ import okhttp3.Request;
import okhttp3.Response;
public class StripchatFollowedUpdateService extends PaginatedScheduledService {
private static final int PAGE_SIZE = 30;
private Stripchat stripchat;
private boolean showOnline = true;
public StripchatFollowedUpdateService(Stripchat stripchat) {
this.stripchat = stripchat;
@ -36,22 +36,33 @@ public class StripchatFollowedUpdateService extends PaginatedScheduledService {
return new Task<List<Model>>() {
@Override
public List<Model> call() throws IOException {
int startIndex = (getPage() - 1) * PAGE_SIZE;
JSONArray favoriteModelIds = loadFavoriteModelIds();
List<Model> models = loadModels(favoriteModelIds);
List<Integer> modelIdsToLoad = new ArrayList<>(PAGE_SIZE);
List<Model> models;
if (startIndex < favoriteModelIds.length()) {
int modelsOnPage = Math.min(PAGE_SIZE, favoriteModelIds.length() - startIndex - 1);
for (int i = 0; i < modelsOnPage; i++) {
modelIdsToLoad.add(favoriteModelIds.getInt(startIndex + i));
}
models = loadModels(modelIdsToLoad);
} else {
models = Collections.emptyList();
}
return models;
}
private List<Model> loadModels(JSONArray favoriteModelIds) throws IOException {
private List<Model> loadModels(List<Integer> modelIdsToLoad) throws IOException {
List<Model> models = new ArrayList<>();
HttpUrl.Builder urlBuilder = HttpUrl.parse(stripchat.getBaseUrl() + "/api/front/models/list").newBuilder();
for (int i = 0; i < favoriteModelIds.length(); i++) {
urlBuilder.addQueryParameter("modelIds["+i+"]", Integer.toString(favoriteModelIds.getInt(i)));
for (int i = 0; i < modelIdsToLoad.size(); i++) {
urlBuilder.addQueryParameter("modelIds["+i+"]", modelIdsToLoad.get(i).toString());
}
Request request = new Request.Builder()
.url(urlBuilder.build())
.header(ACCEPT, "*/*")
.header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
.header(REFERER, Stripchat.BASE_URI + "/favorites")
.header(REFERER, Stripchat.baseUri + "/favorites")
.header(CONTENT_TYPE, MIMETYPE_APPLICATION_JSON)
.build();
try (Response response = stripchat.getHttpClient().execute(request)) {
@ -64,10 +75,7 @@ public class StripchatFollowedUpdateService extends PaginatedScheduledService {
StripchatModel model = stripchat.createModel(user.optString("username"));
model.setDescription(user.optString("description"));
model.setPreview(user.optString("previewUrlThumbBig"));
boolean online = Objects.equals(user.optString("status"), "public");
if (showOnline == online) {
models.add(model);
}
models.add(model);
}
}
} else {
@ -79,21 +87,22 @@ public class StripchatFollowedUpdateService extends PaginatedScheduledService {
private JSONArray loadFavoriteModelIds() throws IOException {
SiteUiFactory.getUi(stripchat).login();
stripchat.getHttpClient().login();
long userId = ((StripchatHttpClient) stripchat.getHttpClient()).getUserId();
String url = stripchat.getBaseUrl() + "/api/front/users/" + userId + "/favorites";
Request request = new Request.Builder()
.url(url)
.header(ACCEPT, MIMETYPE_APPLICATION_JSON)
.header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
.header(ORIGIN, Stripchat.BASE_URI)
.header(REFERER, Stripchat.BASE_URI + "/favorites")
.header(ORIGIN, Stripchat.baseUri)
.header(REFERER, Stripchat.baseUri + "/favorites")
.header(CONTENT_TYPE, MIMETYPE_APPLICATION_JSON)
.build();
try (Response response = stripchat.getHttpClient().execute(request)) {
if (response.isSuccessful()) {
JSONObject json = new JSONObject(response.body().string());
if(json.has("userIds")) {
JSONArray userIds = json.getJSONArray("userIds");
if (json.has("modelIds")) {
JSONArray userIds = json.getJSONArray("modelIds");
return userIds;
} else {
return new JSONArray();
@ -105,8 +114,4 @@ public class StripchatFollowedUpdateService extends PaginatedScheduledService {
}
};
}
void showOnline(boolean online) {
this.showOnline = online;
}
}

51
client/src/main/java/ctbrec/ui/sites/stripchat/StripchatSiteUi.java
View File

@ -1,20 +1,30 @@
package ctbrec.ui.sites.stripchat;
import java.io.IOException;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import ctbrec.sites.stripchat.Stripchat;
import ctbrec.sites.stripchat.StripchatHttpClient;
import ctbrec.ui.controls.Dialogs;
import ctbrec.ui.sites.AbstractSiteUi;
import ctbrec.ui.sites.ConfigUI;
import ctbrec.ui.tabs.TabProvider;
import javafx.application.Platform;
public class StripchatSiteUi extends AbstractSiteUi {
private static final Logger LOG = LoggerFactory.getLogger(StripchatSiteUi.class);
private StripchatTabProvider tabProvider;
private StripchatConfigUI configUi;
private Stripchat stripchat;
private Stripchat site;
public StripchatSiteUi(Stripchat stripchat) {
this.stripchat = stripchat;
this.site = stripchat;
tabProvider = new StripchatTabProvider(stripchat);
configUi = new StripchatConfigUI(stripchat);
}
@ -31,7 +41,40 @@ public class StripchatSiteUi extends AbstractSiteUi {
@Override
public synchronized boolean login() throws IOException {
boolean automaticLogin = stripchat.login();
return automaticLogin;
boolean automaticLogin = site.login();
if (automaticLogin) {
return true;
} else {
BlockingQueue<Boolean> queue = new LinkedBlockingQueue<>();
Runnable showDialog = () -> {
// login with external browser
try {
new StripchatElectronLoginDialog(site.getHttpClient().getCookieJar());
} catch (Exception e1) {
LOG.error("Error logging in with external browser", e1);
Dialogs.showError("Login error", "Couldn't login to " + site.getName(), e1);
}
try {
queue.put(true);
} catch (InterruptedException e) {
LOG.error("Error while signaling termination", e);
}
};
Platform.runLater(showDialog);
try {
queue.take();
} catch (InterruptedException e) {
LOG.error("Error while waiting for login dialog to close", e);
throw new IOException(e);
}
StripchatHttpClient httpClient = (StripchatHttpClient) site.getHttpClient();
boolean loggedIn = httpClient.checkLoginSuccess();
return loggedIn;
}
}
}

52
client/src/main/java/ctbrec/ui/tabs/RecordedModelsTab.java
View File

@ -1,9 +1,13 @@
package ctbrec.ui.tabs;
import static ctbrec.SubsequentAction.*;
import java.io.IOException;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import java.time.Instant;
import java.time.LocalDate;
import java.time.ZoneId;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Iterator;
@ -25,6 +29,7 @@ import ctbrec.Config;
import ctbrec.Model;
import ctbrec.Recording;
import ctbrec.StringUtil;
import ctbrec.SubsequentAction;
import ctbrec.recorder.Recorder;
import ctbrec.sites.Site;
import ctbrec.ui.AutosizeAlert;
@ -59,8 +64,10 @@ import javafx.geometry.Pos;
import javafx.scene.control.Alert;
import javafx.scene.control.Button;
import javafx.scene.control.ContextMenu;
import javafx.scene.control.DatePicker;
import javafx.scene.control.Label;
import javafx.scene.control.MenuItem;
import javafx.scene.control.RadioButton;
import javafx.scene.control.ScrollPane;
import javafx.scene.control.SelectionMode;
import javafx.scene.control.Tab;
@ -71,6 +78,7 @@ import javafx.scene.control.TableColumn.SortType;
import javafx.scene.control.TableRow;
import javafx.scene.control.TableView;
import javafx.scene.control.TextField;
import javafx.scene.control.ToggleGroup;
import javafx.scene.control.Tooltip;
import javafx.scene.control.cell.CheckBoxTableCell;
import javafx.scene.control.cell.PropertyValueFactory;
@ -84,6 +92,7 @@ import javafx.scene.input.MouseButton;
import javafx.scene.input.MouseEvent;
import javafx.scene.layout.BorderPane;
import javafx.scene.layout.FlowPane;
import javafx.scene.layout.GridPane;
import javafx.scene.layout.HBox;
import javafx.scene.layout.Priority;
import javafx.util.Callback;
@ -612,6 +621,8 @@ public class RecordedModelsTab extends Tab implements TabSelectionListener {
pauseRecording.setOnAction(e -> pauseRecording(selectedModels));
MenuItem resumeRecording = new MenuItem("Resume Recording");
resumeRecording.setOnAction(e -> resumeRecording(selectedModels));
MenuItem stopRecordingAt = new MenuItem("Stop Recording at Date");
stopRecordingAt.setOnAction(e -> setStopDate(selectedModels.get(0)));
MenuItem openInBrowser = new MenuItem("Open in Browser");
openInBrowser.setOnAction(e -> DesktopIntegration.open(selectedModels.get(0).getUrl()));
MenuItem openInPlayer = new MenuItem("Open in Player");
@ -630,6 +641,7 @@ public class RecordedModelsTab extends Tab implements TabSelectionListener {
ContextMenu menu = new ContextMenu(stop);
if (selectedModels.size() == 1) {
menu.getItems().add(selectedModels.get(0).isSuspended() ? resumeRecording : pauseRecording);
menu.getItems().add(stopRecordingAt);
} else {
menu.getItems().addAll(resumeRecording, pauseRecording);
}
@ -646,6 +658,46 @@ public class RecordedModelsTab extends Tab implements TabSelectionListener {
return menu;
}
private void setStopDate(JavaFxModel model) {
DatePicker datePicker = new DatePicker();
GridPane grid = new GridPane();
grid.setHgap(10);
grid.setVgap(10);
grid.setPadding(new Insets(20, 150, 10, 10));
grid.add(new Label("Stop at"), 0, 0);
grid.add(datePicker, 1, 0);
grid.add(new Label("And then"), 0, 1);
ToggleGroup toggleGroup = new ToggleGroup();
RadioButton pauseButton = new RadioButton("pause recording");
pauseButton.setSelected(model.getRecordUntilSubsequentAction() == PAUSE);
pauseButton.setToggleGroup(toggleGroup);
RadioButton removeButton = new RadioButton("remove model");
removeButton.setSelected(model.getRecordUntilSubsequentAction() == REMOVE);
removeButton.setToggleGroup(toggleGroup);
HBox row = new HBox();
row.getChildren().addAll(pauseButton, removeButton);
HBox.setMargin(pauseButton, new Insets(5));
HBox.setMargin(removeButton, new Insets(5));
grid.add(row, 1, 1);
if (model.getRecordUntil().toEpochMilli() != Model.RECORD_INDEFINITELY) {
LocalDate localDate = LocalDate.ofInstant(model.getRecordUntil(), ZoneId.systemDefault());
datePicker.setValue(localDate);
}
boolean userClickedOk = Dialogs.showCustomInput(getTabPane().getScene(), "Stop Recording at", grid);
if (userClickedOk) {
SubsequentAction action = pauseButton.isSelected() ? PAUSE : REMOVE;
LOG.info("Stop at {} and {}", datePicker.getValue(), action);
Instant stopAt = Instant.from(datePicker.getValue().atStartOfDay().atZone(ZoneId.systemDefault()));
model.setRecordUntil(stopAt);
model.setRecordUntilSubsequentAction(action);
try {
recorder.stopRecordingAt(model.getDelegate());
} catch (InvalidKeyException | NoSuchAlgorithmException | IOException e) {
Dialogs.showError(getTabPane().getScene(), "Error", "Couln't set stop date", e);
}
}
}
private void ignore(ObservableList<JavaFxModel> selectedModels) {
for (JavaFxModel fxModel : selectedModels) {
Model modelToIgnore = fxModel.getDelegate();

26
client/src/main/java/ctbrec/ui/tabs/RecordingsTab.java
View File

@ -31,6 +31,8 @@ import ctbrec.Config;
import ctbrec.Recording;
import ctbrec.Recording.State;
import ctbrec.StringUtil;
import ctbrec.event.EventBusHolder;
import ctbrec.event.RecordingStateChangedEvent;
import ctbrec.recorder.ProgressListener;
import ctbrec.recorder.Recorder;
import ctbrec.recorder.RecordingPinnedException;
@ -437,15 +439,13 @@ public class RecordingsTab extends Tab implements TabSelectionListener {
}
MenuItem rerunPostProcessing = new MenuItem("Rerun Post-Processing");
rerunPostProcessing.setOnAction(e -> triggerPostProcessing(first));
if (first.getStatus() == FAILED || first.getStatus() == WAITING || first.getStatus() == FINISHED) {
contextMenu.getItems().add(rerunPostProcessing);
}
rerunPostProcessing.setOnAction(e -> triggerPostProcessing(recordings));
contextMenu.getItems().add(rerunPostProcessing);
rerunPostProcessing.setDisable(!recordings.stream().allMatch(Recording::canBePostProcessed));
if(recordings.size() > 1) {
openInPlayer.setDisable(true);
openDir.setDisable(true);
rerunPostProcessing.setDisable(true);
}
return contextMenu;
@ -565,13 +565,15 @@ public class RecordingsTab extends Tab implements TabSelectionListener {
new Thread(() -> DesktopIntegration.open(tsFile.getParent())).start();
}
private void triggerPostProcessing(JavaFxRecording first) {
private void triggerPostProcessing(List<JavaFxRecording> recs) {
new Thread(() -> {
try {
recorder.rerunPostProcessing(first.getDelegate());
} catch (IOException | InvalidKeyException | NoSuchAlgorithmException | IllegalStateException e1) {
showErrorDialog("Error while starting post-processing", "The post-processing could not be started", e1);
LOG.error("Error while starting post-processing", e1);
for (JavaFxRecording rec : recs) {
try {
recorder.rerunPostProcessing(rec.getDelegate());
} catch (IOException | InvalidKeyException | NoSuchAlgorithmException | IllegalStateException e1) {
showErrorDialog("Error while starting post-processing", "The post-processing could not be started", e1);
LOG.error("Error while starting post-processing", e1);
}
}
}).start();
}
@ -633,6 +635,8 @@ public class RecordingsTab extends Tab implements TabSelectionListener {
Platform.runLater(() -> {
recording.setStatus(FINISHED);
recording.setProgress(-1);
RecordingStateChangedEvent evt = new RecordingStateChangedEvent(target, recording.getStatus(), recording.getModel(), recording.getStartDate());
EventBusHolder.BUS.post(evt);
});
}
});

4
client/src/main/java/ctbrec/ui/tabs/ThumbCell.java
View File

@ -644,4 +644,8 @@ public class ThumbCell extends StackPane {
return new int[2];
}
}
public void releaseResources() {
iv.setImage(null);
}
}

9
client/src/main/java/ctbrec/ui/tabs/ThumbOverviewTab.java
View File

@ -872,6 +872,15 @@ public class ThumbOverviewTab extends Tab implements TabSelectionListener {
updateService.cancel();
}
queue.clear();
for (Iterator<Node> iterator = grid.getChildren().iterator(); iterator.hasNext();) {
Node node = iterator.next();
if(node instanceof ThumbCell) {
ThumbCell thumbCell = (ThumbCell) node;
thumbCell.releaseResources();
iterator.remove();
}
}
}
void suspendUpdates(boolean suspend) {

13
client/src/main/java/ctbrec/ui/tabs/logging/CtbrecAppender.java Normal file
View File

@ -0,0 +1,13 @@
package ctbrec.ui.tabs.logging;
import ch.qos.logback.classic.spi.LoggingEvent;
import ch.qos.logback.core.ConsoleAppender;
import ctbrec.event.EventBusHolder;
public class CtbrecAppender extends ConsoleAppender<LoggingEvent> {
@Override
protected void append(LoggingEvent event) {
EventBusHolder.BUS.post(event);
}
}

194
client/src/main/java/ctbrec/ui/tabs/logging/LoggingTab.java Normal file
View File

@ -0,0 +1,194 @@
package ctbrec.ui.tabs.logging;
import java.time.Instant;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.util.Collections;
import java.util.LinkedList;
import java.util.stream.Collectors;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.eventbus.Subscribe;
import ch.qos.logback.classic.LoggerContext;
import ch.qos.logback.classic.encoder.PatternLayoutEncoder;
import ch.qos.logback.classic.spi.IThrowableProxy;
import ch.qos.logback.classic.spi.LoggingEvent;
import ch.qos.logback.classic.spi.StackTraceElementProxy;
import ctbrec.event.EventBusHolder;
import ctbrec.ui.controls.SearchBox;
import javafx.application.Platform;
import javafx.beans.property.SimpleStringProperty;
import javafx.collections.FXCollections;
import javafx.collections.ObservableList;
import javafx.geometry.Insets;
import javafx.scene.control.ContextMenu;
import javafx.scene.control.MenuItem;
import javafx.scene.control.SelectionMode;
import javafx.scene.control.Tab;
import javafx.scene.control.TableColumn;
import javafx.scene.control.TableView;
import javafx.scene.input.Clipboard;
import javafx.scene.input.ClipboardContent;
import javafx.scene.input.ContextMenuEvent;
import javafx.scene.input.MouseEvent;
import javafx.scene.layout.BorderPane;
public class LoggingTab extends Tab {
private static final int HISTORY_LENGTH = 10_000;
private SearchBox filter = new SearchBox();
private TableView<LoggingEvent> table = new TableView<>();
private ObservableList<LoggingEvent> history = FXCollections.observableList(Collections.synchronizedList(new LinkedList<>()));
private ObservableList<LoggingEvent> filteredEvents = FXCollections.observableArrayList();
private DateTimeFormatter timeFormatter = DateTimeFormatter.ISO_LOCAL_DATE_TIME;
private volatile boolean tabClosed = false;
private ContextMenu popup;
private Object eventBustSubscriber = new Object() {
@Subscribe
public void publishLoggingEevent(LoggingEvent event) {
if (!tabClosed) {
Platform.runLater(() -> {
history.add(event);
if (history.size() > HISTORY_LENGTH - 1) {
history.remove(0);
}
filter();
});
}
}
};
private void filter() {
filteredEvents.clear();
filteredEvents.addAll(history.stream().filter(evt -> {
String q = filter.getText().toLowerCase();
return evt.getLevel().toString().toLowerCase().contains(q)
|| createLogMessage(evt).toLowerCase().contains(q);
}).collect(Collectors.toList()));
}
public LoggingTab() {
setText("Logging");
subscribeToEventBus();
table = new TableView<>(filteredEvents);
int idx = 0;
TableColumn<LoggingEvent, String> level = createTableColumn("Level", 65, idx++);
level.setCellValueFactory(cdf -> new SimpleStringProperty(cdf.getValue().getLevel().toString()));
table.getColumns().add(level);
TableColumn<LoggingEvent, String> time = createTableColumn("Timestamp", 200, idx++);
time.setCellValueFactory(cdf -> {
Instant instant = Instant.ofEpochMilli(cdf.getValue().getTimeStamp());
return new SimpleStringProperty(instant.atZone(ZoneId.systemDefault()).format(timeFormatter));
});
table.getColumns().add(time);
TableColumn<LoggingEvent, String> location = createTableColumn("Location", 250, idx++);
location.setCellValueFactory(cdf -> {
StackTraceElement loc = cdf.getValue().getCallerData()[0];
String l = loc.getFileName() + ":" + loc.getLineNumber();
return new SimpleStringProperty(l);
});
table.getColumns().add(location);
TableColumn<LoggingEvent, String> msg = createTableColumn("Message", 2000, idx++);
msg.setCellValueFactory(cdf -> new SimpleStringProperty(createLogMessage(cdf.getValue())));
table.getColumns().add(msg);
BorderPane layout = new BorderPane();
BorderPane.setMargin(table, new Insets(10));
BorderPane.setMargin(filter, new Insets(10, 10, 0, 10));
layout.setCenter(table);
layout.setTop(filter);
setContent(layout);
setOnClosed(evt -> {
EventBusHolder.BUS.unregister(eventBustSubscriber);
tabClosed = true;
});
filter.setPromptText("Search");
filter.textProperty().addListener( (observableValue, oldValue, newValue) -> filter());
table.getSelectionModel().setSelectionMode(SelectionMode.MULTIPLE);
table.addEventHandler(ContextMenuEvent.CONTEXT_MENU_REQUESTED, event -> {
popup = createContextMenu();
if (popup != null) {
popup.show(table, event.getScreenX(), event.getScreenY());
}
event.consume();
});
table.addEventHandler(MouseEvent.MOUSE_PRESSED, event -> {
if (popup != null) {
popup.hide();
}
});
}
private ContextMenu createContextMenu() {
final ObservableList<LoggingEvent> selectedEvents = table.getSelectionModel().getSelectedItems();
if (selectedEvents.isEmpty()) {
return null;
}
MenuItem copy = new MenuItem("Copy");
copy.setOnAction(e -> {
Platform.runLater(() -> {
String formattedMessages = getFormattedMessages(selectedEvents);
final Clipboard clipboard = Clipboard.getSystemClipboard();
final ClipboardContent content = new ClipboardContent();
content.putString(formattedMessages);
clipboard.setContent(content);
});
});
ContextMenu menu = new ContextMenu(copy);
return menu;
}
private String getFormattedMessages(ObservableList<LoggingEvent> selectedEvents) {
StringBuilder sb = new StringBuilder();
ch.qos.logback.classic.Logger rootLogger = (ch.qos.logback.classic.Logger)LoggerFactory.getLogger(Logger.ROOT_LOGGER_NAME);
LoggerContext loggerContext = rootLogger.getLoggerContext();
loggerContext.reset();
PatternLayoutEncoder encoder = new PatternLayoutEncoder();
encoder.setContext(loggerContext);
encoder.setPattern("%date %level [%thread] %logger{10} [%file:%line] %msg%n");
encoder.start();
for (LoggingEvent evt : selectedEvents) {
byte[] encode = encoder.encode(evt);
sb.append(new String(encode));
}
return sb.toString();
}
private String createLogMessage(LoggingEvent evt) {
StringBuilder sb = new StringBuilder(evt.getFormattedMessage());
if(evt.getThrowableProxy() != null) {
IThrowableProxy throwableProxy = evt.getThrowableProxy();
sb.append('\n').append(throwableProxy.getClassName()).append(':').append(' ').append(throwableProxy.getMessage());
for (StackTraceElementProxy step : throwableProxy.getStackTraceElementProxyArray()) {
sb.append('\n').append('\t').append(step.getSTEAsString());
}
}
return sb.toString();
}
private <T> TableColumn<LoggingEvent, T> createTableColumn(String text, int width, int idx) {
TableColumn<LoggingEvent, T> tc = new TableColumn<>(text);
tc.setPrefWidth(width);
tc.setUserData(idx);
return tc;
}
private void subscribeToEventBus() {
EventBusHolder.BUS.register(eventBustSubscriber);
}
}

6
client/src/main/resources/html/docs/ConfigurationFile.md
View File

@ -41,13 +41,17 @@ the port ctbrec tries to connect to, if it is run in remote mode.
- **livePreviews** (app only) - Enables the live preview feature in the app.
- **minimumResolution** - [1 - 2147483647]. Sets the minimum video height for a recording. ctbrec tries to find a stream quality, which is higher than or equal to this value. If the only provided stream quality is below this threshold, ctbrec won't record the stream.
- **maximumResolution** - [1 - 2147483647]. Sets the maximum video height for a recording. ctbrec tries to find a stream quality, which is lower than or equal to this value. If the only provided stream quality is above this threshold, ctbrec won't record the stream.
- **minimumLengthInSeconds** - [0 - 2147483647] Automatically delete recordings, which are shorter than this amount of seconds. 0 disables this feature.
- **minimumSpaceLeftInBytes** - [0 - 9223372036854775807] The space in bytes ctbrec should conserve on the hard drive. 1 GiB = 1024 MiB = 1048576 KiB = 1073741824 bytes
- **onlineCheckIntervalInSecs** - [1 - 2147483647] How often ctbrec checks, if a model is online. This is not a guaranteed interval: If you record many models, the online check for all models can take longer than this interval. A minute is a reasonable value, but you can go lower, if you don't want to miss a anything. But don't go too low, or you risk to do too many requests in a short amount of time and get banned by some sites.
- **onlineCheckIntervalInSecs** - [1 - 2147483647] How often ctbrec checks, if a model is online. This is not a guaranteed interval: If you record many models, the online check for all models can take longer than this interval. A minute is a reasonable value, but you can go lower, if you don't want to miss a anything. But don't go too low, or you risk to do too many requests in a short amount of time and get banned by some sites.
- **onlineCheckSkipsPausedModels** - [`true`,`false`] Skip the online check for paused models. If you have many models in the recording list, this can reduce the delay when a recording starts after a model came online.
- **postProcessing** - Absolute path to a script, which is executed once a recording is finished. See [Post-Processing](/docs/PostProcessing.md).

7
client/src/main/resources/logback.xml
View File

@ -8,6 +8,12 @@
</pattern>
</encoder>
</appender>
<appender name="GUI" class="ctbrec.ui.tabs.logging.CtbrecAppender">
<encoder>
<pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern>
</encoder>
</appender>
<appender name="FILE"
class="ch.qos.logback.core.rolling.RollingFileAppender">
@ -32,6 +38,7 @@
<root level="DEBUG">
<appender-ref ref="STDOUT" />
<appender-ref ref="FILE" />
<appender-ref ref="GUI" />
</root>
<logger name="ctbrec.LoggingInterceptor" level="info"/>

2
common/pom.xml
View File

@ -8,7 +8,7 @@
<parent>
<groupId>ctbrec</groupId>
<artifactId>master</artifactId>
<version>3.8.1</version>
<version>3.8.6</version>
<relativePath>../master</relativePath>
</parent>

22
common/src/main/java/ctbrec/AbstractModel.java
View File

@ -33,6 +33,8 @@ public abstract class AbstractModel implements Model {
protected State onlineState = State.UNKNOWN;
private Instant lastSeen;
private Instant lastRecorded;
private Instant recordUntil;
private SubsequentAction recordUntilSubsequentAction;
@Override
public boolean isOnline() throws IOException, ExecutionException, InterruptedException {
@ -231,6 +233,26 @@ public abstract class AbstractModel implements Model {
this.lastRecorded = lastRecorded;
}
@Override
public Instant getRecordUntil() {
return Optional.ofNullable(recordUntil).orElse(Instant.ofEpochMilli(RECORD_INDEFINITELY));
}
@Override
public void setRecordUntil(Instant recordUntil) {
this.recordUntil = recordUntil;
}
@Override
public SubsequentAction getRecordUntilSubsequentAction() {
return Optional.ofNullable(recordUntilSubsequentAction).orElse(SubsequentAction.PAUSE);
}
@Override
public void setRecordUntilSubsequentAction(SubsequentAction recordUntilSubsequentAction) {
this.recordUntilSubsequentAction = recordUntilSubsequentAction;
}
@Override
public Download createDownload() {
if (Config.isServerMode() && !Config.getInstance().getSettings().recordSingleFile) {

8
common/src/main/java/ctbrec/Model.java
View File

@ -20,6 +20,8 @@ import ctbrec.sites.Site;
public interface Model extends Comparable<Model>, Serializable {
public static final long RECORD_INDEFINITELY = 9000000000000000000l;
public enum State {
ONLINE("online"),
OFFLINE("offline"),
@ -128,4 +130,10 @@ public interface Model extends Comparable<Model>, Serializable {
public HttpHeaderFactory getHttpHeaderFactory();
public Instant getRecordUntil();
public void setRecordUntil(Instant instant);
public SubsequentAction getRecordUntilSubsequentAction();
public void setRecordUntilSubsequentAction(SubsequentAction action);
}

6
common/src/main/java/ctbrec/Recording.java
View File

@ -1,5 +1,7 @@
package ctbrec;
import static ctbrec.Recording.State.*;
import java.io.File;
import java.io.IOException;
import java.io.Serializable;
@ -272,4 +274,8 @@ public class Recording implements Serializable {
public void refresh() {
sizeInByte = getSize();
}
public boolean canBePostProcessed() {
return getStatus() == FAILED || getStatus() == WAITING || getStatus() == FINISHED;
}
}

2
common/src/main/java/ctbrec/Settings.java
View File

@ -89,6 +89,7 @@ public class Settings {
public List<Model> models = new ArrayList<>();
public List<Model> modelsIgnored = new ArrayList<>();
public int onlineCheckIntervalInSecs = 60;
public boolean onlineCheckSkipsPausedModels = false;
public int overviewUpdateIntervalInSecs = 10;
public String password = ""; // chaturbate password TODO maybe rename this onetime
public String postProcessing = "";
@ -122,6 +123,7 @@ public class Settings {
public String streamateUsername = "";
public String stripchatUsername = "";
public String stripchatPassword = "";
public boolean stripchatUseXhamster = false;
public boolean transportLayerSecurity = true;
public int thumbWidth = 180;
public boolean updateThumbnails = true;

6
common/src/main/java/ctbrec/SubsequentAction.java Normal file
View File

@ -0,0 +1,6 @@
package ctbrec;
public enum SubsequentAction {
PAUSE,
REMOVE
}

2
common/src/main/java/ctbrec/event/EventBusHolder.java
View File

@ -17,7 +17,7 @@ public class EventBusHolder {
private EventBusHolder() {}
public static final EventBus BUS = new AsyncEventBus(Executors.newFixedThreadPool(10, r -> {
public static final EventBus BUS = new AsyncEventBus(Executors.newFixedThreadPool(2, r -> {
Thread t = new Thread(r);
t.setName("EventBus-" + UUID.randomUUID().toString().substring(0, 8));
t.setPriority(Thread.NORM_PRIORITY - 1);

9
common/src/main/java/ctbrec/io/HttpClient.java
View File

@ -16,6 +16,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.TimeUnit;
@ -28,7 +29,6 @@ import javax.net.ssl.X509TrustManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.base.Objects;
import com.squareup.moshi.JsonAdapter;
import com.squareup.moshi.Moshi;
@ -127,9 +127,8 @@ public abstract class HttpClient {
.cookieJar(cookieJar)
.connectionPool(GLOBAL_HTTP_CONN_POOL)
.connectTimeout(Config.getInstance().getSettings().httpTimeout, TimeUnit.MILLISECONDS)
.readTimeout(Config.getInstance().getSettings().httpTimeout, TimeUnit.MILLISECONDS)
//.addInterceptor(new LoggingInterceptor())
.connectionPool(new ConnectionPool(50, 10, TimeUnit.MINUTES));
.readTimeout(Config.getInstance().getSettings().httpTimeout, TimeUnit.MILLISECONDS);
//.addInterceptor(new LoggingInterceptor());
ProxyType proxyType = Config.getInstance().getSettings().proxyType;
if (proxyType == ProxyType.HTTP) {
@ -278,7 +277,7 @@ public abstract class HttpClient {
for (List<Cookie> cookieList : cookies.values()) {
for (Cookie cookie : cookieList) {
for (String cookieName : names) {
if (Objects.equal(cookieName, cookie.name())) {
if (Objects.equals(cookieName, cookie.name())) {
result.add(cookie);
}
}

2
common/src/main/java/ctbrec/io/HttpConstants.java
View File

@ -4,9 +4,9 @@ public class HttpConstants {
public static final String ACCEPT = "Accept";
public static final String ACCEPT_LANGUAGE = "Accept-Language";
public static final String COOKIE = "Cookie";
public static final String CONNECTION = "Connection";
public static final String CONTENT_TYPE = "Content-Type";
public static final String COOKIE = "Cookie";
public static final String KEEP_ALIVE = "keep-alive";
public static final String MIMETYPE_APPLICATION_JSON = "application/json";
public static final String ORIGIN = "Origin";

17
common/src/main/java/ctbrec/io/HttpException.java
View File

@ -4,15 +4,24 @@ import java.io.IOException;
public class HttpException extends IOException {
private int code;
private String msg;
private final String url;
private final int code;
private final String msg;
public HttpException(int code, String msg) {
super(code + " - " + msg);
this.url = "";
this.code = code;
this.msg = msg;
}
public HttpException(String url, int code, String msg) {
super(code + " - " + msg + " - " + url);
this.code = code;
this.msg = msg;
this.url = url;
}
public int getResponseCode() {
return code;
}
@ -20,4 +29,8 @@ public class HttpException extends IOException {
public String getResponseMessage() {
return msg;
}
public String getUrl() {
return url;
}
}

7
common/src/main/java/ctbrec/io/ModelJsonAdapter.java
View File

@ -15,6 +15,7 @@ import com.squareup.moshi.JsonReader.Token;
import com.squareup.moshi.JsonWriter;
import ctbrec.Model;
import ctbrec.SubsequentAction;
import ctbrec.sites.Site;
import ctbrec.sites.chaturbate.ChaturbateModel;
@ -74,6 +75,10 @@ public class ModelJsonAdapter extends JsonAdapter<Model> {
model.setLastSeen(Instant.ofEpochMilli(reader.nextLong()));
} else if(key.equals("lastRecorded")) {
model.setLastRecorded(Instant.ofEpochMilli(reader.nextLong()));
} else if(key.equals("recordUntil")) {
model.setRecordUntil(Instant.ofEpochMilli(reader.nextLong()));
} else if(key.equals("recordUntilSubsequentAction")) {
model.setRecordUntilSubsequentAction(SubsequentAction.valueOf(reader.nextString()));
} else if(key.equals("siteSpecific")) {
reader.beginObject();
try {
@ -115,6 +120,8 @@ public class ModelJsonAdapter extends JsonAdapter<Model> {
writer.name("suspended").value(model.isSuspended());
writer.name("lastSeen").value(model.getLastSeen().toEpochMilli());
writer.name("lastRecorded").value(model.getLastRecorded().toEpochMilli());
writer.name("recordUntil").value(model.getRecordUntil().toEpochMilli());
writer.name("recordUntilSubsequentAction").value(model.getRecordUntilSubsequentAction().name());
writer.name("siteSpecific");
writer.beginObject();
model.writeSiteSpecificData(writer);

273
common/src/main/java/ctbrec/recorder/NextGenLocalRecorder.java
View File

@ -1,5 +1,6 @@
package ctbrec.recorder;
import static ctbrec.SubsequentAction.*;
import static ctbrec.event.Event.Type.*;
import java.io.File;
@ -10,6 +11,7 @@ import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import java.time.Duration;
import java.time.Instant;
import java.time.ZoneId;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
@ -20,8 +22,10 @@ import java.util.Objects;
import java.util.Optional;
import java.util.UUID;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorCompletionService;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.LinkedBlockingQueue;
@ -59,7 +63,7 @@ public class NextGenLocalRecorder implements Recorder {
private volatile boolean recording = true;
private ReentrantLock recorderLock = new ReentrantLock();
private RecorderHttpClient client = new RecorderHttpClient();
private long lastSpaceMessage = 0;
private long lastPreconditionMessage = 0;
private Map<Model, Recording> recordingProcesses = Collections.synchronizedMap(new HashMap<>());
private RecordingManager recordingManager;
@ -213,92 +217,145 @@ public class NextGenLocalRecorder implements Recorder {
private void startRecordingProcess(Model model) throws IOException {
recorderLock.lock();
try {
if (!recording) {
// recorder is not in recording mode
return;
}
if (model.isSuspended()) {
LOG.info("Recording for model {} is suspended.", model);
return;
}
if (recordingProcesses.containsKey(model)) {
LOG.error("A recording for model {} is already running", model);
return;
}
if (!models.contains(model)) {
LOG.info("Model {} has been removed. Restarting of recording cancelled.", model);
return;
}
if (!enoughSpaceForRecording()) {
long now = System.currentTimeMillis();
if ((now - lastSpaceMessage) > TimeUnit.MINUTES.toMillis(1)) {
LOG.info("Not enough space for recording, not starting recording for {}", model);
lastSpaceMessage = now;
}
return;
}
if (!downloadSlotAvailable()) {
long now = System.currentTimeMillis();
if ((now - lastSpaceMessage) > TimeUnit.MINUTES.toMillis(1)) {
LOG.info("The number of downloads is maxed out");
}
// check, if we can stop a recording for a model with lower priority
Optional<Recording> lowerPrioRecordingProcess = recordingProcessWithLowerPrio(model.getPriority());
if (lowerPrioRecordingProcess.isPresent()) {
Download download = lowerPrioRecordingProcess.get().getDownload();
Model lowerPrioModel = download.getModel();
LOG.info("Stopping recording for {}. Prio {} < {}", lowerPrioModel.getName(), lowerPrioModel.getPriority(), model.getPriority());
stopRecordingProcess(lowerPrioModel);
} else {
if ((now - lastSpaceMessage) > TimeUnit.MINUTES.toMillis(1)) {
LOG.info("Other models have higher prio, not starting recording for {}", model.getName());
}
return;
}
}
checkRecordingPreconditions(model);
LOG.info("Starting recording for model {}", model.getName());
Download download = model.createDownload();
download.init(config, model, Instant.now());
Objects.requireNonNull(download.getStartTime(),
"At this point the download should have set a startTime. Make sure to set a startTime in " + download.getClass() + ".init()");
LOG.debug("Downloading with {}", download.getClass().getSimpleName());
Recording rec = new Recording();
rec.setDownload(download);
rec.setPath(download.getPath(model).replaceAll("\\\\", "/"));
rec.setModel(model);
rec.setStartDate(download.getStartTime());
rec.setSingleFile(download.isSingleFile());
recordingProcesses.put(model, rec);
recordingManager.add(rec);
completionService.submit(() -> {
try {
setRecordingStatus(rec, State.RECORDING);
model.setLastRecorded(rec.getStartDate());
recordingManager.saveRecording(rec);
download.start();
} catch (Exception e) {
LOG.error("Download for {} failed. Download state: {}", model.getName(), rec.getStatus(), e);
}
boolean deleted = deleteIfEmpty(rec);
setRecordingStatus(rec, deleted ? State.DELETED : State.WAITING);
if (!deleted) {
// only save the status, if the recording has not been deleted, otherwise we recreate the metadata file
recordingManager.saveRecording(rec);
}
return rec;
});
Download download = createDownload(model);
Recording rec = createRecording(download);
completionService.submit(createDownloadJob(rec));
} catch (RecordUntilExpiredException e) {
LOG.info("Precondition not met to record {}: {}", model, e.getLocalizedMessage());
executeRecordUntilSubsequentAction(model);
} catch (PreconditionNotMetException e) {
LOG.info("Precondition not met to record {}: {}", model, e.getLocalizedMessage());
return;
} finally {
recorderLock.unlock();
}
}
private Download createDownload(Model model) {
Download download = model.createDownload();
download.init(config, model, Instant.now());
Objects.requireNonNull(download.getStartTime(),
"At this point the download should have set a startTime. Make sure to set a startTime in " + download.getClass() + ".init()");
LOG.debug("Downloading with {}", download.getClass().getSimpleName());
return download;
}
private Callable<Recording> createDownloadJob(Recording rec) {
return () -> {
try {
setRecordingStatus(rec, State.RECORDING);
rec.getModel().setLastRecorded(rec.getStartDate());
recordingManager.saveRecording(rec);
rec.getDownload().start();
} catch (Exception e) {
LOG.error("Download for {} failed. Download state: {}", rec.getModel().getName(), rec.getStatus(), e);
}
boolean deleted = deleteIfEmpty(rec);
setRecordingStatus(rec, deleted ? State.DELETED : State.WAITING);
if (!deleted) {
// only save the status, if the recording has not been deleted, otherwise we recreate the metadata file
recordingManager.saveRecording(rec);
}
return rec;
};
}
private void executeRecordUntilSubsequentAction(Model model) throws IOException {
if (model.getRecordUntilSubsequentAction() == PAUSE) {
model.setSuspended(true);
} else if (model.getRecordUntilSubsequentAction() == REMOVE) {
try {
LOG.info("Removing {} because the recording timeframe ended at {}", model, model.getRecordUntil().atZone(ZoneId.systemDefault()));
stopRecording(model);
} catch (InvalidKeyException | NoSuchAlgorithmException e1) {
LOG.error("Error while stopping recording", e1);
}
}
// reset values, so that model can be recorded again
model.setRecordUntil(null);
model.setRecordUntilSubsequentAction(PAUSE);
}
private Recording createRecording(Download download) throws IOException {
Model model = download.getModel();
Recording rec = new Recording();
rec.setDownload(download);
rec.setPath(download.getPath(model).replaceAll("\\\\", "/"));
rec.setModel(model);
rec.setStartDate(download.getStartTime());
rec.setSingleFile(download.isSingleFile());
recordingProcesses.put(model, rec);
recordingManager.add(rec);
return rec;
}
private void checkRecordingPreconditions(Model model) throws IOException {
ensureRecorderIsActive();
ensureModelIsNotSuspended(model);
ensureRecordUntilIsInFuture(model);
ensureNoRecordingRunningForModel(model);
ensureModelShouldBeRecorded(model);
ensureEnoughSpaceForRecording();
ensureDownloadSlotAvailable(model);
}
private void ensureRecordUntilIsInFuture(Model model) {
if (Instant.now().isAfter(model.getRecordUntil())) {
throw new RecordUntilExpiredException(model.getRecordUntil());
}
}
private void ensureEnoughSpaceForRecording() throws IOException {
if (!enoughSpaceForRecording()) {
throw new PreconditionNotMetException("Not enough disk space for recording");
}
}
private void ensureDownloadSlotAvailable(Model model) {
if (!downloadSlotAvailable()) {
long now = System.currentTimeMillis();
if ((now - lastPreconditionMessage) > TimeUnit.MINUTES.toMillis(1)) {
LOG.info("The number of downloads is maxed out");
}
// check, if we can stop a recording for a model with lower priority
Optional<Recording> lowerPrioRecordingProcess = recordingProcessWithLowerPrio(model.getPriority());
if (lowerPrioRecordingProcess.isPresent()) {
Download download = lowerPrioRecordingProcess.get().getDownload();
Model lowerPrioModel = download.getModel();
LOG.info("Stopping recording for {}. Prio {} < {}", lowerPrioModel.getName(), lowerPrioModel.getPriority(), model.getPriority());
stopRecordingProcess(lowerPrioModel);
} else {
throw new PreconditionNotMetException("Other models have higher prio, not starting recording for " + model.getName());
}
}
}
private void ensureModelShouldBeRecorded(Model model) {
if (!models.contains(model)) {
throw new PreconditionNotMetException("Model " + model + " has been removed. Restarting of recording cancelled.");
}
}
private void ensureNoRecordingRunningForModel(Model model) {
if (recordingProcesses.containsKey(model)) {
throw new PreconditionNotMetException("A recording for model " + model + " is already running");
}
}
private void ensureModelIsNotSuspended(Model model) {
if (model.isSuspended()) {
throw new PreconditionNotMetException("Recording for model " + model + " is suspended");
}
}
private void ensureRecorderIsActive() {
if (!recording) {
throw new PreconditionNotMetException("Recorder is not in recording mode");
}
}
private Optional<Recording> recordingProcessWithLowerPrio(int priority) {
Model lowest = null;
for (Model m : recordingProcesses.keySet()) {
@ -449,14 +506,26 @@ public class NextGenLocalRecorder implements Recorder {
try {
// make a copy to avoid ConcurrentModificationException
List<Recording> toStop = new ArrayList<>(recordingProcesses.values());
for (Recording rec : toStop) {
Optional.ofNullable(rec.getDownload()).ifPresent(Download::stop);
if (!toStop.isEmpty()) {
ExecutorService shutdownPool = Executors.newFixedThreadPool(toStop.size());
List<Future<?>> shutdownFutures = new ArrayList<>(toStop.size());
for (Recording rec : toStop) {
Optional.ofNullable(rec.getDownload()).ifPresent(d -> {
shutdownFutures.add(shutdownPool.submit(() -> d.stop()));
});
}
shutdownPool.shutdown();
try {
shutdownPool.awaitTermination(10, TimeUnit.MINUTES);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
}
} finally {
recorderLock.unlock();
}
// wait for post-processing to finish
// wait for downloads to finish
LOG.info("Waiting for downloads to finish");
for (int i = 0; i < 60; i++) {
if (!recordingProcesses.isEmpty()) {
@ -471,11 +540,12 @@ public class NextGenLocalRecorder implements Recorder {
// shutdown threadpools
try {
LOG.info("Shutting down pools");
LOG.info("Shutting down download pool");
downloadPool.shutdown();
ppPool.shutdown();
client.shutdown();
downloadPool.awaitTermination(1, TimeUnit.MINUTES);
LOG.info("Shutting down post-processing pool");
ppPool.shutdown();
int minutesToWait = 10;
LOG.info("Waiting {} minutes (max) for post-processing to finish", minutesToWait);
ppPool.awaitTermination(minutesToWait, TimeUnit.MINUTES);
@ -555,11 +625,11 @@ public class NextGenLocalRecorder implements Recorder {
return getModels().stream().filter(m -> {
try {
return m.isOnline();
} catch (IOException | ExecutionException e) {
return false;
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
return false;
} catch (Exception e) {
return false;
}
}).collect(Collectors.toList());
}
@ -700,4 +770,31 @@ public class NextGenLocalRecorder implements Recorder {
rec.setNote(note);
recordingManager.saveRecording(rec);
}
@Override
public void stopRecordingAt(Model model) throws IOException, InvalidKeyException, NoSuchAlgorithmException {
recorderLock.lock();
try {
int index = models.indexOf(model);
if (index >= 0) {
Model m = models.get(index);
m.setRecordUntil(model.getRecordUntil());
m.setRecordUntilSubsequentAction(model.getRecordUntilSubsequentAction());
config.save();
} else {
throw new NoSuchElementException("Model " + model.getName() + " [" + model.getUrl() + "] not found in list of recorded models");
}
if (recordingProcesses.containsKey(model)) {
Recording rec = recordingProcesses.get(model);
rec.getDownload().stop();
}
} finally {
recorderLock.unlock();
}
if (Instant.now().isAfter(model.getRecordUntil())) {
executeRecordUntilSubsequentAction(model);
}
}
}

12
common/src/main/java/ctbrec/recorder/OnlineMonitor.java
View File

@ -38,9 +38,11 @@ public class OnlineMonitor extends Thread {
private Map<Model, Model.State> states = new HashMap<>();
private Map<String, ExecutorService> executors = new HashMap<>();
private Config config;
public OnlineMonitor(Recorder recorder) {
public OnlineMonitor(Recorder recorder, Config config) {
this.recorder = recorder;
this.config = config;
setName("OnlineMonitor");
setDaemon(true);
}
@ -80,7 +82,11 @@ public class OnlineMonitor extends Thread {
// submit online check jobs to the executor for the model's site
List<Future<?>> futures = new LinkedList<>();
for (Model model : models) {
futures.add(updateModel(model));
if (config.getSettings().onlineCheckSkipsPausedModels && model.isSuspended()) {
continue;
} else {
futures.add(updateModel(model));
}
}
// wait for all jobs to finish
for (Future<?> future : futures) {
@ -134,7 +140,7 @@ public class OnlineMonitor extends Thread {
private void suspendUntilNextIteration(List<Model> models, Duration timeCheckTook) {
LOG.trace("Online check for {} models took {} seconds", models.size(), timeCheckTook.getSeconds());
long sleepTime = Config.getInstance().getSettings().onlineCheckIntervalInSecs;
long sleepTime = config.getSettings().onlineCheckIntervalInSecs;
if(timeCheckTook.getSeconds() < sleepTime) {
try {
if (running) {

15
common/src/main/java/ctbrec/recorder/PreconditionNotMetException.java Normal file
View File

@ -0,0 +1,15 @@
package ctbrec.recorder;
public class PreconditionNotMetException extends RuntimeException {
public PreconditionNotMetException() {
super("Precondition not met");
}
public PreconditionNotMetException(String message) {
super(message);
}
public PreconditionNotMetException(String message, Throwable t) {
super(message, t);
}
}

21
common/src/main/java/ctbrec/recorder/RecordUntilExpiredException.java Normal file
View File

@ -0,0 +1,21 @@
package ctbrec.recorder;
import java.time.Instant;
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
public class RecordUntilExpiredException extends PreconditionNotMetException {
private Instant until;
public RecordUntilExpiredException(Instant until) {
this.until = until;
}
@Override
public String getMessage() {
LocalDateTime dateTime = LocalDateTime.ofInstant(until, ZoneId.systemDefault());
String date = DateTimeFormatter.ISO_LOCAL_DATE.format(dateTime);
return "Recording expired at " + date;
}
}

1
common/src/main/java/ctbrec/recorder/Recorder.java
View File

@ -14,6 +14,7 @@ public interface Recorder {
public void startRecording(Model model) throws IOException, InvalidKeyException, NoSuchAlgorithmException;
public void stopRecording(Model model) throws IOException, InvalidKeyException, NoSuchAlgorithmException;
public void stopRecordingAt(Model model) throws IOException, InvalidKeyException, NoSuchAlgorithmException;
public void switchStreamSource(Model model) throws IOException, InvalidKeyException, NoSuchAlgorithmException;

5
common/src/main/java/ctbrec/recorder/RemoteRecorder.java
View File

@ -86,6 +86,11 @@ public class RemoteRecorder implements Recorder {
sendRequest("stop", model);
}
@Override
public void stopRecordingAt(Model model) throws IOException, InvalidKeyException, NoSuchAlgorithmException {
sendRequest("stopAt", model);
}
private void sendRequest(String action, Model model) throws IOException, InvalidKeyException, NoSuchAlgorithmException {
String payload = modelRequestAdapter.toJson(new ModelRequest(action, model));
LOG.debug("Sending request to recording server: {}", payload);

5
common/src/main/java/ctbrec/recorder/download/StreamSource.java
View File

@ -4,6 +4,7 @@ import java.text.DecimalFormat;
public class StreamSource implements Comparable<StreamSource> {
public static final int ORIGIN = Integer.MAX_VALUE - 1;
public static final int UNKNOWN = Integer.MAX_VALUE;
public int bandwidth;
public int width;
public int height;
@ -45,7 +46,7 @@ public class StreamSource implements Comparable<StreamSource> {
public String toString() {
DecimalFormat df = new DecimalFormat("0.00");
float mbit = bandwidth / 1024.0f / 1024.0f;
if (height == Integer.MAX_VALUE) {
if (height == UNKNOWN) {
return "unknown resolution (" + df.format(mbit) + " Mbit/s)";
} else if (height == ORIGIN) {
return "Origin";
@ -61,7 +62,7 @@ public class StreamSource implements Comparable<StreamSource> {
@Override
public int compareTo(StreamSource o) {
int heightDiff = height - o.height;
if(heightDiff != 0 && height != Integer.MAX_VALUE && o.height != Integer.MAX_VALUE) {
if(heightDiff != 0 && height != UNKNOWN && o.height != UNKNOWN) {
return heightDiff;
} else {
return bandwidth - o.bandwidth;

10
common/src/main/java/ctbrec/recorder/download/hls/AbstractHlsDownload.java
View File

@ -1,6 +1,8 @@
package ctbrec.recorder.download.hls;
import static ctbrec.io.HttpConstants.*;
import static ctbrec.io.HttpConstants.ORIGIN;
import static ctbrec.recorder.download.StreamSource.*;
import java.io.ByteArrayInputStream;
import java.io.IOException;
@ -64,7 +66,7 @@ public abstract class AbstractHlsDownload extends AbstractDownload {
protected volatile boolean running = false;
protected Model model = new UnknownModel();
protected transient LinkedBlockingQueue<Runnable> downloadQueue = new LinkedBlockingQueue<>(50);
protected transient ExecutorService downloadThreadPool = new ThreadPoolExecutor(5, 5, 2, TimeUnit.MINUTES, downloadQueue, createThreadFactory());
protected transient ExecutorService downloadThreadPool = new ThreadPoolExecutor(0, 5, 20, TimeUnit.SECONDS, downloadQueue, createThreadFactory());
protected State state = State.UNKNOWN;
private int playlistEmptyCount = 0;
@ -160,12 +162,12 @@ public abstract class AbstractHlsDownload extends AbstractDownload {
LOG.debug("{} selected {}", model.getName(), streamSources.get(model.getStreamUrlIndex()));
url = streamSources.get(model.getStreamUrlIndex()).getMediaPlaylistUrl();
} else {
// filter out stream resolutions, which are too high
// filter out stream resolutions, which are out of range of the configured min and max
int minRes = Config.getInstance().getSettings().minimumResolution;
int maxRes = Config.getInstance().getSettings().maximumResolution;
List<StreamSource> filteredStreamSources = streamSources.stream()
.filter(src -> src.height == 0 || minRes <= src.height)
.filter(src -> src.height == 0 || maxRes >= src.height)
.filter(src -> src.height == 0 || src.height == UNKNOWN || minRes <= src.height)
.filter(src -> src.height == 0 || src.height == UNKNOWN || maxRes >= src.height)
.collect(Collectors.toList());
if (filteredStreamSources.isEmpty()) {

48
common/src/main/java/ctbrec/recorder/download/hls/MergedFfmpegHlsDownload.java
View File

@ -1,11 +1,14 @@
package ctbrec.recorder.download.hls;
import static java.util.Optional.*;
import java.io.EOFException;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.MalformedURLException;
import java.net.URL;
import java.nio.file.Files;
import java.time.Duration;
@ -14,12 +17,13 @@ import java.time.ZonedDateTime;
import java.util.Arrays;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.Optional;
import java.util.Queue;
import java.util.concurrent.Callable;
import java.util.concurrent.CancellationException;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.regex.Pattern;
import org.slf4j.Logger;
@ -56,6 +60,7 @@ public class MergedFfmpegHlsDownload extends AbstractHlsDownload {
private transient OutputStream ffmpegStdIn;
protected transient Thread ffmpegThread;
private transient Object ffmpegStartMonitor = new Object();
private Queue<Future<byte[]>> downloads = new LinkedList<>();
public MergedFfmpegHlsDownload(HttpClient client) {
super(client);
@ -104,6 +109,7 @@ public class MergedFfmpegHlsDownload extends AbstractHlsDownload {
LOG.debug("Starting to download segments");
downloadSegments(segments, true);
ffmpegThread.join();
LOG.debug("FFmpeg thread terminated");
}
} catch (ParseException e) {
throw new IOException("Couldn't parse stream information", e);
@ -178,7 +184,8 @@ public class MergedFfmpegHlsDownload extends AbstractHlsDownload {
}
}
});
ffmpegThread.setName("FFmpeg");
String name = "FFmpeg " + ofNullable(model).map(Model::getName).orElse("").trim();
ffmpegThread.setName(name);
ffmpegThread.start();
}
@ -229,6 +236,9 @@ public class MergedFfmpegHlsDownload extends AbstractHlsDownload {
LOG.info("Unexpected error while downloading {}", model, e);
}
running = false;
} catch (MalformedURLException e) {
LOG.info("Malformed URL {} - {}", model, segmentPlaylistUri, e);
running = false;
} catch (Exception e) {
LOG.info("Unexpected error while downloading {}", model, e);
running = false;
@ -250,7 +260,7 @@ public class MergedFfmpegHlsDownload extends AbstractHlsDownload {
int skip = nextSegment - lsp.seq;
// add segments to download threadpool
Queue<Future<byte[]>> downloads = new LinkedList<>();
downloads.clear();
if (downloadQueue.remainingCapacity() == 0) {
LOG.warn("Download to slow for this stream. Download queue is full. Skipping segment");
} else {
@ -274,11 +284,15 @@ public class MergedFfmpegHlsDownload extends AbstractHlsDownload {
private void writeFinishedSegments(Queue<Future<byte[]>> downloads) throws ExecutionException, IOException {
for (Future<byte[]> downloadFuture : downloads) {
try {
byte[] segmentData = downloadFuture.get();
byte[] segmentData = downloadFuture.get(30, TimeUnit.SECONDS);
writeSegment(segmentData);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
LOG.error("Error while downloading segment", e);
} catch (TimeoutException e) {
LOG.info("Segment download took too long for {}. Not waiting for it any longer", getModel());
} catch (CancellationException e) {
LOG.info("Segment download cancelled");
} catch (ExecutionException e) {
Throwable cause = e.getCause();
if (cause instanceof MissingSegmentException) {
@ -286,7 +300,7 @@ public class MergedFfmpegHlsDownload extends AbstractHlsDownload {
LOG.debug("Error while downloading segment, because model {} is offline. Stopping now", model.getName());
running = false;
} else {
LOG.debug("Segment not available, but model {} still online. Going on", Optional.ofNullable(model).map(Model::getName).orElse("n/a"));
LOG.debug("Segment not available, but model {} still online. Going on", ofNullable(model).map(Model::getName).orElse("n/a"));
}
} else if (cause instanceof HttpException) {
HttpException he = (HttpException) cause;
@ -295,10 +309,10 @@ public class MergedFfmpegHlsDownload extends AbstractHlsDownload {
running = false;
} else {
if (he.getResponseCode() == 404) {
LOG.info("Playlist for {} not found [HTTP 404]. Stopping now", Optional.ofNullable(model).map(Model::getName).orElse("n/a"));
LOG.info("Playlist for {} not found [HTTP 404]. Stopping now", ofNullable(model).map(Model::getName).orElse("n/a"));
running = false;
} else if (he.getResponseCode() == 403) {
LOG.info("Playlist for {} not accessible [HTTP 403]. Stopping now", Optional.ofNullable(model).map(Model::getName).orElse("n/a"));
LOG.info("Playlist for {} not accessible [HTTP 403]. Stopping now", ofNullable(model).map(Model::getName).orElse("n/a"));
running = false;
} else {
throw he;
@ -374,9 +388,23 @@ public class MergedFfmpegHlsDownload extends AbstractHlsDownload {
@Override
synchronized void internalStop() {
running = false;
try {
downloadQueue.clear();
for (Future<?> future : downloads) {
future.cancel(true);
}
downloadThreadPool.shutdownNow();
LOG.debug("Waiting for segment download thread pool to terminate for model {}", getModel());
downloadThreadPool.awaitTermination(60, TimeUnit.SECONDS);
LOG.debug("Segment download thread pool terminated for model {}", getModel());
} catch (InterruptedException e) {
LOG.error("Interrupted while waiting for segment pool to shutdown");
Thread.currentThread().interrupt();
}
if (ffmpegStdIn != null) {
try {
downloadQueue.clear();
ffmpegStdIn.close();
} catch (IOException e) {
LOG.error("Couldn't terminate FFmpeg by closing stdin", e);
@ -385,7 +413,7 @@ public class MergedFfmpegHlsDownload extends AbstractHlsDownload {
if (ffmpeg != null) {
try {
boolean waitFor = ffmpeg.waitFor(5, TimeUnit.MINUTES);
boolean waitFor = ffmpeg.waitFor(45, TimeUnit.SECONDS);
if (!waitFor && ffmpeg.isAlive()) {
LOG.info("FFmpeg didn't terminate. Destroying the process with force!");
ffmpeg.destroyForcibly();
@ -415,7 +443,7 @@ public class MergedFfmpegHlsDownload extends AbstractHlsDownload {
int maxTries = 3;
for (int i = 1; i <= maxTries && running; i++) {
Builder builder = new Request.Builder().url(url);
addHeaders(builder, Optional.ofNullable(model).map(Model::getHttpHeaderFactory).map(HttpHeaderFactory::createSegmentHeaders).orElse(new HashMap<>()));
addHeaders(builder, ofNullable(model).map(Model::getHttpHeaderFactory).map(HttpHeaderFactory::createSegmentHeaders).orElse(new HashMap<>()));
Request request = builder.build();
try (Response response = client.execute(request)) {
if (response.isSuccessful()) {

2
common/src/main/java/ctbrec/sites/cam4/Cam4.java
View File

@ -44,7 +44,7 @@ public class Cam4 extends AbstractSite {
}
@Override
public Model createModel(String name) {
public Cam4Model createModel(String name) {
Cam4Model m = new Cam4Model();
m.setSite(this);
m.setName(name);

2
common/src/main/java/ctbrec/sites/cam4/Cam4Model.java
View File

@ -103,7 +103,7 @@ public class Cam4Model extends AbstractModel {
onlineState = OFFLINE;
break;
default:
LOG.debug("Unknown show type {}", showType);
LOG.debug("Unknown show type [{}]", showType);
onlineState = UNKNOWN;
}

164
common/src/main/java/ctbrec/sites/camsoda/CamsodaModel.java
View File

@ -10,6 +10,7 @@ import java.util.Collections;
import java.util.List;
import java.util.Locale;
import java.util.Objects;
import java.util.Optional;
import java.util.Random;
import java.util.concurrent.ExecutionException;
@ -44,99 +45,124 @@ public class CamsodaModel extends AbstractModel {
private static final String EDGE_SERVERS = "edge_servers";
private static final String STATUS = "status";
private static final Logger LOG = LoggerFactory.getLogger(CamsodaModel.class);
private String streamUrl;
private transient List<StreamSource> streamSources = null;
private transient boolean isNew;
private float sortOrder = 0;
private Random random = new Random();
int[] resolution = new int[2];
boolean oldStreamUrl = true;
public String getStreamUrl() throws IOException {
if (streamUrl == null) {
if(oldStreamUrl) {
loadModel();
} else {
getNewStreamUrl();
}
Request req = createJsonRequest(getTokenInfoUrl());
JSONObject response = executeJsonRequest(req);
if (response.optInt(STATUS) == 1 && response.optJSONArray(EDGE_SERVERS) != null && response.optJSONArray(EDGE_SERVERS).length() > 0) {
String edgeServer = response.getJSONArray(EDGE_SERVERS).getString(0);
String streamName = response.getString(STREAM_NAME);
String token = response.getString("token");
return constructStreamUrl(edgeServer, streamName, token);
} else {
throw new JSONException("JSON response has not the expected structure");
}
return streamUrl;
}
public String getNewStreamUrl() throws IOException {
private String getTokenInfoUrl() {
String guestUsername = "guest_" + 10_000 + random.nextInt(50_000);
String url = site.getBaseUrl() + "/api/v1/video/vtoken/" + getName() + "?username=" + guestUsername;
Request req = new Request.Builder()
.url(url)
String tokenInfoUrl = site.getBaseUrl() + "/api/v1/video/vtoken/" + getName() + "?username=" + guestUsername;
return tokenInfoUrl;
}
private String constructStreamUrl(String edgeServer, String streamName, String token) {
StringBuilder url = new StringBuilder("https://");
url.append(edgeServer).append('/');
if (streamName.contains("-flu")) {
url.append(streamName);
url.append("_h264_aac");
url.append(streamName.contains("-flu-hd") ? "_720p" : "_480p");
url.append("/index.m3u8");
if (!isPublic(streamName)) {
url.append("?token=").append(token);
}
} else {
// https://vide7-ord.camsoda.com/cam/mp4:maxandtokio-enc10-ord_h264_aac_480p/playlist.m3u8
url.append("cam/mp4:");
url.append(streamName);
url.append("_h264_aac_480p/playlist.m3u8");
}
LOG.trace("Stream URL: {}", url);
return url.toString();
}
private Request createJsonRequest(String tokenInfoUrl) {
return new Request.Builder()
.url(tokenInfoUrl)
.header(ACCEPT, MIMETYPE_APPLICATION_JSON)
.header(ACCEPT_LANGUAGE, Locale.ENGLISH.getLanguage())
.header(X_REQUESTED_WITH, XML_HTTP_REQUEST)
.header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
.build();
try (Response response = site.getHttpClient().execute(req)) {
}
private JSONObject executeJsonRequest(Request request) throws IOException {
try (Response response = site.getHttpClient().execute(request)) {
if (response.isSuccessful()) {
JSONObject jsonResponse = new JSONObject(response.body().string());
if (jsonResponse.optInt(STATUS) == 1) {
String edgeServer = jsonResponse.getJSONArray(EDGE_SERVERS).getString(0);
String streamName = jsonResponse.getString(STREAM_NAME);
String token = jsonResponse.getString("token");
streamUrl = "https://" + edgeServer + '/' + streamName + "_h264_aac_480p/index.m3u8?token=" + token;
} else {
throw new JSONException("Response does not contain a token");
}
return jsonResponse;
} else {
throw new HttpException(response.code(), response.message());
}
}
return streamUrl;
}
private boolean isPublic(String streamName) {
return Optional.ofNullable(streamName).orElse("").contains("_public");
}
@Override
public List<StreamSource> getStreamSources() throws IOException, ExecutionException, ParseException, PlaylistException {
String playlistUrl = getStreamUrl();
if (playlistUrl == null) {
return Collections.emptyList();
}
LOG.trace("Loading playlist {}", playlistUrl);
Request req = new Request.Builder()
.url(playlistUrl)
.header(ACCEPT_LANGUAGE, Locale.ENGLISH.getLanguage())
.header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
.build();
try (Response response = site.getHttpClient().execute(req)) {
if (response.isSuccessful()) {
InputStream inputStream = response.body().byteStream();
PlaylistParser parser = new PlaylistParser(inputStream, Format.EXT_M3U, Encoding.UTF_8, ParsingMode.LENIENT);
Playlist playlist = parser.parse();
MasterPlaylist master = playlist.getMasterPlaylist();
PlaylistData playlistData = master.getPlaylists().get(0);
StreamSource streamsource = new StreamSource();
if (oldStreamUrl) {
streamsource.mediaPlaylistUrl = streamUrl.replace("playlist.m3u8", playlistData.getUri());
} else {
int cutOffAt = playlistUrl.indexOf("index.m3u8");
try {
String playlistUrl = getStreamUrl();
if (playlistUrl == null) {
return Collections.emptyList();
}
LOG.trace("Loading playlist {}", playlistUrl);
Request req = new Request.Builder()
.url(playlistUrl)
.header(ACCEPT_LANGUAGE, Locale.ENGLISH.getLanguage())
.header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
.build();
try (Response response = site.getHttpClient().execute(req)) {
if (response.isSuccessful()) {
InputStream inputStream = response.body().byteStream();
PlaylistParser parser = new PlaylistParser(inputStream, Format.EXT_M3U, Encoding.UTF_8, ParsingMode.LENIENT);
Playlist playlist = parser.parse();
MasterPlaylist master = playlist.getMasterPlaylist();
PlaylistData playlistData = master.getPlaylists().get(0);
StreamSource streamsource = new StreamSource();
int cutOffAt = Math.max(playlistUrl.indexOf("index.m3u8"), playlistUrl.indexOf("playlist.m3u8"));
String segmentPlaylistUrl = playlistUrl.substring(0, cutOffAt) + playlistData.getUri();
streamsource.mediaPlaylistUrl = segmentPlaylistUrl;
}
if (playlistData.hasStreamInfo()) {
StreamInfo info = playlistData.getStreamInfo();
streamsource.bandwidth = info.getBandwidth();
streamsource.width = info.hasResolution() ? info.getResolution().width : 0;
streamsource.height = info.hasResolution() ? info.getResolution().height : 0;
if (playlistData.hasStreamInfo()) {
StreamInfo info = playlistData.getStreamInfo();
streamsource.bandwidth = info.getBandwidth();
streamsource.width = info.hasResolution() ? info.getResolution().width : 0;
streamsource.height = info.hasResolution() ? info.getResolution().height : 0;
} else {
streamsource.bandwidth = 0;
streamsource.width = 0;
streamsource.height = 0;
}
streamSources = new ArrayList<>();
streamSources.add(streamsource);
} else {
streamsource.bandwidth = 0;
streamsource.width = 0;
streamsource.height = 0;
LOG.trace("Response: {}", response.body().string());
throw new HttpException(playlistUrl, response.code(), response.message());
}
streamSources = new ArrayList<>();
streamSources.add(streamsource);
} else {
LOG.trace("Response: {}", response.body().string());
throw new HttpException(response.code(), response.message());
}
return streamSources;
} catch (JSONException e) {
return Collections.emptyList();
}
return streamSources;
}
private void loadModel() throws IOException {
@ -151,15 +177,9 @@ public class CamsodaModel extends AbstractModel {
try (Response response = site.getHttpClient().execute(req)) {
if (response.isSuccessful()) {
JSONObject result = new JSONObject(response.body().string());
if (result.getBoolean(STATUS)) {
if (result.optBoolean(STATUS)) {
JSONObject chat = result.getJSONObject("user").getJSONObject("chat");
String status = chat.getString(STATUS);
oldStreamUrl = !chat.getString(STREAM_NAME).contains("/");
if (oldStreamUrl && chat.has(EDGE_SERVERS)) {
String edgeServer = chat.getJSONArray(EDGE_SERVERS).getString(0);
String streamName = chat.getString(STREAM_NAME);
streamUrl = "https://" + edgeServer + "/cam/mp4:" + streamName + "_h264_aac_480p/playlist.m3u8";
}
setOnlineStateByStatus(status);
} else {
throw new IOException("Result was not ok");
@ -222,11 +242,11 @@ public class CamsodaModel extends AbstractModel {
return resolution;
} else {
try {
List<StreamSource> streamSources = getStreamSources();
if (streamSources.isEmpty()) {
List<StreamSource> sources = getStreamSources();
if (sources.isEmpty()) {
return new int[] { 0, 0 };
} else {
StreamSource src = streamSources.get(0);
StreamSource src = sources.get(0);
resolution = new int[] { src.width, src.height };
return resolution;
}
@ -309,10 +329,6 @@ public class CamsodaModel extends AbstractModel {
}
}
public void setStreamUrl(String streamUrl) {
this.streamUrl = streamUrl;
}
public float getSortOrder() {
return sortOrder;
}

56
common/src/main/java/ctbrec/sites/flirt4free/Flirt4FreeModel.java
View File

@ -86,12 +86,11 @@ public class Flirt4FreeModel extends AbstractModel {
return false;
}
JSONObject json = new JSONObject(body);
//LOG.debug("check model status: {}", json.toString(2));
online = Objects.equals(json.optString("status"), "online");
id = String.valueOf(json.get("model_id"));
online = Objects.equals(json.optString("status"), "online"); // online is true, even if the model is in private or away
updateModelId(json);
if (online) {
try {
loadStreamUrl();
loadModelInfo();
} catch (Exception e) {
online = false;
onlineState = Model.State.OFFLINE;
@ -109,6 +108,18 @@ public class Flirt4FreeModel extends AbstractModel {
return online;
}
private void updateModelId(JSONObject json) {
if (json.has("model_id")) {
Object modelId = json.get("model_id");
if (modelId instanceof Number) {
Number n = (Number) modelId;
if (n.intValue() > 0) {
id = String.valueOf(json.get("model_id"));
}
}
}
}
private void loadModelInfo() throws IOException, InterruptedException {
String url = getSite().getBaseUrl() + "/webservices/chat-room-interface.php?a=login_room&model_id=" + id;
LOG.trace("Loading url {}", url);
@ -127,13 +138,15 @@ public class Flirt4FreeModel extends AbstractModel {
// LOG.debug("chat-room-interface {}", json.toString(2));
JSONObject config = json.getJSONObject("config");
JSONObject performer = config.getJSONObject("performer");
setName(performer.optString("name_seo", "n/a"));
setDisplayName(performer.optString("name", "n/a"));
setUrl(getSite().getBaseUrl() + "/rooms/" + getName() + '/');
setDisplayName(performer.optString("name", getName()));
JSONObject room = config.getJSONObject("room");
chatHost = room.getString("host");
chatPort = room.getString("port_to_be");
chatToken = json.getString("token_enc");
String status = room.optString("status");
setOnlineState(mapStatus(status));
online = onlineState == State.ONLINE;
JSONObject user = config.getJSONObject("user");
userIp = user.getString("ip");
} else {
@ -147,6 +160,19 @@ public class Flirt4FreeModel extends AbstractModel {
}
}
private State mapStatus(String status) {
switch (status) {
case "P":
case "F":
return State.PRIVATE;
case "A":
return State.AWAY;
case "O":
default:
return State.ONLINE;
}
}
@Override
public List<StreamSource> getStreamSources() throws IOException, ExecutionException, ParseException, PlaylistException {
return getStreamSources(true);
@ -238,10 +264,9 @@ public class Flirt4FreeModel extends AbstractModel {
streamHost = data.getString("stream_host"); // TODO look, if the stream_host is equal to the one encoded in base64 in some of the ajax requests (parameters)
online = true;
isInteractiveShow = data.optString("devices").equals("1");
if(data.optString("room_state").equals("P")) {
onlineState = Model.State.PRIVATE;
online = false;
}
String roomState = data.optString("room_state");
onlineState = mapStatus(roomState);
online = onlineState == State.ONLINE;
if(data.optString("room_state").equals("0") && data.optString("login_group_id").equals("14")) {
onlineState = Model.State.GROUP;
online = false;
@ -262,6 +287,7 @@ public class Flirt4FreeModel extends AbstractModel {
synchronized (monitor) {
monitor.notify();
}
response.close();
}
@Override
@ -455,8 +481,10 @@ public class Flirt4FreeModel extends AbstractModel {
@Override
public void readSiteSpecificData(JsonReader reader) throws IOException {
reader.nextName();
id = reader.nextString();
if (reader.hasNext()) {
reader.nextName();
id = reader.nextString();
}
}
@Override
@ -481,7 +509,7 @@ public class Flirt4FreeModel extends AbstractModel {
}
private void acquireSlot() throws InterruptedException {
//LOG.debug("Acquire: {}", requestThrottle.availablePermits());
//LOG.debug("Acquire: {} - Queue: {}", requestThrottle.availablePermits(), requestThrottle.getQueueLength());
requestThrottle.acquire();
long now = System.currentTimeMillis();
long millisSinceLastRequest = now - lastRequest;
@ -494,6 +522,6 @@ public class Flirt4FreeModel extends AbstractModel {
private void releaseSlot() {
lastRequest = System.currentTimeMillis();
requestThrottle.release();
//LOG.debug("Release: {}", requestThrottle.availablePermits());
// LOG.debug("Release: {}", requestThrottle.availablePermits());
}
}

9
common/src/main/java/ctbrec/sites/jasmin/LiveJasmin.java
View File

@ -5,6 +5,7 @@ import static ctbrec.io.HttpConstants.*;
import java.io.IOException;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
import java.util.regex.Matcher;
@ -13,6 +14,8 @@ import java.util.regex.Pattern;
import org.json.JSONObject;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import ctbrec.Config;
import ctbrec.Model;
@ -27,6 +30,7 @@ import okhttp3.Response;
public class LiveJasmin extends AbstractSite {
private static final Logger LOG = LoggerFactory.getLogger(LiveJasmin.class);
public static String baseUrl = "";
public static String baseDomain = "";
private HttpClient httpClient;
@ -169,7 +173,8 @@ public class LiveJasmin extends AbstractSite {
}
return models;
} else {
throw new IOException("Response was not successful: " + url + "\n" + body);
LOG.debug("Response was not successful: {}\n{}", url, body);
return Collections.emptyList();
}
} else {
throw new HttpException(response.code(), response.message());
@ -198,7 +203,7 @@ public class LiveJasmin extends AbstractSite {
String name = m.group(1);
return createModel(name);
}
m = Pattern.compile("http.*?livejasmin\\.com.*?/chat-html5/(.*)").matcher(url);
m = Pattern.compile("http.*?livejasmin\\.com.*?/chat(?:-html5)?/(.*)").matcher(url);
if(m.find()) {
String name = m.group(1);
return createModel(name);

7
common/src/main/java/ctbrec/sites/jasmin/LiveJasminModel.java
View File

@ -72,6 +72,8 @@ public class LiveJasminModel extends AbstractModel {
JSONObject config = data.getJSONObject("config");
JSONObject chatRoom = config.getJSONObject("chatRoom");
setId(chatRoom.getString("p_id"));
setName(chatRoom.getString("performer_id"));
setDisplayName(chatRoom.getString("display_name"));
if (chatRoom.has("profile_picture_url")) {
setPreview(chatRoom.getString("profile_picture_url"));
}
@ -80,11 +82,14 @@ public class LiveJasminModel extends AbstractModel {
if (chatRoom.optInt("is_on_private", 0) == 1) {
onlineState = State.PRIVATE;
}
if (chatRoom.optInt("is_video_call_enabled", 0) == 1) {
onlineState = State.PRIVATE;
}
resolution = new int[2];
resolution[0] = config.optInt("streamWidth");
resolution[1] = config.optInt("streamHeight");
online = onlineState == State.ONLINE;
LOG.trace("{} - status:{} {} {} {}", getName(), online, onlineState, Arrays.toString(resolution), getUrl());
LOG.trace("{} - status:{} {} {} {} {}", getName(), online, onlineState, Arrays.toString(resolution), getUrl(), id);
} else {
throw new IOException("Response was not successful: " + body);
}

4
common/src/main/java/ctbrec/sites/mfc/HlsStreamSourceProvider.java
View File

@ -50,8 +50,8 @@ public class HlsStreamSourceProvider implements StreamSourceProvider {
src.width = playlist.getStreamInfo().getResolution().width;
src.height = playlist.getStreamInfo().getResolution().height;
} else {
src.width = Integer.MAX_VALUE;
src.height = Integer.MAX_VALUE;
src.width = StreamSource.UNKNOWN;
src.height = StreamSource.UNKNOWN;
}
String masterUrl = streamUrl;
String baseUrl = masterUrl.substring(0, masterUrl.lastIndexOf('/') + 1);

135
common/src/main/java/ctbrec/sites/mfc/MyFreeCamsClient.java
View File

@ -40,6 +40,7 @@ import com.squareup.moshi.Moshi;
import ctbrec.Config;
import ctbrec.StringUtil;
import ctbrec.io.HttpException;
import okhttp3.Cookie;
import okhttp3.Request;
import okhttp3.Response;
@ -56,6 +57,7 @@ public class MyFreeCamsClient {
private static MyFreeCamsClient instance;
private MyFreeCams mfc;
private WebSocket ws;
private Thread keepAlive;
private Moshi moshi;
private volatile boolean running = false;
@ -92,6 +94,7 @@ public class MyFreeCamsClient {
}
public void start() throws IOException {
requestLandingPage(); // to get some cookies
running = true;
serverConfig = new ServerConfig(mfc);
List<String> websocketServers = new ArrayList<>(serverConfig.wsServers.size());
@ -133,6 +136,21 @@ public class MyFreeCamsClient {
watchDog.start();
}
private void requestLandingPage() throws IOException {
Request req = new Request.Builder()
.url(MyFreeCams.baseUrl)
.header(ACCEPT, "*/*")
.header(ACCEPT_LANGUAGE, Locale.ENGLISH.getLanguage())
.header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
.header(CONNECTION, KEEP_ALIVE)
.build();
try(Response resp = mfc.getHttpClient().execute(req)) {
if(!resp.isSuccessful()) {
throw new HttpException(resp.code(), resp.message());
}
}
}
public void stop() {
running = false;
ws.close(1000, "Good Bye"); // terminate normally (1000)
@ -454,54 +472,6 @@ public class MyFreeCamsClient {
model.update(state, getStreamUrl(state));
}
private Message parseMessage(StringBuilder msgBuffer) throws UnsupportedEncodingException {
int packetLengthBytes = 6;
if (msgBuffer.length() < packetLengthBytes) {
// packet size not transmitted completely
return null;
} else {
try {
int packetLength = Integer.parseInt(msgBuffer.substring(0, packetLengthBytes));
if (packetLength > msgBuffer.length() - packetLengthBytes) {
// packet not complete
return null;
} else {
LOG.trace("<-- {}", msgBuffer);
msgBuffer.delete(0, packetLengthBytes);
StringBuilder rawMessage = new StringBuilder(msgBuffer.substring(0, packetLength));
int type = parseNextInt(rawMessage);
int sender = parseNextInt(rawMessage);
int receiver = parseNextInt(rawMessage);
int arg1 = parseNextInt(rawMessage);
int arg2 = parseNextInt(rawMessage);
Message message = new Message(type, sender, receiver, arg1, arg2, URLDecoder.decode(rawMessage.toString(), "utf-8"));
msgBuffer.delete(0, packetLength);
return message;
}
} catch (Exception e) {
LOG.error("StringBuilder contains invalid data {}", msgBuffer.toString(), e);
String logfile = "mfc_messages.log";
try (FileOutputStream fout = new FileOutputStream(logfile)) {
for (String string : receivedTextHistory) {
fout.write(string.getBytes());
fout.write(10);
}
} catch (Exception e1) {
LOG.error("Couldn't write mfc message history to {}", logfile, e1);
}
msgBuffer.setLength(0);
return null;
}
}
}
private int parseNextInt(StringBuilder s) {
int nextSpace = s.indexOf(" ");
int i = Integer.parseInt(s.substring(0, nextSpace));
s.delete(0, nextSpace + 1);
return i;
}
@Override
public void onMessage(WebSocket webSocket, ByteString bytes) {
super.onMessage(webSocket, bytes);
@ -511,6 +481,54 @@ public class MyFreeCamsClient {
return websocket;
}
private Message parseMessage(StringBuilder msgBuffer) throws UnsupportedEncodingException {
int packetLengthBytes = 6;
if (msgBuffer.length() < packetLengthBytes) {
// packet size not transmitted completely
return null;
} else {
try {
int packetLength = Integer.parseInt(msgBuffer.substring(0, packetLengthBytes));
if (packetLength > msgBuffer.length() - packetLengthBytes) {
// packet not complete
return null;
} else {
LOG.trace("<-- {}", msgBuffer);
msgBuffer.delete(0, packetLengthBytes);
StringBuilder rawMessage = new StringBuilder(msgBuffer.substring(0, packetLength));
int type = parseNextInt(rawMessage);
int sender = parseNextInt(rawMessage);
int receiver = parseNextInt(rawMessage);
int arg1 = parseNextInt(rawMessage);
int arg2 = parseNextInt(rawMessage);
Message message = new Message(type, sender, receiver, arg1, arg2, URLDecoder.decode(rawMessage.toString(), "utf-8"));
msgBuffer.delete(0, packetLength);
return message;
}
} catch (Exception e) {
LOG.error("StringBuilder contains invalid data {}", msgBuffer.toString(), e);
String logfile = "mfc_messages.log";
try (FileOutputStream fout = new FileOutputStream(logfile)) {
for (String string : receivedTextHistory) {
fout.write(string.getBytes());
fout.write(10);
}
} catch (Exception e1) {
LOG.error("Couldn't write mfc message history to {}", logfile, e1);
}
msgBuffer.setLength(0);
return null;
}
}
}
private int parseNextInt(StringBuilder s) {
int nextSpace = s.indexOf(" ");
int i = Integer.parseInt(s.substring(0, nextSpace));
s.delete(0, nextSpace + 1);
return i;
}
protected boolean follow(int uid) {
if (ws != null) {
return ws.send(ADDFRIENDREQ + " " + sessionId + " 0 " + uid + " 1\n");
@ -562,8 +580,11 @@ public class MyFreeCamsClient {
}
private void startKeepAlive(WebSocket ws) {
Thread keepAlive = new Thread(() -> {
while (running) {
if (keepAlive != null) {
keepAlive.interrupt();
}
keepAlive = new Thread(() -> {
while (running && !Thread.currentThread().isInterrupted()) {
try {
if (!connecting) {
LOG.trace("--> NULL to keep the connection alive");
@ -699,4 +720,18 @@ public class MyFreeCamsClient {
public Collection<SessionState> getSessionStates() {
return Collections.unmodifiableCollection(sessionStates.asMap().values());
}
public void joinChannel(MyFreeCamsModel model) {
SessionState state = getSessionState(model);
int userChannel = 100000000 + state.getUid();
LOG.debug("Joining chat channel for model {}", model.getDisplayName());
try {
search(model.getName());
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
ws.send(MessageTypes.ROOMDATA + " " + sessionId + " 0 1 0\n");
ws.send(MessageTypes.UEOPT + " " + sessionId + " 0 66 1 111111\n");
ws.send(MessageTypes.JOINCHAN + " " + sessionId + " 0 " + userChannel + " 9\n");
}
}

13
common/src/main/java/ctbrec/sites/mfc/ServerConfig.java
View File

@ -7,7 +7,7 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import org.json.JSONArray;
import org.json.JSONObject;
@ -74,7 +74,7 @@ public class ServerConfig {
}
public boolean isOnHtml5VideoServer(SessionState state) {
int camserv = Objects.requireNonNull(Objects.requireNonNull(state.getU()).getCamserv());
int camserv = getCamServ(state);
return isOnObsServer(state)
|| h5Servers.containsKey(Integer.toString(camserv))
|| (camserv >= 904 && camserv <= 915
@ -86,12 +86,17 @@ public class ServerConfig {
}
public boolean isOnWzObsVideoServer(SessionState state) {
int camserv = Objects.requireNonNull(Objects.requireNonNull(state.getU()).getCamserv());
int camserv = getCamServ(state);
return wzobsServers.containsKey(Integer.toString(camserv));
}
public boolean isOnNgServer(SessionState state) {
int camserv = Objects.requireNonNull(Objects.requireNonNull(state.getU()).getCamserv());
int camserv = getCamServ(state);
return ngVideoServers.containsKey(Integer.toString(camserv));
}
private static int getCamServ(SessionState state) {
int camserv = Optional.ofNullable(state).map(SessionState::getU).map(User::getCamserv).orElse(-1);
return camserv;
}
}

8
common/src/main/java/ctbrec/sites/streamate/StreamateHttpClient.java
View File

@ -6,6 +6,7 @@ import java.io.IOException;
import java.util.Collections;
import java.util.Locale;
import java.util.NoSuchElementException;
import java.util.UUID;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@ -15,7 +16,6 @@ import org.slf4j.LoggerFactory;
import ctbrec.Config;
import ctbrec.io.HttpClient;
import ctbrec.io.HttpConstants;
import ctbrec.io.HttpException;
import okhttp3.Cookie;
import okhttp3.HttpUrl;
@ -60,8 +60,10 @@ public class StreamateHttpClient extends HttpClient {
private void loadXsrfToken() {
// do a first request to get cookies and stuff
Request req = new Request.Builder() //
.url(Streamate.BASE_URL) //
.header(HttpConstants.USER_AGENT, Config.getInstance().getSettings().httpUserAgent) //
.url(Streamate.BASE_URL + "/initialData.js") //
.header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) //
.header(COOKIE, "smtid="+UUID.randomUUID().toString()+"; Xld_rct=1;") //
.header(REFERER, Streamate.BASE_URL)
.build();
try (Response resp = execute(req)) {
if (resp.code() == 200) {

35
common/src/main/java/ctbrec/sites/stripchat/Stripchat.java
View File

@ -23,9 +23,19 @@ import okhttp3.Response;
public class Stripchat extends AbstractSite {
public static final String BASE_URI = "https://stripchat.com";
public static String domain = "stripchat.com";
public static String baseUri = "https://stripchat.com";
private HttpClient httpClient;
@Override
public void init() throws IOException {
boolean hamster = Config.getInstance().getSettings().stripchatUseXhamster;
if (hamster) {
domain = "xhamsterlive.com";
baseUri = "https://" + domain;
}
}
@Override
public String getName() {
return "Stripchat";
@ -33,7 +43,7 @@ public class Stripchat extends AbstractSite {
@Override
public String getBaseUrl() {
return BASE_URI;
return baseUri;
}
@Override
@ -61,15 +71,15 @@ public class Stripchat extends AbstractSite {
throw new IOException("Account settings not available");
}
String username = Config.getInstance().getSettings().camsodaUsername;
String url = BASE_URI + "/api/v1/user/" + username;
String username = Config.getInstance().getSettings().stripchatPassword;
String url = baseUri + "/api/v1/user/" + username;
Request request = new Request.Builder().url(url).build();
try(Response response = getHttpClient().execute(request)) {
if(response.isSuccessful()) {
try (Response response = getHttpClient().execute(request)) {
if (response.isSuccessful()) {
JSONObject json = new JSONObject(response.body().string());
if(json.has("user")) {
if (json.has("user")) {
JSONObject user = json.getJSONObject("user");
if(user.has("tokens")) {
if (user.has("tokens")) {
return (double) user.getInt("tokens");
}
}
@ -93,11 +103,6 @@ public class Stripchat extends AbstractSite {
return httpClient;
}
@Override
public void init() throws IOException {
// noop
}
@Override
public void shutdown() {
if (httpClient != null) {
@ -122,7 +127,7 @@ public class Stripchat extends AbstractSite {
@Override
public List<Model> search(String q) throws IOException, InterruptedException {
String url = BASE_URI + "/api/front/v2/models/search?limit=20&query=" + URLEncoder.encode(q, "utf-8");
String url = baseUri + "/api/front/v2/models/search?limit=20&query=" + URLEncoder.encode(q, "utf-8");
Request req = new Request.Builder()
.url(url)
.header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
@ -162,7 +167,7 @@ public class Stripchat extends AbstractSite {
@Override
public Model createModelFromUrl(String url) {
Matcher m = Pattern.compile("https?://(?:.*?\\.)?stripchat.com/([^/]*?)/?").matcher(url);
Matcher m = Pattern.compile("https?://(?:.*?\\.)?(?:stripchat.com|xhamsterlive.com)/([^/]*?)/?").matcher(url);
if (m.matches()) {
String modelName = m.group(1);
return createModel(modelName);

69
common/src/main/java/ctbrec/sites/stripchat/StripchatHttpClient.java
View File

@ -4,6 +4,7 @@ import static ctbrec.io.HttpConstants.*;
import java.io.IOException;
import org.json.JSONException;
import org.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -31,14 +32,20 @@ public class StripchatHttpClient extends HttpClient {
@Override
public boolean login() throws IOException {
if(loggedIn) {
if (loggedIn) {
if (csrfToken == null) {
loadCsrfToken();
}
return true;
}
// persisted cookies might let us log in
if(checkLoginSuccess()) {
if (checkLoginSuccess()) {
loggedIn = true;
LOG.debug("Logged in with cookies");
if (csrfToken == null) {
loadCsrfToken();
}
return true;
}
@ -46,7 +53,7 @@ public class StripchatHttpClient extends HttpClient {
loadCsrfToken();
}
String url = Stripchat.BASE_URI + "/api/front/auth/login";
String url = Stripchat.baseUri + "/api/front/auth/login";
JSONObject requestParams = new JSONObject();
requestParams.put("loginOrEmail", Config.getInstance().getSettings().stripchatUsername);
requestParams.put("password", Config.getInstance().getSettings().stripchatPassword);
@ -59,8 +66,8 @@ public class StripchatHttpClient extends HttpClient {
.url(url)
.header(ACCEPT, MIMETYPE_APPLICATION_JSON)
.header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
.header(ORIGIN, Stripchat.BASE_URI)
.header(REFERER, Stripchat.BASE_URI)
.header(ORIGIN, Stripchat.baseUri)
.header(REFERER, Stripchat.baseUri)
.header(CONTENT_TYPE, MIMETYPE_APPLICATION_JSON)
.post(body)
.build();
@ -75,19 +82,20 @@ public class StripchatHttpClient extends HttpClient {
return false;
}
} else {
throw new HttpException(response.code(), response.message());
LOG.info("Auto-Login failed: {} {} {}", response.code(), response.message(), url);
return false;
}
}
}
private void loadCsrfToken() throws IOException {
String url = Stripchat.BASE_URI + "/api/front/v2/config/data?requestPath=%2F&timezoneOffset=0";
String url = Stripchat.baseUri + "/api/front/v2/config/data?requestPath=%2F&timezoneOffset=0";
Request request = new Request.Builder()
.url(url)
.header(ACCEPT, MIMETYPE_APPLICATION_JSON)
.header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
.header(ORIGIN, Stripchat.BASE_URI)
.header(REFERER, Stripchat.BASE_URI)
.header(ORIGIN, Stripchat.baseUri)
.header(REFERER, Stripchat.baseUri)
.header(CONTENT_TYPE, MIMETYPE_APPLICATION_JSON)
.build();
try (Response response = execute(request)) {
@ -107,11 +115,48 @@ public class StripchatHttpClient extends HttpClient {
* check, if the login worked
* @throws IOException
*/
public boolean checkLoginSuccess() {
return userId > 0;
public boolean checkLoginSuccess() throws IOException {
long userId = getUserId();
String url = Stripchat.baseUri + "/api/front/users/" + userId + "/favorites";
Request request = new Request.Builder()
.url(url)
.header(ACCEPT, MIMETYPE_APPLICATION_JSON)
.header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
.header(ORIGIN, Stripchat.baseUri)
.header(REFERER, Stripchat.baseUri + "/favorites")
.header(CONTENT_TYPE, MIMETYPE_APPLICATION_JSON)
.build();
try (Response response = execute(request)) {
if (response.isSuccessful()) {
return true;
}
} catch (Exception e) {
LOG.info("Login check returned unsuccessful: {}", e.getLocalizedMessage());
}
return false;
}
public long getUserId() {
public long getUserId() throws JSONException, IOException {
if (userId == 0) {
String url = Stripchat.baseUri + "/api/front/users/username/" + Config.getInstance().getSettings().stripchatUsername;
Request request = new Request.Builder()
.url(url)
.header(ACCEPT, MIMETYPE_APPLICATION_JSON)
.header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
.header(ORIGIN, Stripchat.baseUri)
.header(REFERER, Stripchat.baseUri)
.header(CONTENT_TYPE, MIMETYPE_APPLICATION_JSON)
.build();
try (Response response = execute(request)) {
if (response.isSuccessful()) {
JSONObject resp = new JSONObject(response.body().string());
JSONObject user = resp.getJSONObject("user");
userId = user.optLong("id");
} else {
throw new HttpException(url, response.code(), response.message());
}
}
}
return userId;
}

19
common/src/main/java/ctbrec/sites/stripchat/StripchatModel.java
View File

@ -95,9 +95,8 @@ public class StripchatModel extends AbstractModel {
best.width = broadcastSettings.optInt("width");
best.mediaPlaylistUrl = "https://b-" + serverName + ".stripst.com/hls/" + streamName + "/" + streamName + ".m3u8";
sources.add(best);
Object resolutionObject = broadcastSettings.get("resolutions");
if (resolutionObject instanceof JSONObject) {
JSONObject resolutions = (JSONObject) resolutionObject;
JSONObject resolutions = broadcastSettings.optJSONObject("resolutions");
if (resolutions instanceof JSONObject) {
JSONArray heights = resolutions.names();
for (int i = 0; i < heights.length(); i++) {
String h = heights.getString(i);
@ -145,12 +144,13 @@ public class StripchatModel extends AbstractModel {
@Override
public boolean follow() throws IOException {
getSite().getHttpClient().login();
JSONObject modelInfo = loadModelInfo();
JSONObject user = modelInfo.getJSONObject("user");
long modelId = user.optLong("id");
StripchatHttpClient client = (StripchatHttpClient) getSite().getHttpClient();
String url = Stripchat.BASE_URI + "/api/front/users/" + client.getUserId() + "/favorites/" + modelId;
String url = Stripchat.baseUri + "/api/front/users/" + client.getUserId() + "/favorites/" + modelId;
JSONObject requestParams = new JSONObject();
requestParams.put("csrfToken", client.getCsrfToken());
requestParams.put("csrfTimestamp", client.getCsrfTimestamp());
@ -160,8 +160,8 @@ public class StripchatModel extends AbstractModel {
.url(url)
.header(ACCEPT, "*/*")
.header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
.header(ORIGIN, Stripchat.BASE_URI)
.header(REFERER, Stripchat.BASE_URI + '/' + getName())
.header(ORIGIN, Stripchat.baseUri)
.header(REFERER, Stripchat.baseUri + '/' + getName())
.header(CONTENT_TYPE, MIMETYPE_APPLICATION_JSON)
.put(body)
.build();
@ -176,6 +176,7 @@ public class StripchatModel extends AbstractModel {
@Override
public boolean unfollow() throws IOException {
getSite().getHttpClient().login();
JSONObject modelInfo = loadModelInfo();
JSONObject user = modelInfo.getJSONObject("user");
long modelId = user.optLong("id");
@ -183,7 +184,7 @@ public class StripchatModel extends AbstractModel {
favoriteIds.put(modelId);
StripchatHttpClient client = (StripchatHttpClient) getSite().getHttpClient();
String url = Stripchat.BASE_URI + "/api/front/users/" + client.getUserId() + "/favorites";
String url = Stripchat.baseUri + "/api/front/users/" + client.getUserId() + "/favorites";
JSONObject requestParams = new JSONObject();
requestParams.put("favoriteIds", favoriteIds);
requestParams.put("csrfToken", client.getCsrfToken());
@ -194,8 +195,8 @@ public class StripchatModel extends AbstractModel {
.url(url)
.header(ACCEPT, MIMETYPE_APPLICATION_JSON)
.header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
.header(ORIGIN, Stripchat.BASE_URI)
.header(REFERER, Stripchat.BASE_URI)
.header(ORIGIN, Stripchat.baseUri)
.header(REFERER, Stripchat.baseUri)
.header(CONTENT_TYPE, MIMETYPE_APPLICATION_JSON)
.delete(body)
.build();

21
common/src/test/java/ctbrec/ReflectionUtil.java Normal file
View File

@ -0,0 +1,21 @@
package ctbrec;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.util.Arrays;
import java.util.List;
import java.util.stream.Collectors;
public class ReflectionUtil {
private ReflectionUtil () {}
@SuppressWarnings("unchecked")
public static <T> T call(Object target, String methodName, Object...args) throws NoSuchMethodException, SecurityException, IllegalAccessException, IllegalArgumentException, InvocationTargetException {
List<Class<?>> argTypes = Arrays.stream(args).map(arg -> arg.getClass()).collect(Collectors.toList());
Class<?>[] argTypeArray = argTypes.toArray(new Class[0]);
Method method = target.getClass().getDeclaredMethod(methodName, argTypeArray);
method.setAccessible(true);
return (T) method.invoke(target, args);
}
}

36
common/src/test/java/ctbrec/sites/mfc/MyFreeCamsClientTest.java Normal file
View File

@ -0,0 +1,36 @@
package ctbrec.sites.mfc;
import static org.junit.Assert.*;
import java.lang.reflect.InvocationTargetException;
import org.json.JSONObject;
import org.junit.Test;
import ctbrec.ReflectionUtil;
public class MyFreeCamsClientTest {
@Test
public void testMessageParsing() throws NoSuchMethodException, SecurityException, IllegalAccessException, IllegalArgumentException, InvocationTargetException {
MyFreeCamsClient client = MyFreeCamsClient.getInstance();
StringBuilder input = new StringBuilder("00001933 0 439895060 1 0 00001933 0 439895060 1 0 0000208 0 439895060 0 112 00038...");
Message msg = ReflectionUtil.call(client, "parseMessage", input);
assertEquals(33, msg.getType());
assertEquals(0, msg.getSender());
assertEquals(439895060, msg.getReceiver());
assertEquals(1, msg.getArg1());
assertEquals(0, msg.getArg2());
assertTrue(msg.getMessage().isBlank());
input = new StringBuilder("00207820 439461784 439895060 12 507930 %7B%22lv%22%3A4%2C%22nm%22%3A%22Nivea%22%2C%22pid%22%3A1%2C%22sid%22%3A439461784%2C%22uid%22%3A507930%2C%22vs%22%3A12%2C%22u%22%3A%7B%22age%22%3A33%2C%22avatar%22%3A1%2C%22blurb%22%3A%22I%20love%20when%20my%20nose%20touch%20your%20belly%20when%20I%20do%20you%20a%20blowjob!%20When%20I%20look%20into%20your%20horny%20eyes%20when%22%2C%22camserv%22%3A1367%2C%22chat_color%22%3A%22FF0000%22%2C%22chat_font%22%3A0%2C%22chat_opt%22%3A1%2C%22ethnic%22%3A%22Caucasian%22%2C%22photos%22%3A74%2C%22profile%22%3A1%2C%22status%22%3A%22%22%7D%2C%22m%22%3A%7B%22camscore%22%3A7505.700%2C%22continent%22%3A%22EU%22%2C%22flags%22%3A605224%2C%22hidecs%22%3Atrue%2C%22kbit%22%3A0%2C%22lastnews%22%3A0%2C%22mg%22%3A0%2C%22missmfc%22%3A2%2C%22new_model%22%3A0%2C%22rank%22%3A0%2C%22rc%22%3A20%2C%22sfw%22%3A0%2C%22topic%22%3A%22hi%253A)%255Bnone%255D-Topless%252C500-snap4life%252C50-spanks%252C180-flash%252C700-10%2520mins%2520of%2520Nora%2520fun%252C666-shot%252C27%252C270%2520%253C3%22%7D%2C%22x%22%3A%7B%22fcext%22%3A%7B%22sm%22%3A%22%22%2C%22sfw%22%3A0%7D%2C%22share%22%3A%7B%22follows%22%3A11%2C%22albums%22%3A0%2C%22clubs%22%3A0%2C%22tm_album%22%3A0%2C%22collections%22%3A0%2C%22stores%22%3A0%2C%22goals%22%3A0%2C%22polls%22%3A0%2C%22things%22%3A0%2C%22recent_album_tm%22%3A0%2C%22recent_club_tm%22%3A0%2C%22recent_collection_tm%22%3A0%2C%22recent_goal_tm%22%3A0%2C%22recent_item_tm%22%3A0%2C%22recent_poll_tm%22%3A0%2C%22recent_story_tm%22%3A0%2C%22recent_album_thumb%22%3A%22%22%2C%22recent_club_thumb%22%3A%22%22%2C%22recent_collection_thumb%22%3A%22%22%2C%22recent_goal_thumb%22%3A%22%22%2C%22recent_item_thumb%22%3A%22%22%2C%22recent_poll_thumb%22%3A%22%22%2C%22recent_story_thumb%22%3A%22%22%2C%22recent_album_title%22%3A%22%22%2C%22recent_club_title%22%3A%22%22%2C%22recent_collection_title%22%3A%22%22%2C%22recent_goal_title%22%3A%22%22%2C%22recent_item_title%22%3A%22%22%2C%22recent_poll_title%22%3A%22%22%2C%22recent_story_title%22%3A%22%22%2C%22recent_album_slug%22%3A%22%22%2C%22recent_collection_slug%22%3A%22%22%2C%22tipmenus%22%3A0%7D%7D%7D");
msg = ReflectionUtil.call(client, "parseMessage", input);
assertEquals(20, msg.getType());
assertEquals(439461784, msg.getSender());
assertEquals(439895060, msg.getReceiver());
assertEquals(12, msg.getArg1());
assertEquals(507930, msg.getArg2());
assertEquals(JSONObject.class, new JSONObject(msg.getMessage()).getClass()); // make sure we have a parsable JSON message
}
}

4
master/pom.xml
View File

@ -6,7 +6,7 @@
<groupId>ctbrec</groupId>
<artifactId>master</artifactId>
<packaging>pom</packaging>
<version>3.8.1</version>
<version>3.8.6</version>
<modules>
<module>../common</module>
@ -16,7 +16,7 @@
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<version.javafx>14-ea+4</version.javafx>
<version.javafx>14.0.2.1</version.javafx>
</properties>
<build>

2
server/pom.xml
View File

@ -8,7 +8,7 @@
<parent>
<groupId>ctbrec</groupId>
<artifactId>master</artifactId>
<version>3.8.1</version>
<version>3.8.6</version>
<relativePath>../master</relativePath>
</parent>

2
server/src/main/java/ctbrec/recorder/server/HttpServer.java
View File

@ -106,7 +106,7 @@ public class HttpServer {
safeLogin(site);
}
}
onlineMonitor = new OnlineMonitor(recorder);
onlineMonitor = new OnlineMonitor(recorder, config);
onlineMonitor.start();
startHttpServer();
}

11
server/src/main/java/ctbrec/recorder/server/RecorderServlet.java
View File

@ -96,6 +96,17 @@ public class RecorderServlet extends AbstractCtbrecServlet {
response = "{\"status\": \"success\", \"msg\": \"Stopping recording\"}";
resp.getWriter().write(response);
break;
case "stopAt":
new Thread(() -> {
try {
recorder.stopRecordingAt(request.model);
} catch (InvalidKeyException | NoSuchAlgorithmException | IllegalStateException | IOException e) {
LOG.error("Couldn't stop recording for model {}", request.model, e);
}
}).start();
response = "{\"status\": \"success\", \"msg\": \"Stopping recording\"}";
resp.getWriter().write(response);
break;
case "list":
resp.getWriter().write("{\"status\": \"success\", \"msg\": \"List of models\", \"models\": [");
JsonAdapter<Model> modelAdapter = new ModelJsonAdapter();

8
server/src/main/resources/html/static/custom.css
View File

@ -58,6 +58,10 @@ th a:hover {
text-decoration: none;
}
.checkmark-green {
color: #28a745;
}
.red {
color: #dc4444;
}

15
server/src/main/resources/html/static/index.html
View File

@ -96,19 +96,22 @@
<thead>
<tr>
<th><a href="#" data-bind="orderable: {collection: 'models', field: 'ko_name'}">Model</a></th>
<th><a href="#" data-bind="orderable: {collection: 'models', field: 'ko_suspended'}">Paused</a></th>
<th><a href="#" data-bind="orderable: {collection: 'models', field: 'ko_online'}">Online</a></th>
<th><a href="#" data-bind="orderable: {collection: 'models', field: 'ko_recording'}">Recording</a></th>
<th>Actions</th>
<th></th>
<th></th>
</tr>
</thead>
<tbody data-bind="foreach: models">
<tr>
<td><a data-bind="attr: { href: ko_url, title: ko_name }, text: ko_name"></a></td>
<td><input type="checkbox" data-bind="checked: ko_suspended" /></td>
<td><input type="checkbox" disabled data-bind="checked: ko_online" /></td>
<td><input type="checkbox" disabled data-bind="checked: ko_recording" /></td>
<td><button class="btn btn-secondary fa fa-minus-circle" title="Stop recording" data-bind="click: ctbrec.stop"></button></td>
<td><span data-bind="checked: ko_online, class: ko_online() ? `fa fa-check-square checkmark-green` : ``" style="font-size: 2em"></span></td>
<td><span data-bind="checked: ko_recording, class: ko_recording() ? `fa fa-circle red` : ``" style="font-size: 2em"></span></td>
<td>
<button class="btn btn-secondary fa fa-play" title="Resume recording" data-bind="click: ctbrec.resume, visible: ko_suspended"></button>
<button class="btn btn-secondary fa fa-pause" title="Pause recording" data-bind="click: ctbrec.suspend, hidden: ko_suspended"></button>
</td>
<td><button class="btn btn-secondary fa fa-trash" title="Remove model" data-bind="click: ctbrec.stop"></button></td>
</tr>
</tbody>
</table>

2
server/src/main/resources/html/static/models.js
View File

@ -69,6 +69,7 @@ function syncModels(models) {
}
}
model.ko_recording = ko.observable(model.online && !model.suspended);
//model.ko_recording_class = ko.observable( (model.online && !model.suspended) ? 'fa fa-circle red' : '' );
model.ko_suspended = ko.observable(model.suspended);
model.swallowEvents = false;
model.ko_suspended.subscribe(function(checked) {
@ -102,6 +103,7 @@ function syncModels(models) {
}
}
m.ko_online(onlineState);
//m.ko_recording_class( (model.online && !model.suspended) ? 'fa fa-circle red' : '');
m.swallowEvents = true;
m.ko_suspended(model.suspended);
m.swallowEvents = false;

1
server/src/main/resources/html/static/recordings.js
View File

@ -195,7 +195,6 @@ function updateDiskSpace() {
throughput.bytes(data.throughput);
throughput.timeframe(data.throughputTimeframe);
let bytesPerSecond = data.throughput / data.throughputTimeframe;
console.log(data.throughput, data.throughputTimeframe, bytesPerSecond, calculateSize(bytesPerSecond) + '/s');
throughput.text(calculateSize(bytesPerSecond) + '/s');
} else {
if (console)