diff --git a/CHANGELOG.md b/CHANGELOG.md
index ff02c63e..c956f655 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,17 @@
+1.12.0
+========================
+* Added threshold setting to keep free space on the recording device.
+ This is useful, if you don't want to use up all of your storage.
+ The free space is also shown on the recordings tab
+* Tweaked the download internals a lot. Downloads should not hang
+ in RECORDING state without actually recording. Downloads should
+ be more robust in general.
+* Fixed and improved split recordings
+* Improved detection of online state for Cam4 models
+* Accelerated the initial loading of the "Recording" tab for many
+ Chaturbate models
+* Recordings tab now shows smaller size units (Bytes, KiB, MiB, GiB)
+
1.11.0
========================
* Added model search function
diff --git a/client/.gitignore b/client/.gitignore
index fc247909..2c4e8e27 100644
--- a/client/.gitignore
+++ b/client/.gitignore
@@ -2,7 +2,7 @@
/target/
*~
*.bak
-/ctbrec.log
+/*.log
/ctbrec-tunnel.sh
/jre/
/server-local.sh
diff --git a/client/pom.xml b/client/pom.xml
index c0e4bb57..021c748b 100644
--- a/client/pom.xml
+++ b/client/pom.xml
@@ -8,7 +8,7 @@
ctbrec
master
- 1.11.0
+ 1.12.0
../master
diff --git a/client/src/main/java/ctbrec/ui/JavaFxRecording.java b/client/src/main/java/ctbrec/ui/JavaFxRecording.java
index 313a7e75..e44f36c0 100644
--- a/client/src/main/java/ctbrec/ui/JavaFxRecording.java
+++ b/client/src/main/java/ctbrec/ui/JavaFxRecording.java
@@ -1,10 +1,11 @@
package ctbrec.ui;
-import java.text.DecimalFormat;
import java.time.Instant;
import ctbrec.Config;
import ctbrec.Recording;
+import javafx.beans.property.LongProperty;
+import javafx.beans.property.SimpleLongProperty;
import javafx.beans.property.SimpleStringProperty;
import javafx.beans.property.StringProperty;
@@ -12,9 +13,10 @@ public class JavaFxRecording extends Recording {
private transient StringProperty statusProperty = new SimpleStringProperty();
private transient StringProperty progressProperty = new SimpleStringProperty();
- private transient StringProperty sizeProperty = new SimpleStringProperty();
+ private transient LongProperty sizeProperty = new SimpleLongProperty();
private Recording delegate;
+ private long lastValue = 0;
public JavaFxRecording(Recording recording) {
this.delegate = recording;
@@ -89,9 +91,7 @@ public class JavaFxRecording extends Recording {
@Override
public void setSizeInByte(long sizeInByte) {
delegate.setSizeInByte(sizeInByte);
- double sizeInGiB = sizeInByte / 1024.0 / 1024 / 1024;
- DecimalFormat df = new DecimalFormat("0.00");
- sizeProperty.setValue(df.format(sizeInGiB) + " GiB");
+ sizeProperty.set(sizeInByte);
}
public StringProperty getProgressProperty() {
@@ -151,8 +151,13 @@ public class JavaFxRecording extends Recording {
return delegate.getSizeInByte();
}
- public StringProperty getSizeProperty() {
+ public LongProperty getSizeProperty() {
return sizeProperty;
}
+ public boolean valueChanged() {
+ boolean changed = getSizeInByte() != lastValue;
+ lastValue = getSizeInByte();
+ return changed;
+ }
}
diff --git a/client/src/main/java/ctbrec/ui/RecordingsTab.java b/client/src/main/java/ctbrec/ui/RecordingsTab.java
index 2c7f4fb2..f8fef9dd 100644
--- a/client/src/main/java/ctbrec/ui/RecordingsTab.java
+++ b/client/src/main/java/ctbrec/ui/RecordingsTab.java
@@ -8,6 +8,7 @@ import java.io.IOException;
import java.net.URL;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
+import java.text.DecimalFormat;
import java.time.Instant;
import java.time.ZoneId;
import java.time.ZonedDateTime;
@@ -47,13 +48,16 @@ import javafx.scene.Cursor;
import javafx.scene.control.Alert.AlertType;
import javafx.scene.control.ButtonType;
import javafx.scene.control.ContextMenu;
+import javafx.scene.control.Label;
import javafx.scene.control.MenuItem;
+import javafx.scene.control.ProgressBar;
import javafx.scene.control.ScrollPane;
import javafx.scene.control.Tab;
import javafx.scene.control.TableCell;
import javafx.scene.control.TableColumn;
import javafx.scene.control.TableColumn.SortType;
import javafx.scene.control.TableView;
+import javafx.scene.control.Tooltip;
import javafx.scene.control.cell.PropertyValueFactory;
import javafx.scene.input.ContextMenuEvent;
import javafx.scene.input.KeyCode;
@@ -62,6 +66,9 @@ import javafx.scene.input.MouseButton;
import javafx.scene.input.MouseEvent;
import javafx.scene.layout.BorderPane;
import javafx.scene.layout.FlowPane;
+import javafx.scene.layout.HBox;
+import javafx.scene.layout.StackPane;
+import javafx.scene.text.Font;
import javafx.stage.FileChooser;
import javafx.util.Callback;
import javafx.util.Duration;
@@ -74,12 +81,16 @@ public class RecordingsTab extends Tab implements TabSelectionListener {
private Recorder recorder;
@SuppressWarnings("unused")
private List sites;
+ private long spaceTotal = -1;
+ private long spaceFree = -1;
FlowPane grid = new FlowPane();
ScrollPane scrollPane = new ScrollPane();
TableView table = new TableView();
ObservableList observableRecordings = FXCollections.observableArrayList();
ContextMenu popup;
+ ProgressBar spaceLeft;
+ Label spaceLabel;
public RecordingsTab(String title, Recorder recorder, Config config, List sites) {
super(title);
@@ -136,9 +147,37 @@ public class RecordingsTab extends Tab implements TabSelectionListener {
TableColumn progress = new TableColumn<>("Progress");
progress.setCellValueFactory((cdf) -> cdf.getValue().getProgressProperty());
progress.setPrefWidth(100);
- TableColumn size = new TableColumn<>("Size");
- size.setCellValueFactory((cdf) -> cdf.getValue().getSizeProperty());
+ TableColumn size = new TableColumn<>("Size");
+ size.setStyle("-fx-alignment: CENTER-RIGHT;");
size.setPrefWidth(100);
+ size.setCellValueFactory(cdf -> cdf.getValue().getSizeProperty());
+ size.setCellFactory(new Callback, TableCell>() {
+ @Override
+ public TableCell call(TableColumn param) {
+ TableCell cell = new TableCell() {
+ @Override
+ protected void updateItem(Number sizeInByte, boolean empty) {
+ if(empty || sizeInByte == null) {
+ setText(null);
+ setStyle(null);
+ } else {
+ setText(StringUtil.formatSize(sizeInByte));
+ if(Objects.equals(System.getenv("CTBREC_DEV"), "1")) {
+ int row = this.getTableRow().getIndex();
+ JavaFxRecording rec = tableViewProperty().get().getItems().get(row);
+ if(!rec.valueChanged() && rec.getStatus() == STATUS.RECORDING) {
+ setStyle("-fx-alignment: CENTER-RIGHT; -fx-background-color: red");
+ } else {
+ setStyle("-fx-alignment: CENTER-RIGHT;");
+ //setStyle(null);
+ }
+ }
+ }
+ }
+ };
+ return cell;
+ }
+ });
table.getColumns().addAll(name, date, status, progress, size);
table.setItems(observableRecordings);
@@ -179,8 +218,21 @@ public class RecordingsTab extends Tab implements TabSelectionListener {
});
scrollPane.setContent(table);
+ HBox spaceBox = new HBox(5);
+ Label l = new Label("Space left on device");
+ HBox.setMargin(l, new Insets(2, 0, 0, 0));
+ spaceBox.getChildren().add(l);
+ spaceLeft = new ProgressBar(0);
+ spaceLeft.setPrefSize(200, 22);
+ spaceLabel = new Label();
+ spaceLabel.setFont(Font.font(11));
+ StackPane stack = new StackPane(spaceLeft, spaceLabel);
+ spaceBox.getChildren().add(stack);
+ BorderPane.setMargin(spaceBox, new Insets(5));
+
BorderPane root = new BorderPane();
root.setPadding(new Insets(5));
+ root.setTop(spaceBox);
root.setCenter(scrollPane);
setContent(root);
@@ -191,30 +243,8 @@ public class RecordingsTab extends Tab implements TabSelectionListener {
updateService = createUpdateService();
updateService.setPeriod(new Duration(TimeUnit.SECONDS.toMillis(2)));
updateService.setOnSucceeded((event) -> {
- List recordings = updateService.getValue();
- if (recordings == null) {
- return;
- }
-
- for (Iterator iterator = observableRecordings.iterator(); iterator.hasNext();) {
- JavaFxRecording old = iterator.next();
- if (!recordings.contains(old)) {
- // remove deleted recordings
- iterator.remove();
- }
- }
- for (JavaFxRecording recording : recordings) {
- if (!observableRecordings.contains(recording)) {
- // add new recordings
- observableRecordings.add(recording);
- } else {
- // update existing ones
- int index = observableRecordings.indexOf(recording);
- JavaFxRecording old = observableRecordings.get(index);
- old.update(recording);
- }
- }
- table.sort();
+ updateRecordingsTable();
+ updateFreeSpaceDisplay();
});
updateService.setOnFailed((event) -> {
LOG.info("Couldn't get list of recordings from recorder", event.getSource().getException());
@@ -226,6 +256,46 @@ public class RecordingsTab extends Tab implements TabSelectionListener {
});
}
+ private void updateFreeSpaceDisplay() {
+ if(spaceTotal != -1 && spaceFree != -1) {
+ double free = ((double)spaceFree) / spaceTotal;
+ spaceLeft.setProgress(free);
+ double totalGiB = ((double) spaceTotal) / 1024 / 1024 / 1024;
+ double freeGiB = ((double) spaceFree) / 1024 / 1024 / 1024;
+ DecimalFormat df = new DecimalFormat("0.00");
+ String tt = df.format(freeGiB) + " / " + df.format(totalGiB) + " GiB";
+ spaceLeft.setTooltip(new Tooltip(tt));
+ spaceLabel.setText(tt);
+ }
+ }
+
+ private void updateRecordingsTable() {
+ List recordings = updateService.getValue();
+ if (recordings == null) {
+ return;
+ }
+
+ for (Iterator iterator = observableRecordings.iterator(); iterator.hasNext();) {
+ JavaFxRecording old = iterator.next();
+ if (!recordings.contains(old)) {
+ // remove deleted recordings
+ iterator.remove();
+ }
+ }
+ for (JavaFxRecording recording : recordings) {
+ if (!observableRecordings.contains(recording)) {
+ // add new recordings
+ observableRecordings.add(recording);
+ } else {
+ // update existing ones
+ int index = observableRecordings.indexOf(recording);
+ JavaFxRecording old = observableRecordings.get(index);
+ old.update(recording);
+ }
+ }
+ table.sort();
+ }
+
private ScheduledService> createUpdateService() {
ScheduledService> updateService = new ScheduledService>() {
@Override
@@ -233,12 +303,23 @@ public class RecordingsTab extends Tab implements TabSelectionListener {
return new Task>() {
@Override
public List call() throws IOException, InvalidKeyException, NoSuchAlgorithmException, IllegalStateException {
+ updateSpace();
+
List recordings = new ArrayList<>();
for (Recording rec : recorder.getRecordings()) {
recordings.add(new JavaFxRecording(rec));
}
return recordings;
}
+
+ private void updateSpace() {
+ try {
+ spaceTotal = recorder.getTotalSpaceBytes();
+ spaceFree = recorder.getFreeSpaceBytes();
+ } catch (IOException e) {
+ LOG.error("Couldn't update free space", e);
+ }
+ }
};
}
};
diff --git a/client/src/main/java/ctbrec/ui/SettingsTab.java b/client/src/main/java/ctbrec/ui/SettingsTab.java
index 54091aa9..d1f816cc 100644
--- a/client/src/main/java/ctbrec/ui/SettingsTab.java
+++ b/client/src/main/java/ctbrec/ui/SettingsTab.java
@@ -55,6 +55,7 @@ import javafx.stage.FileChooser;;
public class SettingsTab extends Tab implements TabSelectionListener {
private static final transient Logger LOG = LoggerFactory.getLogger(SettingsTab.class);
+ private static final int ONE_GiB_IN_BYTES = 1024 * 1024 * 1024;
public static final int CHECKBOX_MARGIN = 6;
private TextField recordingsDirectory;
@@ -65,6 +66,7 @@ public class SettingsTab extends Tab implements TabSelectionListener {
private TextField server;
private TextField port;
private TextField onlineCheckIntervalInSecs;
+ private TextField leaveSpaceOnDevice;
private CheckBox loadResolution;
private CheckBox secureCommunication = new CheckBox();
private CheckBox chooseStreamQuality = new CheckBox();
@@ -120,7 +122,7 @@ public class SettingsTab extends Tab implements TabSelectionListener {
// left side
leftSide.getChildren().add(createGeneralPanel());
- leftSide.getChildren().add(createLocationsPanel());
+ leftSide.getChildren().add(createRecorderPanel());
leftSide.getChildren().add(createRecordLocationPanel());
//right side
@@ -253,9 +255,20 @@ public class SettingsTab extends Tab implements TabSelectionListener {
return recordLocation;
}
- private Node createLocationsPanel() {
+ private Node createRecorderPanel() {
int row = 0;
GridPane layout = createGridLayout();
+ layout.add(new Label("Post-Processing"), 0, row);
+ postProcessing = new TextField(Config.getInstance().getSettings().postProcessing);
+ postProcessing.focusedProperty().addListener(createPostProcessingFocusListener());
+ GridPane.setFillWidth(postProcessing, true);
+ GridPane.setHgrow(postProcessing, Priority.ALWAYS);
+ GridPane.setColumnSpan(postProcessing, 2);
+ GridPane.setMargin(postProcessing, new Insets(0, 0, 0, CHECKBOX_MARGIN));
+ layout.add(postProcessing, 1, row);
+ postProcessingDirectoryButton = createPostProcessingBrowseButton();
+ layout.add(postProcessingDirectoryButton, 3, row++);
+
layout.add(new Label("Recordings Directory"), 0, row);
recordingsDirectory = new TextField(Config.getInstance().getSettings().recordingsDir);
recordingsDirectory.focusedProperty().addListener(createRecordingsDirectoryFocusListener());
@@ -283,16 +296,97 @@ public class SettingsTab extends Tab implements TabSelectionListener {
GridPane.setMargin(directoryStructure, new Insets(0, 0, 0, CHECKBOX_MARGIN));
layout.add(directoryStructure, 1, row++);
- layout.add(new Label("Post-Processing"), 0, row);
- postProcessing = new TextField(Config.getInstance().getSettings().postProcessing);
- postProcessing.focusedProperty().addListener(createPostProcessingFocusListener());
- GridPane.setFillWidth(postProcessing, true);
- GridPane.setHgrow(postProcessing, Priority.ALWAYS);
- GridPane.setColumnSpan(postProcessing, 2);
- GridPane.setMargin(postProcessing, new Insets(0, 0, 0, CHECKBOX_MARGIN));
- layout.add(postProcessing, 1, row);
- postProcessingDirectoryButton = createPostProcessingBrowseButton();
- layout.add(postProcessingDirectoryButton, 3, row++);
+ Label l = new Label("Maximum resolution (0 = unlimited)");
+ layout.add(l, 0, row);
+ List resolutionOptions = new ArrayList<>();
+ resolutionOptions.add(1080);
+ resolutionOptions.add(720);
+ resolutionOptions.add(600);
+ resolutionOptions.add(480);
+ resolutionOptions.add(0);
+ maxResolution = new ComboBox<>(FXCollections.observableList(resolutionOptions));
+ setMaxResolutionValue();
+ maxResolution.setOnAction((e) -> {
+ Config.getInstance().getSettings().maximumResolution = maxResolution.getSelectionModel().getSelectedItem();
+ saveConfig();
+ });
+ maxResolution.prefWidthProperty().bind(directoryStructure.widthProperty());
+ layout.add(maxResolution, 1, row++);
+ GridPane.setMargin(l, new Insets(0, 0, 0, 0));
+ GridPane.setMargin(maxResolution, new Insets(0, 0, 0, CHECKBOX_MARGIN));
+
+ l = new Label("Split recordings after (minutes)");
+ layout.add(l, 0, row);
+ List splitOptions = new ArrayList<>();
+ splitOptions.add(new SplitAfterOption("disabled", 0));
+ if(Config.isDevMode()) {
+ splitOptions.add(new SplitAfterOption( "1 min", 1 * 60));
+ splitOptions.add(new SplitAfterOption( "3 min", 3 * 60));
+ splitOptions.add(new SplitAfterOption( "5 min", 5 * 60));
+ }
+ splitOptions.add(new SplitAfterOption("10 min", 10 * 60));
+ splitOptions.add(new SplitAfterOption("15 min", 15 * 60));
+ splitOptions.add(new SplitAfterOption("20 min", 20 * 60));
+ splitOptions.add(new SplitAfterOption("30 min", 30 * 60));
+ splitOptions.add(new SplitAfterOption("60 min", 60 * 60));
+ splitAfter = new ComboBox<>(FXCollections.observableList(splitOptions));
+ layout.add(splitAfter, 1, row++);
+ setSplitAfterValue();
+ splitAfter.setOnAction((e) -> {
+ Config.getInstance().getSettings().splitRecordings = splitAfter.getSelectionModel().getSelectedItem().getValue();
+ saveConfig();
+ });
+ splitAfter.prefWidthProperty().bind(directoryStructure.widthProperty());
+ GridPane.setMargin(l, new Insets(0, 0, 0, 0));
+ GridPane.setMargin(splitAfter, new Insets(0, 0, 0, CHECKBOX_MARGIN));
+
+ Tooltip tt = new Tooltip("Check every x seconds, if a model came online");
+ l = new Label("Check online state every (seconds)");
+ l.setTooltip(tt);
+ layout.add(l, 0, row);
+ onlineCheckIntervalInSecs = new TextField(Integer.toString(Config.getInstance().getSettings().onlineCheckIntervalInSecs));
+ onlineCheckIntervalInSecs.setTooltip(tt);
+ onlineCheckIntervalInSecs.textProperty().addListener((observable, oldValue, newValue) -> {
+ if (!newValue.matches("\\d*")) {
+ onlineCheckIntervalInSecs.setText(newValue.replaceAll("[^\\d]", ""));
+ }
+ if(!onlineCheckIntervalInSecs.getText().isEmpty()) {
+ Config.getInstance().getSettings().onlineCheckIntervalInSecs = Integer.parseInt(onlineCheckIntervalInSecs.getText());
+ saveConfig();
+ }
+ });
+ GridPane.setMargin(onlineCheckIntervalInSecs, new Insets(0, 0, 0, CHECKBOX_MARGIN));
+ layout.add(onlineCheckIntervalInSecs, 1, row++);
+
+ tt = new Tooltip("Stop recording, if the free space on the device gets below this threshold");
+ l = new Label("Leave space on device (GiB)");
+ l.setTooltip(tt);
+ layout.add(l, 0, row);
+ long minimumSpaceLeftInBytes = Config.getInstance().getSettings().minimumSpaceLeftInBytes;
+ int minimumSpaceLeftInGiB = (int) (minimumSpaceLeftInBytes / ONE_GiB_IN_BYTES);
+ leaveSpaceOnDevice = new TextField(Integer.toString(minimumSpaceLeftInGiB));
+ leaveSpaceOnDevice.setTooltip(tt);
+ leaveSpaceOnDevice.textProperty().addListener((observable, oldValue, newValue) -> {
+ if (!newValue.matches("\\d*")) {
+ leaveSpaceOnDevice.setText(newValue.replaceAll("[^\\d]", ""));
+ }
+ if(!leaveSpaceOnDevice.getText().isEmpty()) {
+ long spaceLeftInGiB = Long.parseLong(leaveSpaceOnDevice.getText());
+ Config.getInstance().getSettings().minimumSpaceLeftInBytes = spaceLeftInGiB * ONE_GiB_IN_BYTES;
+ saveConfig();
+ }
+ });
+ GridPane.setMargin(leaveSpaceOnDevice, new Insets(0, 0, 0, CHECKBOX_MARGIN));
+ layout.add(leaveSpaceOnDevice, 1, row++);
+
+ TitledPane locations = new TitledPane("Recorder", layout);
+ locations.setCollapsible(false);
+ return locations;
+ }
+
+ private Node createGeneralPanel() {
+ GridPane layout = createGridLayout();
+ int row = 0;
layout.add(new Label("Player"), 0, row);
mediaPlayer = new TextField(Config.getInstance().getSettings().mediaPlayer);
@@ -304,15 +398,18 @@ public class SettingsTab extends Tab implements TabSelectionListener {
layout.add(mediaPlayer, 1, row);
layout.add(createMpvBrowseButton(), 3, row++);
- TitledPane locations = new TitledPane("Locations", layout);
- locations.setCollapsible(false);
- return locations;
- }
+ Label l = new Label("Allow multiple players");
+ layout.add(l, 0, row);
+ multiplePlayers.setSelected(!Config.getInstance().getSettings().singlePlayer);
+ multiplePlayers.setOnAction((e) -> {
+ Config.getInstance().getSettings().singlePlayer = !multiplePlayers.isSelected();
+ saveConfig();
+ });
+ GridPane.setMargin(l, new Insets(3, 0, 0, 0));
+ GridPane.setMargin(multiplePlayers, new Insets(CHECKBOX_MARGIN, 0, 0, CHECKBOX_MARGIN));
+ layout.add(multiplePlayers, 1, row++);
- private Node createGeneralPanel() {
- GridPane layout = createGridLayout();
- int row = 0;
- Label l = new Label("Display stream resolution in overview");
+ l = new Label("Display stream resolution in overview");
layout.add(l, 0, row);
loadResolution = new CheckBox();
loadResolution.setSelected(Config.getInstance().getSettings().determineResolution);
@@ -323,20 +420,10 @@ public class SettingsTab extends Tab implements TabSelectionListener {
ThumbOverviewTab.queue.clear();
}
});
- //GridPane.setMargin(l, new Insets(CHECKBOX_MARGIN, 0, 0, 0));
- GridPane.setMargin(loadResolution, new Insets(0, 0, 0, CHECKBOX_MARGIN));
+ GridPane.setMargin(l, new Insets(CHECKBOX_MARGIN, 0, 0, 0));
+ GridPane.setMargin(loadResolution, new Insets(CHECKBOX_MARGIN, 0, 0, CHECKBOX_MARGIN));
layout.add(loadResolution, 1, row++);
- l = new Label("Allow multiple players");
- layout.add(l, 0, row);
- multiplePlayers.setSelected(!Config.getInstance().getSettings().singlePlayer);
- multiplePlayers.setOnAction((e) -> {
- Config.getInstance().getSettings().singlePlayer = !multiplePlayers.isSelected();
- saveConfig();
- });
- GridPane.setMargin(l, new Insets(3, 0, 0, 0));
- GridPane.setMargin(multiplePlayers, new Insets(CHECKBOX_MARGIN, 0, 0, CHECKBOX_MARGIN));
- layout.add(multiplePlayers, 1, row++);
l = new Label("Manually select stream quality");
layout.add(l, 0, row);
@@ -357,60 +444,9 @@ public class SettingsTab extends Tab implements TabSelectionListener {
saveConfig();
});
GridPane.setMargin(l, new Insets(CHECKBOX_MARGIN, 0, 0, 0));
- GridPane.setMargin(updateThumbnails, new Insets(CHECKBOX_MARGIN, 0, 0, CHECKBOX_MARGIN));
+ GridPane.setMargin(updateThumbnails, new Insets(CHECKBOX_MARGIN, 0, CHECKBOX_MARGIN, CHECKBOX_MARGIN));
layout.add(updateThumbnails, 1, row++);
- l = new Label("Maximum resolution (0 = unlimited)");
- layout.add(l, 0, row);
- List resolutionOptions = new ArrayList<>();
- resolutionOptions.add(1080);
- resolutionOptions.add(720);
- resolutionOptions.add(600);
- resolutionOptions.add(480);
- resolutionOptions.add(0);
- maxResolution = new ComboBox<>(FXCollections.observableList(resolutionOptions));
- setMaxResolutionValue();
- maxResolution.setOnAction((e) -> {
- Config.getInstance().getSettings().maximumResolution = maxResolution.getSelectionModel().getSelectedItem();
- saveConfig();
- });
- layout.add(maxResolution, 1, row++);
- GridPane.setMargin(l, new Insets(CHECKBOX_MARGIN, 0, 0, 0));
- GridPane.setMargin(maxResolution, new Insets(CHECKBOX_MARGIN, 0, 0, CHECKBOX_MARGIN));
-
- l = new Label("Split recordings after (minutes)");
- layout.add(l, 0, row);
- List options = new ArrayList<>();
- options.add(new SplitAfterOption("disabled", 0));
- options.add(new SplitAfterOption("10 min", 10 * 60));
- options.add(new SplitAfterOption("15 min", 15 * 60));
- options.add(new SplitAfterOption("20 min", 20 * 60));
- options.add(new SplitAfterOption("30 min", 30 * 60));
- options.add(new SplitAfterOption("60 min", 60 * 60));
- splitAfter = new ComboBox<>(FXCollections.observableList(options));
- layout.add(splitAfter, 1, row++);
- setSplitAfterValue();
- splitAfter.setOnAction((e) -> {
- Config.getInstance().getSettings().splitRecordings = splitAfter.getSelectionModel().getSelectedItem().getValue();
- saveConfig();
- });
- GridPane.setMargin(l, new Insets(0, 0, 0, 0));
- GridPane.setMargin(splitAfter, new Insets(0, 0, 0, CHECKBOX_MARGIN));
-
- layout.add(new Label("Check online state every (seconds)"), 0, row);
- onlineCheckIntervalInSecs = new TextField(Integer.toString(Config.getInstance().getSettings().onlineCheckIntervalInSecs));
- onlineCheckIntervalInSecs.textProperty().addListener((observable, oldValue, newValue) -> {
- if (!newValue.matches("\\d*")) {
- onlineCheckIntervalInSecs.setText(newValue.replaceAll("[^\\d]", ""));
- }
- if(!onlineCheckIntervalInSecs.getText().isEmpty()) {
- Config.getInstance().getSettings().onlineCheckIntervalInSecs = Integer.parseInt(onlineCheckIntervalInSecs.getText());
- saveConfig();
- }
- });
- GridPane.setMargin(onlineCheckIntervalInSecs, new Insets(0, 0, 0, CHECKBOX_MARGIN));
- layout.add(onlineCheckIntervalInSecs, 1, row++);
-
l = new Label("Start Tab");
layout.add(l, 0, row);
startTab = new ComboBox<>();
@@ -429,12 +465,6 @@ public class SettingsTab extends Tab implements TabSelectionListener {
GridPane.setMargin(l, new Insets(0, 0, 0, 0));
GridPane.setMargin(colorSettingsPane, new Insets(CHECKBOX_MARGIN, 0, 0, CHECKBOX_MARGIN));
-
- splitAfter.prefWidthProperty().bind(startTab.widthProperty());
- maxResolution.prefWidthProperty().bind(startTab.widthProperty());
- onlineCheckIntervalInSecs.prefWidthProperty().bind(startTab.widthProperty());
- onlineCheckIntervalInSecs.maxWidthProperty().bind(startTab.widthProperty());
-
TitledPane general = new TitledPane("General", layout);
general.setCollapsible(false);
return general;
@@ -481,6 +511,8 @@ public class SettingsTab extends Tab implements TabSelectionListener {
postProcessing.setDisable(!local);
postProcessingDirectoryButton.setDisable(!local);
directoryStructure.setDisable(!local);
+ onlineCheckIntervalInSecs.setDisable(!local);
+ leaveSpaceOnDevice.setDisable(!local);
}
private ChangeListener createRecordingsDirectoryFocusListener() {
@@ -653,9 +685,10 @@ public class SettingsTab extends Tab implements TabSelectionListener {
@Override
public void selected() {
- startTab.getItems().clear();
- for(Tab tab : getTabPane().getTabs()) {
- startTab.getItems().add(tab.getText());
+ if(startTab.getItems().isEmpty()) {
+ for(Tab tab : getTabPane().getTabs()) {
+ startTab.getItems().add(tab.getText());
+ }
}
String startTabName = Config.getInstance().getSettings().startTab;
if(StringUtil.isNotBlank(startTabName)) {
diff --git a/common/pom.xml b/common/pom.xml
index 54db7db8..cb1e5152 100644
--- a/common/pom.xml
+++ b/common/pom.xml
@@ -8,7 +8,7 @@
ctbrec
master
- 1.11.0
+ 1.12.0
../master
diff --git a/common/src/main/java/ctbrec/Config.java b/common/src/main/java/ctbrec/Config.java
index 865f6bc1..871c36ff 100644
--- a/common/src/main/java/ctbrec/Config.java
+++ b/common/src/main/java/ctbrec/Config.java
@@ -100,10 +100,14 @@ public class Config {
Files.write(configFile.toPath(), json.getBytes("utf-8"), CREATE, WRITE, TRUNCATE_EXISTING);
}
- public boolean isServerMode() {
+ public static boolean isServerMode() {
return Objects.equals(System.getProperty("ctbrec.server.mode"), "1");
}
+ public static boolean isDevMode() {
+ return Objects.equals(System.getenv("CTBREC_DEV"), "1");
+ }
+
public File getConfigDir() {
return configDir;
}
@@ -113,10 +117,6 @@ public class Config {
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd_HH-mm");
String startTime = sdf.format(new Date());
File targetFile = new File(dirForRecording, model.getName() + '_' + startTime + ".ts");
- if(getSettings().splitRecordings > 0) {
- LOG.debug("Splitting recordings every {} seconds", getSettings().splitRecordings);
- targetFile = new File(targetFile.getAbsolutePath().replaceAll("\\.ts", "-00000.ts"));
- }
return targetFile;
}
diff --git a/common/src/main/java/ctbrec/Settings.java b/common/src/main/java/ctbrec/Settings.java
index a34e24e9..3b613845 100644
--- a/common/src/main/java/ctbrec/Settings.java
+++ b/common/src/main/java/ctbrec/Settings.java
@@ -37,6 +37,7 @@ public class Settings {
public String httpServer = "localhost";
public String recordingsDir = System.getProperty("user.home") + File.separator + "ctbrec";
public DirectoryStructure recordingsDirStructure = DirectoryStructure.FLAT;
+ public long minimumSpaceLeftInBytes = 0;
public String mediaPlayer = "/usr/bin/mpv";
public String postProcessing = "";
public String username = ""; // chaturbate username TODO maybe rename this onetime
diff --git a/common/src/main/java/ctbrec/StringUtil.java b/common/src/main/java/ctbrec/StringUtil.java
index 229e44b1..d9ae9796 100644
--- a/common/src/main/java/ctbrec/StringUtil.java
+++ b/common/src/main/java/ctbrec/StringUtil.java
@@ -1,5 +1,7 @@
package ctbrec;
+import java.text.DecimalFormat;
+
public class StringUtil {
public static boolean isBlank(String s) {
return s == null || s.trim().isEmpty();
@@ -8,4 +10,21 @@ public class StringUtil {
public static boolean isNotBlank(String s) {
return !isBlank(s);
}
+
+ public static String formatSize(Number sizeInByte) {
+ DecimalFormat df = new DecimalFormat("0.00");
+ String unit = "Bytes";
+ double size = sizeInByte.doubleValue();
+ if(size > 1024.0 * 1024 * 1024) {
+ size = size / 1024.0 / 1024 / 1024;
+ unit = "GiB";
+ } else if(size > 1024.0 * 1024) {
+ size = size / 1024.0 / 1024;
+ unit = "MiB";
+ } else if(size > 1024.0) {
+ size = size / 1024.0;
+ unit = "KiB";
+ }
+ return df.format(size) + ' ' + unit;
+ }
}
diff --git a/common/src/main/java/ctbrec/recorder/LocalRecorder.java b/common/src/main/java/ctbrec/recorder/LocalRecorder.java
index 3afb3ee4..82c5219c 100644
--- a/common/src/main/java/ctbrec/recorder/LocalRecorder.java
+++ b/common/src/main/java/ctbrec/recorder/LocalRecorder.java
@@ -5,6 +5,7 @@ import static ctbrec.Recording.STATUS.*;
import java.io.File;
import java.io.FilenameFilter;
import java.io.IOException;
+import java.nio.file.FileStore;
import java.nio.file.Files;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
@@ -63,6 +64,7 @@ public class LocalRecorder implements Recorder {
private List deleteInProgress = Collections.synchronizedList(new ArrayList<>());
private RecorderHttpClient client = new RecorderHttpClient();
private ReentrantLock lock = new ReentrantLock();
+ private long lastSpaceMessage = 0;
public LocalRecorder(Config config) {
this.config = config;
@@ -81,7 +83,7 @@ public class LocalRecorder implements Recorder {
onlineMonitor.start();
postProcessingTrigger = new PostProcessingTrigger();
- if(Config.getInstance().isServerMode()) {
+ if(Config.isServerMode()) {
postProcessingTrigger.start();
}
@@ -133,7 +135,6 @@ public class LocalRecorder implements Recorder {
return;
}
- LOG.debug("Starting recording for model {}", model.getName());
if (recordingProcesses.containsKey(model)) {
LOG.error("A recording for model {} is already running", model);
return;
@@ -149,8 +150,18 @@ public class LocalRecorder implements Recorder {
lock.unlock();
}
+ if(!enoughSpaceForRecording()) {
+ long now = System.currentTimeMillis();
+ if( (now - lastSpaceMessage) > TimeUnit.MINUTES.toMillis(1)) {
+ LOG.info("Not enough space for recording, not starting recording for {}", model);
+ lastSpaceMessage = now;
+ }
+ return;
+ }
+
+ LOG.debug("Starting recording for model {}", model.getName());
Download download;
- if (Config.getInstance().isServerMode()) {
+ if (Config.isServerMode()) {
download = new HlsDownload(client);
} else {
download = new MergedHlsDownload(client);
@@ -173,7 +184,7 @@ public class LocalRecorder implements Recorder {
Download download = recordingProcesses.get(model);
download.stop();
recordingProcesses.remove(model);
- if(!Config.getInstance().isServerMode()) {
+ if(!Config.isServerMode()) {
postprocess(download);
}
}
@@ -329,6 +340,15 @@ public class LocalRecorder implements Recorder {
public void run() {
running = true;
while (running) {
+ try {
+ if(!enoughSpaceForRecording() && !recordingProcesses.isEmpty()) {
+ LOG.info("No space left -> Stopping all recordings");
+ stopRecordingProcesses();
+ }
+ } catch (IOException e1) {
+ LOG.warn("Couldn't check free space left", e1);
+ }
+
List restart = new ArrayList<>();
for (Iterator> iterator = recordingProcesses.entrySet().iterator(); iterator.hasNext();) {
Entry entry = iterator.next();
@@ -338,7 +358,7 @@ public class LocalRecorder implements Recorder {
LOG.debug("Recording terminated for model {}", m.getName());
iterator.remove();
restart.add(m);
- if(config.isServerMode()) {
+ if(Config.isServerMode()) {
try {
finishRecording(d.getTarget());
} catch(Exception e) {
@@ -365,7 +385,7 @@ public class LocalRecorder implements Recorder {
}
private void finishRecording(File directory) {
- if(Config.getInstance().isServerMode()) {
+ if(Config.isServerMode()) {
Thread t = new Thread() {
@Override
public void run() {
@@ -415,7 +435,7 @@ public class LocalRecorder implements Recorder {
boolean isOnline = model.isOnline(IGNORE_CACHE);
LOG.trace("Checking online state for {}: {}", model, (isOnline ? "online" : "offline"));
if (isOnline && !isSuspended(model) && !recordingProcesses.containsKey(model)) {
- LOG.info("Model {}'s room back to public. Starting recording", model);
+ LOG.info("Model {}'s room back to public", model);
startRecordingProcess(model);
}
} catch (HttpException e) {
@@ -493,7 +513,7 @@ public class LocalRecorder implements Recorder {
@Override
public List getRecordings() {
- if(Config.getInstance().isServerMode()) {
+ if(Config.isServerMode()) {
return listSegmentedRecordings();
} else {
return listMergedRecordings();
@@ -538,7 +558,7 @@ public class LocalRecorder implements Recorder {
return GENERATING_PLAYLIST;
}
- if (config.isServerMode()) {
+ if (Config.isServerMode()) {
if (recording.hasPlaylist()) {
return FINISHED;
} else {
@@ -745,4 +765,25 @@ public class LocalRecorder implements Recorder {
public HttpClient getHttpClient() {
return client;
}
+
+ @Override
+ public long getTotalSpaceBytes() throws IOException {
+ return getRecordingsFileStore().getTotalSpace();
+ }
+
+ @Override
+ public long getFreeSpaceBytes() throws IOException {
+ return getRecordingsFileStore().getUsableSpace();
+ }
+
+ private FileStore getRecordingsFileStore() throws IOException {
+ File recordingsDir = new File(config.getSettings().recordingsDir);
+ FileStore store = Files.getFileStore(recordingsDir.toPath());
+ return store;
+ }
+
+ private boolean enoughSpaceForRecording() throws IOException {
+ long minimum = config.getSettings().minimumSpaceLeftInBytes;
+ return getFreeSpaceBytes() > minimum;
+ }
}
diff --git a/common/src/main/java/ctbrec/recorder/Recorder.java b/common/src/main/java/ctbrec/recorder/Recorder.java
index bc4e60cf..1a9bf682 100644
--- a/common/src/main/java/ctbrec/recorder/Recorder.java
+++ b/common/src/main/java/ctbrec/recorder/Recorder.java
@@ -42,4 +42,18 @@ public interface Recorder {
public List getOnlineModels();
public HttpClient getHttpClient();
+
+ /**
+ * Get the total size of the filesystem we are recording to
+ * @return the total size in bytes
+ * @throws IOException
+ */
+ public long getTotalSpaceBytes() throws IOException;
+
+ /**
+ * Get the free space left on the filesystem we are recording to
+ * @return the free space in bytes
+ * @throws IOException
+ */
+ public long getFreeSpaceBytes() throws IOException;
}
diff --git a/common/src/main/java/ctbrec/recorder/RemoteRecorder.java b/common/src/main/java/ctbrec/recorder/RemoteRecorder.java
index dd648301..7c3ced0a 100644
--- a/common/src/main/java/ctbrec/recorder/RemoteRecorder.java
+++ b/common/src/main/java/ctbrec/recorder/RemoteRecorder.java
@@ -8,6 +8,7 @@ import java.time.Instant;
import java.util.Collections;
import java.util.List;
+import org.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -45,6 +46,8 @@ public class RemoteRecorder implements Recorder {
private List models = Collections.emptyList();
private List onlineModels = Collections.emptyList();
private List sites;
+ private long spaceTotal = -1;
+ private long spaceFree = -1;
private Config config;
private HttpClient client;
@@ -150,10 +153,35 @@ public class RemoteRecorder implements Recorder {
while(running) {
syncModels();
syncOnlineModels();
+ syncSpace();
sleep();
}
}
+ private void syncSpace() {
+ try {
+ String msg = "{\"action\": \"space\"}";
+ RequestBody body = RequestBody.create(JSON, msg);
+ Request.Builder builder = new Request.Builder()
+ .url("http://" + config.getSettings().httpServer + ":" + config.getSettings().httpPort + "/rec")
+ .post(body);
+ addHmacIfNeeded(msg, builder);
+ Request request = builder.build();
+ try(Response response = client.execute(request)) {
+ String json = response.body().string();
+ if(response.isSuccessful()) {
+ JSONObject resp = new JSONObject(json);
+ spaceTotal = resp.getLong("spaceTotal");
+ spaceFree = resp.getLong("spaceFree");
+ } else {
+ LOG.error("Couldn't synchronize with server. HTTP status: {} - {}", response.code(), json);
+ }
+ }
+ } catch (IOException | InvalidKeyException | NoSuchAlgorithmException | IllegalStateException e) {
+ LOG.error("Couldn't synchronize with server", e);
+ }
+ }
+
private void syncModels() {
try {
String msg = "{\"action\": \"list\"}";
@@ -362,4 +390,14 @@ public class RemoteRecorder implements Recorder {
public HttpClient getHttpClient() {
return client;
}
+
+ @Override
+ public long getTotalSpaceBytes() throws IOException {
+ return spaceTotal;
+ }
+
+ @Override
+ public long getFreeSpaceBytes() {
+ return spaceFree;
+ }
}
diff --git a/common/src/main/java/ctbrec/recorder/download/MergedHlsDownload.java b/common/src/main/java/ctbrec/recorder/download/MergedHlsDownload.java
index 84c4c7d6..c9cd0010 100644
--- a/common/src/main/java/ctbrec/recorder/download/MergedHlsDownload.java
+++ b/common/src/main/java/ctbrec/recorder/download/MergedHlsDownload.java
@@ -15,11 +15,19 @@ import java.nio.file.LinkOption;
import java.nio.file.Path;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
-import java.text.DecimalFormat;
import java.time.Duration;
import java.time.Instant;
import java.time.ZonedDateTime;
+import java.util.LinkedList;
+import java.util.Queue;
+import java.util.concurrent.BlockingQueue;
import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -48,11 +56,12 @@ public class MergedHlsDownload extends AbstractHlsDownload {
private BlockingMultiMTSSource multiSource;
private Thread mergeThread;
private Streamer streamer;
- private ZonedDateTime startTime;
+ private ZonedDateTime splitRecStartTime;
private Config config;
private File targetFile;
- private DecimalFormat df = new DecimalFormat("00000");
- private int splitCounter = 0;
+ private BlockingQueue downloadQueue = new LinkedBlockingQueue<>(50);
+ private ExecutorService downloadThreadPool = new ThreadPoolExecutor(5, 5, 2, TimeUnit.MINUTES, downloadQueue);
+ private FileChannel fileChannel = null;
public MergedHlsDownload(HttpClient client) {
super(client);
@@ -67,6 +76,7 @@ public class MergedHlsDownload extends AbstractHlsDownload {
try {
running = true;
super.startTime = Instant.now();
+ splitRecStartTime = ZonedDateTime.now();
mergeThread = createMergeThread(targetFile, progressListener, false);
LOG.debug("Merge thread started");
mergeThread.start();
@@ -81,7 +91,7 @@ public class MergedHlsDownload extends AbstractHlsDownload {
downloadSegments(segmentPlaylistUri, false);
LOG.debug("Waiting for merge thread to finish");
mergeThread.join();
- LOG.debug("Merge thread to finished");
+ LOG.debug("Merge thread finished");
} catch(ParseException e) {
throw new IOException("Couldn't parse stream information", e);
} catch(PlaylistException e) {
@@ -92,7 +102,12 @@ public class MergedHlsDownload extends AbstractHlsDownload {
throw new IOException("Couldn't add HMAC to playlist url", e);
} finally {
alive = false;
- streamer.stop();
+ try {
+ streamer.stop();
+ } catch(Exception e) {
+ LOG.error("Couldn't stop streamer", e);
+ }
+ downloadThreadPool.shutdown();
LOG.debug("Download terminated for {}", segmentPlaylistUri);
}
}
@@ -107,6 +122,7 @@ public class MergedHlsDownload extends AbstractHlsDownload {
running = true;
super.startTime = Instant.now();
+ splitRecStartTime = ZonedDateTime.now();
super.model = model;
targetFile = Config.getInstance().getFileForRecording(model);
String segments = getSegmentPlaylistUrl(model);
@@ -114,6 +130,9 @@ public class MergedHlsDownload extends AbstractHlsDownload {
mergeThread.start();
if(segments != null) {
downloadSegments(segments, true);
+ if(config.getSettings().splitRecordings > 0) {
+ LOG.debug("Splitting recordings every {} seconds", config.getSettings().splitRecordings);
+ }
} else {
throw new IOException("Couldn't determine segments uri");
}
@@ -129,7 +148,11 @@ public class MergedHlsDownload extends AbstractHlsDownload {
} finally {
alive = false;
if(streamer != null) {
- streamer.stop();
+ try {
+ streamer.stop();
+ } catch(Exception e) {
+ LOG.error("Couldn't stop streamer", e);
+ }
}
LOG.debug("Download for {} terminated", model);
}
@@ -138,36 +161,110 @@ public class MergedHlsDownload extends AbstractHlsDownload {
private void downloadSegments(String segmentPlaylistUri, boolean livestreamDownload) throws IOException, ParseException, PlaylistException {
int lastSegment = 0;
int nextSegment = 0;
+ long playlistNotFoundFirstEncounter = -1;
while(running) {
try {
+ if(playlistNotFoundFirstEncounter != -1) {
+ LOG.debug("Downloading playlist {}", segmentPlaylistUri);
+ }
SegmentPlaylist lsp = getNextSegments(segmentPlaylistUri);
+ playlistNotFoundFirstEncounter = -1;
if(!livestreamDownload) {
multiSource.setTotalSegments(lsp.segments.size());
}
- // download segments, which might have been skipped
- downloadMissedSegments(lsp, nextSegment);
-
// download new segments
+ long downloadStart = System.currentTimeMillis();
downloadNewSegments(lsp, nextSegment);
+ long downloadTookMillis = System.currentTimeMillis() - downloadStart;
+
+ // download segments, which might have been skipped
+ //downloadMissedSegments(lsp, nextSegment);
+ if(nextSegment > 0 && lsp.seq > nextSegment) {
+ LOG.warn("Missed segments {} < {} in download for {}. Download took {}ms. Playlist is {}sec", nextSegment, lsp.seq, lsp.url, downloadTookMillis, lsp.totalDuration);
+ }
if(livestreamDownload) {
// split up the recording, if configured
splitRecording();
// wait some time until requesting the segment playlist again to not hammer the server
- waitForNewSegments(lsp, lastSegment);
+ waitForNewSegments(lsp, lastSegment, downloadTookMillis);
lastSegment = lsp.seq;
nextSegment = lastSegment + lsp.segments.size();
} else {
break;
}
- } catch(HttpException e) {
- if(e.getResponseCode() == 404) {
- // playlist is gone -> model probably logged out
- LOG.debug("Playlist not found. Assuming model went offline");
- running = false;
+ } catch(Exception e) {
+ LOG.info("Unexpected error while downloading {}", model.getName(), e);
+ running = false;
+ }
+ }
+ }
+
+ private void downloadNewSegments(SegmentPlaylist lsp, int nextSegment) throws MalformedURLException, MissingSegmentException, ExecutionException, HttpException {
+ int skip = nextSegment - lsp.seq;
+ if(lsp.segments.isEmpty()) {
+ LOG.debug("Empty playlist: {}", lsp.url);
+ }
+
+ // add segments to download threadpool
+ Queue> downloads = new LinkedList<>();
+ if(downloadQueue.remainingCapacity() == 0) {
+ LOG.warn("Download to slow for this stream. Download queue is full. Skipping segment");
+ } else {
+ for (String segment : lsp.segments) {
+ if(!running) {
+ break;
+ }
+ if(skip > 0) {
+ skip--;
+ } else {
+ URL segmentUrl = new URL(segment);
+ Future download = downloadThreadPool.submit(new SegmentDownload(segmentUrl, client));
+ downloads.add(download);
+ }
+ }
+ }
+
+ // get completed downloads and write them to the file
+ // TODO it might be a good idea to do this in a separate thread, so that the main download loop isn't blocked
+ writeFinishedSegments(downloads);
+ }
+
+ private void writeFinishedSegments(Queue> downloads) throws ExecutionException, HttpException {
+ for (Future downloadFuture : downloads) {
+ try {
+ byte[] segmentData = downloadFuture.get();
+ writeSegment(segmentData);
+ } catch (InterruptedException e) {
+ LOG.error("Error while downloading segment", e);
+ } catch (ExecutionException e) {
+ Throwable cause = e.getCause();
+ if(cause instanceof MissingSegmentException) {
+ if(model != null && !isModelOnline()) {
+ LOG.debug("Error while downloading segment, because model {} is offline. Stopping now", model.getName());
+ running = false;
+ } else {
+ LOG.debug("Segment not available, but model {} still online. Going on", model.getName());
+ }
+ } else if(cause instanceof HttpException) {
+ HttpException he = (HttpException) cause;
+ if(model != null && !isModelOnline()) {
+ LOG.debug("Error {} while downloading segment, because model {} is offline. Stopping now", he.getResponseCode(), model.getName());
+ running = false;
+ } else {
+ if(he.getResponseCode() == 404) {
+ LOG.info("Playlist for {} not found [HTTP 404]. Stopping now", model.getName());
+ running = false;
+ } else if(he.getResponseCode() == 403) {
+ LOG.info("Playlist for {} not accessible [HTTP 403]. Stopping now", model.getName());
+ running = false;
+ } else {
+ throw he;
+ }
+ }
} else {
throw e;
}
@@ -175,43 +272,6 @@ public class MergedHlsDownload extends AbstractHlsDownload {
}
}
- private void downloadMissedSegments(SegmentPlaylist lsp, int nextSegment) throws MalformedURLException {
- if(nextSegment > 0 && lsp.seq > nextSegment) {
- LOG.warn("Missed segments {} < {} in download for {}", nextSegment, lsp.seq, lsp.url);
- String first = lsp.segments.get(0);
- int seq = lsp.seq;
- for (int i = nextSegment; i < lsp.seq; i++) {
- URL segmentUrl = new URL(first.replaceAll(Integer.toString(seq), Integer.toString(i)));
- LOG.debug("Loading missed segment {} for model {}", i, lsp.url);
- byte[] segmentData;
- try {
- segmentData = new SegmentDownload(segmentUrl, client).call();
- writeSegment(segmentData);
- } catch (Exception e) {
- LOG.error("Error while downloading segment {}", segmentUrl, e);
- }
- }
- // TODO switch to a lower bitrate/resolution ?!?
- }
- }
-
- private void downloadNewSegments(SegmentPlaylist lsp, int nextSegment) throws MalformedURLException {
- int skip = nextSegment - lsp.seq;
- for (String segment : lsp.segments) {
- if(skip > 0) {
- skip--;
- } else {
- URL segmentUrl = new URL(segment);
- try {
- byte[] segmentData = new SegmentDownload(segmentUrl, client).call();
- writeSegment(segmentData);
- } catch (Exception e) {
- LOG.error("Error while downloading segment {}", segmentUrl, e);
- }
- }
- }
- }
-
private void writeSegment(byte[] segmentData) throws InterruptedException {
InputStream in = new ByteArrayInputStream(segmentData);
InputStreamMTSSource source = InputStreamMTSSource.builder().setInputStream(in).build();
@@ -220,24 +280,36 @@ public class MergedHlsDownload extends AbstractHlsDownload {
private void splitRecording() {
if(config.getSettings().splitRecordings > 0) {
- Duration recordingDuration = Duration.between(startTime, ZonedDateTime.now());
+ Duration recordingDuration = Duration.between(splitRecStartTime, ZonedDateTime.now());
long seconds = recordingDuration.getSeconds();
if(seconds >= config.getSettings().splitRecordings) {
- streamer.stop();
- File target = new File(targetFile.getAbsolutePath().replaceAll("\\.ts", "-"+df.format(++splitCounter)+".ts"));
- mergeThread = createMergeThread(target, null, true);
- mergeThread.start();
- startTime = ZonedDateTime.now();
+ try {
+ targetFile = Config.getInstance().getFileForRecording(model);
+ LOG.debug("Switching to file {}", targetFile.getAbsolutePath());
+ fileChannel = FileChannel.open(targetFile.toPath(), CREATE, WRITE);
+ MTSSink sink = ByteChannelSink.builder().setByteChannel(fileChannel).build();
+ streamer.switchSink(sink);
+ splitRecStartTime = ZonedDateTime.now();
+ } catch (IOException e) {
+ LOG.error("Error while splitting recording", e);
+ running = false;
+ }
}
}
}
- private void waitForNewSegments(SegmentPlaylist lsp, int lastSegment) {
+ private void waitForNewSegments(SegmentPlaylist lsp, int lastSegment, long downloadTookMillis) {
try {
long wait = 0;
if (lastSegment == lsp.seq) {
- // playlist didn't change -> wait for at least half the target duration
- wait = (long) lsp.targetDuration * 1000 / 2;
+ int timeLeftMillis = (int)(lsp.totalDuration * 1000 - downloadTookMillis);
+ if(timeLeftMillis < 3000) { // we have less than 3 seconds to get the new playlist and start downloading it
+ wait = 1;
+ } else {
+ // wait a second to be nice to the server (don't hammer it with requests)
+ // 1 second seems to be a good compromise. every other calculation resulted in more missing segments
+ wait = 1000;
+ }
LOG.trace("Playlist didn't change... waiting for {}ms", wait);
} else {
// playlist did change -> wait for at least last segment duration
@@ -256,7 +328,9 @@ public class MergedHlsDownload extends AbstractHlsDownload {
public void stop() {
running = false;
alive = false;
- streamer.stop();
+ if(streamer != null) {
+ streamer.stop();
+ }
LOG.debug("Download stopped");
}
@@ -267,20 +341,20 @@ public class MergedHlsDownload extends AbstractHlsDownload {
.setProgressListener(listener)
.build();
- FileChannel channel = null;
try {
Path downloadDir = targetFile.getParentFile().toPath();
if (!Files.exists(downloadDir, LinkOption.NOFOLLOW_LINKS)) {
Files.createDirectories(downloadDir);
}
- channel = FileChannel.open(targetFile.toPath(), CREATE, WRITE);
- MTSSink sink = ByteChannelSink.builder().setByteChannel(channel).build();
+ fileChannel = FileChannel.open(targetFile.toPath(), CREATE, WRITE);
+ MTSSink sink = ByteChannelSink.builder().setByteChannel(fileChannel).build();
streamer = Streamer.builder()
.setSource(multiSource)
.setSink(sink)
.setSleepingEnabled(liveStream)
.setBufferSize(10)
+ .setName(model.getName())
.build();
// Start streaming
@@ -293,11 +367,12 @@ public class MergedHlsDownload extends AbstractHlsDownload {
} catch(Exception e) {
LOG.error("Error while saving stream to file", e);
} finally {
- closeFile(channel);
deleteEmptyRecording(targetFile);
+ running = false;
+ closeFile(fileChannel);
}
});
- t.setName("Segment Merger Thread");
+ t.setName("Segment Merger Thread [" + model.getName() + "]");
t.setDaemon(true);
return t;
}
@@ -308,22 +383,22 @@ public class MergedHlsDownload extends AbstractHlsDownload {
Files.delete(targetFile.toPath());
Files.delete(targetFile.getParentFile().toPath());
}
- } catch (IOException e) {
+ } catch (Exception e) {
LOG.error("Error while deleting empty recording {}", targetFile);
}
}
private void closeFile(FileChannel channel) {
try {
- if (channel != null) {
+ if (channel != null && channel.isOpen()) {
channel.close();
}
- } catch (IOException e) {
+ } catch (Exception e) {
LOG.error("Error while closing file channel", e);
}
}
- private static class SegmentDownload implements Callable {
+ private class SegmentDownload implements Callable {
private URL url;
private HttpClient client;
@@ -333,24 +408,38 @@ public class MergedHlsDownload extends AbstractHlsDownload {
}
@Override
- public byte[] call() throws Exception {
+ public byte[] call() throws IOException {
LOG.trace("Downloading segment " + url.getFile());
int maxTries = 3;
- for (int i = 1; i <= maxTries; i++) {
- try {
- Request request = new Request.Builder().url(url).addHeader("connection", "keep-alive").build();
- Response response = client.execute(request);
- byte[] segment = response.body().bytes();
- return segment;
+ for (int i = 1; i <= maxTries && running; i++) {
+ Request request = new Request.Builder().url(url).addHeader("connection", "keep-alive").build();
+ try (Response response = client.execute(request)) {
+ if(response.isSuccessful()) {
+ byte[] segment = response.body().bytes();
+ return segment;
+ } else {
+ throw new HttpException(response.code(), response.message());
+ }
} catch(Exception e) {
if (i == maxTries) {
LOG.warn("Error while downloading segment. Segment {} finally failed", url.getFile());
} else {
- LOG.warn("Error while downloading segment {} on try {}", url.getFile(), i);
+ LOG.warn("Error while downloading segment {} on try {}", url.getFile(), i, e);
+ }
+ if(model != null && !isModelOnline()) {
+ break;
}
}
}
- throw new IOException("Unable to download segment " + url.getFile() + " after " + maxTries + " tries");
+ throw new MissingSegmentException("Unable to download segment " + url.getFile() + " after " + maxTries + " tries");
+ }
+ }
+
+ public boolean isModelOnline() {
+ try {
+ return model.isOnline(IGNORE_CACHE);
+ } catch (IOException | ExecutionException | InterruptedException e) {
+ return false;
}
}
}
diff --git a/common/src/main/java/ctbrec/recorder/download/MissingSegmentException.java b/common/src/main/java/ctbrec/recorder/download/MissingSegmentException.java
new file mode 100644
index 00000000..d6971aab
--- /dev/null
+++ b/common/src/main/java/ctbrec/recorder/download/MissingSegmentException.java
@@ -0,0 +1,11 @@
+package ctbrec.recorder.download;
+
+import java.io.IOException;
+
+public class MissingSegmentException extends IOException {
+
+ public MissingSegmentException(String msg) {
+ super(msg);
+ }
+
+}
diff --git a/common/src/main/java/ctbrec/recorder/download/StreamSource.java b/common/src/main/java/ctbrec/recorder/download/StreamSource.java
index 1968df81..dbb86f03 100644
--- a/common/src/main/java/ctbrec/recorder/download/StreamSource.java
+++ b/common/src/main/java/ctbrec/recorder/download/StreamSource.java
@@ -54,7 +54,7 @@ public class StreamSource implements Comparable {
@Override
public int compareTo(StreamSource o) {
int heightDiff = height - o.height;
- if(heightDiff != 0) {
+ if(heightDiff != 0 && height != Integer.MAX_VALUE && o.height != Integer.MAX_VALUE) {
return heightDiff;
} else {
return bandwidth - o.bandwidth;
diff --git a/common/src/main/java/ctbrec/sites/cam4/Cam4Model.java b/common/src/main/java/ctbrec/sites/cam4/Cam4Model.java
index 30b87e9a..68b24354 100644
--- a/common/src/main/java/ctbrec/sites/cam4/Cam4Model.java
+++ b/common/src/main/java/ctbrec/sites/cam4/Cam4Model.java
@@ -5,6 +5,7 @@ import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
+import java.util.Optional;
import java.util.concurrent.ExecutionException;
import org.json.JSONArray;
@@ -24,6 +25,7 @@ import com.iheartradio.m3u8.data.PlaylistData;
import ctbrec.AbstractModel;
import ctbrec.Config;
+import ctbrec.StringUtil;
import ctbrec.io.HtmlParser;
import ctbrec.io.HttpException;
import ctbrec.recorder.download.StreamSource;
@@ -38,6 +40,7 @@ public class Cam4Model extends AbstractModel {
private String playlistUrl;
private String onlineState = "offline";
private int[] resolution = null;
+ private boolean privateRoom = false;
@Override
public boolean isOnline() throws IOException, ExecutionException, InterruptedException {
@@ -53,7 +56,9 @@ public class Cam4Model extends AbstractModel {
return false;
}
}
- return Objects.equals("NORMAL", onlineState);
+ return (Objects.equals("NORMAL", onlineState) || Objects.equals("GROUP_SHOW_SELLING_TICKETS", onlineState))
+ && StringUtil.isNotBlank(playlistUrl)
+ && !privateRoom;
}
private void loadModelDetails() throws IOException, ModelDetailsEmptyException {
@@ -64,11 +69,13 @@ public class Cam4Model extends AbstractModel {
if(response.isSuccessful()) {
JSONArray json = new JSONArray(response.body().string());
if(json.length() == 0) {
+ onlineState = "offline";
throw new ModelDetailsEmptyException("Model details are empty");
}
JSONObject details = json.getJSONObject(0);
onlineState = details.getString("showType");
playlistUrl = details.getString("hlsPreviewUrl");
+ privateRoom = details.getBoolean("privateRoom");
if(details.has("resolution")) {
String res = details.getString("resolution");
String[] tokens = res.split(":");
@@ -104,7 +111,7 @@ public class Cam4Model extends AbstractModel {
if (playlist.hasStreamInfo()) {
StreamSource src = new StreamSource();
src.bandwidth = playlist.getStreamInfo().getBandwidth();
- src.height = playlist.getStreamInfo().getResolution().height;
+ src.height = Optional.ofNullable(playlist.getStreamInfo()).map(si -> si.getResolution()).map(res -> res.height).orElse(0);
String masterUrl = getPlaylistUrl();
String baseUrl = masterUrl.substring(0, masterUrl.lastIndexOf('/') + 1);
String segmentUri = baseUrl + playlist.getUri();
diff --git a/common/src/main/java/ctbrec/sites/chaturbate/Chaturbate.java b/common/src/main/java/ctbrec/sites/chaturbate/Chaturbate.java
index 36125f85..534a39bb 100644
--- a/common/src/main/java/ctbrec/sites/chaturbate/Chaturbate.java
+++ b/common/src/main/java/ctbrec/sites/chaturbate/Chaturbate.java
@@ -219,7 +219,15 @@ public class Chaturbate extends AbstractSite {
}
StreamInfo getStreamInfo(String modelName) throws IOException, ExecutionException {
- return streamInfoCache.get(modelName);
+ return getStreamInfo(modelName, false);
+ }
+
+ StreamInfo getStreamInfo(String modelName, boolean failFast) throws IOException, ExecutionException {
+ if(failFast) {
+ return streamInfoCache.getIfPresent(modelName);
+ } else {
+ return streamInfoCache.get(modelName);
+ }
}
StreamInfo loadStreamInfo(String modelName) throws HttpException, IOException, InterruptedException {
diff --git a/common/src/main/java/ctbrec/sites/chaturbate/ChaturbateModel.java b/common/src/main/java/ctbrec/sites/chaturbate/ChaturbateModel.java
index 3095c4be..bd17cd23 100644
--- a/common/src/main/java/ctbrec/sites/chaturbate/ChaturbateModel.java
+++ b/common/src/main/java/ctbrec/sites/chaturbate/ChaturbateModel.java
@@ -6,6 +6,7 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
+import java.util.Optional;
import java.util.concurrent.ExecutionException;
import org.slf4j.Logger;
@@ -39,14 +40,16 @@ public class ChaturbateModel extends AbstractModel {
@Override
public boolean isOnline(boolean ignoreCache) throws IOException, ExecutionException, InterruptedException {
- StreamInfo info;
+ String roomStatus;
if(ignoreCache) {
- info = getChaturbate().loadStreamInfo(getName());
+ StreamInfo info = getChaturbate().loadStreamInfo(getName());
+ roomStatus = Optional.ofNullable(info).map(i -> i.room_status).orElse("");
LOG.trace("Model {} room status: {}", getName(), info.room_status);
} else {
- info = getChaturbate().getStreamInfo(getName());
+ StreamInfo info = getChaturbate().getStreamInfo(getName(), true);
+ roomStatus = Optional.ofNullable(info).map(i -> i.room_status).orElse("");
}
- return Objects.equals("public", info.room_status);
+ return Objects.equals("public", roomStatus);
}
@Override
diff --git a/common/src/main/java/ctbrec/sites/mfc/MyFreeCamsModel.java b/common/src/main/java/ctbrec/sites/mfc/MyFreeCamsModel.java
index f0776843..45f13c08 100644
--- a/common/src/main/java/ctbrec/sites/mfc/MyFreeCamsModel.java
+++ b/common/src/main/java/ctbrec/sites/mfc/MyFreeCamsModel.java
@@ -221,7 +221,7 @@ public class MyFreeCamsModel extends AbstractModel {
setName(state.getNm());
setState(State.of(state.getVs()));
setStreamUrl(streamUrl);
- Optional camScore = Optional.of(state.getM()).map(m -> m.getCamscore());
+ Optional camScore = Optional.ofNullable(state.getM()).map(m -> m.getCamscore());
setCamScore(camScore.orElse(0.0));
// preview
diff --git a/common/src/main/java/org/taktik/mpegts/Streamer.java b/common/src/main/java/org/taktik/mpegts/Streamer.java
index d844da92..560bbfc8 100644
--- a/common/src/main/java/org/taktik/mpegts/Streamer.java
+++ b/common/src/main/java/org/taktik/mpegts/Streamer.java
@@ -30,12 +30,14 @@ public class Streamer {
private Thread streamingThread;
private boolean sleepingEnabled;
+ private String name;
- private Streamer(MTSSource source, MTSSink sink, int bufferSize, boolean sleepingEnabled) {
+ private Streamer(MTSSource source, MTSSink sink, int bufferSize, boolean sleepingEnabled, String name) {
this.source = source;
this.sink = sink;
this.bufferSize = bufferSize;
this.sleepingEnabled = sleepingEnabled;
+ this.name = name;
}
public void stream() throws InterruptedException {
@@ -48,20 +50,26 @@ public class Streamer {
try {
preBuffer();
} catch (Exception e) {
- throw new IllegalStateException("Error while bufering", e);
+ throw new IllegalStateException("Error while buffering", e);
}
log.info("Done PreBuffering");
- bufferingThread = new Thread(this::fillBuffer, "buffering");
+ bufferingThread = new Thread(this::fillBuffer, "Buffering ["+name+"]");
bufferingThread.setDaemon(true);
bufferingThread.start();
- streamingThread = new Thread(this::internalStream, "streaming");
+ streamingThread = new Thread(this::internalStream, "Streaming ["+name+"]");
streamingThread.setDaemon(true);
streamingThread.start();
bufferingThread.join();
streamingThread.join();
+
+ try {
+ sink.close();
+ } catch(Exception e) {
+ log.error("Couldn't close sink", e);
+ }
}
public void stop() {
@@ -85,6 +93,16 @@ public class Streamer {
}
}
+ public void switchSink(MTSSink sink) {
+ MTSSink old = this.sink;
+ this.sink = sink;
+ try {
+ old.close();
+ } catch (Exception e) {
+ log.error("Couldn't close old sink while switching sinks", e);
+ }
+ }
+
private void internalStream() {
boolean resetState = false;
MTSPacket packet = null;
@@ -123,7 +141,7 @@ public class Streamer {
}
}
} catch (InterruptedException e1) {
- if(!endOfSourceReached) {
+ if(!endOfSourceReached && !streamingShouldStop) {
log.error("Interrupted while waiting for packet");
continue;
} else {
@@ -240,7 +258,7 @@ public class Streamer {
// Stream packet
// System.out.println("Streaming packet #" + packetCount + ", PID=" + mtsPacket.getPid() + ", pcrCount=" + pcrCount + ", continuityCounter=" + mtsPacket.getContinuityCounter());
- if(!streamingShouldStop) {
+ if(!streamingShouldStop && !Thread.interrupted()) {
try {
sink.send(packet);
} catch (Exception e) {
@@ -275,7 +293,7 @@ public class Streamer {
buffer.put(packet);
put = true;
} catch (InterruptedException ignored) {
-
+ log.error("Error adding packet to buffer", ignored);
}
}
}
@@ -287,7 +305,11 @@ public class Streamer {
log.error("Error reading from source", e);
} finally {
endOfSourceReached = true;
- streamingThread.interrupt();
+ try {
+ streamingThread.interrupt();
+ } catch(Exception e) {
+ log.error("Couldn't interrupt streaming thread", e);
+ }
}
}
@@ -308,6 +330,7 @@ public class Streamer {
private MTSSource source;
private int bufferSize = 1000;
private boolean sleepingEnabled = false;
+ private String name;
public StreamerBuilder setSink(MTSSink sink) {
this.sink = sink;
@@ -329,10 +352,16 @@ public class Streamer {
return this;
}
+ public StreamerBuilder setName(String name) {
+ this.name = name;
+ return this;
+ }
+
public Streamer build() {
Preconditions.checkNotNull(sink);
Preconditions.checkNotNull(source);
- return new Streamer(source, sink, bufferSize, sleepingEnabled);
+ return new Streamer(source, sink, bufferSize, sleepingEnabled, name);
}
+
}
}
\ No newline at end of file
diff --git a/master/pom.xml b/master/pom.xml
index 8d905ac5..3c43cdae 100644
--- a/master/pom.xml
+++ b/master/pom.xml
@@ -6,7 +6,7 @@
ctbrec
master
pom
- 1.11.0
+ 1.12.0
../common
diff --git a/server/pom.xml b/server/pom.xml
index 873084e8..69969b61 100644
--- a/server/pom.xml
+++ b/server/pom.xml
@@ -8,7 +8,7 @@
ctbrec
master
- 1.11.0
+ 1.12.0
../master
diff --git a/server/src/main/java/ctbrec/recorder/server/RecorderServlet.java b/server/src/main/java/ctbrec/recorder/server/RecorderServlet.java
index ad6f81dd..c213ce2f 100644
--- a/server/src/main/java/ctbrec/recorder/server/RecorderServlet.java
+++ b/server/src/main/java/ctbrec/recorder/server/RecorderServlet.java
@@ -137,9 +137,13 @@ public class RecorderServlet extends AbstractCtbrecServlet {
response = "{\"status\": \"success\", \"msg\": \"Recording resumed\"}";
resp.getWriter().write(response);
break;
+ case "space":
+ response = "{\"status\": \"success\", \"spaceTotal\": "+recorder.getTotalSpaceBytes()+", \"spaceFree\": "+recorder.getFreeSpaceBytes()+"}";
+ resp.getWriter().write(response);
+ break;
default:
resp.setStatus(SC_BAD_REQUEST);
- response = "{\"status\": \"error\", \"msg\": \"Unknown action\"}";
+ response = "{\"status\": \"error\", \"msg\": \"Unknown action ["+request.action+"]\"}";
resp.getWriter().write(response);
break;
}