From 91ea7d65a33e63e8766ccad11993b80ac27f58e5 Mon Sep 17 00:00:00 2001 From: 0xboobface <0xboobface@gmail.com> Date: Mon, 26 Nov 2018 00:13:53 +0100 Subject: [PATCH 01/21] Sort by height only if the height is set If the height is not available, it is set to Integer.MAX_VALUE. IT makes not sense to compare by that value. Instead compare the bitrates. --- common/src/main/java/ctbrec/recorder/download/StreamSource.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/src/main/java/ctbrec/recorder/download/StreamSource.java b/common/src/main/java/ctbrec/recorder/download/StreamSource.java index 1968df81..dbb86f03 100644 --- a/common/src/main/java/ctbrec/recorder/download/StreamSource.java +++ b/common/src/main/java/ctbrec/recorder/download/StreamSource.java @@ -54,7 +54,7 @@ public class StreamSource implements Comparable { @Override public int compareTo(StreamSource o) { int heightDiff = height - o.height; - if(heightDiff != 0) { + if(heightDiff != 0 && height != Integer.MAX_VALUE && o.height != Integer.MAX_VALUE) { return heightDiff; } else { return bandwidth - o.bandwidth; From 8fdb24bad18d011e2b638711fe006882eaf1a327 Mon Sep 17 00:00:00 2001 From: 0xboobface <0xboobface@gmail.com> Date: Mon, 26 Nov 2018 13:49:42 +0100 Subject: [PATCH 02/21] Add methods to get the free and total space --- .../java/ctbrec/recorder/LocalRecorder.java | 17 +++++++++ .../main/java/ctbrec/recorder/Recorder.java | 14 +++++++ .../java/ctbrec/recorder/RemoteRecorder.java | 38 +++++++++++++++++++ .../recorder/server/RecorderServlet.java | 6 ++- 4 files changed, 74 insertions(+), 1 deletion(-) diff --git a/common/src/main/java/ctbrec/recorder/LocalRecorder.java b/common/src/main/java/ctbrec/recorder/LocalRecorder.java index 3afb3ee4..f4fb1f1f 100644 --- a/common/src/main/java/ctbrec/recorder/LocalRecorder.java +++ b/common/src/main/java/ctbrec/recorder/LocalRecorder.java @@ -5,6 +5,7 @@ import static ctbrec.Recording.STATUS.*; import java.io.File; import java.io.FilenameFilter; import java.io.IOException; +import java.nio.file.FileStore; import java.nio.file.Files; import java.security.InvalidKeyException; import java.security.NoSuchAlgorithmException; @@ -745,4 +746,20 @@ public class LocalRecorder implements Recorder { public HttpClient getHttpClient() { return client; } + + @Override + public long getTotalSpaceBytes() throws IOException { + return getRecordingsFileStore().getTotalSpace(); + } + + @Override + public long getFreeSpaceBytes() throws IOException { + return getRecordingsFileStore().getUsableSpace(); + } + + private FileStore getRecordingsFileStore() throws IOException { + File recordingsDir = new File(Config.getInstance().getSettings().recordingsDir); + FileStore store = Files.getFileStore(recordingsDir.toPath()); + return store; + } } diff --git a/common/src/main/java/ctbrec/recorder/Recorder.java b/common/src/main/java/ctbrec/recorder/Recorder.java index bc4e60cf..1a9bf682 100644 --- a/common/src/main/java/ctbrec/recorder/Recorder.java +++ b/common/src/main/java/ctbrec/recorder/Recorder.java @@ -42,4 +42,18 @@ public interface Recorder { public List getOnlineModels(); public HttpClient getHttpClient(); + + /** + * Get the total size of the filesystem we are recording to + * @return the total size in bytes + * @throws IOException + */ + public long getTotalSpaceBytes() throws IOException; + + /** + * Get the free space left on the filesystem we are recording to + * @return the free space in bytes + * @throws IOException + */ + public long getFreeSpaceBytes() throws IOException; } diff --git a/common/src/main/java/ctbrec/recorder/RemoteRecorder.java b/common/src/main/java/ctbrec/recorder/RemoteRecorder.java index dd648301..7c3ced0a 100644 --- a/common/src/main/java/ctbrec/recorder/RemoteRecorder.java +++ b/common/src/main/java/ctbrec/recorder/RemoteRecorder.java @@ -8,6 +8,7 @@ import java.time.Instant; import java.util.Collections; import java.util.List; +import org.json.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -45,6 +46,8 @@ public class RemoteRecorder implements Recorder { private List models = Collections.emptyList(); private List onlineModels = Collections.emptyList(); private List sites; + private long spaceTotal = -1; + private long spaceFree = -1; private Config config; private HttpClient client; @@ -150,10 +153,35 @@ public class RemoteRecorder implements Recorder { while(running) { syncModels(); syncOnlineModels(); + syncSpace(); sleep(); } } + private void syncSpace() { + try { + String msg = "{\"action\": \"space\"}"; + RequestBody body = RequestBody.create(JSON, msg); + Request.Builder builder = new Request.Builder() + .url("http://" + config.getSettings().httpServer + ":" + config.getSettings().httpPort + "/rec") + .post(body); + addHmacIfNeeded(msg, builder); + Request request = builder.build(); + try(Response response = client.execute(request)) { + String json = response.body().string(); + if(response.isSuccessful()) { + JSONObject resp = new JSONObject(json); + spaceTotal = resp.getLong("spaceTotal"); + spaceFree = resp.getLong("spaceFree"); + } else { + LOG.error("Couldn't synchronize with server. HTTP status: {} - {}", response.code(), json); + } + } + } catch (IOException | InvalidKeyException | NoSuchAlgorithmException | IllegalStateException e) { + LOG.error("Couldn't synchronize with server", e); + } + } + private void syncModels() { try { String msg = "{\"action\": \"list\"}"; @@ -362,4 +390,14 @@ public class RemoteRecorder implements Recorder { public HttpClient getHttpClient() { return client; } + + @Override + public long getTotalSpaceBytes() throws IOException { + return spaceTotal; + } + + @Override + public long getFreeSpaceBytes() { + return spaceFree; + } } diff --git a/server/src/main/java/ctbrec/recorder/server/RecorderServlet.java b/server/src/main/java/ctbrec/recorder/server/RecorderServlet.java index ad6f81dd..c213ce2f 100644 --- a/server/src/main/java/ctbrec/recorder/server/RecorderServlet.java +++ b/server/src/main/java/ctbrec/recorder/server/RecorderServlet.java @@ -137,9 +137,13 @@ public class RecorderServlet extends AbstractCtbrecServlet { response = "{\"status\": \"success\", \"msg\": \"Recording resumed\"}"; resp.getWriter().write(response); break; + case "space": + response = "{\"status\": \"success\", \"spaceTotal\": "+recorder.getTotalSpaceBytes()+", \"spaceFree\": "+recorder.getFreeSpaceBytes()+"}"; + resp.getWriter().write(response); + break; default: resp.setStatus(SC_BAD_REQUEST); - response = "{\"status\": \"error\", \"msg\": \"Unknown action\"}"; + response = "{\"status\": \"error\", \"msg\": \"Unknown action ["+request.action+"]\"}"; resp.getWriter().write(response); break; } From 5708d7f259428e8917709602150196e69824260f Mon Sep 17 00:00:00 2001 From: 0xboobface <0xboobface@gmail.com> Date: Mon, 26 Nov 2018 14:03:40 +0100 Subject: [PATCH 03/21] Add display to show space left on device --- .../main/java/ctbrec/ui/RecordingsTab.java | 101 +++++++++++++----- 1 file changed, 77 insertions(+), 24 deletions(-) diff --git a/client/src/main/java/ctbrec/ui/RecordingsTab.java b/client/src/main/java/ctbrec/ui/RecordingsTab.java index 2c7f4fb2..7efba008 100644 --- a/client/src/main/java/ctbrec/ui/RecordingsTab.java +++ b/client/src/main/java/ctbrec/ui/RecordingsTab.java @@ -8,6 +8,7 @@ import java.io.IOException; import java.net.URL; import java.security.InvalidKeyException; import java.security.NoSuchAlgorithmException; +import java.text.DecimalFormat; import java.time.Instant; import java.time.ZoneId; import java.time.ZonedDateTime; @@ -47,13 +48,16 @@ import javafx.scene.Cursor; import javafx.scene.control.Alert.AlertType; import javafx.scene.control.ButtonType; import javafx.scene.control.ContextMenu; +import javafx.scene.control.Label; import javafx.scene.control.MenuItem; +import javafx.scene.control.ProgressBar; import javafx.scene.control.ScrollPane; import javafx.scene.control.Tab; import javafx.scene.control.TableCell; import javafx.scene.control.TableColumn; import javafx.scene.control.TableColumn.SortType; import javafx.scene.control.TableView; +import javafx.scene.control.Tooltip; import javafx.scene.control.cell.PropertyValueFactory; import javafx.scene.input.ContextMenuEvent; import javafx.scene.input.KeyCode; @@ -62,6 +66,9 @@ import javafx.scene.input.MouseButton; import javafx.scene.input.MouseEvent; import javafx.scene.layout.BorderPane; import javafx.scene.layout.FlowPane; +import javafx.scene.layout.HBox; +import javafx.scene.layout.StackPane; +import javafx.scene.text.Font; import javafx.stage.FileChooser; import javafx.util.Callback; import javafx.util.Duration; @@ -74,12 +81,16 @@ public class RecordingsTab extends Tab implements TabSelectionListener { private Recorder recorder; @SuppressWarnings("unused") private List sites; + private long spaceTotal = -1; + private long spaceFree = -1; FlowPane grid = new FlowPane(); ScrollPane scrollPane = new ScrollPane(); TableView table = new TableView(); ObservableList observableRecordings = FXCollections.observableArrayList(); ContextMenu popup; + ProgressBar spaceLeft; + Label spaceLabel; public RecordingsTab(String title, Recorder recorder, Config config, List sites) { super(title); @@ -179,8 +190,21 @@ public class RecordingsTab extends Tab implements TabSelectionListener { }); scrollPane.setContent(table); + HBox spaceBox = new HBox(5); + Label l = new Label("Space left on device"); + HBox.setMargin(l, new Insets(2, 0, 0, 0)); + spaceBox.getChildren().add(l); + spaceLeft = new ProgressBar(0); + spaceLeft.setPrefSize(200, 22); + spaceLabel = new Label(); + spaceLabel.setFont(Font.font(11)); + StackPane stack = new StackPane(spaceLeft, spaceLabel); + spaceBox.getChildren().add(stack); + BorderPane.setMargin(spaceBox, new Insets(5)); + BorderPane root = new BorderPane(); root.setPadding(new Insets(5)); + root.setTop(spaceBox); root.setCenter(scrollPane); setContent(root); @@ -191,30 +215,8 @@ public class RecordingsTab extends Tab implements TabSelectionListener { updateService = createUpdateService(); updateService.setPeriod(new Duration(TimeUnit.SECONDS.toMillis(2))); updateService.setOnSucceeded((event) -> { - List recordings = updateService.getValue(); - if (recordings == null) { - return; - } - - for (Iterator iterator = observableRecordings.iterator(); iterator.hasNext();) { - JavaFxRecording old = iterator.next(); - if (!recordings.contains(old)) { - // remove deleted recordings - iterator.remove(); - } - } - for (JavaFxRecording recording : recordings) { - if (!observableRecordings.contains(recording)) { - // add new recordings - observableRecordings.add(recording); - } else { - // update existing ones - int index = observableRecordings.indexOf(recording); - JavaFxRecording old = observableRecordings.get(index); - old.update(recording); - } - } - table.sort(); + updateRecordingsTable(); + updateFreeSpaceDisplay(); }); updateService.setOnFailed((event) -> { LOG.info("Couldn't get list of recordings from recorder", event.getSource().getException()); @@ -226,6 +228,46 @@ public class RecordingsTab extends Tab implements TabSelectionListener { }); } + private void updateFreeSpaceDisplay() { + if(spaceTotal != -1 && spaceFree != -1) { + double free = ((double)spaceFree) / spaceTotal; + spaceLeft.setProgress(free); + double totalGiB = ((double) spaceTotal) / 1024 / 1024 / 1024; + double freeGiB = ((double) spaceFree) / 1024 / 1024 / 1024; + DecimalFormat df = new DecimalFormat("0.00"); + String tt = df.format(freeGiB) + " / " + df.format(totalGiB) + " GiB"; + spaceLeft.setTooltip(new Tooltip(tt)); + spaceLabel.setText(tt); + } + } + + private void updateRecordingsTable() { + List recordings = updateService.getValue(); + if (recordings == null) { + return; + } + + for (Iterator iterator = observableRecordings.iterator(); iterator.hasNext();) { + JavaFxRecording old = iterator.next(); + if (!recordings.contains(old)) { + // remove deleted recordings + iterator.remove(); + } + } + for (JavaFxRecording recording : recordings) { + if (!observableRecordings.contains(recording)) { + // add new recordings + observableRecordings.add(recording); + } else { + // update existing ones + int index = observableRecordings.indexOf(recording); + JavaFxRecording old = observableRecordings.get(index); + old.update(recording); + } + } + table.sort(); + } + private ScheduledService> createUpdateService() { ScheduledService> updateService = new ScheduledService>() { @Override @@ -233,12 +275,23 @@ public class RecordingsTab extends Tab implements TabSelectionListener { return new Task>() { @Override public List call() throws IOException, InvalidKeyException, NoSuchAlgorithmException, IllegalStateException { + updateSpace(); + List recordings = new ArrayList<>(); for (Recording rec : recorder.getRecordings()) { recordings.add(new JavaFxRecording(rec)); } return recordings; } + + private void updateSpace() { + try { + spaceTotal = recorder.getTotalSpaceBytes(); + spaceFree = recorder.getFreeSpaceBytes(); + } catch (IOException e) { + LOG.error("Couldn't update free space", e); + } + } }; } }; From cc2aa3c8d5f4ec6a65f1a1c85de0cecd352f79e4 Mon Sep 17 00:00:00 2001 From: 0xboobface <0xboobface@gmail.com> Date: Mon, 26 Nov 2018 15:28:44 +0100 Subject: [PATCH 04/21] Add threshold setting for minimum space on disk If there is less space left on the device than specified by the setting, the recorder will stop all recordings and don't start new ones until the free space rises above this threshold again. --- common/src/main/java/ctbrec/Settings.java | 1 + .../java/ctbrec/recorder/LocalRecorder.java | 30 +++++++++++++++++-- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/common/src/main/java/ctbrec/Settings.java b/common/src/main/java/ctbrec/Settings.java index a34e24e9..3b613845 100644 --- a/common/src/main/java/ctbrec/Settings.java +++ b/common/src/main/java/ctbrec/Settings.java @@ -37,6 +37,7 @@ public class Settings { public String httpServer = "localhost"; public String recordingsDir = System.getProperty("user.home") + File.separator + "ctbrec"; public DirectoryStructure recordingsDirStructure = DirectoryStructure.FLAT; + public long minimumSpaceLeftInBytes = 0; public String mediaPlayer = "/usr/bin/mpv"; public String postProcessing = ""; public String username = ""; // chaturbate username TODO maybe rename this onetime diff --git a/common/src/main/java/ctbrec/recorder/LocalRecorder.java b/common/src/main/java/ctbrec/recorder/LocalRecorder.java index f4fb1f1f..14aca3a7 100644 --- a/common/src/main/java/ctbrec/recorder/LocalRecorder.java +++ b/common/src/main/java/ctbrec/recorder/LocalRecorder.java @@ -64,6 +64,7 @@ public class LocalRecorder implements Recorder { private List deleteInProgress = Collections.synchronizedList(new ArrayList<>()); private RecorderHttpClient client = new RecorderHttpClient(); private ReentrantLock lock = new ReentrantLock(); + private long lastSpaceMessage = 0; public LocalRecorder(Config config) { this.config = config; @@ -134,7 +135,6 @@ public class LocalRecorder implements Recorder { return; } - LOG.debug("Starting recording for model {}", model.getName()); if (recordingProcesses.containsKey(model)) { LOG.error("A recording for model {} is already running", model); return; @@ -150,6 +150,16 @@ public class LocalRecorder implements Recorder { lock.unlock(); } + if(!enoughSpaceForRecording()) { + long now = System.currentTimeMillis(); + if( (now - lastSpaceMessage) > TimeUnit.MINUTES.toMillis(1)) { + LOG.info("Not enough space for recording, not starting recording for {}", model); + lastSpaceMessage = now; + } + return; + } + + LOG.debug("Starting recording for model {}", model.getName()); Download download; if (Config.getInstance().isServerMode()) { download = new HlsDownload(client); @@ -330,6 +340,15 @@ public class LocalRecorder implements Recorder { public void run() { running = true; while (running) { + try { + if(!enoughSpaceForRecording() && !recordingProcesses.isEmpty()) { + LOG.info("No space left -> Stopping all recordings"); + stopRecordingProcesses(); + } + } catch (IOException e1) { + LOG.warn("Couldn't check free space left", e1); + } + List restart = new ArrayList<>(); for (Iterator> iterator = recordingProcesses.entrySet().iterator(); iterator.hasNext();) { Entry entry = iterator.next(); @@ -416,7 +435,7 @@ public class LocalRecorder implements Recorder { boolean isOnline = model.isOnline(IGNORE_CACHE); LOG.trace("Checking online state for {}: {}", model, (isOnline ? "online" : "offline")); if (isOnline && !isSuspended(model) && !recordingProcesses.containsKey(model)) { - LOG.info("Model {}'s room back to public. Starting recording", model); + LOG.info("Model {}'s room back to public", model); startRecordingProcess(model); } } catch (HttpException e) { @@ -758,8 +777,13 @@ public class LocalRecorder implements Recorder { } private FileStore getRecordingsFileStore() throws IOException { - File recordingsDir = new File(Config.getInstance().getSettings().recordingsDir); + File recordingsDir = new File(config.getSettings().recordingsDir); FileStore store = Files.getFileStore(recordingsDir.toPath()); return store; } + + private boolean enoughSpaceForRecording() throws IOException { + long minimum = config.getSettings().minimumSpaceLeftInBytes; + return getFreeSpaceBytes() > minimum; + } } From a7cc0882f6af347fb7541768bdff8c3518decf05 Mon Sep 17 00:00:00 2001 From: 0xboobface <0xboobface@gmail.com> Date: Mon, 26 Nov 2018 15:45:12 +0100 Subject: [PATCH 05/21] Reorganize settings tab --- .../src/main/java/ctbrec/ui/SettingsTab.java | 178 +++++++++--------- 1 file changed, 88 insertions(+), 90 deletions(-) diff --git a/client/src/main/java/ctbrec/ui/SettingsTab.java b/client/src/main/java/ctbrec/ui/SettingsTab.java index 54091aa9..b5d74b22 100644 --- a/client/src/main/java/ctbrec/ui/SettingsTab.java +++ b/client/src/main/java/ctbrec/ui/SettingsTab.java @@ -120,7 +120,7 @@ public class SettingsTab extends Tab implements TabSelectionListener { // left side leftSide.getChildren().add(createGeneralPanel()); - leftSide.getChildren().add(createLocationsPanel()); + leftSide.getChildren().add(createRecorderPanel()); leftSide.getChildren().add(createRecordLocationPanel()); //right side @@ -253,9 +253,20 @@ public class SettingsTab extends Tab implements TabSelectionListener { return recordLocation; } - private Node createLocationsPanel() { + private Node createRecorderPanel() { int row = 0; GridPane layout = createGridLayout(); + layout.add(new Label("Post-Processing"), 0, row); + postProcessing = new TextField(Config.getInstance().getSettings().postProcessing); + postProcessing.focusedProperty().addListener(createPostProcessingFocusListener()); + GridPane.setFillWidth(postProcessing, true); + GridPane.setHgrow(postProcessing, Priority.ALWAYS); + GridPane.setColumnSpan(postProcessing, 2); + GridPane.setMargin(postProcessing, new Insets(0, 0, 0, CHECKBOX_MARGIN)); + layout.add(postProcessing, 1, row); + postProcessingDirectoryButton = createPostProcessingBrowseButton(); + layout.add(postProcessingDirectoryButton, 3, row++); + layout.add(new Label("Recordings Directory"), 0, row); recordingsDirectory = new TextField(Config.getInstance().getSettings().recordingsDir); recordingsDirectory.focusedProperty().addListener(createRecordingsDirectoryFocusListener()); @@ -283,16 +294,67 @@ public class SettingsTab extends Tab implements TabSelectionListener { GridPane.setMargin(directoryStructure, new Insets(0, 0, 0, CHECKBOX_MARGIN)); layout.add(directoryStructure, 1, row++); - layout.add(new Label("Post-Processing"), 0, row); - postProcessing = new TextField(Config.getInstance().getSettings().postProcessing); - postProcessing.focusedProperty().addListener(createPostProcessingFocusListener()); - GridPane.setFillWidth(postProcessing, true); - GridPane.setHgrow(postProcessing, Priority.ALWAYS); - GridPane.setColumnSpan(postProcessing, 2); - GridPane.setMargin(postProcessing, new Insets(0, 0, 0, CHECKBOX_MARGIN)); - layout.add(postProcessing, 1, row); - postProcessingDirectoryButton = createPostProcessingBrowseButton(); - layout.add(postProcessingDirectoryButton, 3, row++); + Label l = new Label("Maximum resolution (0 = unlimited)"); + layout.add(l, 0, row); + List resolutionOptions = new ArrayList<>(); + resolutionOptions.add(1080); + resolutionOptions.add(720); + resolutionOptions.add(600); + resolutionOptions.add(480); + resolutionOptions.add(0); + maxResolution = new ComboBox<>(FXCollections.observableList(resolutionOptions)); + setMaxResolutionValue(); + maxResolution.setOnAction((e) -> { + Config.getInstance().getSettings().maximumResolution = maxResolution.getSelectionModel().getSelectedItem(); + saveConfig(); + }); + maxResolution.prefWidthProperty().bind(directoryStructure.widthProperty()); + layout.add(maxResolution, 1, row++); + GridPane.setMargin(l, new Insets(0, 0, 0, 0)); + GridPane.setMargin(maxResolution, new Insets(0, 0, 0, CHECKBOX_MARGIN)); + + l = new Label("Split recordings after (minutes)"); + layout.add(l, 0, row); + List splitOptions = new ArrayList<>(); + splitOptions.add(new SplitAfterOption("disabled", 0)); + splitOptions.add(new SplitAfterOption("10 min", 10 * 60)); + splitOptions.add(new SplitAfterOption("15 min", 15 * 60)); + splitOptions.add(new SplitAfterOption("20 min", 20 * 60)); + splitOptions.add(new SplitAfterOption("30 min", 30 * 60)); + splitOptions.add(new SplitAfterOption("60 min", 60 * 60)); + splitAfter = new ComboBox<>(FXCollections.observableList(splitOptions)); + layout.add(splitAfter, 1, row++); + setSplitAfterValue(); + splitAfter.setOnAction((e) -> { + Config.getInstance().getSettings().splitRecordings = splitAfter.getSelectionModel().getSelectedItem().getValue(); + saveConfig(); + }); + splitAfter.prefWidthProperty().bind(directoryStructure.widthProperty()); + GridPane.setMargin(l, new Insets(0, 0, 0, 0)); + GridPane.setMargin(splitAfter, new Insets(0, 0, 0, CHECKBOX_MARGIN)); + + layout.add(new Label("Check online state every (seconds)"), 0, row); + onlineCheckIntervalInSecs = new TextField(Integer.toString(Config.getInstance().getSettings().onlineCheckIntervalInSecs)); + onlineCheckIntervalInSecs.textProperty().addListener((observable, oldValue, newValue) -> { + if (!newValue.matches("\\d*")) { + onlineCheckIntervalInSecs.setText(newValue.replaceAll("[^\\d]", "")); + } + if(!onlineCheckIntervalInSecs.getText().isEmpty()) { + Config.getInstance().getSettings().onlineCheckIntervalInSecs = Integer.parseInt(onlineCheckIntervalInSecs.getText()); + saveConfig(); + } + }); + GridPane.setMargin(onlineCheckIntervalInSecs, new Insets(0, 0, 0, CHECKBOX_MARGIN)); + layout.add(onlineCheckIntervalInSecs, 1, row++); + + TitledPane locations = new TitledPane("Recorder", layout); + locations.setCollapsible(false); + return locations; + } + + private Node createGeneralPanel() { + GridPane layout = createGridLayout(); + int row = 0; layout.add(new Label("Player"), 0, row); mediaPlayer = new TextField(Config.getInstance().getSettings().mediaPlayer); @@ -304,15 +366,18 @@ public class SettingsTab extends Tab implements TabSelectionListener { layout.add(mediaPlayer, 1, row); layout.add(createMpvBrowseButton(), 3, row++); - TitledPane locations = new TitledPane("Locations", layout); - locations.setCollapsible(false); - return locations; - } + Label l = new Label("Allow multiple players"); + layout.add(l, 0, row); + multiplePlayers.setSelected(!Config.getInstance().getSettings().singlePlayer); + multiplePlayers.setOnAction((e) -> { + Config.getInstance().getSettings().singlePlayer = !multiplePlayers.isSelected(); + saveConfig(); + }); + GridPane.setMargin(l, new Insets(3, 0, 0, 0)); + GridPane.setMargin(multiplePlayers, new Insets(CHECKBOX_MARGIN, 0, 0, CHECKBOX_MARGIN)); + layout.add(multiplePlayers, 1, row++); - private Node createGeneralPanel() { - GridPane layout = createGridLayout(); - int row = 0; - Label l = new Label("Display stream resolution in overview"); + l = new Label("Display stream resolution in overview"); layout.add(l, 0, row); loadResolution = new CheckBox(); loadResolution.setSelected(Config.getInstance().getSettings().determineResolution); @@ -323,20 +388,10 @@ public class SettingsTab extends Tab implements TabSelectionListener { ThumbOverviewTab.queue.clear(); } }); - //GridPane.setMargin(l, new Insets(CHECKBOX_MARGIN, 0, 0, 0)); - GridPane.setMargin(loadResolution, new Insets(0, 0, 0, CHECKBOX_MARGIN)); + GridPane.setMargin(l, new Insets(CHECKBOX_MARGIN, 0, 0, 0)); + GridPane.setMargin(loadResolution, new Insets(CHECKBOX_MARGIN, 0, 0, CHECKBOX_MARGIN)); layout.add(loadResolution, 1, row++); - l = new Label("Allow multiple players"); - layout.add(l, 0, row); - multiplePlayers.setSelected(!Config.getInstance().getSettings().singlePlayer); - multiplePlayers.setOnAction((e) -> { - Config.getInstance().getSettings().singlePlayer = !multiplePlayers.isSelected(); - saveConfig(); - }); - GridPane.setMargin(l, new Insets(3, 0, 0, 0)); - GridPane.setMargin(multiplePlayers, new Insets(CHECKBOX_MARGIN, 0, 0, CHECKBOX_MARGIN)); - layout.add(multiplePlayers, 1, row++); l = new Label("Manually select stream quality"); layout.add(l, 0, row); @@ -357,60 +412,9 @@ public class SettingsTab extends Tab implements TabSelectionListener { saveConfig(); }); GridPane.setMargin(l, new Insets(CHECKBOX_MARGIN, 0, 0, 0)); - GridPane.setMargin(updateThumbnails, new Insets(CHECKBOX_MARGIN, 0, 0, CHECKBOX_MARGIN)); + GridPane.setMargin(updateThumbnails, new Insets(CHECKBOX_MARGIN, 0, CHECKBOX_MARGIN, CHECKBOX_MARGIN)); layout.add(updateThumbnails, 1, row++); - l = new Label("Maximum resolution (0 = unlimited)"); - layout.add(l, 0, row); - List resolutionOptions = new ArrayList<>(); - resolutionOptions.add(1080); - resolutionOptions.add(720); - resolutionOptions.add(600); - resolutionOptions.add(480); - resolutionOptions.add(0); - maxResolution = new ComboBox<>(FXCollections.observableList(resolutionOptions)); - setMaxResolutionValue(); - maxResolution.setOnAction((e) -> { - Config.getInstance().getSettings().maximumResolution = maxResolution.getSelectionModel().getSelectedItem(); - saveConfig(); - }); - layout.add(maxResolution, 1, row++); - GridPane.setMargin(l, new Insets(CHECKBOX_MARGIN, 0, 0, 0)); - GridPane.setMargin(maxResolution, new Insets(CHECKBOX_MARGIN, 0, 0, CHECKBOX_MARGIN)); - - l = new Label("Split recordings after (minutes)"); - layout.add(l, 0, row); - List options = new ArrayList<>(); - options.add(new SplitAfterOption("disabled", 0)); - options.add(new SplitAfterOption("10 min", 10 * 60)); - options.add(new SplitAfterOption("15 min", 15 * 60)); - options.add(new SplitAfterOption("20 min", 20 * 60)); - options.add(new SplitAfterOption("30 min", 30 * 60)); - options.add(new SplitAfterOption("60 min", 60 * 60)); - splitAfter = new ComboBox<>(FXCollections.observableList(options)); - layout.add(splitAfter, 1, row++); - setSplitAfterValue(); - splitAfter.setOnAction((e) -> { - Config.getInstance().getSettings().splitRecordings = splitAfter.getSelectionModel().getSelectedItem().getValue(); - saveConfig(); - }); - GridPane.setMargin(l, new Insets(0, 0, 0, 0)); - GridPane.setMargin(splitAfter, new Insets(0, 0, 0, CHECKBOX_MARGIN)); - - layout.add(new Label("Check online state every (seconds)"), 0, row); - onlineCheckIntervalInSecs = new TextField(Integer.toString(Config.getInstance().getSettings().onlineCheckIntervalInSecs)); - onlineCheckIntervalInSecs.textProperty().addListener((observable, oldValue, newValue) -> { - if (!newValue.matches("\\d*")) { - onlineCheckIntervalInSecs.setText(newValue.replaceAll("[^\\d]", "")); - } - if(!onlineCheckIntervalInSecs.getText().isEmpty()) { - Config.getInstance().getSettings().onlineCheckIntervalInSecs = Integer.parseInt(onlineCheckIntervalInSecs.getText()); - saveConfig(); - } - }); - GridPane.setMargin(onlineCheckIntervalInSecs, new Insets(0, 0, 0, CHECKBOX_MARGIN)); - layout.add(onlineCheckIntervalInSecs, 1, row++); - l = new Label("Start Tab"); layout.add(l, 0, row); startTab = new ComboBox<>(); @@ -429,12 +433,6 @@ public class SettingsTab extends Tab implements TabSelectionListener { GridPane.setMargin(l, new Insets(0, 0, 0, 0)); GridPane.setMargin(colorSettingsPane, new Insets(CHECKBOX_MARGIN, 0, 0, CHECKBOX_MARGIN)); - - splitAfter.prefWidthProperty().bind(startTab.widthProperty()); - maxResolution.prefWidthProperty().bind(startTab.widthProperty()); - onlineCheckIntervalInSecs.prefWidthProperty().bind(startTab.widthProperty()); - onlineCheckIntervalInSecs.maxWidthProperty().bind(startTab.widthProperty()); - TitledPane general = new TitledPane("General", layout); general.setCollapsible(false); return general; From ba4ac952e2ba9844b7512ab4500c2d40c95f4018 Mon Sep 17 00:00:00 2001 From: 0xboobface <0xboobface@gmail.com> Date: Mon, 26 Nov 2018 16:04:08 +0100 Subject: [PATCH 06/21] Add threshold for space left on device to SettingsTab --- .../src/main/java/ctbrec/ui/SettingsTab.java | 31 ++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/client/src/main/java/ctbrec/ui/SettingsTab.java b/client/src/main/java/ctbrec/ui/SettingsTab.java index b5d74b22..e20f1e5b 100644 --- a/client/src/main/java/ctbrec/ui/SettingsTab.java +++ b/client/src/main/java/ctbrec/ui/SettingsTab.java @@ -55,6 +55,7 @@ import javafx.stage.FileChooser;; public class SettingsTab extends Tab implements TabSelectionListener { private static final transient Logger LOG = LoggerFactory.getLogger(SettingsTab.class); + private static final int ONE_GiB_IN_BYTES = 1024 * 1024 * 1024; public static final int CHECKBOX_MARGIN = 6; private TextField recordingsDirectory; @@ -65,6 +66,7 @@ public class SettingsTab extends Tab implements TabSelectionListener { private TextField server; private TextField port; private TextField onlineCheckIntervalInSecs; + private TextField leaveSpaceOnDevice; private CheckBox loadResolution; private CheckBox secureCommunication = new CheckBox(); private CheckBox chooseStreamQuality = new CheckBox(); @@ -333,8 +335,12 @@ public class SettingsTab extends Tab implements TabSelectionListener { GridPane.setMargin(l, new Insets(0, 0, 0, 0)); GridPane.setMargin(splitAfter, new Insets(0, 0, 0, CHECKBOX_MARGIN)); - layout.add(new Label("Check online state every (seconds)"), 0, row); + Tooltip tt = new Tooltip("Check every x seconds, if a model came online"); + l = new Label("Check online state every (seconds)"); + l.setTooltip(tt); + layout.add(l, 0, row); onlineCheckIntervalInSecs = new TextField(Integer.toString(Config.getInstance().getSettings().onlineCheckIntervalInSecs)); + onlineCheckIntervalInSecs.setTooltip(tt); onlineCheckIntervalInSecs.textProperty().addListener((observable, oldValue, newValue) -> { if (!newValue.matches("\\d*")) { onlineCheckIntervalInSecs.setText(newValue.replaceAll("[^\\d]", "")); @@ -347,6 +353,27 @@ public class SettingsTab extends Tab implements TabSelectionListener { GridPane.setMargin(onlineCheckIntervalInSecs, new Insets(0, 0, 0, CHECKBOX_MARGIN)); layout.add(onlineCheckIntervalInSecs, 1, row++); + tt = new Tooltip("Stop recording, if the free space on the device gets below this threshold"); + l = new Label("Leave space on device (GiB)"); + l.setTooltip(tt); + layout.add(l, 0, row); + long minimumSpaceLeftInBytes = Config.getInstance().getSettings().minimumSpaceLeftInBytes; + int minimumSpaceLeftInGiB = (int) (minimumSpaceLeftInBytes / ONE_GiB_IN_BYTES); + leaveSpaceOnDevice = new TextField(Integer.toString(minimumSpaceLeftInGiB)); + leaveSpaceOnDevice.setTooltip(tt); + leaveSpaceOnDevice.textProperty().addListener((observable, oldValue, newValue) -> { + if (!newValue.matches("\\d*")) { + leaveSpaceOnDevice.setText(newValue.replaceAll("[^\\d]", "")); + } + if(!leaveSpaceOnDevice.getText().isEmpty()) { + long spaceLeftInGiB = Long.parseLong(leaveSpaceOnDevice.getText()); + Config.getInstance().getSettings().minimumSpaceLeftInBytes = spaceLeftInGiB * ONE_GiB_IN_BYTES; + saveConfig(); + } + }); + GridPane.setMargin(leaveSpaceOnDevice, new Insets(0, 0, 0, CHECKBOX_MARGIN)); + layout.add(leaveSpaceOnDevice, 1, row++); + TitledPane locations = new TitledPane("Recorder", layout); locations.setCollapsible(false); return locations; @@ -479,6 +506,8 @@ public class SettingsTab extends Tab implements TabSelectionListener { postProcessing.setDisable(!local); postProcessingDirectoryButton.setDisable(!local); directoryStructure.setDisable(!local); + onlineCheckIntervalInSecs.setDisable(!local); + leaveSpaceOnDevice.setDisable(!local); } private ChangeListener createRecordingsDirectoryFocusListener() { From bf8a3c72400babb4b4473e340c9c33e0f0bc0143 Mon Sep 17 00:00:00 2001 From: 0xboobface <0xboobface@gmail.com> Date: Tue, 27 Nov 2018 13:25:30 +0100 Subject: [PATCH 07/21] Fill start tab combobox only once --- client/src/main/java/ctbrec/ui/SettingsTab.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/client/src/main/java/ctbrec/ui/SettingsTab.java b/client/src/main/java/ctbrec/ui/SettingsTab.java index e20f1e5b..0d2722e3 100644 --- a/client/src/main/java/ctbrec/ui/SettingsTab.java +++ b/client/src/main/java/ctbrec/ui/SettingsTab.java @@ -680,9 +680,10 @@ public class SettingsTab extends Tab implements TabSelectionListener { @Override public void selected() { - startTab.getItems().clear(); - for(Tab tab : getTabPane().getTabs()) { - startTab.getItems().add(tab.getText()); + if(startTab.getItems().isEmpty()) { + for(Tab tab : getTabPane().getTabs()) { + startTab.getItems().add(tab.getText()); + } } String startTabName = Config.getInstance().getSettings().startTab; if(StringUtil.isNotBlank(startTabName)) { From 8826de38b2c51c2949c4e6d191c35b09f3812bc2 Mon Sep 17 00:00:00 2001 From: 0xboobface <0xboobface@gmail.com> Date: Tue, 27 Nov 2018 13:36:00 +0100 Subject: [PATCH 08/21] Improve calculation of size property --- .../main/java/ctbrec/ui/JavaFxRecording.java | 11 +++--- .../main/java/ctbrec/ui/RecordingsTab.java | 34 +++++++++++++++++-- 2 files changed, 37 insertions(+), 8 deletions(-) diff --git a/client/src/main/java/ctbrec/ui/JavaFxRecording.java b/client/src/main/java/ctbrec/ui/JavaFxRecording.java index 313a7e75..af9f0241 100644 --- a/client/src/main/java/ctbrec/ui/JavaFxRecording.java +++ b/client/src/main/java/ctbrec/ui/JavaFxRecording.java @@ -1,10 +1,11 @@ package ctbrec.ui; -import java.text.DecimalFormat; import java.time.Instant; import ctbrec.Config; import ctbrec.Recording; +import javafx.beans.property.LongProperty; +import javafx.beans.property.SimpleLongProperty; import javafx.beans.property.SimpleStringProperty; import javafx.beans.property.StringProperty; @@ -12,7 +13,7 @@ public class JavaFxRecording extends Recording { private transient StringProperty statusProperty = new SimpleStringProperty(); private transient StringProperty progressProperty = new SimpleStringProperty(); - private transient StringProperty sizeProperty = new SimpleStringProperty(); + private transient LongProperty sizeProperty = new SimpleLongProperty(); private Recording delegate; @@ -89,9 +90,7 @@ public class JavaFxRecording extends Recording { @Override public void setSizeInByte(long sizeInByte) { delegate.setSizeInByte(sizeInByte); - double sizeInGiB = sizeInByte / 1024.0 / 1024 / 1024; - DecimalFormat df = new DecimalFormat("0.00"); - sizeProperty.setValue(df.format(sizeInGiB) + " GiB"); + sizeProperty.set(sizeInByte); } public StringProperty getProgressProperty() { @@ -151,7 +150,7 @@ public class JavaFxRecording extends Recording { return delegate.getSizeInByte(); } - public StringProperty getSizeProperty() { + public LongProperty getSizeProperty() { return sizeProperty; } diff --git a/client/src/main/java/ctbrec/ui/RecordingsTab.java b/client/src/main/java/ctbrec/ui/RecordingsTab.java index 7efba008..6581bcb5 100644 --- a/client/src/main/java/ctbrec/ui/RecordingsTab.java +++ b/client/src/main/java/ctbrec/ui/RecordingsTab.java @@ -147,9 +147,39 @@ public class RecordingsTab extends Tab implements TabSelectionListener { TableColumn progress = new TableColumn<>("Progress"); progress.setCellValueFactory((cdf) -> cdf.getValue().getProgressProperty()); progress.setPrefWidth(100); - TableColumn size = new TableColumn<>("Size"); - size.setCellValueFactory((cdf) -> cdf.getValue().getSizeProperty()); + TableColumn size = new TableColumn<>("Size"); + size.setStyle("-fx-alignment: CENTER-RIGHT;"); size.setPrefWidth(100); + size.setCellValueFactory(cdf -> cdf.getValue().getSizeProperty()); + size.setCellFactory(new Callback, TableCell>() { + @Override + public TableCell call(TableColumn param) { + TableCell cell = new TableCell() { + @Override + protected void updateItem(Number sizeInByte, boolean empty) { + if(empty || sizeInByte == null) { + setText(null); + } else { + DecimalFormat df = new DecimalFormat("0.00"); + String unit = "Bytes"; + double size = sizeInByte.doubleValue(); + if(size > 1024.0 * 1024 * 1024) { + size = size / 1024.0 / 1024 / 1024; + unit = "GiB"; + } else if(size > 1024.0 * 1024) { + size = size / 1024.0 / 1024; + unit = "MiB"; + } else if(size > 1024.0) { + size = size / 1024.0; + unit = "KiB"; + } + setText(df.format(size) + ' ' + unit); + } + } + }; + return cell; + } + }); table.getColumns().addAll(name, date, status, progress, size); table.setItems(observableRecordings); From 55b219d27122325d4ef265a823921299a9d2fa9c Mon Sep 17 00:00:00 2001 From: 0xboobface <0xboobface@gmail.com> Date: Tue, 27 Nov 2018 14:11:52 +0100 Subject: [PATCH 09/21] Move size formatting code to StringUtil --- .../main/java/ctbrec/ui/RecordingsTab.java | 15 +-------------- common/src/main/java/ctbrec/StringUtil.java | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/client/src/main/java/ctbrec/ui/RecordingsTab.java b/client/src/main/java/ctbrec/ui/RecordingsTab.java index 6581bcb5..b60d439b 100644 --- a/client/src/main/java/ctbrec/ui/RecordingsTab.java +++ b/client/src/main/java/ctbrec/ui/RecordingsTab.java @@ -160,20 +160,7 @@ public class RecordingsTab extends Tab implements TabSelectionListener { if(empty || sizeInByte == null) { setText(null); } else { - DecimalFormat df = new DecimalFormat("0.00"); - String unit = "Bytes"; - double size = sizeInByte.doubleValue(); - if(size > 1024.0 * 1024 * 1024) { - size = size / 1024.0 / 1024 / 1024; - unit = "GiB"; - } else if(size > 1024.0 * 1024) { - size = size / 1024.0 / 1024; - unit = "MiB"; - } else if(size > 1024.0) { - size = size / 1024.0; - unit = "KiB"; - } - setText(df.format(size) + ' ' + unit); + setText(StringUtil.formatSize(sizeInByte)); } } }; diff --git a/common/src/main/java/ctbrec/StringUtil.java b/common/src/main/java/ctbrec/StringUtil.java index 229e44b1..d9ae9796 100644 --- a/common/src/main/java/ctbrec/StringUtil.java +++ b/common/src/main/java/ctbrec/StringUtil.java @@ -1,5 +1,7 @@ package ctbrec; +import java.text.DecimalFormat; + public class StringUtil { public static boolean isBlank(String s) { return s == null || s.trim().isEmpty(); @@ -8,4 +10,21 @@ public class StringUtil { public static boolean isNotBlank(String s) { return !isBlank(s); } + + public static String formatSize(Number sizeInByte) { + DecimalFormat df = new DecimalFormat("0.00"); + String unit = "Bytes"; + double size = sizeInByte.doubleValue(); + if(size > 1024.0 * 1024 * 1024) { + size = size / 1024.0 / 1024 / 1024; + unit = "GiB"; + } else if(size > 1024.0 * 1024) { + size = size / 1024.0 / 1024; + unit = "MiB"; + } else if(size > 1024.0) { + size = size / 1024.0; + unit = "KiB"; + } + return df.format(size) + ' ' + unit; + } } From c543af64291f941814aaa7a93fd7153cc8721248 Mon Sep 17 00:00:00 2001 From: 0xboobface <0xboobface@gmail.com> Date: Tue, 27 Nov 2018 17:53:37 +0100 Subject: [PATCH 10/21] Set online state to offline if model details cannot be loaded --- common/src/main/java/ctbrec/sites/cam4/Cam4Model.java | 1 + 1 file changed, 1 insertion(+) diff --git a/common/src/main/java/ctbrec/sites/cam4/Cam4Model.java b/common/src/main/java/ctbrec/sites/cam4/Cam4Model.java index 30b87e9a..8bcd8d8e 100644 --- a/common/src/main/java/ctbrec/sites/cam4/Cam4Model.java +++ b/common/src/main/java/ctbrec/sites/cam4/Cam4Model.java @@ -64,6 +64,7 @@ public class Cam4Model extends AbstractModel { if(response.isSuccessful()) { JSONArray json = new JSONArray(response.body().string()); if(json.length() == 0) { + onlineState = "offline"; throw new ModelDetailsEmptyException("Model details are empty"); } JSONObject details = json.getJSONObject(0); From e29634001684e3fea93bc5b942abf189c2f6e79c Mon Sep 17 00:00:00 2001 From: 0xboobface <0xboobface@gmail.com> Date: Tue, 27 Nov 2018 17:54:43 +0100 Subject: [PATCH 11/21] Ignore log files --- client/.gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/.gitignore b/client/.gitignore index fc247909..2c4e8e27 100644 --- a/client/.gitignore +++ b/client/.gitignore @@ -2,7 +2,7 @@ /target/ *~ *.bak -/ctbrec.log +/*.log /ctbrec-tunnel.sh /jre/ /server-local.sh From 33642705a05121c3e7e3d3f0225463f55a9af9ad Mon Sep 17 00:00:00 2001 From: 0xboobface <0xboobface@gmail.com> Date: Tue, 27 Nov 2018 18:53:22 +0100 Subject: [PATCH 12/21] Check playlistUrl in isOnline If the playlistUrl is empty, we cannot record, so the model is offline --- common/src/main/java/ctbrec/sites/cam4/Cam4Model.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/common/src/main/java/ctbrec/sites/cam4/Cam4Model.java b/common/src/main/java/ctbrec/sites/cam4/Cam4Model.java index 8bcd8d8e..2afa1100 100644 --- a/common/src/main/java/ctbrec/sites/cam4/Cam4Model.java +++ b/common/src/main/java/ctbrec/sites/cam4/Cam4Model.java @@ -24,6 +24,7 @@ import com.iheartradio.m3u8.data.PlaylistData; import ctbrec.AbstractModel; import ctbrec.Config; +import ctbrec.StringUtil; import ctbrec.io.HtmlParser; import ctbrec.io.HttpException; import ctbrec.recorder.download.StreamSource; @@ -53,7 +54,7 @@ public class Cam4Model extends AbstractModel { return false; } } - return Objects.equals("NORMAL", onlineState); + return Objects.equals("NORMAL", onlineState) && StringUtil.isNotBlank(playlistUrl); } private void loadModelDetails() throws IOException, ModelDetailsEmptyException { From 7edc79b0e335267a37d516d0675bf61ba2d93efd Mon Sep 17 00:00:00 2001 From: 0xboobface <0xboobface@gmail.com> Date: Wed, 28 Nov 2018 11:47:40 +0100 Subject: [PATCH 13/21] Take boolean privateRoom into account for online state --- common/src/main/java/ctbrec/sites/cam4/Cam4Model.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/common/src/main/java/ctbrec/sites/cam4/Cam4Model.java b/common/src/main/java/ctbrec/sites/cam4/Cam4Model.java index 2afa1100..68b24354 100644 --- a/common/src/main/java/ctbrec/sites/cam4/Cam4Model.java +++ b/common/src/main/java/ctbrec/sites/cam4/Cam4Model.java @@ -5,6 +5,7 @@ import java.io.InputStream; import java.util.ArrayList; import java.util.List; import java.util.Objects; +import java.util.Optional; import java.util.concurrent.ExecutionException; import org.json.JSONArray; @@ -39,6 +40,7 @@ public class Cam4Model extends AbstractModel { private String playlistUrl; private String onlineState = "offline"; private int[] resolution = null; + private boolean privateRoom = false; @Override public boolean isOnline() throws IOException, ExecutionException, InterruptedException { @@ -54,7 +56,9 @@ public class Cam4Model extends AbstractModel { return false; } } - return Objects.equals("NORMAL", onlineState) && StringUtil.isNotBlank(playlistUrl); + return (Objects.equals("NORMAL", onlineState) || Objects.equals("GROUP_SHOW_SELLING_TICKETS", onlineState)) + && StringUtil.isNotBlank(playlistUrl) + && !privateRoom; } private void loadModelDetails() throws IOException, ModelDetailsEmptyException { @@ -71,6 +75,7 @@ public class Cam4Model extends AbstractModel { JSONObject details = json.getJSONObject(0); onlineState = details.getString("showType"); playlistUrl = details.getString("hlsPreviewUrl"); + privateRoom = details.getBoolean("privateRoom"); if(details.has("resolution")) { String res = details.getString("resolution"); String[] tokens = res.split(":"); @@ -106,7 +111,7 @@ public class Cam4Model extends AbstractModel { if (playlist.hasStreamInfo()) { StreamSource src = new StreamSource(); src.bandwidth = playlist.getStreamInfo().getBandwidth(); - src.height = playlist.getStreamInfo().getResolution().height; + src.height = Optional.ofNullable(playlist.getStreamInfo()).map(si -> si.getResolution()).map(res -> res.height).orElse(0); String masterUrl = getPlaylistUrl(); String baseUrl = masterUrl.substring(0, masterUrl.lastIndexOf('/') + 1); String segmentUri = baseUrl + playlist.getUri(); From c4c8fe83fa44b9ccdb3506fdf3f1faeab15ee22a Mon Sep 17 00:00:00 2001 From: 0xboobface <0xboobface@gmail.com> Date: Wed, 28 Nov 2018 15:22:42 +0100 Subject: [PATCH 14/21] Improve MergedHlsDownload * Add better exception handling * Check, if the model is still online, when an error occurs * Download segments in parallel, so that less segments are missed --- .../recorder/download/MergedHlsDownload.java | 212 ++++++++++++------ .../download/MissingSegmentException.java | 11 + .../main/java/org/taktik/mpegts/Streamer.java | 31 ++- 3 files changed, 179 insertions(+), 75 deletions(-) create mode 100644 common/src/main/java/ctbrec/recorder/download/MissingSegmentException.java diff --git a/common/src/main/java/ctbrec/recorder/download/MergedHlsDownload.java b/common/src/main/java/ctbrec/recorder/download/MergedHlsDownload.java index 84c4c7d6..d4a935e3 100644 --- a/common/src/main/java/ctbrec/recorder/download/MergedHlsDownload.java +++ b/common/src/main/java/ctbrec/recorder/download/MergedHlsDownload.java @@ -19,7 +19,16 @@ import java.text.DecimalFormat; import java.time.Duration; import java.time.Instant; import java.time.ZonedDateTime; +import java.util.LinkedList; +import java.util.Queue; +import java.util.concurrent.BlockingQueue; import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -53,6 +62,8 @@ public class MergedHlsDownload extends AbstractHlsDownload { private File targetFile; private DecimalFormat df = new DecimalFormat("00000"); private int splitCounter = 0; + private BlockingQueue downloadQueue = new LinkedBlockingQueue<>(50); + private ExecutorService downloadThreadPool = new ThreadPoolExecutor(5, 5, 2, TimeUnit.MINUTES, downloadQueue); public MergedHlsDownload(HttpClient client) { super(client); @@ -81,7 +92,7 @@ public class MergedHlsDownload extends AbstractHlsDownload { downloadSegments(segmentPlaylistUri, false); LOG.debug("Waiting for merge thread to finish"); mergeThread.join(); - LOG.debug("Merge thread to finished"); + LOG.debug("Merge thread finished"); } catch(ParseException e) { throw new IOException("Couldn't parse stream information", e); } catch(PlaylistException e) { @@ -92,7 +103,12 @@ public class MergedHlsDownload extends AbstractHlsDownload { throw new IOException("Couldn't add HMAC to playlist url", e); } finally { alive = false; - streamer.stop(); + try { + streamer.stop(); + } catch(Exception e) { + LOG.error("Couldn't stop streamer", e); + } + downloadThreadPool.shutdown(); LOG.debug("Download terminated for {}", segmentPlaylistUri); } } @@ -129,7 +145,11 @@ public class MergedHlsDownload extends AbstractHlsDownload { } finally { alive = false; if(streamer != null) { - streamer.stop(); + try { + streamer.stop(); + } catch(Exception e) { + LOG.error("Couldn't stop streamer", e); + } } LOG.debug("Download for {} terminated", model); } @@ -138,36 +158,109 @@ public class MergedHlsDownload extends AbstractHlsDownload { private void downloadSegments(String segmentPlaylistUri, boolean livestreamDownload) throws IOException, ParseException, PlaylistException { int lastSegment = 0; int nextSegment = 0; + long playlistNotFoundFirstEncounter = -1; while(running) { try { + if(playlistNotFoundFirstEncounter != -1) { + LOG.debug("Downloading playlist {}", segmentPlaylistUri); + } SegmentPlaylist lsp = getNextSegments(segmentPlaylistUri); + playlistNotFoundFirstEncounter = -1; if(!livestreamDownload) { multiSource.setTotalSegments(lsp.segments.size()); } - // download segments, which might have been skipped - downloadMissedSegments(lsp, nextSegment); - // download new segments + long downloadStart = System.currentTimeMillis(); downloadNewSegments(lsp, nextSegment); + long downloadTookMillis = System.currentTimeMillis() - downloadStart; + + // download segments, which might have been skipped + //downloadMissedSegments(lsp, nextSegment); + if(nextSegment > 0 && lsp.seq > nextSegment) { + LOG.warn("Missed segments {} < {} in download for {}. Download took {}ms. Playlist is {}sec", nextSegment, lsp.seq, lsp.url, downloadTookMillis, lsp.totalDuration); + } if(livestreamDownload) { // split up the recording, if configured splitRecording(); // wait some time until requesting the segment playlist again to not hammer the server - waitForNewSegments(lsp, lastSegment); + waitForNewSegments(lsp, lastSegment, downloadTookMillis); lastSegment = lsp.seq; nextSegment = lastSegment + lsp.segments.size(); } else { break; } - } catch(HttpException e) { - if(e.getResponseCode() == 404) { - // playlist is gone -> model probably logged out - LOG.debug("Playlist not found. Assuming model went offline"); - running = false; + } catch(Exception e) { + LOG.info("Unexpected error while downloading ", model.getName()); + running = false; + } + } + } + + private void downloadNewSegments(SegmentPlaylist lsp, int nextSegment) throws MalformedURLException, MissingSegmentException, ExecutionException, HttpException { + int skip = nextSegment - lsp.seq; + if(lsp.segments.isEmpty()) { + LOG.debug("Empty playlist: {}", lsp.url); + } + + // add segments to download threadpool + Queue> downloads = new LinkedList<>(); + if(downloadQueue.remainingCapacity() == 0) { + LOG.warn("Download to slow for this stream. Download queue is full. Skipping segment"); + } else { + for (String segment : lsp.segments) { + if(!running) { + break; + } + if(skip > 0) { + skip--; + } else { + URL segmentUrl = new URL(segment); + Future download = downloadThreadPool.submit(new SegmentDownload(segmentUrl, client)); + downloads.add(download); + } + } + } + + // get completed downloads and write them to the file + writeFinishedSegments(downloads); + } + + private void writeFinishedSegments(Queue> downloads) throws ExecutionException, HttpException { + for (Future downloadFuture : downloads) { + try { + byte[] segmentData = downloadFuture.get(); + writeSegment(segmentData); + } catch (InterruptedException e) { + LOG.error("Error while downloading segment", e); + } catch (ExecutionException e) { + Throwable cause = e.getCause(); + if(cause instanceof MissingSegmentException) { + if(model != null && !isModelOnline()) { + LOG.debug("Error while downloading segment, because model {} is offline. Stopping now", model.getName()); + running = false; + } else { + LOG.debug("Segment not available, but model {} still online. Going on", model.getName()); + } + } else if(cause instanceof HttpException) { + HttpException he = (HttpException) cause; + if(model != null && !isModelOnline()) { + LOG.debug("Error {} while downloading segment, because model {} is offline. Stopping now", he.getResponseCode(), model.getName()); + running = false; + } else { + if(he.getResponseCode() == 404) { + LOG.info("Playlist for {} not found [HTTP 404]. Stopping now", model.getName()); + running = false; + } else if(he.getResponseCode() == 403) { + LOG.info("Playlist for {} not accessible [HTTP 403]. Stopping now", model.getName()); + running = false; + } else { + throw he; + } + } } else { throw e; } @@ -175,43 +268,6 @@ public class MergedHlsDownload extends AbstractHlsDownload { } } - private void downloadMissedSegments(SegmentPlaylist lsp, int nextSegment) throws MalformedURLException { - if(nextSegment > 0 && lsp.seq > nextSegment) { - LOG.warn("Missed segments {} < {} in download for {}", nextSegment, lsp.seq, lsp.url); - String first = lsp.segments.get(0); - int seq = lsp.seq; - for (int i = nextSegment; i < lsp.seq; i++) { - URL segmentUrl = new URL(first.replaceAll(Integer.toString(seq), Integer.toString(i))); - LOG.debug("Loading missed segment {} for model {}", i, lsp.url); - byte[] segmentData; - try { - segmentData = new SegmentDownload(segmentUrl, client).call(); - writeSegment(segmentData); - } catch (Exception e) { - LOG.error("Error while downloading segment {}", segmentUrl, e); - } - } - // TODO switch to a lower bitrate/resolution ?!? - } - } - - private void downloadNewSegments(SegmentPlaylist lsp, int nextSegment) throws MalformedURLException { - int skip = nextSegment - lsp.seq; - for (String segment : lsp.segments) { - if(skip > 0) { - skip--; - } else { - URL segmentUrl = new URL(segment); - try { - byte[] segmentData = new SegmentDownload(segmentUrl, client).call(); - writeSegment(segmentData); - } catch (Exception e) { - LOG.error("Error while downloading segment {}", segmentUrl, e); - } - } - } - } - private void writeSegment(byte[] segmentData) throws InterruptedException { InputStream in = new ByteArrayInputStream(segmentData); InputStreamMTSSource source = InputStreamMTSSource.builder().setInputStream(in).build(); @@ -232,12 +288,18 @@ public class MergedHlsDownload extends AbstractHlsDownload { } } - private void waitForNewSegments(SegmentPlaylist lsp, int lastSegment) { + private void waitForNewSegments(SegmentPlaylist lsp, int lastSegment, long downloadTookMillis) { try { long wait = 0; if (lastSegment == lsp.seq) { - // playlist didn't change -> wait for at least half the target duration - wait = (long) lsp.targetDuration * 1000 / 2; + int timeLeftMillis = (int)(lsp.totalDuration * 1000 - downloadTookMillis); + if(timeLeftMillis < 3000) { // we have less than 3 seconds to get the new playlist and start downloading it + wait = 1; + } else { + // wait a second to be nice to the server (don't hammer it with requests) + // 1 second seems to be a good compromise. every other calculation resulted in more missing segments + wait = 1000; + } LOG.trace("Playlist didn't change... waiting for {}ms", wait); } else { // playlist did change -> wait for at least last segment duration @@ -256,7 +318,9 @@ public class MergedHlsDownload extends AbstractHlsDownload { public void stop() { running = false; alive = false; - streamer.stop(); + if(streamer != null) { + streamer.stop(); + } LOG.debug("Download stopped"); } @@ -281,6 +345,7 @@ public class MergedHlsDownload extends AbstractHlsDownload { .setSink(sink) .setSleepingEnabled(liveStream) .setBufferSize(10) + .setName(model.getName()) .build(); // Start streaming @@ -295,9 +360,10 @@ public class MergedHlsDownload extends AbstractHlsDownload { } finally { closeFile(channel); deleteEmptyRecording(targetFile); + running = false; } }); - t.setName("Segment Merger Thread"); + t.setName("Segment Merger Thread [" + model.getName() + "]"); t.setDaemon(true); return t; } @@ -308,7 +374,7 @@ public class MergedHlsDownload extends AbstractHlsDownload { Files.delete(targetFile.toPath()); Files.delete(targetFile.getParentFile().toPath()); } - } catch (IOException e) { + } catch (Exception e) { LOG.error("Error while deleting empty recording {}", targetFile); } } @@ -318,12 +384,12 @@ public class MergedHlsDownload extends AbstractHlsDownload { if (channel != null) { channel.close(); } - } catch (IOException e) { + } catch (Exception e) { LOG.error("Error while closing file channel", e); } } - private static class SegmentDownload implements Callable { + private class SegmentDownload implements Callable { private URL url; private HttpClient client; @@ -333,24 +399,38 @@ public class MergedHlsDownload extends AbstractHlsDownload { } @Override - public byte[] call() throws Exception { + public byte[] call() throws IOException { LOG.trace("Downloading segment " + url.getFile()); int maxTries = 3; - for (int i = 1; i <= maxTries; i++) { - try { - Request request = new Request.Builder().url(url).addHeader("connection", "keep-alive").build(); - Response response = client.execute(request); - byte[] segment = response.body().bytes(); - return segment; + for (int i = 1; i <= maxTries && running; i++) { + Request request = new Request.Builder().url(url).addHeader("connection", "keep-alive").build(); + try (Response response = client.execute(request)) { + if(response.isSuccessful()) { + byte[] segment = response.body().bytes(); + return segment; + } else { + throw new HttpException(response.code(), response.message()); + } } catch(Exception e) { if (i == maxTries) { LOG.warn("Error while downloading segment. Segment {} finally failed", url.getFile()); } else { - LOG.warn("Error while downloading segment {} on try {}", url.getFile(), i); + LOG.warn("Error while downloading segment {} on try {}", url.getFile(), i, e); + } + if(model != null && !isModelOnline()) { + break; } } } - throw new IOException("Unable to download segment " + url.getFile() + " after " + maxTries + " tries"); + throw new MissingSegmentException("Unable to download segment " + url.getFile() + " after " + maxTries + " tries"); + } + } + + public boolean isModelOnline() { + try { + return model.isOnline(IGNORE_CACHE); + } catch (IOException | ExecutionException | InterruptedException e) { + return false; } } } diff --git a/common/src/main/java/ctbrec/recorder/download/MissingSegmentException.java b/common/src/main/java/ctbrec/recorder/download/MissingSegmentException.java new file mode 100644 index 00000000..d6971aab --- /dev/null +++ b/common/src/main/java/ctbrec/recorder/download/MissingSegmentException.java @@ -0,0 +1,11 @@ +package ctbrec.recorder.download; + +import java.io.IOException; + +public class MissingSegmentException extends IOException { + + public MissingSegmentException(String msg) { + super(msg); + } + +} diff --git a/common/src/main/java/org/taktik/mpegts/Streamer.java b/common/src/main/java/org/taktik/mpegts/Streamer.java index d844da92..3ae5312d 100644 --- a/common/src/main/java/org/taktik/mpegts/Streamer.java +++ b/common/src/main/java/org/taktik/mpegts/Streamer.java @@ -30,12 +30,14 @@ public class Streamer { private Thread streamingThread; private boolean sleepingEnabled; + private String name; - private Streamer(MTSSource source, MTSSink sink, int bufferSize, boolean sleepingEnabled) { + private Streamer(MTSSource source, MTSSink sink, int bufferSize, boolean sleepingEnabled, String name) { this.source = source; this.sink = sink; this.bufferSize = bufferSize; this.sleepingEnabled = sleepingEnabled; + this.name = name; } public void stream() throws InterruptedException { @@ -48,15 +50,15 @@ public class Streamer { try { preBuffer(); } catch (Exception e) { - throw new IllegalStateException("Error while bufering", e); + throw new IllegalStateException("Error while buffering", e); } log.info("Done PreBuffering"); - bufferingThread = new Thread(this::fillBuffer, "buffering"); + bufferingThread = new Thread(this::fillBuffer, "Buffering ["+name+"]"); bufferingThread.setDaemon(true); bufferingThread.start(); - streamingThread = new Thread(this::internalStream, "streaming"); + streamingThread = new Thread(this::internalStream, "Streaming ["+name+"]"); streamingThread.setDaemon(true); streamingThread.start(); @@ -123,7 +125,7 @@ public class Streamer { } } } catch (InterruptedException e1) { - if(!endOfSourceReached) { + if(!endOfSourceReached && !streamingShouldStop) { log.error("Interrupted while waiting for packet"); continue; } else { @@ -240,7 +242,7 @@ public class Streamer { // Stream packet // System.out.println("Streaming packet #" + packetCount + ", PID=" + mtsPacket.getPid() + ", pcrCount=" + pcrCount + ", continuityCounter=" + mtsPacket.getContinuityCounter()); - if(!streamingShouldStop) { + if(!streamingShouldStop && !Thread.interrupted()) { try { sink.send(packet); } catch (Exception e) { @@ -275,7 +277,7 @@ public class Streamer { buffer.put(packet); put = true; } catch (InterruptedException ignored) { - + log.error("Error adding packet to buffer", ignored); } } } @@ -287,7 +289,11 @@ public class Streamer { log.error("Error reading from source", e); } finally { endOfSourceReached = true; - streamingThread.interrupt(); + try { + streamingThread.interrupt(); + } catch(Exception e) { + log.error("Couldn't interrupt streaming thread", e); + } } } @@ -308,6 +314,7 @@ public class Streamer { private MTSSource source; private int bufferSize = 1000; private boolean sleepingEnabled = false; + private String name; public StreamerBuilder setSink(MTSSink sink) { this.sink = sink; @@ -329,10 +336,16 @@ public class Streamer { return this; } + public StreamerBuilder setName(String name) { + this.name = name; + return this; + } + public Streamer build() { Preconditions.checkNotNull(sink); Preconditions.checkNotNull(source); - return new Streamer(source, sink, bufferSize, sleepingEnabled); + return new Streamer(source, sink, bufferSize, sleepingEnabled, name); } + } } \ No newline at end of file From e9909fe11aba3e2f9048781d080ec000785e0852 Mon Sep 17 00:00:00 2001 From: 0xboobface <0xboobface@gmail.com> Date: Wed, 28 Nov 2018 15:23:56 +0100 Subject: [PATCH 15/21] Add debug styling Display size cell red, if the size didn't change. This is only done when run in DEV mode. This makes it easier to debug freezing / hanging downloads --- client/src/main/java/ctbrec/ui/JavaFxRecording.java | 6 ++++++ client/src/main/java/ctbrec/ui/RecordingsTab.java | 11 +++++++++++ 2 files changed, 17 insertions(+) diff --git a/client/src/main/java/ctbrec/ui/JavaFxRecording.java b/client/src/main/java/ctbrec/ui/JavaFxRecording.java index af9f0241..e44f36c0 100644 --- a/client/src/main/java/ctbrec/ui/JavaFxRecording.java +++ b/client/src/main/java/ctbrec/ui/JavaFxRecording.java @@ -16,6 +16,7 @@ public class JavaFxRecording extends Recording { private transient LongProperty sizeProperty = new SimpleLongProperty(); private Recording delegate; + private long lastValue = 0; public JavaFxRecording(Recording recording) { this.delegate = recording; @@ -154,4 +155,9 @@ public class JavaFxRecording extends Recording { return sizeProperty; } + public boolean valueChanged() { + boolean changed = getSizeInByte() != lastValue; + lastValue = getSizeInByte(); + return changed; + } } diff --git a/client/src/main/java/ctbrec/ui/RecordingsTab.java b/client/src/main/java/ctbrec/ui/RecordingsTab.java index b60d439b..f8fef9dd 100644 --- a/client/src/main/java/ctbrec/ui/RecordingsTab.java +++ b/client/src/main/java/ctbrec/ui/RecordingsTab.java @@ -159,8 +159,19 @@ public class RecordingsTab extends Tab implements TabSelectionListener { protected void updateItem(Number sizeInByte, boolean empty) { if(empty || sizeInByte == null) { setText(null); + setStyle(null); } else { setText(StringUtil.formatSize(sizeInByte)); + if(Objects.equals(System.getenv("CTBREC_DEV"), "1")) { + int row = this.getTableRow().getIndex(); + JavaFxRecording rec = tableViewProperty().get().getItems().get(row); + if(!rec.valueChanged() && rec.getStatus() == STATUS.RECORDING) { + setStyle("-fx-alignment: CENTER-RIGHT; -fx-background-color: red"); + } else { + setStyle("-fx-alignment: CENTER-RIGHT;"); + //setStyle(null); + } + } } } }; From 3a7f2ceca630a8a3c7561938db771475aeb5d5af Mon Sep 17 00:00:00 2001 From: 0xboobface <0xboobface@gmail.com> Date: Wed, 28 Nov 2018 15:37:35 +0100 Subject: [PATCH 16/21] Add convenience method Config.isDevMode() Also made isServerMode() static --- client/src/main/java/ctbrec/ui/SettingsTab.java | 5 +++++ common/src/main/java/ctbrec/Config.java | 6 +++++- .../main/java/ctbrec/recorder/LocalRecorder.java | 14 +++++++------- 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/client/src/main/java/ctbrec/ui/SettingsTab.java b/client/src/main/java/ctbrec/ui/SettingsTab.java index 0d2722e3..d1f816cc 100644 --- a/client/src/main/java/ctbrec/ui/SettingsTab.java +++ b/client/src/main/java/ctbrec/ui/SettingsTab.java @@ -319,6 +319,11 @@ public class SettingsTab extends Tab implements TabSelectionListener { layout.add(l, 0, row); List splitOptions = new ArrayList<>(); splitOptions.add(new SplitAfterOption("disabled", 0)); + if(Config.isDevMode()) { + splitOptions.add(new SplitAfterOption( "1 min", 1 * 60)); + splitOptions.add(new SplitAfterOption( "3 min", 3 * 60)); + splitOptions.add(new SplitAfterOption( "5 min", 5 * 60)); + } splitOptions.add(new SplitAfterOption("10 min", 10 * 60)); splitOptions.add(new SplitAfterOption("15 min", 15 * 60)); splitOptions.add(new SplitAfterOption("20 min", 20 * 60)); diff --git a/common/src/main/java/ctbrec/Config.java b/common/src/main/java/ctbrec/Config.java index 865f6bc1..20821ea8 100644 --- a/common/src/main/java/ctbrec/Config.java +++ b/common/src/main/java/ctbrec/Config.java @@ -100,10 +100,14 @@ public class Config { Files.write(configFile.toPath(), json.getBytes("utf-8"), CREATE, WRITE, TRUNCATE_EXISTING); } - public boolean isServerMode() { + public static boolean isServerMode() { return Objects.equals(System.getProperty("ctbrec.server.mode"), "1"); } + public static boolean isDevMode() { + return Objects.equals(System.getenv("CTBREC_DEV"), "1"); + } + public File getConfigDir() { return configDir; } diff --git a/common/src/main/java/ctbrec/recorder/LocalRecorder.java b/common/src/main/java/ctbrec/recorder/LocalRecorder.java index 14aca3a7..82c5219c 100644 --- a/common/src/main/java/ctbrec/recorder/LocalRecorder.java +++ b/common/src/main/java/ctbrec/recorder/LocalRecorder.java @@ -83,7 +83,7 @@ public class LocalRecorder implements Recorder { onlineMonitor.start(); postProcessingTrigger = new PostProcessingTrigger(); - if(Config.getInstance().isServerMode()) { + if(Config.isServerMode()) { postProcessingTrigger.start(); } @@ -161,7 +161,7 @@ public class LocalRecorder implements Recorder { LOG.debug("Starting recording for model {}", model.getName()); Download download; - if (Config.getInstance().isServerMode()) { + if (Config.isServerMode()) { download = new HlsDownload(client); } else { download = new MergedHlsDownload(client); @@ -184,7 +184,7 @@ public class LocalRecorder implements Recorder { Download download = recordingProcesses.get(model); download.stop(); recordingProcesses.remove(model); - if(!Config.getInstance().isServerMode()) { + if(!Config.isServerMode()) { postprocess(download); } } @@ -358,7 +358,7 @@ public class LocalRecorder implements Recorder { LOG.debug("Recording terminated for model {}", m.getName()); iterator.remove(); restart.add(m); - if(config.isServerMode()) { + if(Config.isServerMode()) { try { finishRecording(d.getTarget()); } catch(Exception e) { @@ -385,7 +385,7 @@ public class LocalRecorder implements Recorder { } private void finishRecording(File directory) { - if(Config.getInstance().isServerMode()) { + if(Config.isServerMode()) { Thread t = new Thread() { @Override public void run() { @@ -513,7 +513,7 @@ public class LocalRecorder implements Recorder { @Override public List getRecordings() { - if(Config.getInstance().isServerMode()) { + if(Config.isServerMode()) { return listSegmentedRecordings(); } else { return listMergedRecordings(); @@ -558,7 +558,7 @@ public class LocalRecorder implements Recorder { return GENERATING_PLAYLIST; } - if (config.isServerMode()) { + if (Config.isServerMode()) { if (recording.hasPlaylist()) { return FINISHED; } else { From 403c1ed2d020b731c6b3ec7794242e18fdd01a27 Mon Sep 17 00:00:00 2001 From: 0xboobface <0xboobface@gmail.com> Date: Wed, 28 Nov 2018 16:03:21 +0100 Subject: [PATCH 17/21] Fix split recordings Split recordings didn't work, because splitRecStartTime had been removed by accident. Also the splitting now does not start a new recording, but switches the output file in Streamer. This is a much cleaner and smoother approach, because it is much faster and no segments are missed --- common/src/main/java/ctbrec/Config.java | 4 -- .../recorder/download/MergedHlsDownload.java | 41 +++++++++++-------- .../main/java/org/taktik/mpegts/Streamer.java | 16 ++++++++ 3 files changed, 41 insertions(+), 20 deletions(-) diff --git a/common/src/main/java/ctbrec/Config.java b/common/src/main/java/ctbrec/Config.java index 20821ea8..871c36ff 100644 --- a/common/src/main/java/ctbrec/Config.java +++ b/common/src/main/java/ctbrec/Config.java @@ -117,10 +117,6 @@ public class Config { SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd_HH-mm"); String startTime = sdf.format(new Date()); File targetFile = new File(dirForRecording, model.getName() + '_' + startTime + ".ts"); - if(getSettings().splitRecordings > 0) { - LOG.debug("Splitting recordings every {} seconds", getSettings().splitRecordings); - targetFile = new File(targetFile.getAbsolutePath().replaceAll("\\.ts", "-00000.ts")); - } return targetFile; } diff --git a/common/src/main/java/ctbrec/recorder/download/MergedHlsDownload.java b/common/src/main/java/ctbrec/recorder/download/MergedHlsDownload.java index d4a935e3..c9cd0010 100644 --- a/common/src/main/java/ctbrec/recorder/download/MergedHlsDownload.java +++ b/common/src/main/java/ctbrec/recorder/download/MergedHlsDownload.java @@ -15,7 +15,6 @@ import java.nio.file.LinkOption; import java.nio.file.Path; import java.security.InvalidKeyException; import java.security.NoSuchAlgorithmException; -import java.text.DecimalFormat; import java.time.Duration; import java.time.Instant; import java.time.ZonedDateTime; @@ -57,13 +56,12 @@ public class MergedHlsDownload extends AbstractHlsDownload { private BlockingMultiMTSSource multiSource; private Thread mergeThread; private Streamer streamer; - private ZonedDateTime startTime; + private ZonedDateTime splitRecStartTime; private Config config; private File targetFile; - private DecimalFormat df = new DecimalFormat("00000"); - private int splitCounter = 0; private BlockingQueue downloadQueue = new LinkedBlockingQueue<>(50); private ExecutorService downloadThreadPool = new ThreadPoolExecutor(5, 5, 2, TimeUnit.MINUTES, downloadQueue); + private FileChannel fileChannel = null; public MergedHlsDownload(HttpClient client) { super(client); @@ -78,6 +76,7 @@ public class MergedHlsDownload extends AbstractHlsDownload { try { running = true; super.startTime = Instant.now(); + splitRecStartTime = ZonedDateTime.now(); mergeThread = createMergeThread(targetFile, progressListener, false); LOG.debug("Merge thread started"); mergeThread.start(); @@ -123,6 +122,7 @@ public class MergedHlsDownload extends AbstractHlsDownload { running = true; super.startTime = Instant.now(); + splitRecStartTime = ZonedDateTime.now(); super.model = model; targetFile = Config.getInstance().getFileForRecording(model); String segments = getSegmentPlaylistUrl(model); @@ -130,6 +130,9 @@ public class MergedHlsDownload extends AbstractHlsDownload { mergeThread.start(); if(segments != null) { downloadSegments(segments, true); + if(config.getSettings().splitRecordings > 0) { + LOG.debug("Splitting recordings every {} seconds", config.getSettings().splitRecordings); + } } else { throw new IOException("Couldn't determine segments uri"); } @@ -194,7 +197,7 @@ public class MergedHlsDownload extends AbstractHlsDownload { break; } } catch(Exception e) { - LOG.info("Unexpected error while downloading ", model.getName()); + LOG.info("Unexpected error while downloading {}", model.getName(), e); running = false; } } @@ -226,6 +229,7 @@ public class MergedHlsDownload extends AbstractHlsDownload { } // get completed downloads and write them to the file + // TODO it might be a good idea to do this in a separate thread, so that the main download loop isn't blocked writeFinishedSegments(downloads); } @@ -276,14 +280,20 @@ public class MergedHlsDownload extends AbstractHlsDownload { private void splitRecording() { if(config.getSettings().splitRecordings > 0) { - Duration recordingDuration = Duration.between(startTime, ZonedDateTime.now()); + Duration recordingDuration = Duration.between(splitRecStartTime, ZonedDateTime.now()); long seconds = recordingDuration.getSeconds(); if(seconds >= config.getSettings().splitRecordings) { - streamer.stop(); - File target = new File(targetFile.getAbsolutePath().replaceAll("\\.ts", "-"+df.format(++splitCounter)+".ts")); - mergeThread = createMergeThread(target, null, true); - mergeThread.start(); - startTime = ZonedDateTime.now(); + try { + targetFile = Config.getInstance().getFileForRecording(model); + LOG.debug("Switching to file {}", targetFile.getAbsolutePath()); + fileChannel = FileChannel.open(targetFile.toPath(), CREATE, WRITE); + MTSSink sink = ByteChannelSink.builder().setByteChannel(fileChannel).build(); + streamer.switchSink(sink); + splitRecStartTime = ZonedDateTime.now(); + } catch (IOException e) { + LOG.error("Error while splitting recording", e); + running = false; + } } } } @@ -331,14 +341,13 @@ public class MergedHlsDownload extends AbstractHlsDownload { .setProgressListener(listener) .build(); - FileChannel channel = null; try { Path downloadDir = targetFile.getParentFile().toPath(); if (!Files.exists(downloadDir, LinkOption.NOFOLLOW_LINKS)) { Files.createDirectories(downloadDir); } - channel = FileChannel.open(targetFile.toPath(), CREATE, WRITE); - MTSSink sink = ByteChannelSink.builder().setByteChannel(channel).build(); + fileChannel = FileChannel.open(targetFile.toPath(), CREATE, WRITE); + MTSSink sink = ByteChannelSink.builder().setByteChannel(fileChannel).build(); streamer = Streamer.builder() .setSource(multiSource) @@ -358,9 +367,9 @@ public class MergedHlsDownload extends AbstractHlsDownload { } catch(Exception e) { LOG.error("Error while saving stream to file", e); } finally { - closeFile(channel); deleteEmptyRecording(targetFile); running = false; + closeFile(fileChannel); } }); t.setName("Segment Merger Thread [" + model.getName() + "]"); @@ -381,7 +390,7 @@ public class MergedHlsDownload extends AbstractHlsDownload { private void closeFile(FileChannel channel) { try { - if (channel != null) { + if (channel != null && channel.isOpen()) { channel.close(); } } catch (Exception e) { diff --git a/common/src/main/java/org/taktik/mpegts/Streamer.java b/common/src/main/java/org/taktik/mpegts/Streamer.java index 3ae5312d..560bbfc8 100644 --- a/common/src/main/java/org/taktik/mpegts/Streamer.java +++ b/common/src/main/java/org/taktik/mpegts/Streamer.java @@ -64,6 +64,12 @@ public class Streamer { bufferingThread.join(); streamingThread.join(); + + try { + sink.close(); + } catch(Exception e) { + log.error("Couldn't close sink", e); + } } public void stop() { @@ -87,6 +93,16 @@ public class Streamer { } } + public void switchSink(MTSSink sink) { + MTSSink old = this.sink; + this.sink = sink; + try { + old.close(); + } catch (Exception e) { + log.error("Couldn't close old sink while switching sinks", e); + } + } + private void internalStream() { boolean resetState = false; MTSPacket packet = null; From ef9566999a5894863bcd8fb5945e1dd4e3507164 Mon Sep 17 00:00:00 2001 From: 0xboobface <0xboobface@gmail.com> Date: Wed, 28 Nov 2018 16:43:30 +0100 Subject: [PATCH 18/21] Fixed possible NPE in update method --- common/src/main/java/ctbrec/sites/mfc/MyFreeCamsModel.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/src/main/java/ctbrec/sites/mfc/MyFreeCamsModel.java b/common/src/main/java/ctbrec/sites/mfc/MyFreeCamsModel.java index f0776843..45f13c08 100644 --- a/common/src/main/java/ctbrec/sites/mfc/MyFreeCamsModel.java +++ b/common/src/main/java/ctbrec/sites/mfc/MyFreeCamsModel.java @@ -221,7 +221,7 @@ public class MyFreeCamsModel extends AbstractModel { setName(state.getNm()); setState(State.of(state.getVs())); setStreamUrl(streamUrl); - Optional camScore = Optional.of(state.getM()).map(m -> m.getCamscore()); + Optional camScore = Optional.ofNullable(state.getM()).map(m -> m.getCamscore()); setCamScore(camScore.orElse(0.0)); // preview From cbb6f3f45a9709ee9a0dd4b7816185eff0ceb922 Mon Sep 17 00:00:00 2001 From: 0xboobface <0xboobface@gmail.com> Date: Wed, 28 Nov 2018 16:45:42 +0100 Subject: [PATCH 19/21] Add failFast version of getStreamInfo for faster startup With many chaturbate models, the loading of the recording tab took a long time, because for each model the online state was loaded by the loading cache. The failFast version just returns null and makes the inital loading of recorder.getOnlineModels() much faster. --- .../main/java/ctbrec/sites/chaturbate/Chaturbate.java | 10 +++++++++- .../java/ctbrec/sites/chaturbate/ChaturbateModel.java | 11 +++++++---- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/common/src/main/java/ctbrec/sites/chaturbate/Chaturbate.java b/common/src/main/java/ctbrec/sites/chaturbate/Chaturbate.java index 36125f85..534a39bb 100644 --- a/common/src/main/java/ctbrec/sites/chaturbate/Chaturbate.java +++ b/common/src/main/java/ctbrec/sites/chaturbate/Chaturbate.java @@ -219,7 +219,15 @@ public class Chaturbate extends AbstractSite { } StreamInfo getStreamInfo(String modelName) throws IOException, ExecutionException { - return streamInfoCache.get(modelName); + return getStreamInfo(modelName, false); + } + + StreamInfo getStreamInfo(String modelName, boolean failFast) throws IOException, ExecutionException { + if(failFast) { + return streamInfoCache.getIfPresent(modelName); + } else { + return streamInfoCache.get(modelName); + } } StreamInfo loadStreamInfo(String modelName) throws HttpException, IOException, InterruptedException { diff --git a/common/src/main/java/ctbrec/sites/chaturbate/ChaturbateModel.java b/common/src/main/java/ctbrec/sites/chaturbate/ChaturbateModel.java index 3095c4be..bd17cd23 100644 --- a/common/src/main/java/ctbrec/sites/chaturbate/ChaturbateModel.java +++ b/common/src/main/java/ctbrec/sites/chaturbate/ChaturbateModel.java @@ -6,6 +6,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.Objects; +import java.util.Optional; import java.util.concurrent.ExecutionException; import org.slf4j.Logger; @@ -39,14 +40,16 @@ public class ChaturbateModel extends AbstractModel { @Override public boolean isOnline(boolean ignoreCache) throws IOException, ExecutionException, InterruptedException { - StreamInfo info; + String roomStatus; if(ignoreCache) { - info = getChaturbate().loadStreamInfo(getName()); + StreamInfo info = getChaturbate().loadStreamInfo(getName()); + roomStatus = Optional.ofNullable(info).map(i -> i.room_status).orElse(""); LOG.trace("Model {} room status: {}", getName(), info.room_status); } else { - info = getChaturbate().getStreamInfo(getName()); + StreamInfo info = getChaturbate().getStreamInfo(getName(), true); + roomStatus = Optional.ofNullable(info).map(i -> i.room_status).orElse(""); } - return Objects.equals("public", info.room_status); + return Objects.equals("public", roomStatus); } @Override From ab49ac414ab0e220ba057aa9a0f26fd5af2c0ec9 Mon Sep 17 00:00:00 2001 From: 0xboobface <0xboobface@gmail.com> Date: Wed, 28 Nov 2018 17:37:23 +0100 Subject: [PATCH 20/21] Update changelog --- CHANGELOG.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ff02c63e..c956f655 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,17 @@ +1.12.0 +======================== +* Added threshold setting to keep free space on the recording device. + This is useful, if you don't want to use up all of your storage. + The free space is also shown on the recordings tab +* Tweaked the download internals a lot. Downloads should not hang + in RECORDING state without actually recording. Downloads should + be more robust in general. +* Fixed and improved split recordings +* Improved detection of online state for Cam4 models +* Accelerated the initial loading of the "Recording" tab for many + Chaturbate models +* Recordings tab now shows smaller size units (Bytes, KiB, MiB, GiB) + 1.11.0 ======================== * Added model search function From b7711456125578224f11635dfa6c5de97ed73c70 Mon Sep 17 00:00:00 2001 From: 0xboobface <0xboobface@gmail.com> Date: Wed, 28 Nov 2018 17:52:30 +0100 Subject: [PATCH 21/21] Bump version to 1.12.0 --- client/pom.xml | 2 +- common/pom.xml | 2 +- master/pom.xml | 2 +- server/pom.xml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/client/pom.xml b/client/pom.xml index c0e4bb57..021c748b 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -8,7 +8,7 @@ ctbrec master - 1.11.0 + 1.12.0 ../master diff --git a/common/pom.xml b/common/pom.xml index 54db7db8..cb1e5152 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -8,7 +8,7 @@ ctbrec master - 1.11.0 + 1.12.0 ../master diff --git a/master/pom.xml b/master/pom.xml index 8d905ac5..3c43cdae 100644 --- a/master/pom.xml +++ b/master/pom.xml @@ -6,7 +6,7 @@ ctbrec master pom - 1.11.0 + 1.12.0 ../common diff --git a/server/pom.xml b/server/pom.xml index 873084e8..69969b61 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -8,7 +8,7 @@ ctbrec master - 1.11.0 + 1.12.0 ../master