From 063395bc747133475cb417a0830b38f7a022481e Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Wed, 19 Aug 2020 19:17:52 +0200 Subject: [PATCH 01/33] Increase log level for update check --- common/src/main/java/ctbrec/recorder/OnlineMonitor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/src/main/java/ctbrec/recorder/OnlineMonitor.java b/common/src/main/java/ctbrec/recorder/OnlineMonitor.java index d3f10744..166fdf32 100644 --- a/common/src/main/java/ctbrec/recorder/OnlineMonitor.java +++ b/common/src/main/java/ctbrec/recorder/OnlineMonitor.java @@ -139,7 +139,7 @@ public class OnlineMonitor extends Thread { } private void suspendUntilNextIteration(List models, Duration timeCheckTook) { - LOG.trace("Online check for {} models took {} seconds", models.size(), timeCheckTook.getSeconds()); + LOG.debug("Online check for {} models took {} seconds", models.size(), timeCheckTook.getSeconds()); long sleepTime = config.getSettings().onlineCheckIntervalInSecs; if(timeCheckTook.getSeconds() < sleepTime) { try { From 17a32cd928933d90f2540929e5c031a44c1326f5 Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Sat, 22 Aug 2020 18:31:25 +0200 Subject: [PATCH 02/33] Add first configurable PP step --- .../AbstractPostProcessingPaneFactory.java | 52 ++++++ .../settings/PostProcessingDialogFactory.java | 57 ++++++ .../ui/settings/PostProcessingStepPanel.java | 173 ++++++++++++++++++ .../ui/settings/RemuxerPaneFactory.java | 27 +++ .../java/ctbrec/ui/settings/SettingsTab.java | 3 +- .../ctbrec/ui/settings/api/Preferences.java | 29 ++- common/src/main/java/ctbrec/Config.java | 7 + common/src/main/java/ctbrec/Recording.java | 19 ++ common/src/main/java/ctbrec/Settings.java | 2 + common/src/main/java/ctbrec/io/DevNull.java | 18 ++ .../main/java/ctbrec/io/FileJsonAdapter.java | 30 +++ .../ctbrec/io/PostProcessorJsonAdapter.java | 62 +++++++ .../ctbrec/recorder/NextGenLocalRecorder.java | 6 + .../ctbrec/recorder/RecordingManager.java | 24 ++- .../postprocessing/AbstractPostProcessor.java | 24 +++ .../postprocessing/PostProcessor.java | 16 ++ .../postprocessing/RecordingRenamer.java | 16 ++ .../recorder/postprocessing/Remuxer.java | 82 +++++++++ 18 files changed, 634 insertions(+), 13 deletions(-) create mode 100644 client/src/main/java/ctbrec/ui/settings/AbstractPostProcessingPaneFactory.java create mode 100644 client/src/main/java/ctbrec/ui/settings/PostProcessingDialogFactory.java create mode 100644 client/src/main/java/ctbrec/ui/settings/PostProcessingStepPanel.java create mode 100644 client/src/main/java/ctbrec/ui/settings/RemuxerPaneFactory.java create mode 100644 common/src/main/java/ctbrec/io/DevNull.java create mode 100644 common/src/main/java/ctbrec/io/FileJsonAdapter.java create mode 100644 common/src/main/java/ctbrec/io/PostProcessorJsonAdapter.java create mode 100644 common/src/main/java/ctbrec/recorder/postprocessing/AbstractPostProcessor.java create mode 100644 common/src/main/java/ctbrec/recorder/postprocessing/PostProcessor.java create mode 100644 common/src/main/java/ctbrec/recorder/postprocessing/RecordingRenamer.java create mode 100644 common/src/main/java/ctbrec/recorder/postprocessing/Remuxer.java diff --git a/client/src/main/java/ctbrec/ui/settings/AbstractPostProcessingPaneFactory.java b/client/src/main/java/ctbrec/ui/settings/AbstractPostProcessingPaneFactory.java new file mode 100644 index 00000000..37515cd6 --- /dev/null +++ b/client/src/main/java/ctbrec/ui/settings/AbstractPostProcessingPaneFactory.java @@ -0,0 +1,52 @@ +package ctbrec.ui.settings; + +import java.io.IOException; +import java.util.HashSet; +import java.util.Set; + +import ctbrec.recorder.postprocessing.PostProcessor; +import ctbrec.ui.settings.api.Preferences; +import ctbrec.ui.settings.api.PreferencesStorage; +import ctbrec.ui.settings.api.Setting; +import javafx.beans.property.Property; +import javafx.scene.Node; +import javafx.scene.control.TextField; + +public abstract class AbstractPostProcessingPaneFactory { + + private PostProcessor pp; + Set> properties = new HashSet<>(); + + public abstract Preferences doCreatePostProcessorPane(PostProcessor pp); + + public Preferences createPostProcessorPane(PostProcessor pp) { + this.pp = pp; + return doCreatePostProcessorPane(pp); + } + + class MapPreferencesStorage implements PreferencesStorage { + + @Override + public void save(Preferences preferences) throws IOException { + for (Property property : properties) { + String key = property.getName(); + Object value = preferences.getSetting(key).get().getProperty().getValue(); + pp.getConfig().put(key, value.toString()); + } + } + + @Override + public void load(Preferences preferences) { + // no op + } + + @SuppressWarnings("unchecked") + @Override + public Node createGui(Setting setting) throws Exception { + TextField input = new TextField(); + input.textProperty().bindBidirectional(setting.getProperty()); + return input; + } + + } +} diff --git a/client/src/main/java/ctbrec/ui/settings/PostProcessingDialogFactory.java b/client/src/main/java/ctbrec/ui/settings/PostProcessingDialogFactory.java new file mode 100644 index 00000000..f3bb74ed --- /dev/null +++ b/client/src/main/java/ctbrec/ui/settings/PostProcessingDialogFactory.java @@ -0,0 +1,57 @@ +package ctbrec.ui.settings; + +import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.util.HashMap; +import java.util.Map; + +import ctbrec.Config; +import ctbrec.recorder.postprocessing.PostProcessor; +import ctbrec.recorder.postprocessing.Remuxer; +import ctbrec.ui.controls.Dialogs; +import ctbrec.ui.settings.api.Preferences; +import javafx.collections.ObservableList; +import javafx.scene.Scene; + +public class PostProcessingDialogFactory { + + static Map, Class> ppToDialogMap = new HashMap<>(); + static { + ppToDialogMap.put(Remuxer.class, RemuxerPaneFactory.class); + } + + private PostProcessingDialogFactory() { + } + + public static void openNewDialog(PostProcessor pp, Config config, Scene scene, ObservableList stepList) { + openDialog(pp, config, scene, stepList, true); + } + + public static void openEditDialog(PostProcessor pp, Config config, Scene scene, ObservableList stepList) { + openDialog(pp, config, scene, stepList, false); + } + + private static void openDialog(PostProcessor pp, Config config, Scene scene, ObservableList stepList, boolean newEntry) { + boolean ok; + try { + Preferences preferences = createPreferences(pp); + ok = Dialogs.showCustomInput(scene, "Configure " + pp.getName(), preferences.getView(false)); + if (ok) { + preferences.save(); + if (newEntry) { + stepList.add(pp); + } + } + } catch (NoSuchMethodException | SecurityException | IllegalAccessException | IllegalArgumentException | InvocationTargetException + | InstantiationException | IOException e) { + Dialogs.showError("New post-processing step", "Couldn't create dialog for " + pp.getName(), e); + } + } + + private static Preferences createPreferences(PostProcessor pp) throws InstantiationException, IllegalAccessException, IllegalArgumentException, + InvocationTargetException, NoSuchMethodException, SecurityException { + Class paneFactoryClass = ppToDialogMap.get(pp.getClass()); + AbstractPostProcessingPaneFactory factory = (AbstractPostProcessingPaneFactory) paneFactoryClass.getDeclaredConstructor().newInstance(); + return factory.createPostProcessorPane(pp); + } +} diff --git a/client/src/main/java/ctbrec/ui/settings/PostProcessingStepPanel.java b/client/src/main/java/ctbrec/ui/settings/PostProcessingStepPanel.java new file mode 100644 index 00000000..207515e7 --- /dev/null +++ b/client/src/main/java/ctbrec/ui/settings/PostProcessingStepPanel.java @@ -0,0 +1,173 @@ +package ctbrec.ui.settings; + +import java.io.IOException; +import java.io.InputStream; +import java.lang.reflect.InvocationTargetException; +import java.util.Optional; + +import ctbrec.Config; +import ctbrec.recorder.postprocessing.PostProcessor; +import ctbrec.recorder.postprocessing.RecordingRenamer; +import ctbrec.recorder.postprocessing.Remuxer; +import ctbrec.ui.controls.Dialogs; +import javafx.collections.FXCollections; +import javafx.collections.ListChangeListener; +import javafx.collections.ObservableList; +import javafx.scene.control.Button; +import javafx.scene.control.ChoiceDialog; +import javafx.scene.control.ListView; +import javafx.scene.control.Tooltip; +import javafx.scene.image.Image; +import javafx.scene.layout.GridPane; +import javafx.scene.layout.Priority; +import javafx.scene.layout.VBox; +import javafx.stage.Stage; + +public class PostProcessingStepPanel extends GridPane { + + private Config config; + + private static final Class[] POST_PROCESSOR_CLASSES = new Class[] { Remuxer.class, RecordingRenamer.class }; + + ListView stepListView; + ObservableList stepList; + + Button up; + Button down; + + Button add; + Button remove; + Button edit; + + public PostProcessingStepPanel(Config config) { + this.config = config; + initGui(); + } + + private void initGui() { + setHgap(5); + vgapProperty().bind(hgapProperty()); + + up = createUpButton(); + down = createDownButton(); + add = createAddButton(); + remove = createRemoveButton(); + edit = createEditButton(); + VBox buttons = new VBox(5, add, edit, up, down, remove); + + stepList = FXCollections.observableList(config.getSettings().postProcessors); + stepList.addListener((ListChangeListener) change -> { + try { + config.save(); + } catch (IOException e) { + Dialogs.showError(getScene(), "Couldn't save configuration", "An error occurred while saving the configuration", e); + } + }); + stepListView = new ListView<>(stepList); + GridPane.setHgrow(stepListView, Priority.ALWAYS); + + add(stepListView, 0, 0); + add(buttons, 1, 0); + + stepListView.getSelectionModel().selectedIndexProperty().addListener((obs, oldV, newV) -> { + int idx = newV.intValue(); + boolean noSelection = idx == -1; + up.setDisable(noSelection || idx == 0); + down.setDisable(noSelection || idx == stepList.size() - 1); + edit.setDisable(noSelection); + remove.setDisable(noSelection); + }); + } + + private Button createUpButton() { + Button up = createButton("\u25B4", "Move step up"); + up.setOnAction(evt -> { + int idx = stepListView.getSelectionModel().getSelectedIndex(); + PostProcessor selectedItem = stepListView.getSelectionModel().getSelectedItem(); + stepList.remove(idx); + stepList.add(idx - 1, selectedItem); + stepListView.getSelectionModel().select(idx - 1); + }); + return up; + } + + private Button createDownButton() { + Button down = createButton("\u25BE", "Move step down"); + down.setOnAction(evt -> { + int idx = stepListView.getSelectionModel().getSelectedIndex(); + PostProcessor selectedItem = stepListView.getSelectionModel().getSelectedItem(); + stepList.remove(idx); + stepList.add(idx + 1, selectedItem); + stepListView.getSelectionModel().select(idx + 1); + }); + return down; + } + + private Button createAddButton() { + Button add = createButton("+", "Add a new step"); + add.setDisable(false); + add.setOnAction(evt -> { + PostProcessor[] options = createOptions(); + ChoiceDialog choice = new ChoiceDialog<>(options[0], options); + choice.setTitle("New Post-Processing Step"); + choice.setHeaderText("Select the new step type"); + choice.setResizable(true); + choice.setWidth(600); + choice.getDialogPane().setMinWidth(400); + Stage stage = (Stage) choice.getDialogPane().getScene().getWindow(); + stage.getScene().getStylesheets().addAll(getScene().getStylesheets()); + InputStream icon = Dialogs.class.getResourceAsStream("/icon.png"); + stage.getIcons().add(new Image(icon)); + + Optional result = choice.showAndWait(); + result.ifPresent(pp -> PostProcessingDialogFactory.openNewDialog(pp, config, getScene(), stepList)); + }); + return add; + } + + private PostProcessor[] createOptions() { + try { + PostProcessor[] options = new PostProcessor[POST_PROCESSOR_CLASSES.length]; + for (int i = 0; i < POST_PROCESSOR_CLASSES.length; i++) { + Class cls = POST_PROCESSOR_CLASSES[i]; + PostProcessor pp; + pp = (PostProcessor) cls.getDeclaredConstructor().newInstance(); + options[i] = pp; + } + return options; + } catch (InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException | NoSuchMethodException + | SecurityException e) { + Dialogs.showError(getScene(), "Create post-processor selection", "Error while reaing in post-processing options", e); + return new PostProcessor[0]; + } + } + + private Button createRemoveButton() { + Button remove = createButton("-", "Remove selected step"); + remove.setOnAction(evt -> { + PostProcessor selectedItem = stepListView.getSelectionModel().getSelectedItem(); + if (selectedItem != null) { + stepList.remove(selectedItem); + } + }); + return remove; + } + + private Button createEditButton() { + Button edit = createButton("\u270E", "Edit selected step"); + edit.setOnAction(evt -> { + PostProcessor selectedItem = stepListView.getSelectionModel().getSelectedItem(); + PostProcessingDialogFactory.openEditDialog(selectedItem, config, getScene(), stepList); + stepListView.refresh(); + }); + return edit; + } + + private Button createButton(String text, String tooltip) { + Button b = new Button(text); + b.setTooltip(new Tooltip(tooltip)); + b.setDisable(true); + b.setPrefSize(32, 32); + return b; + } +} diff --git a/client/src/main/java/ctbrec/ui/settings/RemuxerPaneFactory.java b/client/src/main/java/ctbrec/ui/settings/RemuxerPaneFactory.java new file mode 100644 index 00000000..886182ad --- /dev/null +++ b/client/src/main/java/ctbrec/ui/settings/RemuxerPaneFactory.java @@ -0,0 +1,27 @@ +package ctbrec.ui.settings; + +import ctbrec.recorder.postprocessing.PostProcessor; +import ctbrec.recorder.postprocessing.Remuxer; +import ctbrec.ui.settings.api.Category; +import ctbrec.ui.settings.api.Preferences; +import ctbrec.ui.settings.api.Setting; +import javafx.beans.property.SimpleStringProperty; + +public class RemuxerPaneFactory extends AbstractPostProcessingPaneFactory { + + @Override + public Preferences doCreatePostProcessorPane(PostProcessor pp) { + SimpleStringProperty ffmpegParams = new SimpleStringProperty(null, Remuxer.FFMPEG_ARGS, pp.getConfig().getOrDefault(Remuxer.FFMPEG_ARGS, "-c:v copy -c:a copy -movflags faststart -y -f mp4")); + SimpleStringProperty fileExt = new SimpleStringProperty(null, Remuxer.FILE_EXT, pp.getConfig().getOrDefault(Remuxer.FILE_EXT, "mp4")); + properties.add(ffmpegParams); + properties.add(fileExt); + + return Preferences.of(new MapPreferencesStorage(), + Category.of(pp.getName(), + Setting.of("FFmpeg parameters", ffmpegParams), + Setting.of("File extension", fileExt) + ) + ); + } + +} diff --git a/client/src/main/java/ctbrec/ui/settings/SettingsTab.java b/client/src/main/java/ctbrec/ui/settings/SettingsTab.java index f83a38f7..95f78da1 100644 --- a/client/src/main/java/ctbrec/ui/settings/SettingsTab.java +++ b/client/src/main/java/ctbrec/ui/settings/SettingsTab.java @@ -209,7 +209,8 @@ public class SettingsTab extends Tab implements TabSelectionListener { Setting.of("Post-Processing", postProcessing), Setting.of("Threads", postProcessingThreads), Setting.of("Delete recordings shorter than (secs)", minimumLengthInSecs, "Delete recordings, which are shorter than x seconds. 0 to disable"), - Setting.of("Remove recording after post-processing", removeRecordingAfterPp) + Setting.of("Remove recording after post-processing", removeRecordingAfterPp), + Setting.of("Steps", new PostProcessingStepPanel(config)) ) ), Category.of("Events & Actions", new ActionSettingsPanel(recorder)), diff --git a/client/src/main/java/ctbrec/ui/settings/api/Preferences.java b/client/src/main/java/ctbrec/ui/settings/api/Preferences.java index 9b096406..d8c9bb7d 100644 --- a/client/src/main/java/ctbrec/ui/settings/api/Preferences.java +++ b/client/src/main/java/ctbrec/ui/settings/api/Preferences.java @@ -2,6 +2,7 @@ package ctbrec.ui.settings.api; import static java.util.Optional.*; +import java.io.IOException; import java.util.Objects; import java.util.Optional; import java.util.function.Consumer; @@ -21,6 +22,7 @@ import javafx.scene.control.TreeView; import javafx.scene.layout.BorderPane; import javafx.scene.layout.GridPane; import javafx.scene.layout.Priority; +import javafx.scene.layout.Region; import javafx.scene.layout.VBox; public class Preferences { @@ -31,7 +33,10 @@ public class Preferences { private TreeView categoryTree; + private PreferencesStorage preferencesStorage; + private Preferences(PreferencesStorage preferencesStorage, Category...categories) { + this.preferencesStorage = preferencesStorage; this.categories = categories; for (Category category : categories) { assignPreferencesStorage(category, preferencesStorage); @@ -56,15 +61,15 @@ public class Preferences { return new Preferences(preferencesStorage, categories); } - public void save() { - throw new RuntimeException("save not implemented"); + public void save() throws IOException { + preferencesStorage.save(this); } Category[] getCategories() { return categories; } - public Node getView() { + public Region getView(boolean withNavigation) { SearchBox search = new SearchBox(true); search.textProperty().addListener(this::filterTree); TreeItem categoryTreeItems = createCategoryTree(categories, new TreeItem<>(), null); @@ -76,7 +81,9 @@ public class Preferences { VBox.setMargin(categoryTree, new Insets(2)); BorderPane main = new BorderPane(); - main.setLeft(leftSide); + if (withNavigation) { + main.setLeft(leftSide); + } main.setCenter(new Label("Center")); BorderPane.setMargin(leftSide, new Insets(2)); @@ -92,6 +99,10 @@ public class Preferences { return main; } + public Region getView() { + return getView(true); + } + private void filterTree(ObservableValue obs, String oldV, String newV) { String q = ofNullable(newV).orElse("").toLowerCase().trim(); TreeItem filteredCategoryTree = createCategoryTree(categories, new TreeItem<>(), q); @@ -151,6 +162,8 @@ public class Preferences { private Node createGrid(Setting[] settings) throws Exception { GridPane pane = new GridPane(); + pane.setHgap(2); + pane.vgapProperty().bind(pane.hgapProperty()); int row = 0; for (Setting setting : settings) { Node node = setting.getGui(); @@ -198,11 +211,9 @@ public class Preferences { } private void visit(Category cat, Consumer visitor) { - if (cat.hasGroups()) { - for (Group group : cat.getGroups()) { - for (Setting setting : group.getSettings()) { - visitor.accept(setting); - } + for (Group group : cat.getGroups()) { + for (Setting setting : group.getSettings()) { + visitor.accept(setting); } } if (cat.hasSubCategories()) { diff --git a/common/src/main/java/ctbrec/Config.java b/common/src/main/java/ctbrec/Config.java index a7caec1e..c18b3a96 100644 --- a/common/src/main/java/ctbrec/Config.java +++ b/common/src/main/java/ctbrec/Config.java @@ -23,7 +23,10 @@ import org.slf4j.LoggerFactory; import com.squareup.moshi.JsonAdapter; import com.squareup.moshi.Moshi; +import ctbrec.io.FileJsonAdapter; import ctbrec.io.ModelJsonAdapter; +import ctbrec.io.PostProcessorJsonAdapter; +import ctbrec.recorder.postprocessing.PostProcessor; import ctbrec.sites.Site; public class Config { @@ -55,6 +58,8 @@ public class Config { private void load() throws IOException { Moshi moshi = new Moshi.Builder() .add(Model.class, new ModelJsonAdapter(sites)) + .add(PostProcessor.class, new PostProcessorJsonAdapter()) + .add(File.class, new FileJsonAdapter()) .build(); JsonAdapter adapter = moshi.adapter(Settings.class).lenient(); File configFile = new File(configDir, filename); @@ -125,6 +130,8 @@ public class Config { public void save() throws IOException { Moshi moshi = new Moshi.Builder() .add(Model.class, new ModelJsonAdapter()) + .add(PostProcessor.class, new PostProcessorJsonAdapter()) + .add(File.class, new FileJsonAdapter()) .build(); JsonAdapter adapter = moshi.adapter(Settings.class).indent(" "); String json = adapter.toJson(settings); diff --git a/common/src/main/java/ctbrec/Recording.java b/common/src/main/java/ctbrec/Recording.java index e6f60bf8..508aee27 100644 --- a/common/src/main/java/ctbrec/Recording.java +++ b/common/src/main/java/ctbrec/Recording.java @@ -17,6 +17,8 @@ import java.time.LocalDateTime; import java.time.ZoneId; import java.time.format.DateTimeFormatter; import java.util.EnumSet; +import java.util.HashSet; +import java.util.Set; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -39,6 +41,8 @@ public class Recording implements Serializable { private boolean singleFile = false; private boolean pinned = false; private String note; + private Set associatedFiles = new HashSet<>(); + private File postProcessedFile = null; public enum State { RECORDING("recording"), @@ -107,6 +111,17 @@ public class Recording implements Serializable { return recordingsFile; } + public File getPostProcessedFile() { + if (postProcessedFile == null) { + setPostProcessedFile(getAbsoluteFile()); + } + return postProcessedFile; + } + + public void setPostProcessedFile(File postProcessedFile) { + this.postProcessedFile = postProcessedFile; + } + public long getSizeInByte() { return sizeInByte; } @@ -278,4 +293,8 @@ public class Recording implements Serializable { public boolean canBePostProcessed() { return getStatus() == FAILED || getStatus() == WAITING || getStatus() == FINISHED; } + + public Set getAssociatedFiles() { + return associatedFiles; + } } diff --git a/common/src/main/java/ctbrec/Settings.java b/common/src/main/java/ctbrec/Settings.java index e6c7e5fc..ba7feac0 100644 --- a/common/src/main/java/ctbrec/Settings.java +++ b/common/src/main/java/ctbrec/Settings.java @@ -7,6 +7,7 @@ import java.util.List; import java.util.Map; import ctbrec.event.EventHandlerConfiguration; +import ctbrec.recorder.postprocessing.PostProcessor; public class Settings { @@ -94,6 +95,7 @@ public class Settings { public String password = ""; // chaturbate password TODO maybe rename this onetime public String postProcessing = ""; public int postProcessingThreads = 2; + public List postProcessors = new ArrayList<>(); public String proxyHost; public String proxyPassword; public String proxyPort; diff --git a/common/src/main/java/ctbrec/io/DevNull.java b/common/src/main/java/ctbrec/io/DevNull.java new file mode 100644 index 00000000..b40772b9 --- /dev/null +++ b/common/src/main/java/ctbrec/io/DevNull.java @@ -0,0 +1,18 @@ +package ctbrec.io; + +import java.io.IOException; +import java.io.OutputStream; + +public class DevNull extends OutputStream { + @Override + public void write(int b) throws IOException { + } + + @Override + public void write(byte[] b) throws IOException { + } + + @Override + public void write(byte[] b, int off, int len) throws IOException { + } +} diff --git a/common/src/main/java/ctbrec/io/FileJsonAdapter.java b/common/src/main/java/ctbrec/io/FileJsonAdapter.java new file mode 100644 index 00000000..e3d38733 --- /dev/null +++ b/common/src/main/java/ctbrec/io/FileJsonAdapter.java @@ -0,0 +1,30 @@ +package ctbrec.io; + +import java.io.File; +import java.io.IOException; + +import com.squareup.moshi.JsonAdapter; +import com.squareup.moshi.JsonReader; +import com.squareup.moshi.JsonWriter; + +public class FileJsonAdapter extends JsonAdapter { + + @Override + public File fromJson(JsonReader reader) throws IOException { + String path = reader.nextString(); + if (path != null) { + return new File(path); + } else { + return null; + } + } + + @Override + public void toJson(JsonWriter writer, File value) throws IOException { + if (value != null) { + writer.value(value.getCanonicalPath()); + } else { + writer.nullValue(); + } + } +} diff --git a/common/src/main/java/ctbrec/io/PostProcessorJsonAdapter.java b/common/src/main/java/ctbrec/io/PostProcessorJsonAdapter.java new file mode 100644 index 00000000..d5e1f937 --- /dev/null +++ b/common/src/main/java/ctbrec/io/PostProcessorJsonAdapter.java @@ -0,0 +1,62 @@ +package ctbrec.io; + +import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.util.Map.Entry; + +import com.squareup.moshi.JsonAdapter; +import com.squareup.moshi.JsonReader; +import com.squareup.moshi.JsonReader.Token; +import com.squareup.moshi.JsonWriter; + +import ctbrec.recorder.postprocessing.PostProcessor; + +public class PostProcessorJsonAdapter extends JsonAdapter { + + @Override + public PostProcessor fromJson(JsonReader reader) throws IOException { + reader.beginObject(); + Object type = null; + PostProcessor postProcessor = null; + while(reader.hasNext()) { + try { + Token token = reader.peek(); + if(token == Token.NAME) { + String key = reader.nextName(); + if(key.equals("type")) { + type = reader.readJsonValue(); + Class modelClass = Class.forName(type.toString()); + postProcessor = (PostProcessor) modelClass.getDeclaredConstructor().newInstance(); + } else if(key.equals("config")) { + reader.beginObject(); + } else { + String value = reader.nextString(); + postProcessor.getConfig().put(key, value); + } + } else { + reader.skipValue(); + } + } catch (ClassNotFoundException | InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException | NoSuchMethodException | SecurityException e) { + throw new IOException("Couldn't instantiate post-processor class [" + type + "]", e); + } + } + reader.endObject(); + reader.endObject(); + + return postProcessor; + } + + @Override + public void toJson(JsonWriter writer, PostProcessor pp) throws IOException { + writer.beginObject(); + writer.name("type").value(pp.getClass().getName()); + writer.name("config"); + writer.beginObject(); + for (Entry entry : pp.getConfig().entrySet()) { + writer.name(entry.getKey()).value(entry.getValue()); + } + writer.endObject(); + writer.endObject(); + } + +} diff --git a/common/src/main/java/ctbrec/recorder/NextGenLocalRecorder.java b/common/src/main/java/ctbrec/recorder/NextGenLocalRecorder.java index f337c7c6..bffec23e 100644 --- a/common/src/main/java/ctbrec/recorder/NextGenLocalRecorder.java +++ b/common/src/main/java/ctbrec/recorder/NextGenLocalRecorder.java @@ -53,6 +53,7 @@ import ctbrec.event.NoSpaceLeftEvent; import ctbrec.event.RecordingStateChangedEvent; import ctbrec.io.HttpClient; import ctbrec.recorder.download.Download; +import ctbrec.recorder.postprocessing.PostProcessor; import ctbrec.sites.Site; public class NextGenLocalRecorder implements Recorder { @@ -161,6 +162,10 @@ public class NextGenLocalRecorder implements Recorder { setRecordingStatus(recording, State.POST_PROCESSING); recordingManager.saveRecording(recording); recording.postprocess(); + List postProcessors = config.getSettings().postProcessors; + for (PostProcessor postProcessor : postProcessors) { + postProcessor.postprocess(recording); + } setRecordingStatus(recording, State.FINISHED); recordingManager.saveRecording(recording); deleteIfTooShort(recording); @@ -636,6 +641,7 @@ public class NextGenLocalRecorder implements Recorder { @Override public void rerunPostProcessing(Recording recording) { + recording.setPostProcessedFile(null); List recordings = recordingManager.getAll(); for (Recording other : recordings) { if(other.equals(recording)) { diff --git a/common/src/main/java/ctbrec/recorder/RecordingManager.java b/common/src/main/java/ctbrec/recorder/RecordingManager.java index a710344d..e746b011 100644 --- a/common/src/main/java/ctbrec/recorder/RecordingManager.java +++ b/common/src/main/java/ctbrec/recorder/RecordingManager.java @@ -25,6 +25,7 @@ import ctbrec.Config; import ctbrec.Model; import ctbrec.Recording; import ctbrec.Recording.State; +import ctbrec.io.FileJsonAdapter; import ctbrec.io.InstantJsonAdapter; import ctbrec.io.ModelJsonAdapter; import ctbrec.sites.Site; @@ -43,6 +44,7 @@ public class RecordingManager { moshi = new Moshi.Builder() .add(Model.class, new ModelJsonAdapter(sites)) .add(Instant.class, new InstantJsonAdapter()) + .add(File.class, new FileJsonAdapter()) .build(); adapter = moshi.adapter(Recording.class).indent(" "); @@ -122,20 +124,36 @@ public class RecordingManager { recording.setStatus(State.DELETING); File recordingsDir = new File(config.getSettings().recordingsDir); File path = new File(recordingsDir, recording.getPath()); + boolean isFile = path.isFile(); LOG.debug("Deleting {}", path); // delete the video files - if (path.isFile()) { + if (isFile) { Files.delete(path.toPath()); - deleteEmptyParents(path.getParentFile()); } else { deleteDirectory(path); - deleteEmptyParents(path); + } + + // delete files associated with this recording + for (String associated : recording.getAssociatedFiles()) { + File f = new File(associated); + if (f.isFile()) { + Files.delete(f.toPath()); + } else { + deleteDirectory(f); + } } // delete the meta data Files.deleteIfExists(new File(recording.getMetaDataFile()).toPath()); + // delete empty parent files + if (isFile) { + deleteEmptyParents(path.getParentFile()); + } else { + deleteEmptyParents(path); + } + // remove from data structure recordings.remove(recording); recording.setStatus(State.DELETED); diff --git a/common/src/main/java/ctbrec/recorder/postprocessing/AbstractPostProcessor.java b/common/src/main/java/ctbrec/recorder/postprocessing/AbstractPostProcessor.java new file mode 100644 index 00000000..4d26953c --- /dev/null +++ b/common/src/main/java/ctbrec/recorder/postprocessing/AbstractPostProcessor.java @@ -0,0 +1,24 @@ +package ctbrec.recorder.postprocessing; + +import java.util.HashMap; +import java.util.Map; + +public abstract class AbstractPostProcessor implements PostProcessor { + + private Map config = new HashMap<>(); + + @Override + public Map getConfig() { + return config; + } + + @Override + public void setConfig(Map conf) { + this.config = conf; + } + + @Override + public String toString() { + return getName(); + } +} diff --git a/common/src/main/java/ctbrec/recorder/postprocessing/PostProcessor.java b/common/src/main/java/ctbrec/recorder/postprocessing/PostProcessor.java new file mode 100644 index 00000000..cc366520 --- /dev/null +++ b/common/src/main/java/ctbrec/recorder/postprocessing/PostProcessor.java @@ -0,0 +1,16 @@ +package ctbrec.recorder.postprocessing; + +import java.io.IOException; +import java.io.Serializable; +import java.util.Map; + +import ctbrec.Recording; + +public interface PostProcessor extends Serializable { + String getName(); + + void postprocess(Recording rec) throws IOException, InterruptedException; + + Map getConfig(); + void setConfig(Map conf); +} diff --git a/common/src/main/java/ctbrec/recorder/postprocessing/RecordingRenamer.java b/common/src/main/java/ctbrec/recorder/postprocessing/RecordingRenamer.java new file mode 100644 index 00000000..ed771e0d --- /dev/null +++ b/common/src/main/java/ctbrec/recorder/postprocessing/RecordingRenamer.java @@ -0,0 +1,16 @@ +package ctbrec.recorder.postprocessing; + +import ctbrec.Recording; + +public class RecordingRenamer extends AbstractPostProcessor { + + @Override + public String getName() { + return "rename"; + } + + @Override + public void postprocess(Recording rec) { + // TODO rename + } +} diff --git a/common/src/main/java/ctbrec/recorder/postprocessing/Remuxer.java b/common/src/main/java/ctbrec/recorder/postprocessing/Remuxer.java new file mode 100644 index 00000000..4ca87c95 --- /dev/null +++ b/common/src/main/java/ctbrec/recorder/postprocessing/Remuxer.java @@ -0,0 +1,82 @@ +package ctbrec.recorder.postprocessing; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.file.Files; +import java.util.Arrays; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import ctbrec.OS; +import ctbrec.Recording; +import ctbrec.io.StreamRedirectThread; +import ctbrec.recorder.download.ProcessExitedUncleanException; + +public class Remuxer extends AbstractPostProcessor { + + private static final Logger LOG = LoggerFactory.getLogger(Remuxer.class); + + public static final String FFMPEG_ARGS = "ffmpeg.args"; + public static final String FILE_EXT = "file.ext"; + + @Override + public String getName() { + return "remux / transcode"; + } + + @Override + public void postprocess(Recording rec) throws IOException, InterruptedException { + String fileExt = getConfig().get(FILE_EXT); + String[] args = getConfig().get(FFMPEG_ARGS).split(" "); + String[] argsPlusFile = new String[args.length + 3]; + int i = 0; + argsPlusFile[i++] = "-i"; + argsPlusFile[i++] = rec.getPostProcessedFile().getAbsolutePath(); + System.arraycopy(args, 0, argsPlusFile, i, args.length); + File remuxedFile = new File(rec.getPostProcessedFile().getAbsolutePath() + '.' + fileExt); + argsPlusFile[argsPlusFile.length - 1] = remuxedFile.getAbsolutePath(); + String[] cmdline = OS.getFFmpegCommand(argsPlusFile); + LOG.debug(Arrays.toString(cmdline)); + Process ffmpeg = Runtime.getRuntime().exec(cmdline, new String[0], rec.getPostProcessedFile().getParentFile()); + setupLogging(ffmpeg, rec); + rec.setPostProcessedFile(remuxedFile); + rec.getAssociatedFiles().add(remuxedFile.getAbsolutePath()); + } + + private void setupLogging(Process ffmpeg, Recording rec) throws IOException, InterruptedException { + int exitCode = 1; + File video = rec.getPostProcessedFile(); + File ffmpegLog = new File(video.getParentFile(), video.getName() + ".ffmpeg.log"); + try (FileOutputStream mergeLogStream = new FileOutputStream(ffmpegLog)) { + Thread stdout = new Thread(new StreamRedirectThread(ffmpeg.getInputStream(), mergeLogStream)); + Thread stderr = new Thread(new StreamRedirectThread(ffmpeg.getErrorStream(), mergeLogStream)); + stdout.start(); + stderr.start(); + exitCode = ffmpeg.waitFor(); + LOG.debug("FFmpeg exited with code {}", exitCode); + stdout.join(); + stderr.join(); + mergeLogStream.flush(); + } + if (exitCode != 1) { + if (ffmpegLog.exists()) { + Files.delete(ffmpegLog.toPath()); + } + } else { + rec.getAssociatedFiles().add(ffmpegLog.getAbsolutePath()); + LOG.info("FFmpeg exit code was {}. Logfile: {}", exitCode, ffmpegLog.getAbsolutePath()); + throw new ProcessExitedUncleanException("FFmpeg exit code was " + exitCode); + } + } + + @Override + public String toString() { + String s = getName(); + if(getConfig().containsKey(FFMPEG_ARGS)) { + s += " [" + getConfig().get(FFMPEG_ARGS) + ']'; + } + return s; + } +} From bdcf1bee006e046851a4dcbd8d642542b9b8481b Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Sun, 23 Aug 2020 13:37:59 +0200 Subject: [PATCH 03/33] Implement Renamer post-processor --- .../ui/settings/RenamerPaneFactory.java | 24 ++++ .../postprocessing/RecordingRenamer.java | 16 --- .../recorder/postprocessing/Renamer.java | 116 ++++++++++++++++++ 3 files changed, 140 insertions(+), 16 deletions(-) create mode 100644 client/src/main/java/ctbrec/ui/settings/RenamerPaneFactory.java delete mode 100644 common/src/main/java/ctbrec/recorder/postprocessing/RecordingRenamer.java create mode 100644 common/src/main/java/ctbrec/recorder/postprocessing/Renamer.java diff --git a/client/src/main/java/ctbrec/ui/settings/RenamerPaneFactory.java b/client/src/main/java/ctbrec/ui/settings/RenamerPaneFactory.java new file mode 100644 index 00000000..2140b75a --- /dev/null +++ b/client/src/main/java/ctbrec/ui/settings/RenamerPaneFactory.java @@ -0,0 +1,24 @@ +package ctbrec.ui.settings; + +import ctbrec.recorder.postprocessing.PostProcessor; +import ctbrec.recorder.postprocessing.Renamer; +import ctbrec.ui.settings.api.Category; +import ctbrec.ui.settings.api.Preferences; +import ctbrec.ui.settings.api.Setting; +import javafx.beans.property.SimpleStringProperty; + +public class RenamerPaneFactory extends AbstractPostProcessingPaneFactory { + + @Override + public Preferences doCreatePostProcessorPane(PostProcessor pp) { + SimpleStringProperty fileTemplate = new SimpleStringProperty(null, Renamer.FILE_NAME_TEMPLATE, pp.getConfig().getOrDefault(Renamer.FILE_NAME_TEMPLATE, Renamer.DEFAULT)); + properties.add(fileTemplate); + + return Preferences.of(new MapPreferencesStorage(), + Category.of(pp.getName(), + Setting.of("File name", fileTemplate) + ) + ); + } + +} diff --git a/common/src/main/java/ctbrec/recorder/postprocessing/RecordingRenamer.java b/common/src/main/java/ctbrec/recorder/postprocessing/RecordingRenamer.java deleted file mode 100644 index ed771e0d..00000000 --- a/common/src/main/java/ctbrec/recorder/postprocessing/RecordingRenamer.java +++ /dev/null @@ -1,16 +0,0 @@ -package ctbrec.recorder.postprocessing; - -import ctbrec.Recording; - -public class RecordingRenamer extends AbstractPostProcessor { - - @Override - public String getName() { - return "rename"; - } - - @Override - public void postprocess(Recording rec) { - // TODO rename - } -} diff --git a/common/src/main/java/ctbrec/recorder/postprocessing/Renamer.java b/common/src/main/java/ctbrec/recorder/postprocessing/Renamer.java new file mode 100644 index 00000000..af98ad55 --- /dev/null +++ b/common/src/main/java/ctbrec/recorder/postprocessing/Renamer.java @@ -0,0 +1,116 @@ +package ctbrec.recorder.postprocessing; +import static java.util.Optional.*; + +import java.io.File; +import java.io.IOException; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.util.Locale; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.io.Files; + +import ctbrec.Recording; + +public class Renamer extends AbstractPostProcessor { + + private static final Logger LOG = LoggerFactory.getLogger(Renamer.class); + public static final String FILE_NAME_TEMPLATE = "filename.template"; + public static final String DEFAULT = "${modelSanitizedName}_${localDateTime}.${fileSuffix}"; + + @SuppressWarnings("unused") + private String[] placeHolders = { + "${modelName}", + "${modelDisplayName}", + "${modelSanitizedName}", + "${siteName}", + "${siteSanitizedName}", + "${utcDateTime}", + "${localDateTime}", + "${epochSeconds}", + "${fileSuffix}", + "${modelNotes}" + }; + + @Override + public String getName() { + return "rename"; + } + + @Override + public void postprocess(Recording rec) throws IOException { + String filenameTemplate = getConfig().getOrDefault(FILE_NAME_TEMPLATE, DEFAULT); + String filename = filenameTemplate + .replace("${modelName}", ofNullable(rec.getModel().getName()).orElse("modelName")) + .replace("${modelDisplayName}", ofNullable(rec.getModel().getDisplayName()).orElse("displayName")) + .replace("${modelSanitizedName}", ofNullable(rec.getModel().getSanitizedNamed()).orElse("sanitizedName")) + .replace("${siteName}", rec.getModel().getSite().getName()) + .replace("${siteSanitizedName}", getSanitizedSiteName(rec)) + .replace("${fileSuffix}", getFileSuffix(rec)) + .replace("${epochSeconds}", Long.toString(rec.getStartDate().getEpochSecond())) + ; + + filename = replaceUtcDateTime(rec, filename); + filename = replaceLocalDateTime(rec, filename); + + File src = rec.getPostProcessedFile(); + File target = new File(src.getParentFile(), filename); + LOG.info("Renaming {} to {}", src.getName(), target.getName()); + Files.copy(rec.getPostProcessedFile(), target); + //Files.move(rec.getPostProcessedFile(), target); + rec.setPostProcessedFile(target); + rec.getAssociatedFiles().add(target.getAbsolutePath()); + } + + private String replaceUtcDateTime(Recording rec, String filename) { + return replaceDateTime(rec, filename, "utcDateTime", ZoneOffset.UTC); + } + + private String replaceLocalDateTime(Recording rec, String filename) { + return replaceDateTime(rec, filename, "localDateTime", ZoneId.systemDefault()); + } + + private String replaceDateTime(Recording rec, String filename, String placeHolder, ZoneId zone) { + String pattern = "yyyy-mm-dd_HH-mm-ss"; + Matcher m = Pattern.compile("\\$\\{" + placeHolder + "(?:\\((.*?)\\))?\\}").matcher(filename); + if (m.find()) { + String p = m.group(1); + if (p != null) { + pattern = p; + } + } + String formattedDate = getDateTime(rec, pattern, zone); + return m.replaceAll(formattedDate); + } + + private String getDateTime(Recording rec, String pattern, ZoneId zone) { + return DateTimeFormatter.ofPattern(pattern) + .withLocale(Locale.getDefault()) + .withZone(zone) + .format(rec.getStartDate()); + } + + private CharSequence getFileSuffix(Recording rec) { + String filename = rec.getPostProcessedFile().getName(); + return filename.substring(filename.lastIndexOf('.') + 1); + } + + + private CharSequence getSanitizedSiteName(Recording rec) { + return rec.getModel().getSite().getName().replace(' ', '_').replace('\\', '_').replace('/', '_'); + } + + @Override + public String toString() { + String s = getName(); + if (getConfig().containsKey(FILE_NAME_TEMPLATE)) { + s += " [" + getConfig().get(FILE_NAME_TEMPLATE) + ']'; + } + return s; + } +} From 89fa681a598c619f60ba8f40da2dd18b3523a22e Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Sun, 23 Aug 2020 13:38:18 +0200 Subject: [PATCH 04/33] Increase log level --- .../src/main/java/ctbrec/recorder/postprocessing/Remuxer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/src/main/java/ctbrec/recorder/postprocessing/Remuxer.java b/common/src/main/java/ctbrec/recorder/postprocessing/Remuxer.java index 4ca87c95..0aa5051a 100644 --- a/common/src/main/java/ctbrec/recorder/postprocessing/Remuxer.java +++ b/common/src/main/java/ctbrec/recorder/postprocessing/Remuxer.java @@ -38,7 +38,7 @@ public class Remuxer extends AbstractPostProcessor { File remuxedFile = new File(rec.getPostProcessedFile().getAbsolutePath() + '.' + fileExt); argsPlusFile[argsPlusFile.length - 1] = remuxedFile.getAbsolutePath(); String[] cmdline = OS.getFFmpegCommand(argsPlusFile); - LOG.debug(Arrays.toString(cmdline)); + LOG.info(Arrays.toString(cmdline)); Process ffmpeg = Runtime.getRuntime().exec(cmdline, new String[0], rec.getPostProcessedFile().getParentFile()); setupLogging(ffmpeg, rec); rec.setPostProcessedFile(remuxedFile); From 000ea174c2322b3f8bd0ba22c0c28b9a2937b61a Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Sun, 23 Aug 2020 13:38:43 +0200 Subject: [PATCH 05/33] Add renamer --- .../ctbrec/ui/settings/PostProcessingDialogFactory.java | 7 ++++++- .../java/ctbrec/ui/settings/PostProcessingStepPanel.java | 4 ++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/client/src/main/java/ctbrec/ui/settings/PostProcessingDialogFactory.java b/client/src/main/java/ctbrec/ui/settings/PostProcessingDialogFactory.java index f3bb74ed..2d67dbb5 100644 --- a/client/src/main/java/ctbrec/ui/settings/PostProcessingDialogFactory.java +++ b/client/src/main/java/ctbrec/ui/settings/PostProcessingDialogFactory.java @@ -8,16 +8,19 @@ import java.util.Map; import ctbrec.Config; import ctbrec.recorder.postprocessing.PostProcessor; import ctbrec.recorder.postprocessing.Remuxer; +import ctbrec.recorder.postprocessing.Renamer; import ctbrec.ui.controls.Dialogs; import ctbrec.ui.settings.api.Preferences; import javafx.collections.ObservableList; import javafx.scene.Scene; +import javafx.scene.layout.Region; public class PostProcessingDialogFactory { static Map, Class> ppToDialogMap = new HashMap<>(); static { ppToDialogMap.put(Remuxer.class, RemuxerPaneFactory.class); + ppToDialogMap.put(Renamer.class, RenamerPaneFactory.class); } private PostProcessingDialogFactory() { @@ -35,7 +38,9 @@ public class PostProcessingDialogFactory { boolean ok; try { Preferences preferences = createPreferences(pp); - ok = Dialogs.showCustomInput(scene, "Configure " + pp.getName(), preferences.getView(false)); + Region view = preferences.getView(false); + view.setMinWidth(600); + ok = Dialogs.showCustomInput(scene, "Configure " + pp.getName(), view); if (ok) { preferences.save(); if (newEntry) { diff --git a/client/src/main/java/ctbrec/ui/settings/PostProcessingStepPanel.java b/client/src/main/java/ctbrec/ui/settings/PostProcessingStepPanel.java index 207515e7..121c1ba3 100644 --- a/client/src/main/java/ctbrec/ui/settings/PostProcessingStepPanel.java +++ b/client/src/main/java/ctbrec/ui/settings/PostProcessingStepPanel.java @@ -7,7 +7,7 @@ import java.util.Optional; import ctbrec.Config; import ctbrec.recorder.postprocessing.PostProcessor; -import ctbrec.recorder.postprocessing.RecordingRenamer; +import ctbrec.recorder.postprocessing.Renamer; import ctbrec.recorder.postprocessing.Remuxer; import ctbrec.ui.controls.Dialogs; import javafx.collections.FXCollections; @@ -27,7 +27,7 @@ public class PostProcessingStepPanel extends GridPane { private Config config; - private static final Class[] POST_PROCESSOR_CLASSES = new Class[] { Remuxer.class, RecordingRenamer.class }; + private static final Class[] POST_PROCESSOR_CLASSES = new Class[] { Remuxer.class, Renamer.class }; ListView stepListView; ObservableList stepList; From e3819b823d223ab65f3bab3bded7cf1e1542f1ae Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Sun, 23 Aug 2020 13:38:55 +0200 Subject: [PATCH 06/33] Update changelog --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5fc1f8af..00a88bcb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +3.10.0 +======================== +* New post-processing +* Fix: MV Live models with spaces in the name not indicated as recording + 3.9.0 ======================== * Added support for Manyvids Live. From 1034488e9489ed04438ca6e90c98fe65ca8e9a61 Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Mon, 24 Aug 2020 15:15:34 +0200 Subject: [PATCH 07/33] Make RecorderServlet and RemoteRecorder compatible to new recording fields --- common/src/main/java/ctbrec/recorder/RemoteRecorder.java | 7 ++++++- .../main/java/ctbrec/recorder/postprocessing/Renamer.java | 4 ++-- .../main/java/ctbrec/recorder/server/RecorderServlet.java | 3 +++ 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/common/src/main/java/ctbrec/recorder/RemoteRecorder.java b/common/src/main/java/ctbrec/recorder/RemoteRecorder.java index 5cec3296..37dcb6ec 100644 --- a/common/src/main/java/ctbrec/recorder/RemoteRecorder.java +++ b/common/src/main/java/ctbrec/recorder/RemoteRecorder.java @@ -27,6 +27,7 @@ import ctbrec.event.EventBusHolder; import ctbrec.event.NoSpaceLeftEvent; import ctbrec.event.RecordingStateChangedEvent; import ctbrec.io.BandwidthMeter; +import ctbrec.io.FileJsonAdapter; import ctbrec.io.HttpClient; import ctbrec.io.HttpException; import ctbrec.io.InstantJsonAdapter; @@ -45,7 +46,11 @@ public class RemoteRecorder implements Recorder { private static final Logger LOG = LoggerFactory.getLogger(RemoteRecorder.class); public static final MediaType JSON = MediaType.parse("application/json; charset=utf-8"); - private Moshi moshi = new Moshi.Builder().add(Instant.class, new InstantJsonAdapter()).add(Model.class, new ModelJsonAdapter()).build(); + private Moshi moshi = new Moshi.Builder() + .add(Instant.class, new InstantJsonAdapter()) + .add(Model.class, new ModelJsonAdapter()) + .add(File.class, new FileJsonAdapter()) + .build(); private JsonAdapter modelListResponseAdapter = moshi.adapter(ModelListResponse.class); private JsonAdapter recordingListResponseAdapter = moshi.adapter(RecordingListResponse.class); private JsonAdapter modelRequestAdapter = moshi.adapter(ModelRequest.class); diff --git a/common/src/main/java/ctbrec/recorder/postprocessing/Renamer.java b/common/src/main/java/ctbrec/recorder/postprocessing/Renamer.java index af98ad55..291cecce 100644 --- a/common/src/main/java/ctbrec/recorder/postprocessing/Renamer.java +++ b/common/src/main/java/ctbrec/recorder/postprocessing/Renamer.java @@ -32,7 +32,7 @@ public class Renamer extends AbstractPostProcessor { "${siteSanitizedName}", "${utcDateTime}", "${localDateTime}", - "${epochSeconds}", + "${epochSecond}", "${fileSuffix}", "${modelNotes}" }; @@ -52,7 +52,7 @@ public class Renamer extends AbstractPostProcessor { .replace("${siteName}", rec.getModel().getSite().getName()) .replace("${siteSanitizedName}", getSanitizedSiteName(rec)) .replace("${fileSuffix}", getFileSuffix(rec)) - .replace("${epochSeconds}", Long.toString(rec.getStartDate().getEpochSecond())) + .replace("${epochSecond}", Long.toString(rec.getStartDate().getEpochSecond())) ; filename = replaceUtcDateTime(rec, filename); diff --git a/server/src/main/java/ctbrec/recorder/server/RecorderServlet.java b/server/src/main/java/ctbrec/recorder/server/RecorderServlet.java index d42478bd..a7919586 100644 --- a/server/src/main/java/ctbrec/recorder/server/RecorderServlet.java +++ b/server/src/main/java/ctbrec/recorder/server/RecorderServlet.java @@ -2,6 +2,7 @@ package ctbrec.recorder.server; import static javax.servlet.http.HttpServletResponse.*; +import java.io.File; import java.io.IOException; import java.security.InvalidKeyException; import java.security.NoSuchAlgorithmException; @@ -25,6 +26,7 @@ import ctbrec.Config; import ctbrec.Model; import ctbrec.Recording; import ctbrec.io.BandwidthMeter; +import ctbrec.io.FileJsonAdapter; import ctbrec.io.InstantJsonAdapter; import ctbrec.io.ModelJsonAdapter; import ctbrec.recorder.Recorder; @@ -63,6 +65,7 @@ public class RecorderServlet extends AbstractCtbrecServlet { Moshi moshi = new Moshi.Builder() .add(Instant.class, new InstantJsonAdapter()) .add(Model.class, new ModelJsonAdapter(sites)) + .add(File.class, new FileJsonAdapter()) .build(); JsonAdapter requestAdapter = moshi.adapter(Request.class); Request request = requestAdapter.fromJson(json); From 4f7d080f56a1bff2fe73ade658b196f87728d86f Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Sun, 30 Aug 2020 13:41:03 +0200 Subject: [PATCH 08/33] Add more pp steps --- .../AbstractPostProcessingPaneFactory.java | 150 +++++++++++++++++- .../settings/PostProcessingDialogFactory.java | 31 ++-- .../ui/settings/PostProcessingStepPanel.java | 15 +- .../ctbrec/recorder/postprocessing/Copy.java | 39 +++++ .../recorder/postprocessing/Renamer.java | 3 +- 5 files changed, 219 insertions(+), 19 deletions(-) create mode 100644 common/src/main/java/ctbrec/recorder/postprocessing/Copy.java diff --git a/client/src/main/java/ctbrec/ui/settings/AbstractPostProcessingPaneFactory.java b/client/src/main/java/ctbrec/ui/settings/AbstractPostProcessingPaneFactory.java index 37515cd6..76c2fdc6 100644 --- a/client/src/main/java/ctbrec/ui/settings/AbstractPostProcessingPaneFactory.java +++ b/client/src/main/java/ctbrec/ui/settings/AbstractPostProcessingPaneFactory.java @@ -4,16 +4,38 @@ import java.io.IOException; import java.util.HashSet; import java.util.Set; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import ctbrec.recorder.postprocessing.PostProcessor; +import ctbrec.ui.controls.DirectorySelectionBox; +import ctbrec.ui.controls.ProgramSelectionBox; +import ctbrec.ui.settings.api.ExclusiveSelectionProperty; import ctbrec.ui.settings.api.Preferences; import ctbrec.ui.settings.api.PreferencesStorage; import ctbrec.ui.settings.api.Setting; +import ctbrec.ui.settings.api.SimpleDirectoryProperty; +import ctbrec.ui.settings.api.SimpleFileProperty; +import javafx.beans.property.BooleanProperty; +import javafx.beans.property.IntegerProperty; +import javafx.beans.property.ListProperty; +import javafx.beans.property.LongProperty; import javafx.beans.property.Property; +import javafx.beans.property.StringProperty; +import javafx.geometry.Insets; import javafx.scene.Node; +import javafx.scene.control.CheckBox; +import javafx.scene.control.ComboBox; +import javafx.scene.control.Label; +import javafx.scene.control.RadioButton; import javafx.scene.control.TextField; +import javafx.scene.control.ToggleGroup; +import javafx.scene.layout.HBox; +import javafx.util.converter.NumberStringConverter; public abstract class AbstractPostProcessingPaneFactory { + private static final Logger LOG = LoggerFactory.getLogger(AbstractPostProcessingPaneFactory.class); private PostProcessor pp; Set> properties = new HashSet<>(); @@ -31,6 +53,7 @@ public abstract class AbstractPostProcessingPaneFactory { for (Property property : properties) { String key = property.getName(); Object value = preferences.getSetting(key).get().getProperty().getValue(); + LOG.debug("{}={}", key, value.toString()); pp.getConfig().put(key, value.toString()); } } @@ -40,13 +63,132 @@ public abstract class AbstractPostProcessingPaneFactory { // no op } - @SuppressWarnings("unchecked") @Override public Node createGui(Setting setting) throws Exception { - TextField input = new TextField(); - input.textProperty().bindBidirectional(setting.getProperty()); - return input; + Property prop = setting.getProperty(); + if (prop instanceof ExclusiveSelectionProperty) { + return createRadioGroup(setting); + } else if (prop instanceof SimpleDirectoryProperty) { + return createDirectorySelector(setting); + } else if (prop instanceof SimpleFileProperty) { + return createFileSelector(setting); + } else if (prop instanceof IntegerProperty) { + return createIntegerProperty(setting); + } else if (prop instanceof LongProperty) { + return createLongProperty(setting); + } else if (prop instanceof BooleanProperty) { + return createBooleanProperty(setting); + } else if (prop instanceof ListProperty) { + return createComboBox(setting); + } else if (prop instanceof StringProperty) { + return createStringProperty(setting); + } else { + return new Label("Unsupported Type for key " + setting.getKey() + ": " + setting.getProperty()); + } } + } + private Node createRadioGroup(Setting setting) { + ExclusiveSelectionProperty prop = (ExclusiveSelectionProperty) setting.getProperty(); + ToggleGroup toggleGroup = new ToggleGroup(); + RadioButton optionA = new RadioButton(prop.getOptionA()); + optionA.setSelected(prop.getValue()); + optionA.setToggleGroup(toggleGroup); + RadioButton optionB = new RadioButton(prop.getOptionB()); + optionB.setSelected(!optionA.isSelected()); + optionB.setToggleGroup(toggleGroup); + optionA.selectedProperty().bindBidirectional(prop); + HBox row = new HBox(); + row.getChildren().addAll(optionA, optionB); + HBox.setMargin(optionA, new Insets(5)); + HBox.setMargin(optionB, new Insets(5)); + return row; + } + + private Node createFileSelector(Setting setting) { + ProgramSelectionBox programSelector = new ProgramSelectionBox(""); + // programSelector.fileProperty().addListener((obs, o, n) -> saveValue(() -> { + // String path = n; + // Field field = Settings.class.getField(setting.getKey()); + // String oldValue = (String) field.get(settings); + // if (!Objects.equals(path, oldValue)) { + // field.set(settings, path); + // config.save(); + // } + // })); + StringProperty property = (StringProperty) setting.getProperty(); + programSelector.fileProperty().bindBidirectional(property); + return programSelector; + } + + private Node createDirectorySelector(Setting setting) { + DirectorySelectionBox directorySelector = new DirectorySelectionBox(""); + directorySelector.prefWidth(400); + // directorySelector.fileProperty().addListener((obs, o, n) -> saveValue(() -> { + // String path = n; + // Field field = Settings.class.getField(setting.getKey()); + // String oldValue = (String) field.get(settings); + // if (!Objects.equals(path, oldValue)) { + // field.set(settings, path); + // config.save(); + // } + // })); + StringProperty property = (StringProperty) setting.getProperty(); + directorySelector.fileProperty().bindBidirectional(property); + return directorySelector; + } + + @SuppressWarnings("unchecked") + private Node createStringProperty(Setting setting) { + TextField ctrl = new TextField(); + ctrl.textProperty().bindBidirectional(setting.getProperty()); + return ctrl; + } + + @SuppressWarnings("unchecked") + private Node createIntegerProperty(Setting setting) { + TextField ctrl = new TextField(); + Property prop = setting.getProperty(); + ctrl.textProperty().bindBidirectional(prop, new NumberStringConverter()); + return ctrl; + } + + @SuppressWarnings("unchecked") + private Node createLongProperty(Setting setting) { + TextField ctrl = new TextField(); + Property prop = setting.getProperty(); + ctrl.textProperty().bindBidirectional(prop, new NumberStringConverter()); + return ctrl; + } + + private Node createBooleanProperty(Setting setting) { + CheckBox ctrl = new CheckBox(); + BooleanProperty prop = (BooleanProperty) setting.getProperty(); + ctrl.selectedProperty().bindBidirectional(prop); + return ctrl; + } + + @SuppressWarnings({ "rawtypes", "unchecked" }) + private Node createComboBox(Setting setting) throws NoSuchFieldException, IllegalAccessException { + ListProperty listProp = (ListProperty) setting.getProperty(); + ComboBox comboBox = new ComboBox(listProp); + // Field field = Settings.class.getField(setting.getKey()); + // Object value = field.get(Config.getInstance().getSettings()); + // if (StringUtil.isNotBlank(value.toString())) { + // if (setting.getConverter() != null) { + // comboBox.getSelectionModel().select(setting.getConverter().convertTo(value)); + // } else { + // comboBox.getSelectionModel().select(value); + // } + // } + // comboBox.valueProperty().addListener((obs, oldV, newV) -> saveValue(() -> { + // if (setting.getConverter() != null) { + // field.set(settings, setting.getConverter().convertFrom(newV)); + // } else { + // field.set(settings, newV); + // } + // config.save(); + // })); + return comboBox; } } diff --git a/client/src/main/java/ctbrec/ui/settings/PostProcessingDialogFactory.java b/client/src/main/java/ctbrec/ui/settings/PostProcessingDialogFactory.java index 2d67dbb5..094f4c8c 100644 --- a/client/src/main/java/ctbrec/ui/settings/PostProcessingDialogFactory.java +++ b/client/src/main/java/ctbrec/ui/settings/PostProcessingDialogFactory.java @@ -4,6 +4,7 @@ import java.io.IOException; import java.lang.reflect.InvocationTargetException; import java.util.HashMap; import java.util.Map; +import java.util.Optional; import ctbrec.Config; import ctbrec.recorder.postprocessing.PostProcessor; @@ -37,15 +38,19 @@ public class PostProcessingDialogFactory { private static void openDialog(PostProcessor pp, Config config, Scene scene, ObservableList stepList, boolean newEntry) { boolean ok; try { - Preferences preferences = createPreferences(pp); - Region view = preferences.getView(false); - view.setMinWidth(600); - ok = Dialogs.showCustomInput(scene, "Configure " + pp.getName(), view); - if (ok) { - preferences.save(); - if (newEntry) { - stepList.add(pp); + Optional preferences = createPreferences(pp); + if(preferences.isPresent()) { + Region view = preferences.get().getView(false); + view.setMinWidth(600); + ok = Dialogs.showCustomInput(scene, "Configure " + pp.getName(), view); + if (ok) { + preferences.get().save(); + if (newEntry) { + stepList.add(pp); + } } + } else if (newEntry) { + stepList.add(pp); } } catch (NoSuchMethodException | SecurityException | IllegalAccessException | IllegalArgumentException | InvocationTargetException | InstantiationException | IOException e) { @@ -53,10 +58,14 @@ public class PostProcessingDialogFactory { } } - private static Preferences createPreferences(PostProcessor pp) throws InstantiationException, IllegalAccessException, IllegalArgumentException, + private static Optional createPreferences(PostProcessor pp) throws InstantiationException, IllegalAccessException, IllegalArgumentException, InvocationTargetException, NoSuchMethodException, SecurityException { Class paneFactoryClass = ppToDialogMap.get(pp.getClass()); - AbstractPostProcessingPaneFactory factory = (AbstractPostProcessingPaneFactory) paneFactoryClass.getDeclaredConstructor().newInstance(); - return factory.createPostProcessorPane(pp); + if (paneFactoryClass != null) { + AbstractPostProcessingPaneFactory factory = (AbstractPostProcessingPaneFactory) paneFactoryClass.getDeclaredConstructor().newInstance(); + return Optional.of(factory.createPostProcessorPane(pp)); + } else { + return Optional.empty(); + } } } diff --git a/client/src/main/java/ctbrec/ui/settings/PostProcessingStepPanel.java b/client/src/main/java/ctbrec/ui/settings/PostProcessingStepPanel.java index 121c1ba3..91e5d367 100644 --- a/client/src/main/java/ctbrec/ui/settings/PostProcessingStepPanel.java +++ b/client/src/main/java/ctbrec/ui/settings/PostProcessingStepPanel.java @@ -6,9 +6,10 @@ import java.lang.reflect.InvocationTargetException; import java.util.Optional; import ctbrec.Config; +import ctbrec.recorder.postprocessing.Copy; import ctbrec.recorder.postprocessing.PostProcessor; -import ctbrec.recorder.postprocessing.Renamer; import ctbrec.recorder.postprocessing.Remuxer; +import ctbrec.recorder.postprocessing.Renamer; import ctbrec.ui.controls.Dialogs; import javafx.collections.FXCollections; import javafx.collections.ListChangeListener; @@ -27,7 +28,7 @@ public class PostProcessingStepPanel extends GridPane { private Config config; - private static final Class[] POST_PROCESSOR_CLASSES = new Class[] { Remuxer.class, Renamer.class }; + private static final Class[] POST_PROCESSOR_CLASSES = new Class[] { Copy.class, Remuxer.class, Renamer.class }; ListView stepListView; ObservableList stepList; @@ -121,10 +122,19 @@ public class PostProcessingStepPanel extends GridPane { Optional result = choice.showAndWait(); result.ifPresent(pp -> PostProcessingDialogFactory.openNewDialog(pp, config, getScene(), stepList)); + saveConfig(); }); return add; } + private void saveConfig() { + try { + config.save(); + } catch (IOException e) { + Dialogs.showError("Post-Processing", "Couldn't save post-processing step", e); + } + } + private PostProcessor[] createOptions() { try { PostProcessor[] options = new PostProcessor[POST_PROCESSOR_CLASSES.length]; @@ -159,6 +169,7 @@ public class PostProcessingStepPanel extends GridPane { PostProcessor selectedItem = stepListView.getSelectionModel().getSelectedItem(); PostProcessingDialogFactory.openEditDialog(selectedItem, config, getScene(), stepList); stepListView.refresh(); + saveConfig(); }); return edit; } diff --git a/common/src/main/java/ctbrec/recorder/postprocessing/Copy.java b/common/src/main/java/ctbrec/recorder/postprocessing/Copy.java new file mode 100644 index 00000000..88dcf452 --- /dev/null +++ b/common/src/main/java/ctbrec/recorder/postprocessing/Copy.java @@ -0,0 +1,39 @@ +package ctbrec.recorder.postprocessing; + +import java.io.File; +import java.io.IOException; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.io.Files; + +import ctbrec.Recording; + +public class Copy extends AbstractPostProcessor { + + private static final transient Logger LOG = LoggerFactory.getLogger(Copy.class); + + @Override + public String getName() { + return "create a copy"; + } + + @Override + public void postprocess(Recording rec) throws IOException, InterruptedException { + File orig = rec.getPostProcessedFile(); + String copyFilename = getFilenameForCopy(orig); + File copy = new File(orig.getParentFile(), copyFilename); + LOG.info("Creating a copy {}", copy); + Files.copy(rec.getPostProcessedFile(), copy); + rec.setPostProcessedFile(copy); + rec.getAssociatedFiles().add(copy.getAbsolutePath()); + } + + private String getFilenameForCopy(File orig) { + String filename = orig.getName(); + String name = filename.substring(0, filename.lastIndexOf('.')); + String ext = filename.substring(filename.lastIndexOf('.') + 1); + return name + "_copy." + ext; + } +} diff --git a/common/src/main/java/ctbrec/recorder/postprocessing/Renamer.java b/common/src/main/java/ctbrec/recorder/postprocessing/Renamer.java index 291cecce..b542cb24 100644 --- a/common/src/main/java/ctbrec/recorder/postprocessing/Renamer.java +++ b/common/src/main/java/ctbrec/recorder/postprocessing/Renamer.java @@ -61,8 +61,7 @@ public class Renamer extends AbstractPostProcessor { File src = rec.getPostProcessedFile(); File target = new File(src.getParentFile(), filename); LOG.info("Renaming {} to {}", src.getName(), target.getName()); - Files.copy(rec.getPostProcessedFile(), target); - //Files.move(rec.getPostProcessedFile(), target); + Files.move(rec.getPostProcessedFile(), target); rec.setPostProcessedFile(target); rec.getAssociatedFiles().add(target.getAbsolutePath()); } From 4f526fd13e2f21cecf4d4191b15c68485e16b5f0 Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Sun, 13 Sep 2020 17:34:58 +0200 Subject: [PATCH 09/33] Store absolute path in metadata file instead of path relative to rec dir --- .../main/java/ctbrec/ui/JavaFxRecording.java | 18 +- client/src/main/java/ctbrec/ui/Player.java | 4 +- .../java/ctbrec/ui/tabs/RecordingsTab.java | 15 +- common/src/main/java/ctbrec/Recording.java | 43 +++- .../ctbrec/recorder/NextGenLocalRecorder.java | 4 +- .../ctbrec/recorder/RecordingFileMonitor.java | 236 ------------------ .../ctbrec/recorder/RecordingManager.java | 9 +- .../java/ctbrec/recorder/RemoteRecorder.java | 4 +- .../recorder/download/dash/DashDownload.java | 3 +- 9 files changed, 65 insertions(+), 271 deletions(-) delete mode 100644 common/src/main/java/ctbrec/recorder/RecordingFileMonitor.java diff --git a/client/src/main/java/ctbrec/ui/JavaFxRecording.java b/client/src/main/java/ctbrec/ui/JavaFxRecording.java index 0009d9d2..5ba78e6b 100644 --- a/client/src/main/java/ctbrec/ui/JavaFxRecording.java +++ b/client/src/main/java/ctbrec/ui/JavaFxRecording.java @@ -1,5 +1,6 @@ package ctbrec.ui; +import java.io.File; import java.time.Instant; import ctbrec.Config; @@ -158,11 +159,6 @@ public class JavaFxRecording extends Recording { setSizeInByte(updated.getSizeInByte()); } - @Override - public String getPath() { - return delegate.getPath(); - } - @Override public void setPath(String path) { delegate.setPath(path); @@ -223,4 +219,16 @@ public class JavaFxRecording extends Recording { public StringProperty getNoteProperty() { return notesProperty; } + + @Override + public File getAbsoluteFile() { + return delegate.getAbsoluteFile(); + } + + @Override + public void setAbsoluteFile(File absoluteFile) { + delegate.setAbsoluteFile(absoluteFile); + } + + } diff --git a/client/src/main/java/ctbrec/ui/Player.java b/client/src/main/java/ctbrec/ui/Player.java index 96df79dd..af88b945 100644 --- a/client/src/main/java/ctbrec/ui/Player.java +++ b/client/src/main/java/ctbrec/ui/Player.java @@ -152,7 +152,7 @@ public class Player { Config cfg = Config.getInstance(); try { if (cfg.getSettings().localRecording && rec != null) { - File file = new File(cfg.getSettings().recordingsDir, rec.getPath()); + File file = rec.getAbsoluteFile(); String[] cmdline = createCmdline(file.getAbsolutePath()); playerProcess = rt.exec(cmdline, OS.getEnvironment(), file.getParentFile()); } else { @@ -206,7 +206,7 @@ public class Player { private String getRemoteRecordingUrl(Recording rec, Config cfg) throws MalformedURLException, InvalidKeyException, NoSuchAlgorithmException, UnsupportedEncodingException { String hlsBase = Config.getInstance().getServerUrl() + "/hls"; - String recUrl = hlsBase + rec.getPath() + (rec.isSingleFile() ? "" : "/playlist.m3u8"); + String recUrl = hlsBase + '/' + rec.getId() + (rec.isSingleFile() ? "" : "/playlist.m3u8"); if (cfg.getSettings().requireAuthentication) { URL u = new URL(recUrl); String path = u.getPath(); diff --git a/client/src/main/java/ctbrec/ui/tabs/RecordingsTab.java b/client/src/main/java/ctbrec/ui/tabs/RecordingsTab.java index 5fff704d..c48b41ad 100644 --- a/client/src/main/java/ctbrec/ui/tabs/RecordingsTab.java +++ b/client/src/main/java/ctbrec/ui/tabs/RecordingsTab.java @@ -560,8 +560,7 @@ public class RecordingsTab extends Tab implements TabSelectionListener { private void onOpenDirectory(JavaFxRecording first) { String recordingsDir = Config.getInstance().getSettings().recordingsDir; - String path = first.getPath(); - File tsFile = new File(recordingsDir, path); + File tsFile = first.getAbsoluteFile(); new Thread(() -> DesktopIntegration.open(tsFile.getParent())).start(); } @@ -579,7 +578,7 @@ public class RecordingsTab extends Tab implements TabSelectionListener { } private void download(Recording recording) { - LOG.debug("Path {}", recording.getPath()); + LOG.debug("Path {}", recording.getAbsoluteFile()); String filename = proposeTargetFilename(recording); FileChooser chooser = new FileChooser(); chooser.setInitialFileName(filename); @@ -600,7 +599,7 @@ public class RecordingsTab extends Tab implements TabSelectionListener { } private String proposeTargetFilename(Recording recording) { - String path = recording.getPath().substring(1); + String path = recording.getAbsoluteFile().getAbsolutePath().substring(1); if(recording.isSingleFile()) { return new File(path).getName(); } else { @@ -615,11 +614,11 @@ public class RecordingsTab extends Tab implements TabSelectionListener { try { String hlsBase = config.getServerUrl() + "/hls"; if (recording.isSingleFile()) { - URL url = new URL(hlsBase + recording.getPath()); + URL url = new URL(hlsBase + '/' + recording.getId()); FileDownload download = new FileDownload(CamrecApplication.httpClient, createDownloadListener(recording)); download.start(url, target); } else { - URL url = new URL(hlsBase + recording.getPath() + "/playlist.m3u8"); + URL url = new URL(hlsBase + '/' + recording.getId() + "/playlist.m3u8"); MergedFfmpegHlsDownload download = new MergedFfmpegHlsDownload(CamrecApplication.httpClient); download.init(config, recording.getModel(), Instant.now()); LOG.info("Downloading {}", url); @@ -641,7 +640,7 @@ public class RecordingsTab extends Tab implements TabSelectionListener { } }); t.setDaemon(true); - t.setName("Download Thread " + recording.getPath()); + t.setName("Download Thread " + recording.getAbsoluteFile().toString()); t.start(); } @@ -650,7 +649,7 @@ public class RecordingsTab extends Tab implements TabSelectionListener { if (progress == 100) { recording.setStatus(FINISHED); recording.setProgress(-1); - LOG.debug("Download finished for recording {}", recording.getPath()); + LOG.debug("Download finished for recording {} - {}", recording.getId(), recording.getAbsoluteFile()); } else { recording.setStatus(DOWNLOADING); recording.setProgress(progress); diff --git a/common/src/main/java/ctbrec/Recording.java b/common/src/main/java/ctbrec/Recording.java index 508aee27..5ab665cd 100644 --- a/common/src/main/java/ctbrec/Recording.java +++ b/common/src/main/java/ctbrec/Recording.java @@ -19,6 +19,7 @@ import java.time.format.DateTimeFormatter; import java.util.EnumSet; import java.util.HashSet; import java.util.Set; +import java.util.UUID; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -30,6 +31,7 @@ import ctbrec.recorder.download.Download; public class Recording implements Serializable { private static final transient Logger LOG = LoggerFactory.getLogger(Recording.class); + private String id; private Model model; private transient Download download; private Instant startDate; @@ -42,6 +44,7 @@ public class Recording implements Serializable { private boolean pinned = false; private String note; private Set associatedFiles = new HashSet<>(); + private File absoluteFile = null; private File postProcessedFile = null; public enum State { @@ -68,6 +71,17 @@ public class Recording implements Serializable { } } + public String getId() { + if (id == null) { + id = UUID.randomUUID().toString(); + } + return id; + } + + public void setId(String id) { + this.id = id; + } + public Instant getStartDate() { return startDate; } @@ -97,18 +111,27 @@ public class Recording implements Serializable { this.progress = progress; } - public String getPath() { - return path; - } + // public String getPath() { + // return path; + // } public void setPath(String path) { this.path = path; } public File getAbsoluteFile() { - String recordingsDir = Config.getInstance().getSettings().recordingsDir; - File recordingsFile = new File(recordingsDir, getPath()); - return recordingsFile; + if (absoluteFile == null) { + String recordingsDir = Config.getInstance().getSettings().recordingsDir; + File recordingsFile = new File(recordingsDir, path); + absoluteFile = recordingsFile; + return absoluteFile; + } else { + return absoluteFile; + } + } + + public void setAbsoluteFile(File absoluteFile) { + this.absoluteFile = absoluteFile; } public File getPostProcessedFile() { @@ -222,11 +245,11 @@ public class Recording implements Serializable { } else if (!getModel().equals(other.getModel())) { return false; } - if (getPath() == null) { - if (other.getPath() != null) { + if (path == null) { + if (other.path != null) { return false; } - } else if (!getPath().equals(other.getPath())) { + } else if (!path.equals(other.path)) { return false; } if (getStartDate() == null) { @@ -247,7 +270,7 @@ public class Recording implements Serializable { } private long getSize() { - File rec = new File(Config.getInstance().getSettings().recordingsDir, getPath()); + File rec = getAbsoluteFile(); if (rec.isDirectory()) { return getDirectorySize(rec); } else { diff --git a/common/src/main/java/ctbrec/recorder/NextGenLocalRecorder.java b/common/src/main/java/ctbrec/recorder/NextGenLocalRecorder.java index bffec23e..7825781d 100644 --- a/common/src/main/java/ctbrec/recorder/NextGenLocalRecorder.java +++ b/common/src/main/java/ctbrec/recorder/NextGenLocalRecorder.java @@ -282,7 +282,9 @@ public class NextGenLocalRecorder implements Recorder { Model model = download.getModel(); Recording rec = new Recording(); rec.setDownload(download); - rec.setPath(download.getPath(model).replaceAll("\\\\", "/")); + String recordingFile = download.getPath(model).replaceAll("\\\\", "/"); + File absoluteFile = new File(config.getSettings().recordingsDir, recordingFile); + rec.setAbsoluteFile(absoluteFile); rec.setModel(model); rec.setStartDate(download.getStartTime()); rec.setSingleFile(download.isSingleFile()); diff --git a/common/src/main/java/ctbrec/recorder/RecordingFileMonitor.java b/common/src/main/java/ctbrec/recorder/RecordingFileMonitor.java deleted file mode 100644 index 0a5fe253..00000000 --- a/common/src/main/java/ctbrec/recorder/RecordingFileMonitor.java +++ /dev/null @@ -1,236 +0,0 @@ -package ctbrec.recorder; - -import static java.nio.file.StandardWatchEventKinds.*; - -import java.io.File; -import java.io.IOException; -import java.nio.file.ClosedWatchServiceException; -import java.nio.file.FileSystems; -import java.nio.file.FileVisitResult; -import java.nio.file.Files; -import java.nio.file.LinkOption; -import java.nio.file.Path; -import java.nio.file.SimpleFileVisitor; -import java.nio.file.WatchEvent; -import java.nio.file.WatchKey; -import java.nio.file.WatchService; -import java.nio.file.attribute.BasicFileAttributes; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.regex.Pattern; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import ctbrec.Config; -import ctbrec.Recording; - -public class RecordingFileMonitor { - - private static final transient Logger LOG = LoggerFactory.getLogger(RecordingFileMonitor.class); - private WatchService watcher; - private Map keys; - private boolean running = true; - private RecordingManager manager; - - public RecordingFileMonitor(RecordingManager manager) throws IOException { - this.manager = manager; - this.watcher = FileSystems.getDefault().newWatchService(); - this.keys = new HashMap<>(); - registerAll(new File(Config.getInstance().getSettings().recordingsDir).toPath()); - } - - void processEvents() { - while (running) { - // wait for key to be signalled - WatchKey key; - try { - key = watcher.take(); - } catch (InterruptedException | ClosedWatchServiceException x) { - return; - } - - Path dir = keys.get(key); - if (dir == null) { - LOG.error("WatchKey not recognized!!"); - continue; - } - - List> events = key.pollEvents(); - LOG.debug("Size: {}", events.size()); - if (isRenameProcess(events)) { - handleRename(dir, events); - } else { - for (WatchEvent event : events) { - WatchEvent.Kind kind = event.kind(); - - // TBD - provide example of how OVERFLOW event is handled - if (kind == OVERFLOW) { - continue; - } - - // Context for directory entry event is the file name of entry - WatchEvent ev = cast(event); - Path name = ev.context(); - Path child = dir.resolve(name); - - if(Files.isRegularFile(child)) { - if (kind == ENTRY_CREATE) { - handleFileCreation(child); - } else if (kind == ENTRY_DELETE) { - handleFileDeletion(child); - } - } else { - if (kind == ENTRY_CREATE) { - handleDirCreation(child); - } else if (kind == ENTRY_DELETE) { - handleDirDeletion(child); - } - } - } - } - - // reset key and remove from set if directory no longer accessible - boolean valid = key.reset(); - if (!valid) { - keys.remove(key); - - // all directories are inaccessible - if (keys.isEmpty()) { - break; - } - } - } - } - - private void handleRename(Path dir, List> events) { - WatchEvent deleteEvent = cast(events.get(0)); - WatchEvent createEvent = cast(events.get(1)); - Path from = dir.resolve(deleteEvent.context()); - Path to = dir.resolve(createEvent.context()); - LOG.debug("{} -> {}", from, to); - List affectedRecordings = getAffectedRecordings(from); - adjustPaths(affectedRecordings, from, to); - if (Files.isDirectory(to, LinkOption.NOFOLLOW_LINKS)) { - unregister(from); - try { - registerAll(to); - } catch (IOException e) { - e.printStackTrace(); - } - } - } - - private List getAffectedRecordings(Path from) { - String f = from.toAbsolutePath().toString(); - List affected = new ArrayList<>(); - for (Recording rec : manager.getAll()) { - String r = rec.getAbsoluteFile().getAbsolutePath(); - if (r.startsWith(f)) { - affected.add(rec); - } - } - return affected; - } - - private void adjustPaths(List affectedRecordings, Path from, Path to) { - for (Recording rec : affectedRecordings) { - String oldPath = rec.getAbsoluteFile().getAbsolutePath(); - String newPath = oldPath.replace(from.toString(), to.toString()); - String recordingsDir = Config.getInstance().getSettings().recordingsDir; - String relativePath = newPath.replaceFirst(Pattern.quote(recordingsDir), ""); - LOG.debug("Recording path has changed {} -> {}", rec.getPath(), relativePath); - rec.setPath(relativePath); - try { - manager.saveRecording(rec); - } catch (IOException e) { - LOG.error("Couldn't update recording path in meta data file", e); - } - } - } - - private void handleFileCreation(Path child) { - LOG.trace("File created {}", child); - } - - private void handleFileDeletion(Path child) { - LOG.trace("File deleted {}", child); - } - - private void handleDirCreation(Path dir) { - try { - registerAll(dir); - LOG.trace("Directory added {}", dir); - } catch (IOException x) { - // ignore to keep sample readbale - } - } - - private void handleDirDeletion(Path dir) { - // TODO unregister key ?!? - - // only delete directories, which have actually been deleted - if(Files.notExists(dir, LinkOption.NOFOLLOW_LINKS)) { - LOG.trace("Directory Deleted {}", dir); - } - } - - private boolean isRenameProcess(List> events) { - if(events.size() == 2) { - boolean deleteFirst = events.get(0).kind() == ENTRY_DELETE; - boolean createSecond = events.get(1).kind() == ENTRY_CREATE; - return deleteFirst && createSecond; - } else { - return false; - } - } - - /** - * Register the given directory, and all its sub-directories, with the - * WatchService. - */ - private void registerAll(final Path start) throws IOException { - // register directory and sub-directories - Files.walkFileTree(start, new SimpleFileVisitor() { - @Override - public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) { - register(dir); - return FileVisitResult.CONTINUE; - } - }); - } - - /** - * Register the given directory with the WatchService - */ - void register(Path dir) { - try { - WatchKey key = dir.register(watcher, ENTRY_CREATE, ENTRY_DELETE); - keys.put(key, dir); - LOG.debug("Monitor {}", dir); - } catch(IOException e) { - LOG.warn("Couldn't register directory monitor for directory {}", dir, e); - } - } - - public void unregister(Path path) { - - } - - @SuppressWarnings("unchecked") - static WatchEvent cast(WatchEvent event) { - return (WatchEvent) event; - } - - public void addDirectory(Path dir) throws IOException { - LOG.info("Adding monitor for {}", dir); - registerAll(dir); - } - - public void stop() throws IOException { - running = false; - watcher.close(); - } -} diff --git a/common/src/main/java/ctbrec/recorder/RecordingManager.java b/common/src/main/java/ctbrec/recorder/RecordingManager.java index e746b011..4678bb2d 100644 --- a/common/src/main/java/ctbrec/recorder/RecordingManager.java +++ b/common/src/main/java/ctbrec/recorder/RecordingManager.java @@ -96,8 +96,7 @@ public class RecordingManager { } private boolean recordingExists(Recording recording) { - File rec = new File(config.getSettings().recordingsDir, recording.getPath()); - return rec.exists(); + return recording.getAbsoluteFile().exists(); } private File getDir() { @@ -122,8 +121,7 @@ public class RecordingManager { recording = recordings.get(idx); recording.setStatus(State.DELETING); - File recordingsDir = new File(config.getSettings().recordingsDir); - File path = new File(recordingsDir, recording.getPath()); + File path = recording.getAbsoluteFile(); boolean isFile = path.isFile(); LOG.debug("Deleting {}", path); @@ -172,8 +170,7 @@ public class RecordingManager { try { int idx = recordings.indexOf(recording); recording = recordings.get(idx); - File recordingsDir = new File(config.getSettings().recordingsDir); - File path = new File(recordingsDir, recording.getPath()); + File path = recording.getAbsoluteFile(); deleteEmptyParents(path.getParentFile()); // delete the meta data Files.deleteIfExists(new File(recording.getMetaDataFile()).toPath()); diff --git a/common/src/main/java/ctbrec/recorder/RemoteRecorder.java b/common/src/main/java/ctbrec/recorder/RemoteRecorder.java index 37dcb6ec..321bf46c 100644 --- a/common/src/main/java/ctbrec/recorder/RemoteRecorder.java +++ b/common/src/main/java/ctbrec/recorder/RemoteRecorder.java @@ -330,7 +330,7 @@ public class RemoteRecorder implements Recorder { int idx = newRecordings.indexOf(recording); Recording newRecording = newRecordings.get(idx); if (newRecording.getStatus() != recording.getStatus()) { - File file = new File(recording.getPath()); + File file = recording.getAbsoluteFile(); RecordingStateChangedEvent evt = new RecordingStateChangedEvent(file, newRecording.getStatus(), recording.getModel(), recording.getStartDate()); EventBusHolder.BUS.post(evt); @@ -342,7 +342,7 @@ public class RemoteRecorder implements Recorder { justStarted.removeAll(recordings); for (Recording recording : justStarted) { if (recording.getStatus() == Recording.State.RECORDING) { - File file = new File(recording.getPath()); + File file = recording.getAbsoluteFile(); RecordingStateChangedEvent evt = new RecordingStateChangedEvent(file, recording.getStatus(), recording.getModel(), recording.getStartDate()); EventBusHolder.BUS.post(evt); diff --git a/common/src/main/java/ctbrec/recorder/download/dash/DashDownload.java b/common/src/main/java/ctbrec/recorder/download/dash/DashDownload.java index a5c21743..d6203b2a 100644 --- a/common/src/main/java/ctbrec/recorder/download/dash/DashDownload.java +++ b/common/src/main/java/ctbrec/recorder/download/dash/DashDownload.java @@ -391,7 +391,8 @@ public class DashDownload extends AbstractDownload { try { Thread.currentThread().setName("PP " + model.getName()); recording.setStatus(POST_PROCESSING); - String path = recording.getPath(); + // FIXME this was recording.getPath() before and is currently not working. This has to be fixed once DASH is used for a download again + String path = recording.getAbsoluteFile().getAbsolutePath(); File dir = new File(Config.getInstance().getSettings().recordingsDir, path); File file = new File(dir.getParentFile(), dir.getName().substring(0, dir.getName().length() - 5)); new FfmpegMuxer(dir, file); From 90192d9b8f06c44255cdf7651f10e0044cd36aac Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Sun, 13 Sep 2020 19:54:43 +0200 Subject: [PATCH 10/33] Fix server stuff for new recording path handling --- .../main/java/ctbrec/ui/JavaFxRecording.java | 10 +++ common/src/main/java/ctbrec/Recording.java | 4 - .../ctbrec/recorder/NextGenLocalRecorder.java | 1 + .../ctbrec/recorder/RecordingManager.java | 5 ++ .../ctbrec/recorder/server/HlsServlet.java | 75 +++++++++++++------ .../ctbrec/recorder/server/HttpServer.java | 2 +- .../main/resources/html/static/recordings.js | 14 ++-- 7 files changed, 75 insertions(+), 36 deletions(-) diff --git a/client/src/main/java/ctbrec/ui/JavaFxRecording.java b/client/src/main/java/ctbrec/ui/JavaFxRecording.java index 5ba78e6b..e5ecf139 100644 --- a/client/src/main/java/ctbrec/ui/JavaFxRecording.java +++ b/client/src/main/java/ctbrec/ui/JavaFxRecording.java @@ -230,5 +230,15 @@ public class JavaFxRecording extends Recording { delegate.setAbsoluteFile(absoluteFile); } + @Override + public String getId() { + return delegate.getId(); + } + + @Override + public void setId(String id) { + delegate.setId(id); + } + } diff --git a/common/src/main/java/ctbrec/Recording.java b/common/src/main/java/ctbrec/Recording.java index 5ab665cd..12559935 100644 --- a/common/src/main/java/ctbrec/Recording.java +++ b/common/src/main/java/ctbrec/Recording.java @@ -19,7 +19,6 @@ import java.time.format.DateTimeFormatter; import java.util.EnumSet; import java.util.HashSet; import java.util.Set; -import java.util.UUID; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -72,9 +71,6 @@ public class Recording implements Serializable { } public String getId() { - if (id == null) { - id = UUID.randomUUID().toString(); - } return id; } diff --git a/common/src/main/java/ctbrec/recorder/NextGenLocalRecorder.java b/common/src/main/java/ctbrec/recorder/NextGenLocalRecorder.java index 7825781d..d410edee 100644 --- a/common/src/main/java/ctbrec/recorder/NextGenLocalRecorder.java +++ b/common/src/main/java/ctbrec/recorder/NextGenLocalRecorder.java @@ -281,6 +281,7 @@ public class NextGenLocalRecorder implements Recorder { private Recording createRecording(Download download) throws IOException { Model model = download.getModel(); Recording rec = new Recording(); + rec.setId(UUID.randomUUID().toString()); rec.setDownload(download); String recordingFile = download.getPath(model).replaceAll("\\\\", "/"); File absoluteFile = new File(config.getSettings().recordingsDir, recordingFile); diff --git a/common/src/main/java/ctbrec/recorder/RecordingManager.java b/common/src/main/java/ctbrec/recorder/RecordingManager.java index 4678bb2d..90072924 100644 --- a/common/src/main/java/ctbrec/recorder/RecordingManager.java +++ b/common/src/main/java/ctbrec/recorder/RecordingManager.java @@ -13,6 +13,7 @@ import java.util.ArrayList; import java.util.List; import java.util.NoSuchElementException; import java.util.Objects; +import java.util.UUID; import java.util.concurrent.locks.ReentrantLock; import org.slf4j.Logger; @@ -82,6 +83,10 @@ public class RecordingManager { if (recording.getStatus() == RECORDING || recording.getStatus() == GENERATING_PLAYLIST || recording.getStatus() == POST_PROCESSING) { recording.setStatus(WAITING); } + if (recording.getId() == null) { + recording.setId(UUID.randomUUID().toString()); + saveRecording(recording); + } if (recordingExists(recording)) { recordings.add(recording); } else { diff --git a/server/src/main/java/ctbrec/recorder/server/HlsServlet.java b/server/src/main/java/ctbrec/recorder/server/HlsServlet.java index 7919cea6..9498aede 100644 --- a/server/src/main/java/ctbrec/recorder/server/HlsServlet.java +++ b/server/src/main/java/ctbrec/recorder/server/HlsServlet.java @@ -9,6 +9,8 @@ import java.nio.file.Paths; import java.security.InvalidKeyException; import java.security.NoSuchAlgorithmException; import java.util.Enumeration; +import java.util.Objects; +import java.util.Optional; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -20,6 +22,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import ctbrec.Config; +import ctbrec.Recording; +import ctbrec.recorder.Recorder; public class HlsServlet extends AbstractCtbrecServlet { @@ -27,50 +31,69 @@ public class HlsServlet extends AbstractCtbrecServlet { private final Config config; - public HlsServlet(Config config) { + private Recorder recorder; + + public HlsServlet(Config config, Recorder recorder) { this.config = config; + this.recorder = recorder; } @Override protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { + + String contextPath = getServletContext().getContextPath(); String request = req.getRequestURI().substring(contextPath.length() + 5); Path recordingsDirPath = Paths.get(config.getSettings().recordingsDir).toAbsolutePath().normalize(); Path requestedFilePath = recordingsDirPath.resolve(request).toAbsolutePath().normalize(); - boolean isValidRequestedPath = requestedFilePath.startsWith(recordingsDirPath); - if (isValidRequestedPath) { - File requestedFile = requestedFilePath.toFile(); + File requestedFile = requestedFilePath.toFile(); + try { if (requestedFile.getName().equals("playlist.m3u8")) { - try { - boolean isRequestAuthenticated = checkAuthentication(req, req.getRequestURI()); - if (!isRequestAuthenticated) { - writeResponse(resp, SC_UNAUTHORIZED, "{\"status\": \"error\", \"msg\": \"HMAC does not match\"}"); - return; - } - } catch (InvalidKeyException | NoSuchAlgorithmException | IllegalStateException e1) { - writeResponse(resp, SC_UNAUTHORIZED, "{\"status\": \"error\", \"msg\": \"Authentication failed\"}"); + boolean isRequestAuthenticated = checkAuthentication(req, req.getRequestURI()); + if (!isRequestAuthenticated) { + writeResponse(resp, SC_UNAUTHORIZED, "{\"status\": \"error\", \"msg\": \"HMAC does not match\"}"); return; } - servePlaylist(req, resp, requestedFile); - } else { - if (requestedFile.exists()) { - Enumeration headerNames = req.getHeaderNames(); - while(headerNames.hasMoreElements()) { - String header = headerNames.nextElement(); - LOG.trace("{}: {}", header, req.getHeader(header)); - } - serveSegment(req, resp, requestedFile); + String id = request.substring(0, request.indexOf('/')); + Optional rec = getRecordingById(id); + if (rec.isPresent()) { + servePlaylist(req, resp, rec.get().getAbsoluteFile()); } else { error404(req, resp); + return; + } + } else { + String id = request.split("/")[0]; + Optional rec = getRecordingById(id); + if (rec.isPresent()) { + File file = rec.get().getAbsoluteFile(); + if (LOG.isTraceEnabled()) { + Enumeration headerNames = req.getHeaderNames(); + while (headerNames.hasMoreElements()) { + String header = headerNames.nextElement(); + LOG.trace("{}: {}", header, req.getHeader(header)); + } + } + serveSegment(req, resp, file); + } else { + error404(req, resp); + return; } } - } else { - writeResponse(resp, SC_FORBIDDEN, "Stop it!"); + } catch (InvalidKeyException | NoSuchAlgorithmException | IllegalStateException e1) { + writeResponse(resp, SC_UNAUTHORIZED, "{\"status\": \"error\", \"msg\": \"Authentication failed\"}"); + return; } } + private Optional getRecordingById(String id) throws InvalidKeyException, NoSuchAlgorithmException, IOException { + return recorder.getRecordings().stream() + .filter(r -> Objects.equals(id, r.getId())) + .findFirst(); + } + private void writeResponse(HttpServletResponse resp, int code, String body) { try { resp.setStatus(code); @@ -81,6 +104,7 @@ public class HlsServlet extends AbstractCtbrecServlet { } private void error404(HttpServletRequest req, HttpServletResponse resp) { + writeResponse(resp, SC_NOT_FOUND, "{\"status\": \"error\", \"msg\": \"Recording not found\"}"); resp.setStatus(HttpServletResponse.SC_NOT_FOUND); } @@ -90,7 +114,9 @@ public class HlsServlet extends AbstractCtbrecServlet { } private void servePlaylist(HttpServletRequest req, HttpServletResponse resp, File requestedFile) throws IOException { - serveFile(req, resp, requestedFile, "application/x-mpegURL"); + LOG.debug("Serving playlist {}", requestedFile); + File playlist = new File(requestedFile, "playlist.m3u8"); + serveFile(req, resp, playlist, "application/x-mpegURL"); } private void serveFile(HttpServletRequest req, HttpServletResponse resp, File file, String contentType) throws IOException { @@ -103,6 +129,7 @@ public class HlsServlet extends AbstractCtbrecServlet { byte[] buffer = new byte[1024 * 100]; long bytesLeft = range.to - range.from; resp.setContentLengthLong(bytesLeft); + resp.addHeader("Content-Disposition", "attachment; filename=\"" + file.getName() + "\""); if (range.set) { resp.setHeader("Content-Range", "bytes " + range.from + '-' + range.to + '/' + file.length()); } diff --git a/server/src/main/java/ctbrec/recorder/server/HttpServer.java b/server/src/main/java/ctbrec/recorder/server/HttpServer.java index 56126c9b..fc354de5 100644 --- a/server/src/main/java/ctbrec/recorder/server/HttpServer.java +++ b/server/src/main/java/ctbrec/recorder/server/HttpServer.java @@ -213,7 +213,7 @@ public class HttpServer { holder = new ServletHolder(configServlet); defaultContext.addServlet(holder, "/config"); - HlsServlet hlsServlet = new HlsServlet(this.config); + HlsServlet hlsServlet = new HlsServlet(this.config, recorder); holder = new ServletHolder(hlsServlet); defaultContext.addServlet(holder, "/hls/*"); diff --git a/server/src/main/resources/html/static/recordings.js b/server/src/main/resources/html/static/recordings.js index 6c446355..f41a28e7 100644 --- a/server/src/main/resources/html/static/recordings.js +++ b/server/src/main/resources/html/static/recordings.js @@ -1,5 +1,5 @@ function play(recording) { - let src = recording.singleFile ? '/hls' + recording.path : recording.playlist; + let src = recording.singleFile ? '/hls/' + recording.id : recording.playlist; let hmacOfPath = CryptoJS.HmacSHA256(src, hmac); src = '..' + src; if(console) console.log("Path", src, "HMAC", hmacOfPath); @@ -48,7 +48,7 @@ function play(recording) { } function download(recording) { - let src = recording.singleFile ? '/hls' + recording.path : recording.playlist; + let src = recording.singleFile ? '/hls/' + recording.id : recording.playlist; let hmacOfPath = CryptoJS.HmacSHA256(src, hmac); src = '..' + src; if(console) console.log("Path", src, "HMAC", hmacOfPath); @@ -77,7 +77,7 @@ function calculateSize(sizeInByte) { function isRecordingInArray(array, recording) { for ( let idx in array) { let r = array[idx]; - if (r.path === recording.path) { + if (r.id === recording.id) { return true; } } @@ -115,10 +115,10 @@ function syncRecordings(recordings) { recording.ko_progressString = ko.observable(recording.progress === -1 ? '' : recording.progress); recording.ko_size = ko.observable(calculateSize(recording.sizeInByte)); recording.ko_status = ko.observable(recording.status); - if (recording.path.endsWith('.mp4')) { - recording.playlist = '/hls' + recording.path; + if (recording.singleFile) { + recording.playlist = '/hls/' + recording.id; } else { - recording.playlist = '/hls' + recording.path + '/playlist.m3u8'; + recording.playlist = '/hls/' + recording.id + '/playlist.m3u8'; } observableRecordingsArray.push(recording); } @@ -129,7 +129,7 @@ function syncRecordings(recordings) { let recording = recordings[i]; for ( let j in observableRecordingsArray()) { let r = observableRecordingsArray()[j]; - if (recording.path === r.path) { + if (recording.id === r.id) { r.progress = recording.progress; r.sizeInByte = recording.sizeInByte; r.status = recording.status; From bf39d9a639a78509bfda38301b41a3df1a0da822 Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Sun, 20 Sep 2020 18:14:24 +0200 Subject: [PATCH 11/33] Implement proper handling of the orignal and pp files If a copy is created, the original file is not touched anymore. Otherwise the original file is used and the post-processing process is not repeatable anymore, or at least the results might get unpredictable --- .../ctbrec/ui/settings/MoverPaneFactory.java | 24 ++++++++ .../settings/PostProcessingDialogFactory.java | 10 ++-- .../ui/settings/PostProcessingStepPanel.java | 15 ++++- .../ui/settings/RemuxerPaneFactory.java | 6 +- .../ui/settings/RenamerPaneFactory.java | 4 +- .../ctbrec/ui/tabs/RecordedModelsTab.java | 2 +- common/src/main/java/ctbrec/Config.java | 4 ++ common/src/main/java/ctbrec/io/IoUtils.java | 51 +++++++++++++++++ .../ctbrec/recorder/NextGenLocalRecorder.java | 1 + .../ctbrec/recorder/RecordingManager.java | 37 +----------- ...bstractPlaceholderAwarePostProcessor.java} | 53 +++++------------ .../ctbrec/recorder/postprocessing/Copy.java | 2 +- .../postprocessing/DeleteOriginal.java | 29 ++++++++++ .../ctbrec/recorder/postprocessing/Move.java | 57 +++++++++++++++++++ .../{Remuxer.java => Remux.java} | 10 +++- .../recorder/postprocessing/Rename.java | 48 ++++++++++++++++ 16 files changed, 263 insertions(+), 90 deletions(-) create mode 100644 client/src/main/java/ctbrec/ui/settings/MoverPaneFactory.java create mode 100644 common/src/main/java/ctbrec/io/IoUtils.java rename common/src/main/java/ctbrec/recorder/postprocessing/{Renamer.java => AbstractPlaceholderAwarePostProcessor.java} (65%) create mode 100644 common/src/main/java/ctbrec/recorder/postprocessing/DeleteOriginal.java create mode 100644 common/src/main/java/ctbrec/recorder/postprocessing/Move.java rename common/src/main/java/ctbrec/recorder/postprocessing/{Remuxer.java => Remux.java} (89%) create mode 100644 common/src/main/java/ctbrec/recorder/postprocessing/Rename.java diff --git a/client/src/main/java/ctbrec/ui/settings/MoverPaneFactory.java b/client/src/main/java/ctbrec/ui/settings/MoverPaneFactory.java new file mode 100644 index 00000000..c45bc1c9 --- /dev/null +++ b/client/src/main/java/ctbrec/ui/settings/MoverPaneFactory.java @@ -0,0 +1,24 @@ +package ctbrec.ui.settings; + +import ctbrec.recorder.postprocessing.Move; +import ctbrec.recorder.postprocessing.PostProcessor; +import ctbrec.ui.settings.api.Category; +import ctbrec.ui.settings.api.Preferences; +import ctbrec.ui.settings.api.Setting; +import javafx.beans.property.SimpleStringProperty; + +public class MoverPaneFactory extends AbstractPostProcessingPaneFactory { + + @Override + public Preferences doCreatePostProcessorPane(PostProcessor pp) { + SimpleStringProperty pathTemplate = new SimpleStringProperty(null, Move.PATH_TEMPLATE, pp.getConfig().getOrDefault(Move.PATH_TEMPLATE, Move.DEFAULT)); + properties.add(pathTemplate); + + return Preferences.of(new MapPreferencesStorage(), + Category.of(pp.getName(), + Setting.of("Directory", pathTemplate) + ) + ); + } + +} diff --git a/client/src/main/java/ctbrec/ui/settings/PostProcessingDialogFactory.java b/client/src/main/java/ctbrec/ui/settings/PostProcessingDialogFactory.java index 094f4c8c..eccad370 100644 --- a/client/src/main/java/ctbrec/ui/settings/PostProcessingDialogFactory.java +++ b/client/src/main/java/ctbrec/ui/settings/PostProcessingDialogFactory.java @@ -7,9 +7,10 @@ import java.util.Map; import java.util.Optional; import ctbrec.Config; +import ctbrec.recorder.postprocessing.Move; import ctbrec.recorder.postprocessing.PostProcessor; -import ctbrec.recorder.postprocessing.Remuxer; -import ctbrec.recorder.postprocessing.Renamer; +import ctbrec.recorder.postprocessing.Remux; +import ctbrec.recorder.postprocessing.Rename; import ctbrec.ui.controls.Dialogs; import ctbrec.ui.settings.api.Preferences; import javafx.collections.ObservableList; @@ -20,8 +21,9 @@ public class PostProcessingDialogFactory { static Map, Class> ppToDialogMap = new HashMap<>(); static { - ppToDialogMap.put(Remuxer.class, RemuxerPaneFactory.class); - ppToDialogMap.put(Renamer.class, RenamerPaneFactory.class); + ppToDialogMap.put(Remux.class, RemuxerPaneFactory.class); + ppToDialogMap.put(Rename.class, RenamerPaneFactory.class); + ppToDialogMap.put(Move.class, MoverPaneFactory.class); } private PostProcessingDialogFactory() { diff --git a/client/src/main/java/ctbrec/ui/settings/PostProcessingStepPanel.java b/client/src/main/java/ctbrec/ui/settings/PostProcessingStepPanel.java index 91e5d367..05a18f92 100644 --- a/client/src/main/java/ctbrec/ui/settings/PostProcessingStepPanel.java +++ b/client/src/main/java/ctbrec/ui/settings/PostProcessingStepPanel.java @@ -7,9 +7,11 @@ import java.util.Optional; import ctbrec.Config; import ctbrec.recorder.postprocessing.Copy; +import ctbrec.recorder.postprocessing.DeleteOriginal; +import ctbrec.recorder.postprocessing.Move; import ctbrec.recorder.postprocessing.PostProcessor; -import ctbrec.recorder.postprocessing.Remuxer; -import ctbrec.recorder.postprocessing.Renamer; +import ctbrec.recorder.postprocessing.Remux; +import ctbrec.recorder.postprocessing.Rename; import ctbrec.ui.controls.Dialogs; import javafx.collections.FXCollections; import javafx.collections.ListChangeListener; @@ -28,7 +30,14 @@ public class PostProcessingStepPanel extends GridPane { private Config config; - private static final Class[] POST_PROCESSOR_CLASSES = new Class[] { Copy.class, Remuxer.class, Renamer.class }; + + private static final Class[] POST_PROCESSOR_CLASSES = new Class[] { // @formatter: off + Copy.class, + Remux.class, + Rename.class, + Move.class, + DeleteOriginal.class + }; // @formatter: on ListView stepListView; ObservableList stepList; diff --git a/client/src/main/java/ctbrec/ui/settings/RemuxerPaneFactory.java b/client/src/main/java/ctbrec/ui/settings/RemuxerPaneFactory.java index 886182ad..64a9c0fb 100644 --- a/client/src/main/java/ctbrec/ui/settings/RemuxerPaneFactory.java +++ b/client/src/main/java/ctbrec/ui/settings/RemuxerPaneFactory.java @@ -1,7 +1,7 @@ package ctbrec.ui.settings; import ctbrec.recorder.postprocessing.PostProcessor; -import ctbrec.recorder.postprocessing.Remuxer; +import ctbrec.recorder.postprocessing.Remux; import ctbrec.ui.settings.api.Category; import ctbrec.ui.settings.api.Preferences; import ctbrec.ui.settings.api.Setting; @@ -11,8 +11,8 @@ public class RemuxerPaneFactory extends AbstractPostProcessingPaneFactory { @Override public Preferences doCreatePostProcessorPane(PostProcessor pp) { - SimpleStringProperty ffmpegParams = new SimpleStringProperty(null, Remuxer.FFMPEG_ARGS, pp.getConfig().getOrDefault(Remuxer.FFMPEG_ARGS, "-c:v copy -c:a copy -movflags faststart -y -f mp4")); - SimpleStringProperty fileExt = new SimpleStringProperty(null, Remuxer.FILE_EXT, pp.getConfig().getOrDefault(Remuxer.FILE_EXT, "mp4")); + SimpleStringProperty ffmpegParams = new SimpleStringProperty(null, Remux.FFMPEG_ARGS, pp.getConfig().getOrDefault(Remux.FFMPEG_ARGS, "-c:v copy -c:a copy -movflags faststart -y -f mp4")); + SimpleStringProperty fileExt = new SimpleStringProperty(null, Remux.FILE_EXT, pp.getConfig().getOrDefault(Remux.FILE_EXT, "mp4")); properties.add(ffmpegParams); properties.add(fileExt); diff --git a/client/src/main/java/ctbrec/ui/settings/RenamerPaneFactory.java b/client/src/main/java/ctbrec/ui/settings/RenamerPaneFactory.java index 2140b75a..4c887403 100644 --- a/client/src/main/java/ctbrec/ui/settings/RenamerPaneFactory.java +++ b/client/src/main/java/ctbrec/ui/settings/RenamerPaneFactory.java @@ -1,7 +1,7 @@ package ctbrec.ui.settings; import ctbrec.recorder.postprocessing.PostProcessor; -import ctbrec.recorder.postprocessing.Renamer; +import ctbrec.recorder.postprocessing.Rename; import ctbrec.ui.settings.api.Category; import ctbrec.ui.settings.api.Preferences; import ctbrec.ui.settings.api.Setting; @@ -11,7 +11,7 @@ public class RenamerPaneFactory extends AbstractPostProcessingPaneFactory { @Override public Preferences doCreatePostProcessorPane(PostProcessor pp) { - SimpleStringProperty fileTemplate = new SimpleStringProperty(null, Renamer.FILE_NAME_TEMPLATE, pp.getConfig().getOrDefault(Renamer.FILE_NAME_TEMPLATE, Renamer.DEFAULT)); + SimpleStringProperty fileTemplate = new SimpleStringProperty(null, Rename.FILE_NAME_TEMPLATE, pp.getConfig().getOrDefault(Rename.FILE_NAME_TEMPLATE, Rename.DEFAULT)); properties.add(fileTemplate); return Preferences.of(new MapPreferencesStorage(), diff --git a/client/src/main/java/ctbrec/ui/tabs/RecordedModelsTab.java b/client/src/main/java/ctbrec/ui/tabs/RecordedModelsTab.java index 2521d99b..6c4d6613 100644 --- a/client/src/main/java/ctbrec/ui/tabs/RecordedModelsTab.java +++ b/client/src/main/java/ctbrec/ui/tabs/RecordedModelsTab.java @@ -228,7 +228,7 @@ public class RecordedModelsTab extends Tab implements TabSelectionListener { @Override public String get() { - String modelNotes = Config.getInstance().getSettings().modelNotes.getOrDefault(m.getUrl(), ""); + String modelNotes = Config.getInstance().getModelNotes(m); return modelNotes; } }; diff --git a/common/src/main/java/ctbrec/Config.java b/common/src/main/java/ctbrec/Config.java index c18b3a96..fa08993c 100644 --- a/common/src/main/java/ctbrec/Config.java +++ b/common/src/main/java/ctbrec/Config.java @@ -193,4 +193,8 @@ public class Config { } return context; } + + public String getModelNotes(Model m) { + return Config.getInstance().getSettings().modelNotes.getOrDefault(m.getUrl(), ""); + } } diff --git a/common/src/main/java/ctbrec/io/IoUtils.java b/common/src/main/java/ctbrec/io/IoUtils.java new file mode 100644 index 00000000..f3be9bbb --- /dev/null +++ b/common/src/main/java/ctbrec/io/IoUtils.java @@ -0,0 +1,51 @@ +package ctbrec.io; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import ctbrec.Config; + +public class IoUtils { + + private static final Logger LOG = LoggerFactory.getLogger(IoUtils.class); + + private IoUtils() {} + + public static void deleteEmptyParents(File parent) throws IOException { + File recDir = new File(Config.getInstance().getSettings().recordingsDir); + while (parent != null && parent.list() != null && parent.list().length == 0) { + if (parent.equals(recDir)) { + return; + } + LOG.debug("Deleting empty directory {}", parent.getAbsolutePath()); + Files.delete(parent.toPath()); + parent = parent.getParentFile(); + } + } + + public static void deleteDirectory(File directory) throws IOException { + if (!directory.exists()) { + return; + } + + File[] files = directory.listFiles(); + boolean deletedAllFiles = true; + for (File file : files) { + try { + LOG.trace("Deleting {}", file.getAbsolutePath()); + Files.delete(file.toPath()); + } catch (Exception e) { + deletedAllFiles = false; + LOG.debug("Couldn't delete {}", file, e); + } + } + + if (!deletedAllFiles) { + throw new IOException("Couldn't delete all files in " + directory); + } + } +} diff --git a/common/src/main/java/ctbrec/recorder/NextGenLocalRecorder.java b/common/src/main/java/ctbrec/recorder/NextGenLocalRecorder.java index d410edee..3afe4366 100644 --- a/common/src/main/java/ctbrec/recorder/NextGenLocalRecorder.java +++ b/common/src/main/java/ctbrec/recorder/NextGenLocalRecorder.java @@ -164,6 +164,7 @@ public class NextGenLocalRecorder implements Recorder { recording.postprocess(); List postProcessors = config.getSettings().postProcessors; for (PostProcessor postProcessor : postProcessors) { + LOG.debug("Running post-processor: {}", postProcessor.getName()); postProcessor.postprocess(recording); } setRecordingStatus(recording, State.FINISHED); diff --git a/common/src/main/java/ctbrec/recorder/RecordingManager.java b/common/src/main/java/ctbrec/recorder/RecordingManager.java index 90072924..69735f11 100644 --- a/common/src/main/java/ctbrec/recorder/RecordingManager.java +++ b/common/src/main/java/ctbrec/recorder/RecordingManager.java @@ -1,6 +1,7 @@ package ctbrec.recorder; import static ctbrec.Recording.State.*; +import static ctbrec.io.IoUtils.*; import static java.nio.charset.StandardCharsets.*; import static java.nio.file.StandardOpenOption.*; @@ -142,8 +143,10 @@ public class RecordingManager { File f = new File(associated); if (f.isFile()) { Files.delete(f.toPath()); + deleteEmptyParents(f.getParentFile()); } else { deleteDirectory(f); + deleteEmptyParents(f); } } @@ -202,40 +205,6 @@ public class RecordingManager { } } - public static void deleteEmptyParents(File parent) throws IOException { - File recDir = new File(Config.getInstance().getSettings().recordingsDir); - while (parent != null && parent.list() != null && parent.list().length == 0) { - if (parent.equals(recDir)) { - return; - } - LOG.debug("Deleting empty directory {}", parent.getAbsolutePath()); - Files.delete(parent.toPath()); - parent = parent.getParentFile(); - } - } - - private void deleteDirectory(File directory) throws IOException { - if (!directory.exists()) { - return; - } - - File[] files = directory.listFiles(); - boolean deletedAllFiles = true; - for (File file : files) { - try { - LOG.trace("Deleting {}", file.getAbsolutePath()); - Files.delete(file.toPath()); - } catch (Exception e) { - deletedAllFiles = false; - LOG.debug("Couldn't delete {}", file, e); - } - } - - if (!deletedAllFiles) { - throw new IOException("Couldn't delete all files in " + directory); - } - } - public void pin(Recording recording) throws IOException { recordingsLock.lock(); try { diff --git a/common/src/main/java/ctbrec/recorder/postprocessing/Renamer.java b/common/src/main/java/ctbrec/recorder/postprocessing/AbstractPlaceholderAwarePostProcessor.java similarity index 65% rename from common/src/main/java/ctbrec/recorder/postprocessing/Renamer.java rename to common/src/main/java/ctbrec/recorder/postprocessing/AbstractPlaceholderAwarePostProcessor.java index b542cb24..d89d9bf4 100644 --- a/common/src/main/java/ctbrec/recorder/postprocessing/Renamer.java +++ b/common/src/main/java/ctbrec/recorder/postprocessing/AbstractPlaceholderAwarePostProcessor.java @@ -1,8 +1,7 @@ package ctbrec.recorder.postprocessing; + import static java.util.Optional.*; -import java.io.File; -import java.io.IOException; import java.time.ZoneId; import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; @@ -10,18 +9,10 @@ import java.util.Locale; import java.util.regex.Matcher; import java.util.regex.Pattern; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.common.io.Files; - +import ctbrec.Config; import ctbrec.Recording; -public class Renamer extends AbstractPostProcessor { - - private static final Logger LOG = LoggerFactory.getLogger(Renamer.class); - public static final String FILE_NAME_TEMPLATE = "filename.template"; - public static final String DEFAULT = "${modelSanitizedName}_${localDateTime}.${fileSuffix}"; +public abstract class AbstractPlaceholderAwarePostProcessor extends AbstractPostProcessor { @SuppressWarnings("unused") private String[] placeHolders = { @@ -34,18 +25,13 @@ public class Renamer extends AbstractPostProcessor { "${localDateTime}", "${epochSecond}", "${fileSuffix}", - "${modelNotes}" + "${modelNotes}", + "${recordingsDir}" }; - @Override - public String getName() { - return "rename"; - } - - @Override - public void postprocess(Recording rec) throws IOException { - String filenameTemplate = getConfig().getOrDefault(FILE_NAME_TEMPLATE, DEFAULT); - String filename = filenameTemplate + public String fillInPlaceHolders(String input, Recording rec) { + // @formatter:off + String output = input .replace("${modelName}", ofNullable(rec.getModel().getName()).orElse("modelName")) .replace("${modelDisplayName}", ofNullable(rec.getModel().getDisplayName()).orElse("displayName")) .replace("${modelSanitizedName}", ofNullable(rec.getModel().getSanitizedNamed()).orElse("sanitizedName")) @@ -53,17 +39,15 @@ public class Renamer extends AbstractPostProcessor { .replace("${siteSanitizedName}", getSanitizedSiteName(rec)) .replace("${fileSuffix}", getFileSuffix(rec)) .replace("${epochSecond}", Long.toString(rec.getStartDate().getEpochSecond())) + .replace("${modelNotes}", Config.getInstance().getModelNotes(rec.getModel())) + .replace("${recordingsDir}", Config.getInstance().getSettings().recordingsDir) ; - filename = replaceUtcDateTime(rec, filename); - filename = replaceLocalDateTime(rec, filename); + output = replaceUtcDateTime(rec, output); + output = replaceLocalDateTime(rec, output); - File src = rec.getPostProcessedFile(); - File target = new File(src.getParentFile(), filename); - LOG.info("Renaming {} to {}", src.getName(), target.getName()); - Files.move(rec.getPostProcessedFile(), target); - rec.setPostProcessedFile(target); - rec.getAssociatedFiles().add(target.getAbsolutePath()); + return output; + // @formatter:on } private String replaceUtcDateTime(Recording rec, String filename) { @@ -99,17 +83,8 @@ public class Renamer extends AbstractPostProcessor { return filename.substring(filename.lastIndexOf('.') + 1); } - private CharSequence getSanitizedSiteName(Recording rec) { return rec.getModel().getSite().getName().replace(' ', '_').replace('\\', '_').replace('/', '_'); } - @Override - public String toString() { - String s = getName(); - if (getConfig().containsKey(FILE_NAME_TEMPLATE)) { - s += " [" + getConfig().get(FILE_NAME_TEMPLATE) + ']'; - } - return s; - } } diff --git a/common/src/main/java/ctbrec/recorder/postprocessing/Copy.java b/common/src/main/java/ctbrec/recorder/postprocessing/Copy.java index 88dcf452..d77eefcf 100644 --- a/common/src/main/java/ctbrec/recorder/postprocessing/Copy.java +++ b/common/src/main/java/ctbrec/recorder/postprocessing/Copy.java @@ -27,7 +27,7 @@ public class Copy extends AbstractPostProcessor { LOG.info("Creating a copy {}", copy); Files.copy(rec.getPostProcessedFile(), copy); rec.setPostProcessedFile(copy); - rec.getAssociatedFiles().add(copy.getAbsolutePath()); + rec.getAssociatedFiles().add(copy.getCanonicalPath()); } private String getFilenameForCopy(File orig) { diff --git a/common/src/main/java/ctbrec/recorder/postprocessing/DeleteOriginal.java b/common/src/main/java/ctbrec/recorder/postprocessing/DeleteOriginal.java new file mode 100644 index 00000000..49337be3 --- /dev/null +++ b/common/src/main/java/ctbrec/recorder/postprocessing/DeleteOriginal.java @@ -0,0 +1,29 @@ +package ctbrec.recorder.postprocessing; + +import static ctbrec.io.IoUtils.*; + +import java.io.IOException; +import java.nio.file.Files; + +import ctbrec.Recording; + +public class DeleteOriginal extends AbstractPostProcessor { + + @Override + public String getName() { + return "delete original"; + } + + @Override + public void postprocess(Recording rec) throws IOException, InterruptedException { + if (rec.getAbsoluteFile().isFile()) { + Files.deleteIfExists(rec.getAbsoluteFile().toPath()); + deleteEmptyParents(rec.getAbsoluteFile().getParentFile()); + } else { + deleteDirectory(rec.getAbsoluteFile()); + deleteEmptyParents(rec.getAbsoluteFile()); + } + rec.setAbsoluteFile(rec.getPostProcessedFile()); + rec.getAssociatedFiles().remove(rec.getAbsoluteFile().getCanonicalPath()); + } +} diff --git a/common/src/main/java/ctbrec/recorder/postprocessing/Move.java b/common/src/main/java/ctbrec/recorder/postprocessing/Move.java new file mode 100644 index 00000000..498922d3 --- /dev/null +++ b/common/src/main/java/ctbrec/recorder/postprocessing/Move.java @@ -0,0 +1,57 @@ +package ctbrec.recorder.postprocessing; +import static ctbrec.io.IoUtils.*; + +import java.io.File; +import java.io.IOException; +import java.util.Objects; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.io.Files; + +import ctbrec.Recording; + +public class Move extends AbstractPlaceholderAwarePostProcessor { + + private static final Logger LOG = LoggerFactory.getLogger(Rename.class); + public static final String PATH_TEMPLATE = "path.template"; + public static final String DEFAULT = "${modelSanitizedName}" + File.separatorChar + "${localDateTime}"; + + @Override + public String getName() { + return "move"; + } + + @Override + public void postprocess(Recording rec) throws IOException { + String pathTemplate = getConfig().getOrDefault(PATH_TEMPLATE, DEFAULT); + String path = fillInPlaceHolders(pathTemplate, rec); + File src = rec.getPostProcessedFile(); + File target = new File(path, src.getName()); + LOG.info("Moving {} to {}", src.getName(), target.getParentFile().getCanonicalPath()); + Files.createParentDirs(target); + Files.move(rec.getPostProcessedFile(), target); + rec.setPostProcessedFile(target); + if (Objects.equals(src, rec.getAbsoluteFile())) { + rec.setAbsoluteFile(target); + } + rec.getAssociatedFiles().remove(src.getCanonicalPath()); + rec.getAssociatedFiles().add(target.getCanonicalPath()); + + if (src.isFile()) { + deleteEmptyParents(src.getParentFile()); + } else { + deleteEmptyParents(src); + } + } + + @Override + public String toString() { + String s = getName(); + if (getConfig().containsKey(PATH_TEMPLATE)) { + s += " [" + getConfig().get(PATH_TEMPLATE) + ']'; + } + return s; + } +} diff --git a/common/src/main/java/ctbrec/recorder/postprocessing/Remuxer.java b/common/src/main/java/ctbrec/recorder/postprocessing/Remux.java similarity index 89% rename from common/src/main/java/ctbrec/recorder/postprocessing/Remuxer.java rename to common/src/main/java/ctbrec/recorder/postprocessing/Remux.java index 0aa5051a..cd7071bb 100644 --- a/common/src/main/java/ctbrec/recorder/postprocessing/Remuxer.java +++ b/common/src/main/java/ctbrec/recorder/postprocessing/Remux.java @@ -14,9 +14,9 @@ import ctbrec.Recording; import ctbrec.io.StreamRedirectThread; import ctbrec.recorder.download.ProcessExitedUncleanException; -public class Remuxer extends AbstractPostProcessor { +public class Remux extends AbstractPostProcessor { - private static final Logger LOG = LoggerFactory.getLogger(Remuxer.class); + private static final Logger LOG = LoggerFactory.getLogger(Remux.class); public static final String FFMPEG_ARGS = "ffmpeg.args"; public static final String FILE_EXT = "file.ext"; @@ -35,6 +35,7 @@ public class Remuxer extends AbstractPostProcessor { argsPlusFile[i++] = "-i"; argsPlusFile[i++] = rec.getPostProcessedFile().getAbsolutePath(); System.arraycopy(args, 0, argsPlusFile, i, args.length); + File inputFile = rec.getPostProcessedFile(); File remuxedFile = new File(rec.getPostProcessedFile().getAbsolutePath() + '.' + fileExt); argsPlusFile[argsPlusFile.length - 1] = remuxedFile.getAbsolutePath(); String[] cmdline = OS.getFFmpegCommand(argsPlusFile); @@ -42,7 +43,10 @@ public class Remuxer extends AbstractPostProcessor { Process ffmpeg = Runtime.getRuntime().exec(cmdline, new String[0], rec.getPostProcessedFile().getParentFile()); setupLogging(ffmpeg, rec); rec.setPostProcessedFile(remuxedFile); - rec.getAssociatedFiles().add(remuxedFile.getAbsolutePath()); + rec.setAbsoluteFile(remuxedFile); + Files.deleteIfExists(inputFile.toPath()); + rec.getAssociatedFiles().remove(inputFile.getCanonicalPath()); + rec.getAssociatedFiles().add(remuxedFile.getCanonicalPath()); } private void setupLogging(Process ffmpeg, Recording rec) throws IOException, InterruptedException { diff --git a/common/src/main/java/ctbrec/recorder/postprocessing/Rename.java b/common/src/main/java/ctbrec/recorder/postprocessing/Rename.java new file mode 100644 index 00000000..a785be23 --- /dev/null +++ b/common/src/main/java/ctbrec/recorder/postprocessing/Rename.java @@ -0,0 +1,48 @@ +package ctbrec.recorder.postprocessing; +import java.io.File; +import java.io.IOException; +import java.util.Objects; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.io.Files; + +import ctbrec.Recording; + +public class Rename extends AbstractPlaceholderAwarePostProcessor { + + private static final Logger LOG = LoggerFactory.getLogger(Rename.class); + public static final String FILE_NAME_TEMPLATE = "filename.template"; + public static final String DEFAULT = "${modelSanitizedName}_${localDateTime}.${fileSuffix}"; + + @Override + public String getName() { + return "rename"; + } + + @Override + public void postprocess(Recording rec) throws IOException { + String filenameTemplate = getConfig().getOrDefault(FILE_NAME_TEMPLATE, DEFAULT); + String filename = fillInPlaceHolders(filenameTemplate, rec); + File src = rec.getPostProcessedFile(); + File target = new File(src.getParentFile(), filename); + LOG.info("Renaming {} to {}", src.getName(), target.getName()); + Files.move(rec.getPostProcessedFile(), target); + rec.setPostProcessedFile(target); + if (Objects.equals(src, rec.getAbsoluteFile())) { + rec.setAbsoluteFile(target); + } + rec.getAssociatedFiles().remove(src.getCanonicalPath()); + rec.getAssociatedFiles().add(target.getCanonicalPath()); + } + + @Override + public String toString() { + String s = getName(); + if (getConfig().containsKey(FILE_NAME_TEMPLATE)) { + s += " [" + getConfig().get(FILE_NAME_TEMPLATE) + ']'; + } + return s; + } +} From fbf1c8ac1649c4c96739e74498cec1a9cadba5a8 Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Sun, 20 Sep 2020 18:27:26 +0200 Subject: [PATCH 12/33] Add check, if source and target are the same --- common/src/main/java/ctbrec/recorder/postprocessing/Move.java | 4 +++- .../src/main/java/ctbrec/recorder/postprocessing/Rename.java | 3 +++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/common/src/main/java/ctbrec/recorder/postprocessing/Move.java b/common/src/main/java/ctbrec/recorder/postprocessing/Move.java index 498922d3..36dbcbcf 100644 --- a/common/src/main/java/ctbrec/recorder/postprocessing/Move.java +++ b/common/src/main/java/ctbrec/recorder/postprocessing/Move.java @@ -29,6 +29,9 @@ public class Move extends AbstractPlaceholderAwarePostProcessor { String path = fillInPlaceHolders(pathTemplate, rec); File src = rec.getPostProcessedFile(); File target = new File(path, src.getName()); + if (Objects.equals(src, target)) { + return; + } LOG.info("Moving {} to {}", src.getName(), target.getParentFile().getCanonicalPath()); Files.createParentDirs(target); Files.move(rec.getPostProcessedFile(), target); @@ -38,7 +41,6 @@ public class Move extends AbstractPlaceholderAwarePostProcessor { } rec.getAssociatedFiles().remove(src.getCanonicalPath()); rec.getAssociatedFiles().add(target.getCanonicalPath()); - if (src.isFile()) { deleteEmptyParents(src.getParentFile()); } else { diff --git a/common/src/main/java/ctbrec/recorder/postprocessing/Rename.java b/common/src/main/java/ctbrec/recorder/postprocessing/Rename.java index a785be23..441121e9 100644 --- a/common/src/main/java/ctbrec/recorder/postprocessing/Rename.java +++ b/common/src/main/java/ctbrec/recorder/postprocessing/Rename.java @@ -27,6 +27,9 @@ public class Rename extends AbstractPlaceholderAwarePostProcessor { String filename = fillInPlaceHolders(filenameTemplate, rec); File src = rec.getPostProcessedFile(); File target = new File(src.getParentFile(), filename); + if (Objects.equals(src, target)) { + return; + } LOG.info("Renaming {} to {}", src.getName(), target.getName()); Files.move(rec.getPostProcessedFile(), target); rec.setPostProcessedFile(target); From e1bce0acf535b36970abed4af4bd462014e21f74 Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Tue, 22 Sep 2020 12:38:21 +0200 Subject: [PATCH 13/33] Improve handling of the orignal and pp files --- .../main/java/ctbrec/ui/JavaFxRecording.java | 6 ++++ .../java/ctbrec/ui/tabs/RecordingsTab.java | 1 - common/pom.xml | 4 +++ .../java/ctbrec/NotImplementedExcetion.java | 2 +- common/src/main/java/ctbrec/io/IoUtils.java | 8 +++-- .../ctbrec/recorder/NextGenLocalRecorder.java | 3 +- ...AbstractPlaceholderAwarePostProcessor.java | 6 ++-- .../ctbrec/recorder/postprocessing/Copy.java | 23 +++++++++----- .../postprocessing/CreateContactSheet.java | 20 +++++++++++++ .../postprocessing/CreateTimelineThumbs.java | 20 +++++++++++++ .../postprocessing/DeleteOriginal.java | 3 +- .../ctbrec/recorder/postprocessing/Move.java | 12 ++++---- .../postprocessing/PostProcessor.java | 3 +- .../postprocessing/RemoveKeepFile.java | 20 +++++++++++++ .../ctbrec/recorder/postprocessing/Remux.java | 30 +++++++++++++++---- .../recorder/postprocessing/Rename.java | 10 +++---- .../recorder/postprocessing/Script.java | 21 +++++++++++++ master/pom.xml | 5 ++++ .../ctbrec/recorder/server/HlsServlet.java | 9 +++--- 19 files changed, 168 insertions(+), 38 deletions(-) create mode 100644 common/src/main/java/ctbrec/recorder/postprocessing/CreateContactSheet.java create mode 100644 common/src/main/java/ctbrec/recorder/postprocessing/CreateTimelineThumbs.java create mode 100644 common/src/main/java/ctbrec/recorder/postprocessing/RemoveKeepFile.java create mode 100644 common/src/main/java/ctbrec/recorder/postprocessing/Script.java diff --git a/client/src/main/java/ctbrec/ui/JavaFxRecording.java b/client/src/main/java/ctbrec/ui/JavaFxRecording.java index e5ecf139..f74f666c 100644 --- a/client/src/main/java/ctbrec/ui/JavaFxRecording.java +++ b/client/src/main/java/ctbrec/ui/JavaFxRecording.java @@ -157,6 +157,7 @@ public class JavaFxRecording extends Recording { setStatus(updated.getStatus()); setProgress(updated.getProgress()); setSizeInByte(updated.getSizeInByte()); + setSingleFile(updated.isSingleFile()); } @Override @@ -188,6 +189,11 @@ public class JavaFxRecording extends Recording { return delegate.isSingleFile(); } + @Override + public void setSingleFile(boolean singleFile) { + delegate.setSingleFile(singleFile); + } + @Override public boolean isPinned() { return delegate.isPinned(); diff --git a/client/src/main/java/ctbrec/ui/tabs/RecordingsTab.java b/client/src/main/java/ctbrec/ui/tabs/RecordingsTab.java index c48b41ad..1583606e 100644 --- a/client/src/main/java/ctbrec/ui/tabs/RecordingsTab.java +++ b/client/src/main/java/ctbrec/ui/tabs/RecordingsTab.java @@ -559,7 +559,6 @@ public class RecordingsTab extends Tab implements TabSelectionListener { } private void onOpenDirectory(JavaFxRecording first) { - String recordingsDir = Config.getInstance().getSettings().recordingsDir; File tsFile = first.getAbsoluteFile(); new Thread(() -> DesktopIntegration.open(tsFile.getParent())).start(); } diff --git a/common/pom.xml b/common/pom.xml index eaf358d7..541444dc 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -50,6 +50,10 @@ com.google.guava guava + + commons-io + commons-io + org.openjfx javafx-controls diff --git a/common/src/main/java/ctbrec/NotImplementedExcetion.java b/common/src/main/java/ctbrec/NotImplementedExcetion.java index 4fb3d3c9..9ecfda88 100644 --- a/common/src/main/java/ctbrec/NotImplementedExcetion.java +++ b/common/src/main/java/ctbrec/NotImplementedExcetion.java @@ -3,7 +3,7 @@ package ctbrec; public class NotImplementedExcetion extends RuntimeException { public NotImplementedExcetion() { - super(); + super("Not implemented"); } public NotImplementedExcetion(String mesg) { diff --git a/common/src/main/java/ctbrec/io/IoUtils.java b/common/src/main/java/ctbrec/io/IoUtils.java index f3be9bbb..644e540b 100644 --- a/common/src/main/java/ctbrec/io/IoUtils.java +++ b/common/src/main/java/ctbrec/io/IoUtils.java @@ -17,12 +17,14 @@ public class IoUtils { public static void deleteEmptyParents(File parent) throws IOException { File recDir = new File(Config.getInstance().getSettings().recordingsDir); - while (parent != null && parent.list() != null && parent.list().length == 0) { + while (parent != null && (parent.list() != null && parent.list().length == 0 || !parent.exists()) ) { if (parent.equals(recDir)) { return; } - LOG.debug("Deleting empty directory {}", parent.getAbsolutePath()); - Files.delete(parent.toPath()); + if(parent.exists()) { + LOG.debug("Deleting empty directory {}", parent.getAbsolutePath()); + Files.delete(parent.toPath()); + } parent = parent.getParentFile(); } } diff --git a/common/src/main/java/ctbrec/recorder/NextGenLocalRecorder.java b/common/src/main/java/ctbrec/recorder/NextGenLocalRecorder.java index 3afe4366..83a8e484 100644 --- a/common/src/main/java/ctbrec/recorder/NextGenLocalRecorder.java +++ b/common/src/main/java/ctbrec/recorder/NextGenLocalRecorder.java @@ -165,7 +165,7 @@ public class NextGenLocalRecorder implements Recorder { List postProcessors = config.getSettings().postProcessors; for (PostProcessor postProcessor : postProcessors) { LOG.debug("Running post-processor: {}", postProcessor.getName()); - postProcessor.postprocess(recording); + postProcessor.postprocess(recording, config); } setRecordingStatus(recording, State.FINISHED); recordingManager.saveRecording(recording); @@ -652,6 +652,7 @@ public class NextGenLocalRecorder implements Recorder { Download download = other.getModel().createDownload(); download.init(Config.getInstance(), other.getModel(), other.getStartDate()); other.setDownload(download); + other.setPostProcessedFile(null); submitPostProcessingJob(other); return; } diff --git a/common/src/main/java/ctbrec/recorder/postprocessing/AbstractPlaceholderAwarePostProcessor.java b/common/src/main/java/ctbrec/recorder/postprocessing/AbstractPlaceholderAwarePostProcessor.java index d89d9bf4..b790a724 100644 --- a/common/src/main/java/ctbrec/recorder/postprocessing/AbstractPlaceholderAwarePostProcessor.java +++ b/common/src/main/java/ctbrec/recorder/postprocessing/AbstractPlaceholderAwarePostProcessor.java @@ -29,7 +29,7 @@ public abstract class AbstractPlaceholderAwarePostProcessor extends AbstractPost "${recordingsDir}" }; - public String fillInPlaceHolders(String input, Recording rec) { + public String fillInPlaceHolders(String input, Recording rec, Config config) { // @formatter:off String output = input .replace("${modelName}", ofNullable(rec.getModel().getName()).orElse("modelName")) @@ -39,8 +39,8 @@ public abstract class AbstractPlaceholderAwarePostProcessor extends AbstractPost .replace("${siteSanitizedName}", getSanitizedSiteName(rec)) .replace("${fileSuffix}", getFileSuffix(rec)) .replace("${epochSecond}", Long.toString(rec.getStartDate().getEpochSecond())) - .replace("${modelNotes}", Config.getInstance().getModelNotes(rec.getModel())) - .replace("${recordingsDir}", Config.getInstance().getSettings().recordingsDir) + .replace("${modelNotes}", config.getModelNotes(rec.getModel())) + .replace("${recordingsDir}", config.getSettings().recordingsDir) ; output = replaceUtcDateTime(rec, output); diff --git a/common/src/main/java/ctbrec/recorder/postprocessing/Copy.java b/common/src/main/java/ctbrec/recorder/postprocessing/Copy.java index d77eefcf..4dbf0224 100644 --- a/common/src/main/java/ctbrec/recorder/postprocessing/Copy.java +++ b/common/src/main/java/ctbrec/recorder/postprocessing/Copy.java @@ -2,12 +2,13 @@ package ctbrec.recorder.postprocessing; import java.io.File; import java.io.IOException; +import java.nio.file.Files; +import org.apache.commons.io.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.io.Files; - +import ctbrec.Config; import ctbrec.Recording; public class Copy extends AbstractPostProcessor { @@ -20,20 +21,28 @@ public class Copy extends AbstractPostProcessor { } @Override - public void postprocess(Recording rec) throws IOException, InterruptedException { + public void postprocess(Recording rec, Config config) throws IOException, InterruptedException { File orig = rec.getPostProcessedFile(); String copyFilename = getFilenameForCopy(orig); File copy = new File(orig.getParentFile(), copyFilename); LOG.info("Creating a copy {}", copy); - Files.copy(rec.getPostProcessedFile(), copy); + if (orig.isFile()) { + Files.copy(rec.getPostProcessedFile().toPath(), copy.toPath()); + } else { + FileUtils.copyDirectory(orig, copy, true); + } rec.setPostProcessedFile(copy); rec.getAssociatedFiles().add(copy.getCanonicalPath()); } private String getFilenameForCopy(File orig) { String filename = orig.getName(); - String name = filename.substring(0, filename.lastIndexOf('.')); - String ext = filename.substring(filename.lastIndexOf('.') + 1); - return name + "_copy." + ext; + if (orig.isFile()) { + String name = filename.substring(0, filename.lastIndexOf('.')); + String ext = filename.substring(filename.lastIndexOf('.') + 1); + return name + "_copy." + ext; + } else { + return filename + "_copy"; + } } } diff --git a/common/src/main/java/ctbrec/recorder/postprocessing/CreateContactSheet.java b/common/src/main/java/ctbrec/recorder/postprocessing/CreateContactSheet.java new file mode 100644 index 00000000..2f090ad6 --- /dev/null +++ b/common/src/main/java/ctbrec/recorder/postprocessing/CreateContactSheet.java @@ -0,0 +1,20 @@ +package ctbrec.recorder.postprocessing; + +import java.io.IOException; + +import ctbrec.Config; +import ctbrec.NotImplementedExcetion; +import ctbrec.Recording; + +public class CreateContactSheet extends AbstractPlaceholderAwarePostProcessor { + + @Override + public String getName() { + return "create contact sheet"; + } + + @Override + public void postprocess(Recording rec, Config config) throws IOException, InterruptedException { + throw new NotImplementedExcetion(); + } +} diff --git a/common/src/main/java/ctbrec/recorder/postprocessing/CreateTimelineThumbs.java b/common/src/main/java/ctbrec/recorder/postprocessing/CreateTimelineThumbs.java new file mode 100644 index 00000000..c17bcec1 --- /dev/null +++ b/common/src/main/java/ctbrec/recorder/postprocessing/CreateTimelineThumbs.java @@ -0,0 +1,20 @@ +package ctbrec.recorder.postprocessing; + +import java.io.IOException; + +import ctbrec.Config; +import ctbrec.NotImplementedExcetion; +import ctbrec.Recording; + +public class CreateTimelineThumbs extends AbstractPlaceholderAwarePostProcessor { + + @Override + public String getName() { + return "create timeline thumbnails"; + } + + @Override + public void postprocess(Recording rec, Config config) throws IOException, InterruptedException { + throw new NotImplementedExcetion(); + } +} diff --git a/common/src/main/java/ctbrec/recorder/postprocessing/DeleteOriginal.java b/common/src/main/java/ctbrec/recorder/postprocessing/DeleteOriginal.java index 49337be3..71e1af8d 100644 --- a/common/src/main/java/ctbrec/recorder/postprocessing/DeleteOriginal.java +++ b/common/src/main/java/ctbrec/recorder/postprocessing/DeleteOriginal.java @@ -5,6 +5,7 @@ import static ctbrec.io.IoUtils.*; import java.io.IOException; import java.nio.file.Files; +import ctbrec.Config; import ctbrec.Recording; public class DeleteOriginal extends AbstractPostProcessor { @@ -15,7 +16,7 @@ public class DeleteOriginal extends AbstractPostProcessor { } @Override - public void postprocess(Recording rec) throws IOException, InterruptedException { + public void postprocess(Recording rec, Config config) throws IOException, InterruptedException { if (rec.getAbsoluteFile().isFile()) { Files.deleteIfExists(rec.getAbsoluteFile().toPath()); deleteEmptyParents(rec.getAbsoluteFile().getParentFile()); diff --git a/common/src/main/java/ctbrec/recorder/postprocessing/Move.java b/common/src/main/java/ctbrec/recorder/postprocessing/Move.java index 36dbcbcf..40823ddd 100644 --- a/common/src/main/java/ctbrec/recorder/postprocessing/Move.java +++ b/common/src/main/java/ctbrec/recorder/postprocessing/Move.java @@ -3,13 +3,13 @@ import static ctbrec.io.IoUtils.*; import java.io.File; import java.io.IOException; +import java.nio.file.Files; import java.util.Objects; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.io.Files; - +import ctbrec.Config; import ctbrec.Recording; public class Move extends AbstractPlaceholderAwarePostProcessor { @@ -24,17 +24,17 @@ public class Move extends AbstractPlaceholderAwarePostProcessor { } @Override - public void postprocess(Recording rec) throws IOException { + public void postprocess(Recording rec, Config config) throws IOException { String pathTemplate = getConfig().getOrDefault(PATH_TEMPLATE, DEFAULT); - String path = fillInPlaceHolders(pathTemplate, rec); + String path = fillInPlaceHolders(pathTemplate, rec, config); File src = rec.getPostProcessedFile(); File target = new File(path, src.getName()); if (Objects.equals(src, target)) { return; } LOG.info("Moving {} to {}", src.getName(), target.getParentFile().getCanonicalPath()); - Files.createParentDirs(target); - Files.move(rec.getPostProcessedFile(), target); + Files.createDirectories(target.getParentFile().toPath()); + Files.move(rec.getPostProcessedFile().toPath(), target.toPath()); rec.setPostProcessedFile(target); if (Objects.equals(src, rec.getAbsoluteFile())) { rec.setAbsoluteFile(target); diff --git a/common/src/main/java/ctbrec/recorder/postprocessing/PostProcessor.java b/common/src/main/java/ctbrec/recorder/postprocessing/PostProcessor.java index cc366520..9427655d 100644 --- a/common/src/main/java/ctbrec/recorder/postprocessing/PostProcessor.java +++ b/common/src/main/java/ctbrec/recorder/postprocessing/PostProcessor.java @@ -4,12 +4,13 @@ import java.io.IOException; import java.io.Serializable; import java.util.Map; +import ctbrec.Config; import ctbrec.Recording; public interface PostProcessor extends Serializable { String getName(); - void postprocess(Recording rec) throws IOException, InterruptedException; + void postprocess(Recording rec, Config config) throws IOException, InterruptedException; Map getConfig(); void setConfig(Map conf); diff --git a/common/src/main/java/ctbrec/recorder/postprocessing/RemoveKeepFile.java b/common/src/main/java/ctbrec/recorder/postprocessing/RemoveKeepFile.java new file mode 100644 index 00000000..1e76bd41 --- /dev/null +++ b/common/src/main/java/ctbrec/recorder/postprocessing/RemoveKeepFile.java @@ -0,0 +1,20 @@ +package ctbrec.recorder.postprocessing; + +import java.io.IOException; + +import ctbrec.Config; +import ctbrec.NotImplementedExcetion; +import ctbrec.Recording; + +public class RemoveKeepFile extends AbstractPostProcessor { + + @Override + public String getName() { + return "remove recording, but keep the files"; + } + + @Override + public void postprocess(Recording rec, Config config) throws IOException, InterruptedException { + throw new NotImplementedExcetion(); + } +} diff --git a/common/src/main/java/ctbrec/recorder/postprocessing/Remux.java b/common/src/main/java/ctbrec/recorder/postprocessing/Remux.java index cd7071bb..81142ef8 100644 --- a/common/src/main/java/ctbrec/recorder/postprocessing/Remux.java +++ b/common/src/main/java/ctbrec/recorder/postprocessing/Remux.java @@ -5,12 +5,15 @@ import java.io.FileOutputStream; import java.io.IOException; import java.nio.file.Files; import java.util.Arrays; +import java.util.Objects; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import ctbrec.Config; import ctbrec.OS; import ctbrec.Recording; +import ctbrec.io.IoUtils; import ctbrec.io.StreamRedirectThread; import ctbrec.recorder.download.ProcessExitedUncleanException; @@ -27,15 +30,18 @@ public class Remux extends AbstractPostProcessor { } @Override - public void postprocess(Recording rec) throws IOException, InterruptedException { + public void postprocess(Recording rec, Config config) throws IOException, InterruptedException { String fileExt = getConfig().get(FILE_EXT); String[] args = getConfig().get(FFMPEG_ARGS).split(" "); String[] argsPlusFile = new String[args.length + 3]; + File inputFile = rec.getPostProcessedFile(); + if (inputFile.isDirectory()) { + inputFile = new File(inputFile, "playlist.m3u8"); + } int i = 0; argsPlusFile[i++] = "-i"; - argsPlusFile[i++] = rec.getPostProcessedFile().getAbsolutePath(); + argsPlusFile[i++] = inputFile.getCanonicalPath(); System.arraycopy(args, 0, argsPlusFile, i, args.length); - File inputFile = rec.getPostProcessedFile(); File remuxedFile = new File(rec.getPostProcessedFile().getAbsolutePath() + '.' + fileExt); argsPlusFile[argsPlusFile.length - 1] = remuxedFile.getAbsolutePath(); String[] cmdline = OS.getFFmpegCommand(argsPlusFile); @@ -43,16 +49,29 @@ public class Remux extends AbstractPostProcessor { Process ffmpeg = Runtime.getRuntime().exec(cmdline, new String[0], rec.getPostProcessedFile().getParentFile()); setupLogging(ffmpeg, rec); rec.setPostProcessedFile(remuxedFile); - rec.setAbsoluteFile(remuxedFile); - Files.deleteIfExists(inputFile.toPath()); + if (inputFile.getName().equals("playlist.m3u8")) { + IoUtils.deleteDirectory(inputFile.getParentFile()); + if (Objects.equals(inputFile.getParentFile(), rec.getAbsoluteFile())) { + rec.setAbsoluteFile(remuxedFile); + } + } else { + Files.deleteIfExists(inputFile.toPath()); + if (Objects.equals(inputFile, rec.getAbsoluteFile())) { + rec.setAbsoluteFile(remuxedFile); + } + } + IoUtils.deleteEmptyParents(inputFile.getParentFile()); rec.getAssociatedFiles().remove(inputFile.getCanonicalPath()); rec.getAssociatedFiles().add(remuxedFile.getCanonicalPath()); + rec.setSingleFile(true); + rec.setSizeInByte(remuxedFile.length()); } private void setupLogging(Process ffmpeg, Recording rec) throws IOException, InterruptedException { int exitCode = 1; File video = rec.getPostProcessedFile(); File ffmpegLog = new File(video.getParentFile(), video.getName() + ".ffmpeg.log"); + rec.getAssociatedFiles().add(ffmpegLog.getCanonicalPath()); try (FileOutputStream mergeLogStream = new FileOutputStream(ffmpegLog)) { Thread stdout = new Thread(new StreamRedirectThread(ffmpeg.getInputStream(), mergeLogStream)); Thread stderr = new Thread(new StreamRedirectThread(ffmpeg.getErrorStream(), mergeLogStream)); @@ -67,6 +86,7 @@ public class Remux extends AbstractPostProcessor { if (exitCode != 1) { if (ffmpegLog.exists()) { Files.delete(ffmpegLog.toPath()); + rec.getAssociatedFiles().remove(ffmpegLog.getCanonicalPath()); } } else { rec.getAssociatedFiles().add(ffmpegLog.getAbsolutePath()); diff --git a/common/src/main/java/ctbrec/recorder/postprocessing/Rename.java b/common/src/main/java/ctbrec/recorder/postprocessing/Rename.java index 441121e9..4789f029 100644 --- a/common/src/main/java/ctbrec/recorder/postprocessing/Rename.java +++ b/common/src/main/java/ctbrec/recorder/postprocessing/Rename.java @@ -1,13 +1,13 @@ package ctbrec.recorder.postprocessing; import java.io.File; import java.io.IOException; +import java.nio.file.Files; import java.util.Objects; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.io.Files; - +import ctbrec.Config; import ctbrec.Recording; public class Rename extends AbstractPlaceholderAwarePostProcessor { @@ -22,16 +22,16 @@ public class Rename extends AbstractPlaceholderAwarePostProcessor { } @Override - public void postprocess(Recording rec) throws IOException { + public void postprocess(Recording rec, Config config) throws IOException { String filenameTemplate = getConfig().getOrDefault(FILE_NAME_TEMPLATE, DEFAULT); - String filename = fillInPlaceHolders(filenameTemplate, rec); + String filename = fillInPlaceHolders(filenameTemplate, rec, config); File src = rec.getPostProcessedFile(); File target = new File(src.getParentFile(), filename); if (Objects.equals(src, target)) { return; } LOG.info("Renaming {} to {}", src.getName(), target.getName()); - Files.move(rec.getPostProcessedFile(), target); + Files.move(rec.getPostProcessedFile().toPath(), target.toPath()); rec.setPostProcessedFile(target); if (Objects.equals(src, rec.getAbsoluteFile())) { rec.setAbsoluteFile(target); diff --git a/common/src/main/java/ctbrec/recorder/postprocessing/Script.java b/common/src/main/java/ctbrec/recorder/postprocessing/Script.java new file mode 100644 index 00000000..a842867a --- /dev/null +++ b/common/src/main/java/ctbrec/recorder/postprocessing/Script.java @@ -0,0 +1,21 @@ +package ctbrec.recorder.postprocessing; + +import java.io.IOException; + +import ctbrec.Config; +import ctbrec.NotImplementedExcetion; +import ctbrec.Recording; + +public class Script extends AbstractPlaceholderAwarePostProcessor { + + @Override + public String getName() { + return "execute script"; + } + + @Override + public void postprocess(Recording rec, Config config) throws IOException, InterruptedException { + // TODO make it possible to choose, which placeholders to pass to the script + throw new NotImplementedExcetion(); + } +} diff --git a/master/pom.xml b/master/pom.xml index 40e91c83..f55c7825 100644 --- a/master/pom.xml +++ b/master/pom.xml @@ -103,6 +103,11 @@ guava 17.0 + + commons-io + commons-io + 2.8.0 + junit junit diff --git a/server/src/main/java/ctbrec/recorder/server/HlsServlet.java b/server/src/main/java/ctbrec/recorder/server/HlsServlet.java index 9498aede..72dd7858 100644 --- a/server/src/main/java/ctbrec/recorder/server/HlsServlet.java +++ b/server/src/main/java/ctbrec/recorder/server/HlsServlet.java @@ -40,8 +40,6 @@ public class HlsServlet extends AbstractCtbrecServlet { @Override protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { - - String contextPath = getServletContext().getContextPath(); String request = req.getRequestURI().substring(contextPath.length() + 5); Path recordingsDirPath = Paths.get(config.getSettings().recordingsDir).toAbsolutePath().normalize(); @@ -68,7 +66,10 @@ public class HlsServlet extends AbstractCtbrecServlet { String id = request.split("/")[0]; Optional rec = getRecordingById(id); if (rec.isPresent()) { - File file = rec.get().getAbsoluteFile(); + File path = rec.get().getAbsoluteFile(); + if (!path.isFile()) { + path = new File(path, requestedFile.getName()); + } if (LOG.isTraceEnabled()) { Enumeration headerNames = req.getHeaderNames(); while (headerNames.hasMoreElements()) { @@ -76,7 +77,7 @@ public class HlsServlet extends AbstractCtbrecServlet { LOG.trace("{}: {}", header, req.getHeader(header)); } } - serveSegment(req, resp, file); + serveSegment(req, resp, path); } else { error404(req, resp); return; From 4167b222ba36bb16aa343ce671af650dc08e5890 Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Tue, 22 Sep 2020 14:58:20 +0200 Subject: [PATCH 14/33] Add test for Rename post-processor --- common/pom.xml | 5 + ...AbstractPlaceholderAwarePostProcessor.java | 5 +- .../postprocessing/RenameSingleFileTest.java | 127 ++++++++++++++++++ master/pom.xml | 18 +++ 4 files changed, 152 insertions(+), 3 deletions(-) create mode 100644 common/src/test/java/ctbrec/recorder/postprocessing/RenameSingleFileTest.java diff --git a/common/pom.xml b/common/pom.xml index 541444dc..c4e22ca7 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -75,6 +75,11 @@ junit test + + org.mockito + mockito-core + test + javax.xml.bind jaxb-api diff --git a/common/src/main/java/ctbrec/recorder/postprocessing/AbstractPlaceholderAwarePostProcessor.java b/common/src/main/java/ctbrec/recorder/postprocessing/AbstractPlaceholderAwarePostProcessor.java index b790a724..545f795f 100644 --- a/common/src/main/java/ctbrec/recorder/postprocessing/AbstractPlaceholderAwarePostProcessor.java +++ b/common/src/main/java/ctbrec/recorder/postprocessing/AbstractPlaceholderAwarePostProcessor.java @@ -14,8 +14,7 @@ import ctbrec.Recording; public abstract class AbstractPlaceholderAwarePostProcessor extends AbstractPostProcessor { - @SuppressWarnings("unused") - private String[] placeHolders = { + public static final String[] PLACE_HOLDERS = { "${modelName}", "${modelDisplayName}", "${modelSanitizedName}", @@ -59,7 +58,7 @@ public abstract class AbstractPlaceholderAwarePostProcessor extends AbstractPost } private String replaceDateTime(Recording rec, String filename, String placeHolder, ZoneId zone) { - String pattern = "yyyy-mm-dd_HH-mm-ss"; + String pattern = "yyyy-MM-dd_HH-mm-ss"; Matcher m = Pattern.compile("\\$\\{" + placeHolder + "(?:\\((.*?)\\))?\\}").matcher(filename); if (m.find()) { String p = m.group(1); diff --git a/common/src/test/java/ctbrec/recorder/postprocessing/RenameSingleFileTest.java b/common/src/test/java/ctbrec/recorder/postprocessing/RenameSingleFileTest.java new file mode 100644 index 00000000..64399ad4 --- /dev/null +++ b/common/src/test/java/ctbrec/recorder/postprocessing/RenameSingleFileTest.java @@ -0,0 +1,127 @@ +package ctbrec.recorder.postprocessing; + +import static java.nio.charset.StandardCharsets.*; +import static java.nio.file.StandardOpenOption.*; +import static org.junit.Assert.*; +import static org.mockito.ArgumentMatchers.*; +import static org.mockito.Mockito.*; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Instant; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.commons.io.FileUtils; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import ctbrec.Config; +import ctbrec.Model; +import ctbrec.Recording; +import ctbrec.Settings; +import ctbrec.sites.Site; +import ctbrec.sites.chaturbate.Chaturbate; + +public class RenameSingleFileTest { + + private Path baseDir; + private Path recDir; + private File original; + private File postProcessed; + private Instant now = Instant.now(); + + @Before + public void setup() throws IOException { + baseDir = Files.createTempDirectory("ctbrec_test_"); + recDir = baseDir.resolve("recordings"); + original = new File(recDir.toFile(), "original.ts"); + postProcessed = new File(recDir.toFile(), "postProcessed.ts"); + Files.createDirectories(original.getParentFile().toPath()); + Files.write(original.toPath(), "foobar".getBytes(UTF_8), CREATE_NEW, WRITE, TRUNCATE_EXISTING); + Files.write(postProcessed.toPath(), "foobar".getBytes(UTF_8), CREATE_NEW, WRITE, TRUNCATE_EXISTING); + } + + @After + public void teardown() throws IOException { + FileUtils.deleteDirectory(baseDir.toFile()); + } + + @Test + public void testOriginalFileReplacement() throws IOException { + Config config = mockConfig(); + + Recording rec = new Recording(); + rec.setModel(mockModel()); + rec.setAbsoluteFile(original); + rec.setStartDate(now); + Rename pp = new Rename(); + pp.postprocess(rec, config); + + Matcher m = Pattern.compile("Mockita_\\d{4}-\\d{2}-\\d{2}_\\d{2}-\\d{2}-\\d{2}\\.ts").matcher(rec.getAbsoluteFile().getName()); + assertTrue(m.matches()); + assertEquals(rec.getAbsoluteFile(), rec.getPostProcessedFile()); + assertNotEquals(rec.getAbsoluteFile(), original); + } + + @Test + public void testEarlyExit() throws IOException { + Config config = mockConfig(); + Model model = mockModel(); + Recording rec = mock(Recording.class); + when(rec.getModel()).thenReturn(model); + when(rec.getAbsoluteFile()).thenReturn(original); + when(rec.getPostProcessedFile()).thenReturn(original); + when(rec.getStartDate()).thenReturn(now); + doThrow(new RuntimeException("Unexpected call of setAbsoluteFile")).when(rec).setAbsoluteFile(any()); + Rename pp = new Rename(); + pp.getConfig().put(Rename.FILE_NAME_TEMPLATE, "original.ts"); + pp.postprocess(rec, config); + } + + @Test + public void absoluteFileShouldKeepBeingOriginalIfFilesDiffer() throws IOException { + Config config = mockConfig(); + Model model = mockModel(); + Recording rec = mock(Recording.class); + when(rec.getModel()).thenReturn(model); + when(rec.getAbsoluteFile()).thenReturn(original); + when(rec.getPostProcessedFile()).thenReturn(postProcessed); + when(rec.getStartDate()).thenReturn(now); + doThrow(new RuntimeException("Unexpected call of setAbsoluteFile")).when(rec).setAbsoluteFile(any()); + Rename pp = new Rename(); + pp.postprocess(rec, config); + } + + @Test + public void testToString() { + Rename pp = new Rename(); + assertEquals("rename", pp.toString()); + + pp.getConfig().put(Rename.FILE_NAME_TEMPLATE, Rename.DEFAULT); + assertEquals("rename [${modelSanitizedName}_${localDateTime}.${fileSuffix}]", pp.toString()); + } + + Config mockConfig() { + Config config = mock(Config.class); + when(config.getSettings()).thenReturn(mockSettings()); + when(config.getModelNotes(any())).thenReturn("tag, foo, bar"); + return config; + } + + Model mockModel() { + Site site = new Chaturbate(); + Model model = site.createModel("Mockita"); + model.setDisplayName("Mockita Boobilicious"); + return model; + } + + Settings mockSettings() { + Settings settings = new Settings(); + settings.recordingsDir = recDir.toString(); + return settings; + } +} diff --git a/master/pom.xml b/master/pom.xml index f55c7825..68ef277c 100644 --- a/master/pom.xml +++ b/master/pom.xml @@ -26,6 +26,14 @@ maven-assembly-plugin 3.1.0 + + org.apache.maven.plugins + maven-surefire-plugin + 2.22.2 + + true + + @@ -114,6 +122,12 @@ 4.12 test + + org.mockito + mockito-core + 3.5.11 + test + org.eclipse.jetty jetty-server @@ -126,4 +140,8 @@ + + + + From d23f3fea0497cce92cc3272085c639560804a767 Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Tue, 22 Sep 2020 17:40:03 +0200 Subject: [PATCH 15/33] Add test for Rename pp for directories --- ...AbstractPlaceholderAwarePostProcessor.java | 8 ++- .../recorder/postprocessing/Rename.java | 4 +- .../postprocessing/AbstractPpTest.java | 72 +++++++++++++++++++ .../postprocessing/RenameDirectoryTest.java | 52 ++++++++++++++ .../postprocessing/RenameSingleFileTest.java | 60 +--------------- 5 files changed, 136 insertions(+), 60 deletions(-) create mode 100644 common/src/test/java/ctbrec/recorder/postprocessing/AbstractPpTest.java create mode 100644 common/src/test/java/ctbrec/recorder/postprocessing/RenameDirectoryTest.java diff --git a/common/src/main/java/ctbrec/recorder/postprocessing/AbstractPlaceholderAwarePostProcessor.java b/common/src/main/java/ctbrec/recorder/postprocessing/AbstractPlaceholderAwarePostProcessor.java index 545f795f..8831b687 100644 --- a/common/src/main/java/ctbrec/recorder/postprocessing/AbstractPlaceholderAwarePostProcessor.java +++ b/common/src/main/java/ctbrec/recorder/postprocessing/AbstractPlaceholderAwarePostProcessor.java @@ -78,8 +78,12 @@ public abstract class AbstractPlaceholderAwarePostProcessor extends AbstractPost } private CharSequence getFileSuffix(Recording rec) { - String filename = rec.getPostProcessedFile().getName(); - return filename.substring(filename.lastIndexOf('.') + 1); + if(rec.isSingleFile()) { + String filename = rec.getPostProcessedFile().getName(); + return filename.substring(filename.lastIndexOf('.') + 1); + } else { + return ""; + } } private CharSequence getSanitizedSiteName(Recording rec) { diff --git a/common/src/main/java/ctbrec/recorder/postprocessing/Rename.java b/common/src/main/java/ctbrec/recorder/postprocessing/Rename.java index 4789f029..bb1960c5 100644 --- a/common/src/main/java/ctbrec/recorder/postprocessing/Rename.java +++ b/common/src/main/java/ctbrec/recorder/postprocessing/Rename.java @@ -15,6 +15,7 @@ public class Rename extends AbstractPlaceholderAwarePostProcessor { private static final Logger LOG = LoggerFactory.getLogger(Rename.class); public static final String FILE_NAME_TEMPLATE = "filename.template"; public static final String DEFAULT = "${modelSanitizedName}_${localDateTime}.${fileSuffix}"; + public static final String DEFAULT_DIR = "${modelSanitizedName}_${localDateTime}"; @Override public String getName() { @@ -23,7 +24,8 @@ public class Rename extends AbstractPlaceholderAwarePostProcessor { @Override public void postprocess(Recording rec, Config config) throws IOException { - String filenameTemplate = getConfig().getOrDefault(FILE_NAME_TEMPLATE, DEFAULT); + String defaultTemplate = rec.isSingleFile() ? DEFAULT : DEFAULT_DIR; + String filenameTemplate = getConfig().getOrDefault(FILE_NAME_TEMPLATE, defaultTemplate); String filename = fillInPlaceHolders(filenameTemplate, rec, config); File src = rec.getPostProcessedFile(); File target = new File(src.getParentFile(), filename); diff --git a/common/src/test/java/ctbrec/recorder/postprocessing/AbstractPpTest.java b/common/src/test/java/ctbrec/recorder/postprocessing/AbstractPpTest.java new file mode 100644 index 00000000..ba0f11ff --- /dev/null +++ b/common/src/test/java/ctbrec/recorder/postprocessing/AbstractPpTest.java @@ -0,0 +1,72 @@ +package ctbrec.recorder.postprocessing; + +import static java.nio.charset.StandardCharsets.*; +import static java.nio.file.StandardOpenOption.*; +import static org.mockito.ArgumentMatchers.*; +import static org.mockito.Mockito.*; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Instant; + +import org.apache.commons.io.FileUtils; +import org.junit.After; +import org.junit.Before; + +import ctbrec.Config; +import ctbrec.Model; +import ctbrec.Settings; +import ctbrec.sites.Site; +import ctbrec.sites.chaturbate.Chaturbate; + +public abstract class AbstractPpTest { + Path baseDir; + Path recDir; + File original; + File postProcessed; + File originalDir; + File postProcessedDir; + Instant now = Instant.now(); + + @Before + public void setup() throws IOException { + baseDir = Files.createTempDirectory("ctbrec_test_"); + recDir = baseDir.resolve("recordings"); + original = new File(recDir.toFile(), "original.ts"); + postProcessed = new File(recDir.toFile(), "postProcessed.ts"); + originalDir = new File(recDir.toFile(), "original"); + postProcessedDir = new File(recDir.toFile(), "postProcessed"); + Files.createDirectories(original.getParentFile().toPath()); + Files.write(original.toPath(), "foobar".getBytes(UTF_8), CREATE_NEW, WRITE, TRUNCATE_EXISTING); + Files.write(postProcessed.toPath(), "foobar".getBytes(UTF_8), CREATE_NEW, WRITE, TRUNCATE_EXISTING); + Files.createDirectories(originalDir.toPath()); + FileUtils.touch(new File(originalDir, "playlist.m3u8")); + } + + @After + public void teardown() throws IOException { + FileUtils.deleteDirectory(baseDir.toFile()); + } + + Config mockConfig() { + Config config = mock(Config.class); + when(config.getSettings()).thenReturn(mockSettings()); + when(config.getModelNotes(any())).thenReturn("tag, foo, bar"); + return config; + } + + Model mockModel() { + Site site = new Chaturbate(); + Model model = site.createModel("Mockita"); + model.setDisplayName("Mockita Boobilicious"); + return model; + } + + Settings mockSettings() { + Settings settings = new Settings(); + settings.recordingsDir = recDir.toString(); + return settings; + } +} diff --git a/common/src/test/java/ctbrec/recorder/postprocessing/RenameDirectoryTest.java b/common/src/test/java/ctbrec/recorder/postprocessing/RenameDirectoryTest.java new file mode 100644 index 00000000..067bcfbc --- /dev/null +++ b/common/src/test/java/ctbrec/recorder/postprocessing/RenameDirectoryTest.java @@ -0,0 +1,52 @@ +package ctbrec.recorder.postprocessing; + +import static org.junit.Assert.*; +import static org.mockito.ArgumentMatchers.*; +import static org.mockito.Mockito.*; + +import java.io.IOException; +import java.nio.file.Files; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.junit.Test; + +import ctbrec.Config; +import ctbrec.Model; +import ctbrec.Recording; + +public class RenameDirectoryTest extends AbstractPpTest { + + @Test + public void testOriginalFileReplacement() throws IOException { + Config config = mockConfig(); + Recording rec = new Recording(); + rec.setModel(mockModel()); + rec.setAbsoluteFile(originalDir); + rec.setStartDate(now); + rec.setSingleFile(false); + Rename pp = new Rename(); + pp.postprocess(rec, config); + + Matcher m = Pattern.compile("Mockita_\\d{4}-\\d{2}-\\d{2}_\\d{2}-\\d{2}-\\d{2}").matcher(rec.getAbsoluteFile().getName()); + assertTrue(m.matches()); + assertEquals(rec.getAbsoluteFile(), rec.getPostProcessedFile()); + assertNotEquals(rec.getAbsoluteFile(), original); + } + + @Test + public void absoluteFileShouldKeepBeingOriginalIfFilesDiffer() throws IOException { + Config config = mockConfig(); + Model model = mockModel(); + Recording rec = mock(Recording.class); + when(rec.getModel()).thenReturn(model); + when(rec.getAbsoluteFile()).thenReturn(originalDir); + when(rec.getPostProcessedFile()).thenReturn(postProcessedDir); + when(rec.getStartDate()).thenReturn(now); + doThrow(new RuntimeException("Unexpected call of setAbsoluteFile")).when(rec).setAbsoluteFile(any()); + + Files.createDirectories(postProcessedDir.toPath()); + Rename pp = new Rename(); + pp.postprocess(rec, config); + } +} diff --git a/common/src/test/java/ctbrec/recorder/postprocessing/RenameSingleFileTest.java b/common/src/test/java/ctbrec/recorder/postprocessing/RenameSingleFileTest.java index 64399ad4..4b065664 100644 --- a/common/src/test/java/ctbrec/recorder/postprocessing/RenameSingleFileTest.java +++ b/common/src/test/java/ctbrec/recorder/postprocessing/RenameSingleFileTest.java @@ -1,63 +1,29 @@ package ctbrec.recorder.postprocessing; -import static java.nio.charset.StandardCharsets.*; -import static java.nio.file.StandardOpenOption.*; import static org.junit.Assert.*; import static org.mockito.ArgumentMatchers.*; import static org.mockito.Mockito.*; -import java.io.File; import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.time.Instant; import java.util.regex.Matcher; import java.util.regex.Pattern; -import org.apache.commons.io.FileUtils; -import org.junit.After; -import org.junit.Before; import org.junit.Test; import ctbrec.Config; import ctbrec.Model; import ctbrec.Recording; -import ctbrec.Settings; -import ctbrec.sites.Site; -import ctbrec.sites.chaturbate.Chaturbate; -public class RenameSingleFileTest { - - private Path baseDir; - private Path recDir; - private File original; - private File postProcessed; - private Instant now = Instant.now(); - - @Before - public void setup() throws IOException { - baseDir = Files.createTempDirectory("ctbrec_test_"); - recDir = baseDir.resolve("recordings"); - original = new File(recDir.toFile(), "original.ts"); - postProcessed = new File(recDir.toFile(), "postProcessed.ts"); - Files.createDirectories(original.getParentFile().toPath()); - Files.write(original.toPath(), "foobar".getBytes(UTF_8), CREATE_NEW, WRITE, TRUNCATE_EXISTING); - Files.write(postProcessed.toPath(), "foobar".getBytes(UTF_8), CREATE_NEW, WRITE, TRUNCATE_EXISTING); - } - - @After - public void teardown() throws IOException { - FileUtils.deleteDirectory(baseDir.toFile()); - } +public class RenameSingleFileTest extends AbstractPpTest { @Test public void testOriginalFileReplacement() throws IOException { Config config = mockConfig(); - Recording rec = new Recording(); rec.setModel(mockModel()); rec.setAbsoluteFile(original); rec.setStartDate(now); + rec.setSingleFile(true); Rename pp = new Rename(); pp.postprocess(rec, config); @@ -78,7 +44,7 @@ public class RenameSingleFileTest { when(rec.getStartDate()).thenReturn(now); doThrow(new RuntimeException("Unexpected call of setAbsoluteFile")).when(rec).setAbsoluteFile(any()); Rename pp = new Rename(); - pp.getConfig().put(Rename.FILE_NAME_TEMPLATE, "original.ts"); + pp.getConfig().put(Rename.FILE_NAME_TEMPLATE, original.getName()); pp.postprocess(rec, config); } @@ -104,24 +70,4 @@ public class RenameSingleFileTest { pp.getConfig().put(Rename.FILE_NAME_TEMPLATE, Rename.DEFAULT); assertEquals("rename [${modelSanitizedName}_${localDateTime}.${fileSuffix}]", pp.toString()); } - - Config mockConfig() { - Config config = mock(Config.class); - when(config.getSettings()).thenReturn(mockSettings()); - when(config.getModelNotes(any())).thenReturn("tag, foo, bar"); - return config; - } - - Model mockModel() { - Site site = new Chaturbate(); - Model model = site.createModel("Mockita"); - model.setDisplayName("Mockita Boobilicious"); - return model; - } - - Settings mockSettings() { - Settings settings = new Settings(); - settings.recordingsDir = recDir.toString(); - return settings; - } } From 7b1898072f01e20a909b902eea11dff4dc30eaff Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Tue, 22 Sep 2020 20:41:07 +0200 Subject: [PATCH 16/33] Make recording single file after remux Also set the file size to the size of the remuxed one --- .../src/main/java/ctbrec/recorder/postprocessing/Remux.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common/src/main/java/ctbrec/recorder/postprocessing/Remux.java b/common/src/main/java/ctbrec/recorder/postprocessing/Remux.java index 81142ef8..be76c86a 100644 --- a/common/src/main/java/ctbrec/recorder/postprocessing/Remux.java +++ b/common/src/main/java/ctbrec/recorder/postprocessing/Remux.java @@ -60,11 +60,11 @@ public class Remux extends AbstractPostProcessor { rec.setAbsoluteFile(remuxedFile); } } + rec.setSingleFile(true); + rec.setSizeInByte(remuxedFile.length()); IoUtils.deleteEmptyParents(inputFile.getParentFile()); rec.getAssociatedFiles().remove(inputFile.getCanonicalPath()); rec.getAssociatedFiles().add(remuxedFile.getCanonicalPath()); - rec.setSingleFile(true); - rec.setSizeInByte(remuxedFile.length()); } private void setupLogging(Process ffmpeg, Recording rec) throws IOException, InterruptedException { From 67ff48e2dcef554b58e67113d7ea10a3589c22d7 Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Tue, 22 Sep 2020 22:15:28 +0200 Subject: [PATCH 17/33] Pass RecordingManager to post-processors --- .../ui/settings/PostProcessingStepPanel.java | 4 +++- .../ctbrec/recorder/NextGenLocalRecorder.java | 2 +- .../java/ctbrec/recorder/RecordingManager.java | 18 +++++++++++------- .../ctbrec/recorder/postprocessing/Copy.java | 3 ++- .../postprocessing/CreateContactSheet.java | 3 ++- .../postprocessing/CreateTimelineThumbs.java | 3 ++- .../postprocessing/DeleteOriginal.java | 3 ++- .../ctbrec/recorder/postprocessing/Move.java | 3 ++- .../recorder/postprocessing/PostProcessor.java | 3 ++- .../postprocessing/RemoveKeepFile.java | 7 ++++--- .../ctbrec/recorder/postprocessing/Remux.java | 3 ++- .../ctbrec/recorder/postprocessing/Rename.java | 3 ++- .../ctbrec/recorder/postprocessing/Script.java | 3 ++- .../postprocessing/AbstractPpTest.java | 2 ++ .../postprocessing/RenameDirectoryTest.java | 4 ++-- .../postprocessing/RenameSingleFileTest.java | 6 +++--- 16 files changed, 44 insertions(+), 26 deletions(-) diff --git a/client/src/main/java/ctbrec/ui/settings/PostProcessingStepPanel.java b/client/src/main/java/ctbrec/ui/settings/PostProcessingStepPanel.java index 05a18f92..0e2c13e6 100644 --- a/client/src/main/java/ctbrec/ui/settings/PostProcessingStepPanel.java +++ b/client/src/main/java/ctbrec/ui/settings/PostProcessingStepPanel.java @@ -10,6 +10,7 @@ import ctbrec.recorder.postprocessing.Copy; import ctbrec.recorder.postprocessing.DeleteOriginal; import ctbrec.recorder.postprocessing.Move; import ctbrec.recorder.postprocessing.PostProcessor; +import ctbrec.recorder.postprocessing.RemoveKeepFile; import ctbrec.recorder.postprocessing.Remux; import ctbrec.recorder.postprocessing.Rename; import ctbrec.ui.controls.Dialogs; @@ -36,7 +37,8 @@ public class PostProcessingStepPanel extends GridPane { Remux.class, Rename.class, Move.class, - DeleteOriginal.class + DeleteOriginal.class, + RemoveKeepFile.class }; // @formatter: on ListView stepListView; diff --git a/common/src/main/java/ctbrec/recorder/NextGenLocalRecorder.java b/common/src/main/java/ctbrec/recorder/NextGenLocalRecorder.java index 83a8e484..2897e15f 100644 --- a/common/src/main/java/ctbrec/recorder/NextGenLocalRecorder.java +++ b/common/src/main/java/ctbrec/recorder/NextGenLocalRecorder.java @@ -165,7 +165,7 @@ public class NextGenLocalRecorder implements Recorder { List postProcessors = config.getSettings().postProcessors; for (PostProcessor postProcessor : postProcessors) { LOG.debug("Running post-processor: {}", postProcessor.getName()); - postProcessor.postprocess(recording, config); + postProcessor.postprocess(recording, recordingManager, config); } setRecordingStatus(recording, State.FINISHED); recordingManager.saveRecording(recording); diff --git a/common/src/main/java/ctbrec/recorder/RecordingManager.java b/common/src/main/java/ctbrec/recorder/RecordingManager.java index 69735f11..cc7dda57 100644 --- a/common/src/main/java/ctbrec/recorder/RecordingManager.java +++ b/common/src/main/java/ctbrec/recorder/RecordingManager.java @@ -54,6 +54,10 @@ public class RecordingManager { } public void add(Recording rec) throws IOException { + File recordingsMetaDir = getDir(); + String filename = rec.toString() + ".json"; + File recordingMetaData = new File(recordingsMetaDir, filename); + rec.setMetaDataFile(recordingMetaData.getCanonicalPath()); saveRecording(rec); recordingsLock.lock(); try { @@ -64,13 +68,13 @@ public class RecordingManager { } public void saveRecording(Recording rec) throws IOException { - String json = adapter.toJson(rec); - File recordingsMetaDir = getDir(); - String filename = rec.toString() + ".json"; - File recordingMetaData = new File(recordingsMetaDir, filename); - rec.setMetaDataFile(recordingMetaData.getAbsolutePath()); - Files.createDirectories(recordingsMetaDir.toPath()); - Files.write(recordingMetaData.toPath(), json.getBytes(UTF_8), CREATE, WRITE, TRUNCATE_EXISTING); + if (rec.getMetaDataFile() != null) { + File recordingMetaData = new File(rec.getMetaDataFile()); + String json = adapter.toJson(rec); + rec.setMetaDataFile(recordingMetaData.getAbsolutePath()); + Files.createDirectories(recordingMetaData.getParentFile().toPath()); + Files.write(recordingMetaData.toPath(), json.getBytes(UTF_8), CREATE, WRITE, TRUNCATE_EXISTING); + } } private void loadRecordings() throws IOException { diff --git a/common/src/main/java/ctbrec/recorder/postprocessing/Copy.java b/common/src/main/java/ctbrec/recorder/postprocessing/Copy.java index 4dbf0224..a5fe0c54 100644 --- a/common/src/main/java/ctbrec/recorder/postprocessing/Copy.java +++ b/common/src/main/java/ctbrec/recorder/postprocessing/Copy.java @@ -10,6 +10,7 @@ import org.slf4j.LoggerFactory; import ctbrec.Config; import ctbrec.Recording; +import ctbrec.recorder.RecordingManager; public class Copy extends AbstractPostProcessor { @@ -21,7 +22,7 @@ public class Copy extends AbstractPostProcessor { } @Override - public void postprocess(Recording rec, Config config) throws IOException, InterruptedException { + public void postprocess(Recording rec, RecordingManager recordingManager, Config config) throws IOException, InterruptedException { File orig = rec.getPostProcessedFile(); String copyFilename = getFilenameForCopy(orig); File copy = new File(orig.getParentFile(), copyFilename); diff --git a/common/src/main/java/ctbrec/recorder/postprocessing/CreateContactSheet.java b/common/src/main/java/ctbrec/recorder/postprocessing/CreateContactSheet.java index 2f090ad6..d28ed652 100644 --- a/common/src/main/java/ctbrec/recorder/postprocessing/CreateContactSheet.java +++ b/common/src/main/java/ctbrec/recorder/postprocessing/CreateContactSheet.java @@ -5,6 +5,7 @@ import java.io.IOException; import ctbrec.Config; import ctbrec.NotImplementedExcetion; import ctbrec.Recording; +import ctbrec.recorder.RecordingManager; public class CreateContactSheet extends AbstractPlaceholderAwarePostProcessor { @@ -14,7 +15,7 @@ public class CreateContactSheet extends AbstractPlaceholderAwarePostProcessor { } @Override - public void postprocess(Recording rec, Config config) throws IOException, InterruptedException { + public void postprocess(Recording rec, RecordingManager recordingManager, Config config) throws IOException, InterruptedException { throw new NotImplementedExcetion(); } } diff --git a/common/src/main/java/ctbrec/recorder/postprocessing/CreateTimelineThumbs.java b/common/src/main/java/ctbrec/recorder/postprocessing/CreateTimelineThumbs.java index c17bcec1..1adc5f49 100644 --- a/common/src/main/java/ctbrec/recorder/postprocessing/CreateTimelineThumbs.java +++ b/common/src/main/java/ctbrec/recorder/postprocessing/CreateTimelineThumbs.java @@ -5,6 +5,7 @@ import java.io.IOException; import ctbrec.Config; import ctbrec.NotImplementedExcetion; import ctbrec.Recording; +import ctbrec.recorder.RecordingManager; public class CreateTimelineThumbs extends AbstractPlaceholderAwarePostProcessor { @@ -14,7 +15,7 @@ public class CreateTimelineThumbs extends AbstractPlaceholderAwarePostProcessor } @Override - public void postprocess(Recording rec, Config config) throws IOException, InterruptedException { + public void postprocess(Recording rec, RecordingManager recordingManager, Config config) throws IOException, InterruptedException { throw new NotImplementedExcetion(); } } diff --git a/common/src/main/java/ctbrec/recorder/postprocessing/DeleteOriginal.java b/common/src/main/java/ctbrec/recorder/postprocessing/DeleteOriginal.java index 71e1af8d..e73a57a8 100644 --- a/common/src/main/java/ctbrec/recorder/postprocessing/DeleteOriginal.java +++ b/common/src/main/java/ctbrec/recorder/postprocessing/DeleteOriginal.java @@ -7,6 +7,7 @@ import java.nio.file.Files; import ctbrec.Config; import ctbrec.Recording; +import ctbrec.recorder.RecordingManager; public class DeleteOriginal extends AbstractPostProcessor { @@ -16,7 +17,7 @@ public class DeleteOriginal extends AbstractPostProcessor { } @Override - public void postprocess(Recording rec, Config config) throws IOException, InterruptedException { + public void postprocess(Recording rec, RecordingManager recordingManager, Config config) throws IOException, InterruptedException { if (rec.getAbsoluteFile().isFile()) { Files.deleteIfExists(rec.getAbsoluteFile().toPath()); deleteEmptyParents(rec.getAbsoluteFile().getParentFile()); diff --git a/common/src/main/java/ctbrec/recorder/postprocessing/Move.java b/common/src/main/java/ctbrec/recorder/postprocessing/Move.java index 40823ddd..5079385c 100644 --- a/common/src/main/java/ctbrec/recorder/postprocessing/Move.java +++ b/common/src/main/java/ctbrec/recorder/postprocessing/Move.java @@ -11,6 +11,7 @@ import org.slf4j.LoggerFactory; import ctbrec.Config; import ctbrec.Recording; +import ctbrec.recorder.RecordingManager; public class Move extends AbstractPlaceholderAwarePostProcessor { @@ -24,7 +25,7 @@ public class Move extends AbstractPlaceholderAwarePostProcessor { } @Override - public void postprocess(Recording rec, Config config) throws IOException { + public void postprocess(Recording rec, RecordingManager recordingManager, Config config) throws IOException { String pathTemplate = getConfig().getOrDefault(PATH_TEMPLATE, DEFAULT); String path = fillInPlaceHolders(pathTemplate, rec, config); File src = rec.getPostProcessedFile(); diff --git a/common/src/main/java/ctbrec/recorder/postprocessing/PostProcessor.java b/common/src/main/java/ctbrec/recorder/postprocessing/PostProcessor.java index 9427655d..fe484e2f 100644 --- a/common/src/main/java/ctbrec/recorder/postprocessing/PostProcessor.java +++ b/common/src/main/java/ctbrec/recorder/postprocessing/PostProcessor.java @@ -6,11 +6,12 @@ import java.util.Map; import ctbrec.Config; import ctbrec.Recording; +import ctbrec.recorder.RecordingManager; public interface PostProcessor extends Serializable { String getName(); - void postprocess(Recording rec, Config config) throws IOException, InterruptedException; + void postprocess(Recording rec, RecordingManager recordingManager, Config config) throws IOException, InterruptedException; Map getConfig(); void setConfig(Map conf); diff --git a/common/src/main/java/ctbrec/recorder/postprocessing/RemoveKeepFile.java b/common/src/main/java/ctbrec/recorder/postprocessing/RemoveKeepFile.java index 1e76bd41..55fda725 100644 --- a/common/src/main/java/ctbrec/recorder/postprocessing/RemoveKeepFile.java +++ b/common/src/main/java/ctbrec/recorder/postprocessing/RemoveKeepFile.java @@ -3,8 +3,8 @@ package ctbrec.recorder.postprocessing; import java.io.IOException; import ctbrec.Config; -import ctbrec.NotImplementedExcetion; import ctbrec.Recording; +import ctbrec.recorder.RecordingManager; public class RemoveKeepFile extends AbstractPostProcessor { @@ -14,7 +14,8 @@ public class RemoveKeepFile extends AbstractPostProcessor { } @Override - public void postprocess(Recording rec, Config config) throws IOException, InterruptedException { - throw new NotImplementedExcetion(); + public void postprocess(Recording rec, RecordingManager recordingManager, Config config) throws IOException, InterruptedException { + recordingManager.remove(rec); + rec.setMetaDataFile(null); } } diff --git a/common/src/main/java/ctbrec/recorder/postprocessing/Remux.java b/common/src/main/java/ctbrec/recorder/postprocessing/Remux.java index be76c86a..61128568 100644 --- a/common/src/main/java/ctbrec/recorder/postprocessing/Remux.java +++ b/common/src/main/java/ctbrec/recorder/postprocessing/Remux.java @@ -15,6 +15,7 @@ import ctbrec.OS; import ctbrec.Recording; import ctbrec.io.IoUtils; import ctbrec.io.StreamRedirectThread; +import ctbrec.recorder.RecordingManager; import ctbrec.recorder.download.ProcessExitedUncleanException; public class Remux extends AbstractPostProcessor { @@ -30,7 +31,7 @@ public class Remux extends AbstractPostProcessor { } @Override - public void postprocess(Recording rec, Config config) throws IOException, InterruptedException { + public void postprocess(Recording rec, RecordingManager recordingManager, Config config) throws IOException, InterruptedException { String fileExt = getConfig().get(FILE_EXT); String[] args = getConfig().get(FFMPEG_ARGS).split(" "); String[] argsPlusFile = new String[args.length + 3]; diff --git a/common/src/main/java/ctbrec/recorder/postprocessing/Rename.java b/common/src/main/java/ctbrec/recorder/postprocessing/Rename.java index bb1960c5..28e44b86 100644 --- a/common/src/main/java/ctbrec/recorder/postprocessing/Rename.java +++ b/common/src/main/java/ctbrec/recorder/postprocessing/Rename.java @@ -9,6 +9,7 @@ import org.slf4j.LoggerFactory; import ctbrec.Config; import ctbrec.Recording; +import ctbrec.recorder.RecordingManager; public class Rename extends AbstractPlaceholderAwarePostProcessor { @@ -23,7 +24,7 @@ public class Rename extends AbstractPlaceholderAwarePostProcessor { } @Override - public void postprocess(Recording rec, Config config) throws IOException { + public void postprocess(Recording rec, RecordingManager recordingManager, Config config) throws IOException { String defaultTemplate = rec.isSingleFile() ? DEFAULT : DEFAULT_DIR; String filenameTemplate = getConfig().getOrDefault(FILE_NAME_TEMPLATE, defaultTemplate); String filename = fillInPlaceHolders(filenameTemplate, rec, config); diff --git a/common/src/main/java/ctbrec/recorder/postprocessing/Script.java b/common/src/main/java/ctbrec/recorder/postprocessing/Script.java index a842867a..0b278082 100644 --- a/common/src/main/java/ctbrec/recorder/postprocessing/Script.java +++ b/common/src/main/java/ctbrec/recorder/postprocessing/Script.java @@ -5,6 +5,7 @@ import java.io.IOException; import ctbrec.Config; import ctbrec.NotImplementedExcetion; import ctbrec.Recording; +import ctbrec.recorder.RecordingManager; public class Script extends AbstractPlaceholderAwarePostProcessor { @@ -14,7 +15,7 @@ public class Script extends AbstractPlaceholderAwarePostProcessor { } @Override - public void postprocess(Recording rec, Config config) throws IOException, InterruptedException { + public void postprocess(Recording rec, RecordingManager recordingManager, Config config) throws IOException, InterruptedException { // TODO make it possible to choose, which placeholders to pass to the script throw new NotImplementedExcetion(); } diff --git a/common/src/test/java/ctbrec/recorder/postprocessing/AbstractPpTest.java b/common/src/test/java/ctbrec/recorder/postprocessing/AbstractPpTest.java index ba0f11ff..adbd9aad 100644 --- a/common/src/test/java/ctbrec/recorder/postprocessing/AbstractPpTest.java +++ b/common/src/test/java/ctbrec/recorder/postprocessing/AbstractPpTest.java @@ -18,6 +18,7 @@ import org.junit.Before; import ctbrec.Config; import ctbrec.Model; import ctbrec.Settings; +import ctbrec.recorder.RecordingManager; import ctbrec.sites.Site; import ctbrec.sites.chaturbate.Chaturbate; @@ -29,6 +30,7 @@ public abstract class AbstractPpTest { File originalDir; File postProcessedDir; Instant now = Instant.now(); + RecordingManager recordingManager; @Before public void setup() throws IOException { diff --git a/common/src/test/java/ctbrec/recorder/postprocessing/RenameDirectoryTest.java b/common/src/test/java/ctbrec/recorder/postprocessing/RenameDirectoryTest.java index 067bcfbc..abf03070 100644 --- a/common/src/test/java/ctbrec/recorder/postprocessing/RenameDirectoryTest.java +++ b/common/src/test/java/ctbrec/recorder/postprocessing/RenameDirectoryTest.java @@ -26,7 +26,7 @@ public class RenameDirectoryTest extends AbstractPpTest { rec.setStartDate(now); rec.setSingleFile(false); Rename pp = new Rename(); - pp.postprocess(rec, config); + pp.postprocess(rec, recordingManager, config); Matcher m = Pattern.compile("Mockita_\\d{4}-\\d{2}-\\d{2}_\\d{2}-\\d{2}-\\d{2}").matcher(rec.getAbsoluteFile().getName()); assertTrue(m.matches()); @@ -47,6 +47,6 @@ public class RenameDirectoryTest extends AbstractPpTest { Files.createDirectories(postProcessedDir.toPath()); Rename pp = new Rename(); - pp.postprocess(rec, config); + pp.postprocess(rec, recordingManager, config); } } diff --git a/common/src/test/java/ctbrec/recorder/postprocessing/RenameSingleFileTest.java b/common/src/test/java/ctbrec/recorder/postprocessing/RenameSingleFileTest.java index 4b065664..9c52230b 100644 --- a/common/src/test/java/ctbrec/recorder/postprocessing/RenameSingleFileTest.java +++ b/common/src/test/java/ctbrec/recorder/postprocessing/RenameSingleFileTest.java @@ -25,7 +25,7 @@ public class RenameSingleFileTest extends AbstractPpTest { rec.setStartDate(now); rec.setSingleFile(true); Rename pp = new Rename(); - pp.postprocess(rec, config); + pp.postprocess(rec, recordingManager, config); Matcher m = Pattern.compile("Mockita_\\d{4}-\\d{2}-\\d{2}_\\d{2}-\\d{2}-\\d{2}\\.ts").matcher(rec.getAbsoluteFile().getName()); assertTrue(m.matches()); @@ -45,7 +45,7 @@ public class RenameSingleFileTest extends AbstractPpTest { doThrow(new RuntimeException("Unexpected call of setAbsoluteFile")).when(rec).setAbsoluteFile(any()); Rename pp = new Rename(); pp.getConfig().put(Rename.FILE_NAME_TEMPLATE, original.getName()); - pp.postprocess(rec, config); + pp.postprocess(rec, recordingManager, config); } @Test @@ -59,7 +59,7 @@ public class RenameSingleFileTest extends AbstractPpTest { when(rec.getStartDate()).thenReturn(now); doThrow(new RuntimeException("Unexpected call of setAbsoluteFile")).when(rec).setAbsoluteFile(any()); Rename pp = new Rename(); - pp.postprocess(rec, config); + pp.postprocess(rec, recordingManager, config); } @Test From f6afed371734d5c1610a4344fcb1ac48f6042d81 Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Wed, 23 Sep 2020 18:52:30 +0200 Subject: [PATCH 18/33] Add script post-processor --- .../settings/PostProcessingDialogFactory.java | 2 + .../ui/settings/PostProcessingStepPanel.java | 4 +- .../ctbrec/ui/settings/ScriptPaneFactory.java | 27 +++++++++ ...AbstractPlaceholderAwarePostProcessor.java | 6 +- .../recorder/postprocessing/Script.java | 57 ++++++++++++++++++- 5 files changed, 91 insertions(+), 5 deletions(-) create mode 100644 client/src/main/java/ctbrec/ui/settings/ScriptPaneFactory.java diff --git a/client/src/main/java/ctbrec/ui/settings/PostProcessingDialogFactory.java b/client/src/main/java/ctbrec/ui/settings/PostProcessingDialogFactory.java index eccad370..d4b45a64 100644 --- a/client/src/main/java/ctbrec/ui/settings/PostProcessingDialogFactory.java +++ b/client/src/main/java/ctbrec/ui/settings/PostProcessingDialogFactory.java @@ -11,6 +11,7 @@ import ctbrec.recorder.postprocessing.Move; import ctbrec.recorder.postprocessing.PostProcessor; import ctbrec.recorder.postprocessing.Remux; import ctbrec.recorder.postprocessing.Rename; +import ctbrec.recorder.postprocessing.Script; import ctbrec.ui.controls.Dialogs; import ctbrec.ui.settings.api.Preferences; import javafx.collections.ObservableList; @@ -22,6 +23,7 @@ public class PostProcessingDialogFactory { static Map, Class> ppToDialogMap = new HashMap<>(); static { ppToDialogMap.put(Remux.class, RemuxerPaneFactory.class); + ppToDialogMap.put(Script.class, ScriptPaneFactory.class); ppToDialogMap.put(Rename.class, RenamerPaneFactory.class); ppToDialogMap.put(Move.class, MoverPaneFactory.class); } diff --git a/client/src/main/java/ctbrec/ui/settings/PostProcessingStepPanel.java b/client/src/main/java/ctbrec/ui/settings/PostProcessingStepPanel.java index 0e2c13e6..78cb4d12 100644 --- a/client/src/main/java/ctbrec/ui/settings/PostProcessingStepPanel.java +++ b/client/src/main/java/ctbrec/ui/settings/PostProcessingStepPanel.java @@ -13,6 +13,7 @@ import ctbrec.recorder.postprocessing.PostProcessor; import ctbrec.recorder.postprocessing.RemoveKeepFile; import ctbrec.recorder.postprocessing.Remux; import ctbrec.recorder.postprocessing.Rename; +import ctbrec.recorder.postprocessing.Script; import ctbrec.ui.controls.Dialogs; import javafx.collections.FXCollections; import javafx.collections.ListChangeListener; @@ -34,9 +35,10 @@ public class PostProcessingStepPanel extends GridPane { private static final Class[] POST_PROCESSOR_CLASSES = new Class[] { // @formatter: off Copy.class, - Remux.class, Rename.class, Move.class, + Remux.class, + Script.class, DeleteOriginal.class, RemoveKeepFile.class }; // @formatter: on diff --git a/client/src/main/java/ctbrec/ui/settings/ScriptPaneFactory.java b/client/src/main/java/ctbrec/ui/settings/ScriptPaneFactory.java new file mode 100644 index 00000000..1c56cc5b --- /dev/null +++ b/client/src/main/java/ctbrec/ui/settings/ScriptPaneFactory.java @@ -0,0 +1,27 @@ +package ctbrec.ui.settings; + +import ctbrec.recorder.postprocessing.PostProcessor; +import ctbrec.recorder.postprocessing.Script; +import ctbrec.ui.settings.api.Category; +import ctbrec.ui.settings.api.Preferences; +import ctbrec.ui.settings.api.Setting; +import javafx.beans.property.SimpleStringProperty; + +public class ScriptPaneFactory extends AbstractPostProcessingPaneFactory { + + @Override + public Preferences doCreatePostProcessorPane(PostProcessor pp) { + SimpleStringProperty script = new SimpleStringProperty(null, Script.SCRIPT_EXECUTABLE, pp.getConfig().getOrDefault(Script.SCRIPT_EXECUTABLE, "c:\\users\\johndoe\\somescript")); + SimpleStringProperty params = new SimpleStringProperty(null, Script.SCRIPT_PARAMS, pp.getConfig().getOrDefault(Script.SCRIPT_PARAMS, "${absolutePath}")); + properties.add(script); + properties.add(params); + + return Preferences.of(new MapPreferencesStorage(), + Category.of(pp.getName(), + Setting.of("Script", script), + Setting.of("Parameters", params) + ) + ); + } + +} diff --git a/common/src/main/java/ctbrec/recorder/postprocessing/AbstractPlaceholderAwarePostProcessor.java b/common/src/main/java/ctbrec/recorder/postprocessing/AbstractPlaceholderAwarePostProcessor.java index 8831b687..5007d97f 100644 --- a/common/src/main/java/ctbrec/recorder/postprocessing/AbstractPlaceholderAwarePostProcessor.java +++ b/common/src/main/java/ctbrec/recorder/postprocessing/AbstractPlaceholderAwarePostProcessor.java @@ -25,7 +25,9 @@ public abstract class AbstractPlaceholderAwarePostProcessor extends AbstractPost "${epochSecond}", "${fileSuffix}", "${modelNotes}", - "${recordingsDir}" + "${recordingsDir}", + "${absolutePath}", + "${absoluteParentPath}" }; public String fillInPlaceHolders(String input, Recording rec, Config config) { @@ -40,6 +42,8 @@ public abstract class AbstractPlaceholderAwarePostProcessor extends AbstractPost .replace("${epochSecond}", Long.toString(rec.getStartDate().getEpochSecond())) .replace("${modelNotes}", config.getModelNotes(rec.getModel())) .replace("${recordingsDir}", config.getSettings().recordingsDir) + .replace("${absolutePath}", rec.getPostProcessedFile().getAbsolutePath()) + .replace("${absoluteParentPath}", rec.getPostProcessedFile().getParentFile().getAbsolutePath()) ; output = replaceUtcDateTime(rec, output); diff --git a/common/src/main/java/ctbrec/recorder/postprocessing/Script.java b/common/src/main/java/ctbrec/recorder/postprocessing/Script.java index 0b278082..a07b9bfc 100644 --- a/common/src/main/java/ctbrec/recorder/postprocessing/Script.java +++ b/common/src/main/java/ctbrec/recorder/postprocessing/Script.java @@ -1,14 +1,26 @@ package ctbrec.recorder.postprocessing; import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import ctbrec.Config; -import ctbrec.NotImplementedExcetion; +import ctbrec.OS; import ctbrec.Recording; +import ctbrec.io.StreamRedirectThread; import ctbrec.recorder.RecordingManager; +import ctbrec.recorder.download.ProcessExitedUncleanException; public class Script extends AbstractPlaceholderAwarePostProcessor { + private static final Logger LOG = LoggerFactory.getLogger(Script.class); + public static final String SCRIPT_EXECUTABLE = "script.executable"; + public static final String SCRIPT_PARAMS = "script.params"; + @Override public String getName() { return "execute script"; @@ -16,7 +28,46 @@ public class Script extends AbstractPlaceholderAwarePostProcessor { @Override public void postprocess(Recording rec, RecordingManager recordingManager, Config config) throws IOException, InterruptedException { - // TODO make it possible to choose, which placeholders to pass to the script - throw new NotImplementedExcetion(); + List cmdline = buildCommandLine(rec, config); + Runtime rt = Runtime.getRuntime(); + String[] args = cmdline.toArray(new String[0]); + if (LOG.isDebugEnabled()) { + LOG.debug("Running {}", Arrays.toString(args)); + } + Process process = rt.exec(args, OS.getEnvironment()); + startLogging(process); + int exitCode = process.waitFor(); + LOG.debug("Process finished with exit code {}", exitCode); + if (exitCode != 0) { + throw new ProcessExitedUncleanException("Script finished with exit code " + exitCode); + } + } + + private List buildCommandLine(Recording rec, Config config) throws IOException { + String script = getConfig().getOrDefault(SCRIPT_EXECUTABLE, "somescript"); + String params = getConfig().getOrDefault(SCRIPT_PARAMS, "${absolutePath}"); + List cmdline = new ArrayList<>(); + cmdline.add(script); + String replacedParams = fillInPlaceHolders(params, rec, config); + Arrays.stream(replacedParams.split(" ")).forEach(cmdline::add); + return cmdline; + } + + private void startLogging(Process process) { + // TODO maybe write these to a separate log file, e.g. recname.ts.script.log + Thread std = new Thread(new StreamRedirectThread(process.getInputStream(), System.out)); + std.setName("Process stdout pipe"); + std.setDaemon(true); + std.start(); + Thread err = new Thread(new StreamRedirectThread(process.getErrorStream(), System.err)); + err.setName("Process stderr pipe"); + err.setDaemon(true); + err.start(); + } + + @Override + public String toString() { + return (getName() + " " + getConfig().getOrDefault(Script.SCRIPT_EXECUTABLE, "")).trim(); } } + From 30021d184db356f72865f20db154ed7873bd2c13 Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Thu, 24 Sep 2020 11:35:37 +0200 Subject: [PATCH 19/33] Add DeleteTooShort post-processor --- .../settings/DeleteTooShortPaneFactory.java | 24 +++++++++++++ .../settings/PostProcessingDialogFactory.java | 2 ++ .../ui/settings/PostProcessingStepPanel.java | 2 ++ .../java/ctbrec/ui/settings/SettingsTab.java | 9 ----- .../ctbrec/recorder/NextGenLocalRecorder.java | 18 ---------- .../postprocessing/DeleteTooShort.java | 34 +++++++++++++++++++ 6 files changed, 62 insertions(+), 27 deletions(-) create mode 100644 client/src/main/java/ctbrec/ui/settings/DeleteTooShortPaneFactory.java create mode 100644 common/src/main/java/ctbrec/recorder/postprocessing/DeleteTooShort.java diff --git a/client/src/main/java/ctbrec/ui/settings/DeleteTooShortPaneFactory.java b/client/src/main/java/ctbrec/ui/settings/DeleteTooShortPaneFactory.java new file mode 100644 index 00000000..3974ce58 --- /dev/null +++ b/client/src/main/java/ctbrec/ui/settings/DeleteTooShortPaneFactory.java @@ -0,0 +1,24 @@ +package ctbrec.ui.settings; + +import ctbrec.recorder.postprocessing.DeleteTooShort; +import ctbrec.recorder.postprocessing.PostProcessor; +import ctbrec.ui.settings.api.Category; +import ctbrec.ui.settings.api.Preferences; +import ctbrec.ui.settings.api.Setting; +import javafx.beans.property.SimpleStringProperty; + +public class DeleteTooShortPaneFactory extends AbstractPostProcessingPaneFactory { + + @Override + public Preferences doCreatePostProcessorPane(PostProcessor pp) { + SimpleStringProperty minimumLengthInSeconds = new SimpleStringProperty(null, DeleteTooShort.MIN_LEN_IN_SECS, pp.getConfig().getOrDefault(DeleteTooShort.MIN_LEN_IN_SECS, "10")); + properties.add(minimumLengthInSeconds); + + return Preferences.of(new MapPreferencesStorage(), + Category.of(pp.getName(), + Setting.of("Minimum length in seconds", minimumLengthInSeconds) + ) + ); + } + +} diff --git a/client/src/main/java/ctbrec/ui/settings/PostProcessingDialogFactory.java b/client/src/main/java/ctbrec/ui/settings/PostProcessingDialogFactory.java index d4b45a64..c9657160 100644 --- a/client/src/main/java/ctbrec/ui/settings/PostProcessingDialogFactory.java +++ b/client/src/main/java/ctbrec/ui/settings/PostProcessingDialogFactory.java @@ -7,6 +7,7 @@ import java.util.Map; import java.util.Optional; import ctbrec.Config; +import ctbrec.recorder.postprocessing.DeleteTooShort; import ctbrec.recorder.postprocessing.Move; import ctbrec.recorder.postprocessing.PostProcessor; import ctbrec.recorder.postprocessing.Remux; @@ -26,6 +27,7 @@ public class PostProcessingDialogFactory { ppToDialogMap.put(Script.class, ScriptPaneFactory.class); ppToDialogMap.put(Rename.class, RenamerPaneFactory.class); ppToDialogMap.put(Move.class, MoverPaneFactory.class); + ppToDialogMap.put(DeleteTooShort.class, DeleteTooShortPaneFactory.class); } private PostProcessingDialogFactory() { diff --git a/client/src/main/java/ctbrec/ui/settings/PostProcessingStepPanel.java b/client/src/main/java/ctbrec/ui/settings/PostProcessingStepPanel.java index 78cb4d12..3a659888 100644 --- a/client/src/main/java/ctbrec/ui/settings/PostProcessingStepPanel.java +++ b/client/src/main/java/ctbrec/ui/settings/PostProcessingStepPanel.java @@ -8,6 +8,7 @@ import java.util.Optional; import ctbrec.Config; import ctbrec.recorder.postprocessing.Copy; import ctbrec.recorder.postprocessing.DeleteOriginal; +import ctbrec.recorder.postprocessing.DeleteTooShort; import ctbrec.recorder.postprocessing.Move; import ctbrec.recorder.postprocessing.PostProcessor; import ctbrec.recorder.postprocessing.RemoveKeepFile; @@ -40,6 +41,7 @@ public class PostProcessingStepPanel extends GridPane { Remux.class, Script.class, DeleteOriginal.class, + DeleteTooShort.class, RemoveKeepFile.class }; // @formatter: on diff --git a/client/src/main/java/ctbrec/ui/settings/SettingsTab.java b/client/src/main/java/ctbrec/ui/settings/SettingsTab.java index 95f78da1..3f7fdd00 100644 --- a/client/src/main/java/ctbrec/ui/settings/SettingsTab.java +++ b/client/src/main/java/ctbrec/ui/settings/SettingsTab.java @@ -88,7 +88,6 @@ public class SettingsTab extends Tab implements TabSelectionListener { private SimpleIntegerProperty onlineCheckIntervalInSecs; private SimpleBooleanProperty onlineCheckSkipsPausedModels; private SimpleLongProperty leaveSpaceOnDevice; - private SimpleIntegerProperty minimumLengthInSecs; private SimpleStringProperty ffmpegParameters; private SimpleStringProperty fileExtension; private SimpleStringProperty server; @@ -97,9 +96,7 @@ public class SettingsTab extends Tab implements TabSelectionListener { private SimpleBooleanProperty requireAuthentication; private SimpleBooleanProperty transportLayerSecurity; private ExclusiveSelectionProperty recordLocal; - private SimpleFileProperty postProcessing; private SimpleIntegerProperty postProcessingThreads; - private SimpleBooleanProperty removeRecordingAfterPp; private IgnoreList ignoreList; public SettingsTab(List sites, Recorder recorder) { @@ -137,7 +134,6 @@ public class SettingsTab extends Tab implements TabSelectionListener { concurrentRecordings = new SimpleIntegerProperty(null, "concurrentRecordings", settings.concurrentRecordings); onlineCheckIntervalInSecs = new SimpleIntegerProperty(null, "onlineCheckIntervalInSecs", settings.onlineCheckIntervalInSecs); leaveSpaceOnDevice = new SimpleLongProperty(null, "minimumSpaceLeftInBytes", (long) new GigabytesConverter().convertTo(settings.minimumSpaceLeftInBytes)); - minimumLengthInSecs = new SimpleIntegerProperty(null, "minimumLengthInSeconds", settings.minimumLengthInSeconds); ffmpegParameters = new SimpleStringProperty(null, "ffmpegMergedDownloadArgs", settings.ffmpegMergedDownloadArgs); fileExtension = new SimpleStringProperty(null, "ffmpegFileSuffix", settings.ffmpegFileSuffix); server = new SimpleStringProperty(null, "httpServer", settings.httpServer); @@ -147,9 +143,7 @@ public class SettingsTab extends Tab implements TabSelectionListener { requireAuthentication.addListener(this::requireAuthenticationChanged); transportLayerSecurity = new SimpleBooleanProperty(null, "transportLayerSecurity", settings.transportLayerSecurity); recordLocal = new ExclusiveSelectionProperty(null, "localRecording", settings.localRecording, "Local", "Remote"); - postProcessing = new SimpleFileProperty(null, "postProcessing", settings.postProcessing); postProcessingThreads = new SimpleIntegerProperty(null, "postProcessingThreads", settings.postProcessingThreads); - removeRecordingAfterPp = new SimpleBooleanProperty(null, "removeRecordingAfterPostProcessing", settings.removeRecordingAfterPostProcessing); onlineCheckSkipsPausedModels = new SimpleBooleanProperty(null, "onlineCheckSkipsPausedModels", settings.onlineCheckSkipsPausedModels); } @@ -206,10 +200,7 @@ public class SettingsTab extends Tab implements TabSelectionListener { ), Category.of("Post-Processing", Group.of("Post-Processing", - Setting.of("Post-Processing", postProcessing), Setting.of("Threads", postProcessingThreads), - Setting.of("Delete recordings shorter than (secs)", minimumLengthInSecs, "Delete recordings, which are shorter than x seconds. 0 to disable"), - Setting.of("Remove recording after post-processing", removeRecordingAfterPp), Setting.of("Steps", new PostProcessingStepPanel(config)) ) ), diff --git a/common/src/main/java/ctbrec/recorder/NextGenLocalRecorder.java b/common/src/main/java/ctbrec/recorder/NextGenLocalRecorder.java index 2897e15f..3623b5da 100644 --- a/common/src/main/java/ctbrec/recorder/NextGenLocalRecorder.java +++ b/common/src/main/java/ctbrec/recorder/NextGenLocalRecorder.java @@ -9,7 +9,6 @@ import java.nio.file.FileStore; import java.nio.file.Files; import java.security.InvalidKeyException; import java.security.NoSuchAlgorithmException; -import java.time.Duration; import java.time.Instant; import java.time.ZoneId; import java.util.ArrayList; @@ -169,7 +168,6 @@ public class NextGenLocalRecorder implements Recorder { } setRecordingStatus(recording, State.FINISHED); recordingManager.saveRecording(recording); - deleteIfTooShort(recording); LOG.info("Post-processing finished for {}", recording.getModel().getName()); if (config.getSettings().removeRecordingAfterPostProcessing) { recordingManager.remove(recording); @@ -307,22 +305,6 @@ public class NextGenLocalRecorder implements Recorder { } } - private boolean deleteIfTooShort(Recording rec) throws IOException, InvalidKeyException, NoSuchAlgorithmException { - Duration minimumLengthInSeconds = Duration.ofSeconds(Config.getInstance().getSettings().minimumLengthInSeconds); - if (minimumLengthInSeconds.getSeconds() <= 0) { - return false; - } - - Duration recordingLength = rec.getLength(); - if (recordingLength.compareTo(minimumLengthInSeconds) < 0) { - LOG.info("Deleting too short recording {} [{} < {}]", rec, recordingLength, minimumLengthInSeconds); - delete(rec); - return true; - } - - return false; - } - @Override public void stopRecording(Model model) throws IOException, InvalidKeyException, NoSuchAlgorithmException { recorderLock.lock(); diff --git a/common/src/main/java/ctbrec/recorder/postprocessing/DeleteTooShort.java b/common/src/main/java/ctbrec/recorder/postprocessing/DeleteTooShort.java new file mode 100644 index 00000000..bdf38b4f --- /dev/null +++ b/common/src/main/java/ctbrec/recorder/postprocessing/DeleteTooShort.java @@ -0,0 +1,34 @@ +package ctbrec.recorder.postprocessing; + +import java.io.IOException; +import java.time.Duration; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import ctbrec.Config; +import ctbrec.Recording; +import ctbrec.recorder.RecordingManager; + +public class DeleteTooShort extends AbstractPostProcessor { + + private static final transient Logger LOG = LoggerFactory.getLogger(DeleteTooShort.class); + public static final String MIN_LEN_IN_SECS = "minimumLengthInSeconds"; + + @Override + public String getName() { + return "delete too short"; + } + + @Override + public void postprocess(Recording rec, RecordingManager recordingManager, Config config) throws IOException { + Duration minimumLengthInSeconds = Duration.ofSeconds(Integer.parseInt(getConfig().getOrDefault(MIN_LEN_IN_SECS, "0"))); + if (minimumLengthInSeconds.getSeconds() > 0) { + Duration recordingLength = rec.getLength(); + if (recordingLength.compareTo(minimumLengthInSeconds) < 0) { + LOG.info("Deleting too short recording {} [{} < {}]", rec, recordingLength, minimumLengthInSeconds); + recordingManager.delete(rec); + } + } + } +} From 87d88b5bcbe2d54bd032af4e94b73ffa6f8b65ab Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Thu, 24 Sep 2020 11:35:55 +0200 Subject: [PATCH 20/33] Add webhook post-processor skeleton --- .../recorder/postprocessing/Webhook.java | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 common/src/main/java/ctbrec/recorder/postprocessing/Webhook.java diff --git a/common/src/main/java/ctbrec/recorder/postprocessing/Webhook.java b/common/src/main/java/ctbrec/recorder/postprocessing/Webhook.java new file mode 100644 index 00000000..819d2f0b --- /dev/null +++ b/common/src/main/java/ctbrec/recorder/postprocessing/Webhook.java @@ -0,0 +1,37 @@ +package ctbrec.recorder.postprocessing; + +import java.io.IOException; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import ctbrec.Config; +import ctbrec.NotImplementedExcetion; +import ctbrec.Recording; +import ctbrec.recorder.RecordingManager; + +public class Webhook extends AbstractPlaceholderAwarePostProcessor { + + private static final Logger LOG = LoggerFactory.getLogger(Webhook.class); + public static final String URL = "webhook.url"; + public static final String HEADERS = "webhook.headers"; + public static final String METHOD = "webhook.method"; + public static final String DATA = "webhook.data"; + public static final String SECRET = "webhook.secret"; + + @Override + public String getName() { + return "webhook"; + } + + @Override + public void postprocess(Recording rec, RecordingManager recordingManager, Config config) throws IOException, InterruptedException { + throw new NotImplementedExcetion(); + } + + @Override + public String toString() { + return (getName() + " " + getConfig().getOrDefault(Webhook.URL, "")).trim(); + } +} + From b2b6a623efd853d5fd0faf835fc1f8bf8198f8a0 Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Fri, 25 Sep 2020 00:09:23 +0200 Subject: [PATCH 21/33] Add more post-processing tests --- common/pom.xml | 2 +- .../postprocessing/AbstractPpTest.java | 9 +++ .../postprocessing/DeleteOriginalTest.java | 25 ++++++ .../postprocessing/MoveDirectoryTest.java | 55 ++++++++++++++ .../postprocessing/MoveSingleFileTest.java | 76 +++++++++++++++++++ master/pom.xml | 2 +- 6 files changed, 167 insertions(+), 2 deletions(-) create mode 100644 common/src/test/java/ctbrec/recorder/postprocessing/DeleteOriginalTest.java create mode 100644 common/src/test/java/ctbrec/recorder/postprocessing/MoveDirectoryTest.java create mode 100644 common/src/test/java/ctbrec/recorder/postprocessing/MoveSingleFileTest.java diff --git a/common/pom.xml b/common/pom.xml index c4e22ca7..6713bf45 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -77,7 +77,7 @@ org.mockito - mockito-core + mockito-inline test diff --git a/common/src/test/java/ctbrec/recorder/postprocessing/AbstractPpTest.java b/common/src/test/java/ctbrec/recorder/postprocessing/AbstractPpTest.java index adbd9aad..8f299c39 100644 --- a/common/src/test/java/ctbrec/recorder/postprocessing/AbstractPpTest.java +++ b/common/src/test/java/ctbrec/recorder/postprocessing/AbstractPpTest.java @@ -14,6 +14,7 @@ import java.time.Instant; import org.apache.commons.io.FileUtils; import org.junit.After; import org.junit.Before; +import org.mockito.MockedStatic; import ctbrec.Config; import ctbrec.Model; @@ -32,6 +33,8 @@ public abstract class AbstractPpTest { Instant now = Instant.now(); RecordingManager recordingManager; + MockedStatic configStatic; + @Before public void setup() throws IOException { baseDir = Files.createTempDirectory("ctbrec_test_"); @@ -50,12 +53,18 @@ public abstract class AbstractPpTest { @After public void teardown() throws IOException { FileUtils.deleteDirectory(baseDir.toFile()); + if (configStatic != null) { + configStatic.close(); + configStatic = null; + } } Config mockConfig() { Config config = mock(Config.class); when(config.getSettings()).thenReturn(mockSettings()); when(config.getModelNotes(any())).thenReturn("tag, foo, bar"); + configStatic = mockStatic(Config.class); + configStatic.when(Config::getInstance).thenReturn(config); return config; } diff --git a/common/src/test/java/ctbrec/recorder/postprocessing/DeleteOriginalTest.java b/common/src/test/java/ctbrec/recorder/postprocessing/DeleteOriginalTest.java new file mode 100644 index 00000000..0a3e0b4f --- /dev/null +++ b/common/src/test/java/ctbrec/recorder/postprocessing/DeleteOriginalTest.java @@ -0,0 +1,25 @@ +package ctbrec.recorder.postprocessing; + +import java.io.IOException; + +import org.junit.Test; + +import ctbrec.Config; +import ctbrec.Recording; + +public class DeleteOriginalTest extends AbstractPpTest { + + @Test + public void testPostProcessWithSingleFile() throws IOException, InterruptedException { + Recording rec = new Recording(); + rec.setModel(mockModel()); + rec.setAbsoluteFile(original); + rec.setAbsoluteFile(postProcessed); + rec.setStartDate(now); + rec.setSingleFile(true); + + Config config = mockConfig(); + DeleteOriginal pp = new DeleteOriginal(); + pp.postprocess(rec, null, config); + } +} diff --git a/common/src/test/java/ctbrec/recorder/postprocessing/MoveDirectoryTest.java b/common/src/test/java/ctbrec/recorder/postprocessing/MoveDirectoryTest.java new file mode 100644 index 00000000..47ea38ed --- /dev/null +++ b/common/src/test/java/ctbrec/recorder/postprocessing/MoveDirectoryTest.java @@ -0,0 +1,55 @@ +package ctbrec.recorder.postprocessing; + +import static org.junit.Assert.*; +import static org.mockito.ArgumentMatchers.*; +import static org.mockito.Mockito.*; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.junit.Test; + +import ctbrec.Config; +import ctbrec.Model; +import ctbrec.Recording; + +public class MoveDirectoryTest extends AbstractPpTest { + + @Test + public void testOriginalFileReplacement() throws IOException { + Config config = mockConfig(); + Recording rec = new Recording(); + rec.setModel(mockModel()); + rec.setAbsoluteFile(originalDir); + rec.setStartDate(now); + rec.setSingleFile(false); + Move pp = new Move(); + pp.getConfig().put(Move.PATH_TEMPLATE, new File(baseDir.toFile(), Move.DEFAULT).getAbsolutePath()); + pp.postprocess(rec, recordingManager, config); + + Matcher m = Pattern.compile(baseDir.toString() + "/Mockita/\\d{4}-\\d{2}-\\d{2}_\\d{2}-\\d{2}-\\d{2}/original").matcher(rec.getAbsoluteFile().getCanonicalPath()); + assertTrue(m.matches()); + assertEquals(rec.getAbsoluteFile(), rec.getPostProcessedFile()); + assertNotEquals(rec.getAbsoluteFile(), original); + } + + @Test + public void absoluteFileShouldKeepBeingOriginalIfFilesDiffer() throws IOException { + Model model = mockModel(); + Recording rec = mock(Recording.class); + when(rec.getModel()).thenReturn(model); + when(rec.getAbsoluteFile()).thenReturn(originalDir); + when(rec.getPostProcessedFile()).thenReturn(postProcessedDir); + when(rec.getStartDate()).thenReturn(now); + doThrow(new RuntimeException("Unexpected call of setAbsoluteFile")).when(rec).setAbsoluteFile(any()); + + Files.createDirectories(postProcessedDir.toPath()); + Move pp = new Move(); + Config config = mockConfig(); + pp.getConfig().put(Move.PATH_TEMPLATE, new File(baseDir.toFile(), Move.DEFAULT).getAbsolutePath()); + pp.postprocess(rec, recordingManager, config); + } +} diff --git a/common/src/test/java/ctbrec/recorder/postprocessing/MoveSingleFileTest.java b/common/src/test/java/ctbrec/recorder/postprocessing/MoveSingleFileTest.java new file mode 100644 index 00000000..bb560a37 --- /dev/null +++ b/common/src/test/java/ctbrec/recorder/postprocessing/MoveSingleFileTest.java @@ -0,0 +1,76 @@ +package ctbrec.recorder.postprocessing; + +import static org.junit.Assert.*; +import static org.mockito.ArgumentMatchers.*; +import static org.mockito.Mockito.*; + +import java.io.File; +import java.io.IOException; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.junit.Test; + +import ctbrec.Config; +import ctbrec.Model; +import ctbrec.Recording; + +public class MoveSingleFileTest extends AbstractPpTest { + + @Test + public void testOriginalFileReplacement() throws IOException { + Config config = mockConfig(); + Recording rec = new Recording(); + rec.setModel(mockModel()); + rec.setAbsoluteFile(original); + rec.setStartDate(now); + rec.setSingleFile(true); + Move pp = new Move(); + pp.getConfig().put(Move.PATH_TEMPLATE, new File(baseDir.toFile(), Move.DEFAULT).getAbsolutePath()); + pp.postprocess(rec, recordingManager, config); + + Matcher m = Pattern.compile(baseDir.toFile() + "/Mockita/\\d{4}-\\d{2}-\\d{2}_\\d{2}-\\d{2}-\\d{2}/original\\.ts").matcher(rec.getAbsoluteFile().toString()); + assertTrue(m.matches()); + assertEquals(rec.getAbsoluteFile(), rec.getPostProcessedFile()); + assertNotEquals(rec.getAbsoluteFile(), original); + } + + @Test + public void testEarlyExit() throws IOException { + Model model = mockModel(); + Recording rec = mock(Recording.class); + when(rec.getModel()).thenReturn(model); + when(rec.getAbsoluteFile()).thenReturn(original); + when(rec.getPostProcessedFile()).thenReturn(original); + when(rec.getStartDate()).thenReturn(now); + doThrow(new RuntimeException("Unexpected call of setAbsoluteFile")).when(rec).setAbsoluteFile(any()); + Move pp = new Move(); + Config config = mockConfig(); + pp.getConfig().put(Move.PATH_TEMPLATE, original.getParentFile().getCanonicalPath()); + pp.postprocess(rec, recordingManager, config); + } + + @Test + public void absoluteFileShouldKeepBeingOriginalIfFilesDiffer() throws IOException { + Model model = mockModel(); + Recording rec = mock(Recording.class); + when(rec.getModel()).thenReturn(model); + when(rec.getAbsoluteFile()).thenReturn(original); + when(rec.getPostProcessedFile()).thenReturn(postProcessed); + when(rec.getStartDate()).thenReturn(now); + doThrow(new RuntimeException("Unexpected call of setAbsoluteFile")).when(rec).setAbsoluteFile(any()); + Move pp = new Move(); + Config config = mockConfig(); + pp.getConfig().put(Move.PATH_TEMPLATE, new File(baseDir.toFile(), Move.DEFAULT).getAbsolutePath()); + pp.postprocess(rec, recordingManager, config); + } + + @Test + public void testToString() { + Move pp = new Move(); + assertEquals("move", pp.toString()); + + pp.getConfig().put(Move.PATH_TEMPLATE, Move.DEFAULT); + assertEquals("move ["+Move.DEFAULT+"]", pp.toString()); + } +} diff --git a/master/pom.xml b/master/pom.xml index 68ef277c..c81fb52f 100644 --- a/master/pom.xml +++ b/master/pom.xml @@ -124,7 +124,7 @@ org.mockito - mockito-core + mockito-inline 3.5.11 test From 43de2c262037bd1b1a1f6e51891bc13c8bbb7a2d Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Fri, 25 Sep 2020 19:43:00 +0200 Subject: [PATCH 22/33] Add test for DeleteOriginal pp --- .../postprocessing/DeleteOriginalTest.java | 28 ++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/common/src/test/java/ctbrec/recorder/postprocessing/DeleteOriginalTest.java b/common/src/test/java/ctbrec/recorder/postprocessing/DeleteOriginalTest.java index 0a3e0b4f..3b6047ab 100644 --- a/common/src/test/java/ctbrec/recorder/postprocessing/DeleteOriginalTest.java +++ b/common/src/test/java/ctbrec/recorder/postprocessing/DeleteOriginalTest.java @@ -1,6 +1,9 @@ package ctbrec.recorder.postprocessing; +import static org.junit.Assert.*; + import java.io.IOException; +import java.nio.file.Files; import org.junit.Test; @@ -14,12 +17,35 @@ public class DeleteOriginalTest extends AbstractPpTest { Recording rec = new Recording(); rec.setModel(mockModel()); rec.setAbsoluteFile(original); - rec.setAbsoluteFile(postProcessed); + rec.setPostProcessedFile(postProcessed); rec.setStartDate(now); rec.setSingleFile(true); Config config = mockConfig(); DeleteOriginal pp = new DeleteOriginal(); pp.postprocess(rec, null, config); + + assertEquals(postProcessed, rec.getAbsoluteFile()); + assertTrue(rec.getAbsoluteFile().exists()); + assertFalse(original.exists()); + } + + @Test + public void testPostProcessWithDirectory() throws IOException, InterruptedException { + Recording rec = new Recording(); + rec.setModel(mockModel()); + rec.setAbsoluteFile(originalDir); + rec.setPostProcessedFile(postProcessedDir); + rec.setStartDate(now); + rec.setSingleFile(true); + + Config config = mockConfig(); + Files.createDirectories(postProcessedDir.toPath()); + DeleteOriginal pp = new DeleteOriginal(); + pp.postprocess(rec, null, config); + + assertEquals(postProcessedDir, rec.getAbsoluteFile()); + assertTrue(rec.getAbsoluteFile().exists()); + assertFalse(originalDir.exists()); } } From 02814440d258535856555a69d63dea34ce27d576 Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Fri, 25 Sep 2020 21:09:50 +0200 Subject: [PATCH 23/33] Add more tests --- ...ractPlaceholderAwarePostProcessorTest.java | 125 ++++++++++++++++++ .../postprocessing/AbstractPpTest.java | 2 +- .../postprocessing/DeleteOriginalTest.java | 6 + .../postprocessing/MoveDirectoryTest.java | 2 +- .../postprocessing/MoveSingleFileTest.java | 2 +- .../postprocessing/RemoveKeepFileTest.java | 42 ++++++ .../postprocessing/RenameDirectoryTest.java | 2 +- .../postprocessing/RenameSingleFileTest.java | 2 +- 8 files changed, 178 insertions(+), 5 deletions(-) create mode 100644 common/src/test/java/ctbrec/recorder/postprocessing/AbstractPlaceholderAwarePostProcessorTest.java create mode 100644 common/src/test/java/ctbrec/recorder/postprocessing/RemoveKeepFileTest.java diff --git a/common/src/test/java/ctbrec/recorder/postprocessing/AbstractPlaceholderAwarePostProcessorTest.java b/common/src/test/java/ctbrec/recorder/postprocessing/AbstractPlaceholderAwarePostProcessorTest.java new file mode 100644 index 00000000..649c6ec8 --- /dev/null +++ b/common/src/test/java/ctbrec/recorder/postprocessing/AbstractPlaceholderAwarePostProcessorTest.java @@ -0,0 +1,125 @@ +package ctbrec.recorder.postprocessing; + +import static org.junit.Assert.*; + +import java.io.IOException; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.util.Locale; + +import org.junit.Before; +import org.junit.Test; + +import ctbrec.Config; +import ctbrec.Recording; + +public class AbstractPlaceholderAwarePostProcessorTest extends AbstractPpTest { + + Recording rec; + Config config; + Move placeHolderAwarePp; + + @Override + @Before + public void setup() throws IOException { + super.setup(); + rec = new Recording(); + rec.setModel(mockModel()); + rec.setAbsoluteFile(original); + rec.setPostProcessedFile(postProcessed); + rec.setStartDate(now); + rec.setSingleFile(true); + config = mockConfig(); + placeHolderAwarePp = new Move(); + } + + @Test + public void testModelNameReplacement() { + String input = "asdf_${modelName}_asdf"; + assertEquals("asdf_Mockita Boobilicious_asdf", placeHolderAwarePp.fillInPlaceHolders(input, rec, config)); + input = "asdf_${modelDisplayName}_asdf"; + assertEquals("asdf_Mockita Boobilicious_asdf", placeHolderAwarePp.fillInPlaceHolders(input, rec, config)); + input = "asdf_${modelSanitizedName}_asdf"; + assertEquals("asdf_Mockita_Boobilicious_asdf", placeHolderAwarePp.fillInPlaceHolders(input, rec, config)); + } + + @Test + public void testSiteNameReplacement() { + String input = "asdf_${siteName}_asdf"; + assertEquals("asdf_Chaturbate_asdf", placeHolderAwarePp.fillInPlaceHolders(input, rec, config)); + input = "asdf_${siteSanitizedName}_asdf"; + assertEquals("asdf_Chaturbate_asdf", placeHolderAwarePp.fillInPlaceHolders(input, rec, config)); + } + + @Test + public void testUtcTimeReplacement() { + String date = DateTimeFormatter.ofPattern("yyyy-MM-dd_HH-mm-ss") + .withLocale(Locale.US) + .withZone(ZoneOffset.UTC) + .format(rec.getStartDate()); + String input = "asdf_${utcDateTime}_asdf"; + assertEquals("asdf_" + date + "_asdf", placeHolderAwarePp.fillInPlaceHolders(input, rec, config)); + + date = DateTimeFormatter.ofPattern("yyyyMMdd-HHmmss") + .withLocale(Locale.US) + .withZone(ZoneOffset.UTC) + .format(rec.getStartDate()); + input = "asdf_${utcDateTime(yyyyMMdd-HHmmss)}_asdf"; + assertEquals("asdf_" + date + "_asdf", placeHolderAwarePp.fillInPlaceHolders(input, rec, config)); + } + + @Test + public void testLocalTimeReplacement() { + String date = DateTimeFormatter.ofPattern("yyyy-MM-dd_HH-mm-ss") + .withLocale(Locale.US) + .withZone(ZoneId.systemDefault()) + .format(rec.getStartDate()); + String input = "asdf_${localDateTime}_asdf"; + assertEquals("asdf_" + date + "_asdf", placeHolderAwarePp.fillInPlaceHolders(input, rec, config)); + + date = DateTimeFormatter.ofPattern("yyyyMMdd-HHmmss") + .withLocale(Locale.US) + .withZone(ZoneId.systemDefault()) + .format(rec.getStartDate()); + input = "asdf_${localDateTime(yyyyMMdd-HHmmss)}_asdf"; + assertEquals("asdf_" + date + "_asdf", placeHolderAwarePp.fillInPlaceHolders(input, rec, config)); + } + + @Test + public void testEpochReplacement() { + long epoch = now.toEpochMilli() / 1000; + String input = "asdf_${epochSecond}_asdf"; + assertEquals("asdf_" + epoch + "_asdf", placeHolderAwarePp.fillInPlaceHolders(input, rec, config)); + } + + @Test + public void testFileSuffixReplacement() { + String input = "asdf_${fileSuffix}_asdf"; + assertEquals("asdf_ts_asdf", placeHolderAwarePp.fillInPlaceHolders(input, rec, config)); + } + + @Test + public void testRecordingsDirReplacement() { + String input = "asdf_${recordingsDir}_asdf"; + assertEquals("asdf_" + recDir.toString() + "_asdf", placeHolderAwarePp.fillInPlaceHolders(input, rec, config)); + } + + @Test + public void testAbsolutePathReplacement() { + String input = "asdf_${absolutePath}_asdf"; + assertEquals("asdf_" + postProcessed.getAbsolutePath().toString() + "_asdf", placeHolderAwarePp.fillInPlaceHolders(input, rec, config)); + } + + @Test + public void testAbsoluteParentPathReplacement() { + String input = "asdf_${absoluteParentPath}_asdf"; + assertEquals("asdf_" + postProcessed.getParentFile().toString() + "_asdf", placeHolderAwarePp.fillInPlaceHolders(input, rec, config)); + } + + @Test + public void testModelNotesReplacement() { + String input = "asdf_${modelNotes}_asdf"; + assertEquals("asdf_tag, foo, bar_asdf", placeHolderAwarePp.fillInPlaceHolders(input, rec, config)); + } +} diff --git a/common/src/test/java/ctbrec/recorder/postprocessing/AbstractPpTest.java b/common/src/test/java/ctbrec/recorder/postprocessing/AbstractPpTest.java index 8f299c39..2b19e7f7 100644 --- a/common/src/test/java/ctbrec/recorder/postprocessing/AbstractPpTest.java +++ b/common/src/test/java/ctbrec/recorder/postprocessing/AbstractPpTest.java @@ -70,7 +70,7 @@ public abstract class AbstractPpTest { Model mockModel() { Site site = new Chaturbate(); - Model model = site.createModel("Mockita"); + Model model = site.createModel("Mockita Boobilicious"); model.setDisplayName("Mockita Boobilicious"); return model; } diff --git a/common/src/test/java/ctbrec/recorder/postprocessing/DeleteOriginalTest.java b/common/src/test/java/ctbrec/recorder/postprocessing/DeleteOriginalTest.java index 3b6047ab..48e54386 100644 --- a/common/src/test/java/ctbrec/recorder/postprocessing/DeleteOriginalTest.java +++ b/common/src/test/java/ctbrec/recorder/postprocessing/DeleteOriginalTest.java @@ -48,4 +48,10 @@ public class DeleteOriginalTest extends AbstractPpTest { assertTrue(rec.getAbsoluteFile().exists()); assertFalse(originalDir.exists()); } + + @Test + public void testGetName() { + assertEquals("delete original", new DeleteOriginal().getName()); + } + } diff --git a/common/src/test/java/ctbrec/recorder/postprocessing/MoveDirectoryTest.java b/common/src/test/java/ctbrec/recorder/postprocessing/MoveDirectoryTest.java index 47ea38ed..93508e66 100644 --- a/common/src/test/java/ctbrec/recorder/postprocessing/MoveDirectoryTest.java +++ b/common/src/test/java/ctbrec/recorder/postprocessing/MoveDirectoryTest.java @@ -30,7 +30,7 @@ public class MoveDirectoryTest extends AbstractPpTest { pp.getConfig().put(Move.PATH_TEMPLATE, new File(baseDir.toFile(), Move.DEFAULT).getAbsolutePath()); pp.postprocess(rec, recordingManager, config); - Matcher m = Pattern.compile(baseDir.toString() + "/Mockita/\\d{4}-\\d{2}-\\d{2}_\\d{2}-\\d{2}-\\d{2}/original").matcher(rec.getAbsoluteFile().getCanonicalPath()); + Matcher m = Pattern.compile(baseDir.toString() + "/Mockita_Boobilicious/\\d{4}-\\d{2}-\\d{2}_\\d{2}-\\d{2}-\\d{2}/original").matcher(rec.getAbsoluteFile().getCanonicalPath()); assertTrue(m.matches()); assertEquals(rec.getAbsoluteFile(), rec.getPostProcessedFile()); assertNotEquals(rec.getAbsoluteFile(), original); diff --git a/common/src/test/java/ctbrec/recorder/postprocessing/MoveSingleFileTest.java b/common/src/test/java/ctbrec/recorder/postprocessing/MoveSingleFileTest.java index bb560a37..e11cde09 100644 --- a/common/src/test/java/ctbrec/recorder/postprocessing/MoveSingleFileTest.java +++ b/common/src/test/java/ctbrec/recorder/postprocessing/MoveSingleFileTest.java @@ -29,7 +29,7 @@ public class MoveSingleFileTest extends AbstractPpTest { pp.getConfig().put(Move.PATH_TEMPLATE, new File(baseDir.toFile(), Move.DEFAULT).getAbsolutePath()); pp.postprocess(rec, recordingManager, config); - Matcher m = Pattern.compile(baseDir.toFile() + "/Mockita/\\d{4}-\\d{2}-\\d{2}_\\d{2}-\\d{2}-\\d{2}/original\\.ts").matcher(rec.getAbsoluteFile().toString()); + Matcher m = Pattern.compile(baseDir.toFile() + "/Mockita_Boobilicious/\\d{4}-\\d{2}-\\d{2}_\\d{2}-\\d{2}-\\d{2}/original\\.ts").matcher(rec.getAbsoluteFile().toString()); assertTrue(m.matches()); assertEquals(rec.getAbsoluteFile(), rec.getPostProcessedFile()); assertNotEquals(rec.getAbsoluteFile(), original); diff --git a/common/src/test/java/ctbrec/recorder/postprocessing/RemoveKeepFileTest.java b/common/src/test/java/ctbrec/recorder/postprocessing/RemoveKeepFileTest.java new file mode 100644 index 00000000..8909091f --- /dev/null +++ b/common/src/test/java/ctbrec/recorder/postprocessing/RemoveKeepFileTest.java @@ -0,0 +1,42 @@ +package ctbrec.recorder.postprocessing; + +import static org.junit.Assert.*; + +import java.io.IOException; +import java.util.Collections; + +import org.junit.Test; + +import ctbrec.Config; +import ctbrec.Recording; +import ctbrec.recorder.RecordingManager; + +public class RemoveKeepFileTest extends AbstractPpTest { + + @Test + public void testPostProcessWithSingleFile() throws IOException, InterruptedException { + Recording rec = new Recording(); + rec.setModel(mockModel()); + rec.setAbsoluteFile(original); + rec.setPostProcessedFile(postProcessed); + rec.setStartDate(now); + rec.setSingleFile(true); + + Config config = mockConfig(); + RecordingManager rm = new RecordingManager(config, Collections.emptyList()); + rm.add(rec); + assertTrue(rm.getAll().size() == 1); + RemoveKeepFile pp = new RemoveKeepFile(); + pp.postprocess(rec, rm, config); + + assertTrue(rec.getAbsoluteFile().exists()); + assertTrue(rec.getPostProcessedFile().exists()); + assertTrue(rm.getAll().isEmpty()); + } + + @Test + public void testGetName() { + assertEquals("remove recording, but keep the files", new RemoveKeepFile().getName()); + } + +} diff --git a/common/src/test/java/ctbrec/recorder/postprocessing/RenameDirectoryTest.java b/common/src/test/java/ctbrec/recorder/postprocessing/RenameDirectoryTest.java index abf03070..cdd9cfb2 100644 --- a/common/src/test/java/ctbrec/recorder/postprocessing/RenameDirectoryTest.java +++ b/common/src/test/java/ctbrec/recorder/postprocessing/RenameDirectoryTest.java @@ -28,7 +28,7 @@ public class RenameDirectoryTest extends AbstractPpTest { Rename pp = new Rename(); pp.postprocess(rec, recordingManager, config); - Matcher m = Pattern.compile("Mockita_\\d{4}-\\d{2}-\\d{2}_\\d{2}-\\d{2}-\\d{2}").matcher(rec.getAbsoluteFile().getName()); + Matcher m = Pattern.compile("Mockita_Boobilicious_\\d{4}-\\d{2}-\\d{2}_\\d{2}-\\d{2}-\\d{2}").matcher(rec.getAbsoluteFile().getName()); assertTrue(m.matches()); assertEquals(rec.getAbsoluteFile(), rec.getPostProcessedFile()); assertNotEquals(rec.getAbsoluteFile(), original); diff --git a/common/src/test/java/ctbrec/recorder/postprocessing/RenameSingleFileTest.java b/common/src/test/java/ctbrec/recorder/postprocessing/RenameSingleFileTest.java index 9c52230b..de739cd5 100644 --- a/common/src/test/java/ctbrec/recorder/postprocessing/RenameSingleFileTest.java +++ b/common/src/test/java/ctbrec/recorder/postprocessing/RenameSingleFileTest.java @@ -27,7 +27,7 @@ public class RenameSingleFileTest extends AbstractPpTest { Rename pp = new Rename(); pp.postprocess(rec, recordingManager, config); - Matcher m = Pattern.compile("Mockita_\\d{4}-\\d{2}-\\d{2}_\\d{2}-\\d{2}-\\d{2}\\.ts").matcher(rec.getAbsoluteFile().getName()); + Matcher m = Pattern.compile("Mockita_Boobilicious_\\d{4}-\\d{2}-\\d{2}_\\d{2}-\\d{2}-\\d{2}\\.ts").matcher(rec.getAbsoluteFile().getName()); assertTrue(m.matches()); assertEquals(rec.getAbsoluteFile(), rec.getPostProcessedFile()); assertNotEquals(rec.getAbsoluteFile(), original); From 13783140e92948e1154db6c5a4c9a2235ac64275 Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Sat, 26 Sep 2020 14:23:48 +0200 Subject: [PATCH 24/33] Add test for Copy pp --- .../recorder/postprocessing/CopyTest.java | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 common/src/test/java/ctbrec/recorder/postprocessing/CopyTest.java diff --git a/common/src/test/java/ctbrec/recorder/postprocessing/CopyTest.java b/common/src/test/java/ctbrec/recorder/postprocessing/CopyTest.java new file mode 100644 index 00000000..f28f4e26 --- /dev/null +++ b/common/src/test/java/ctbrec/recorder/postprocessing/CopyTest.java @@ -0,0 +1,50 @@ +package ctbrec.recorder.postprocessing; + +import static org.junit.Assert.*; + +import java.io.IOException; + +import org.junit.Test; + +import ctbrec.Config; +import ctbrec.Recording; + +public class CopyTest extends AbstractPpTest { + + @Test + public void testCopySingleFile() throws IOException, InterruptedException { + Config config = mockConfig(); + Recording rec = new Recording(); + rec.setModel(mockModel()); + rec.setAbsoluteFile(original); + rec.setStartDate(now); + rec.setSingleFile(false); + Copy pp = new Copy(); + pp.postprocess(rec, recordingManager, config); + + assertNotEquals(rec.getAbsoluteFile(), rec.getPostProcessedFile()); + assertTrue(original.exists()); + assertTrue(rec.getPostProcessedFile().exists()); + } + + @Test + public void testCopyDirectory() throws IOException, InterruptedException { + Config config = mockConfig(); + Recording rec = new Recording(); + rec.setModel(mockModel()); + rec.setAbsoluteFile(originalDir); + rec.setStartDate(now); + rec.setSingleFile(false); + Copy pp = new Copy(); + pp.postprocess(rec, recordingManager, config); + + assertNotEquals(rec.getAbsoluteFile(), rec.getPostProcessedFile()); + assertTrue(originalDir.exists()); + assertTrue(rec.getPostProcessedFile().exists()); + } + + @Test + public void testGetName() { + assertEquals("create a copy", new Copy().getName()); + } +} From 9806badf57cf314a226e13c1e1feea43d4d16e5c Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Sat, 26 Sep 2020 15:14:54 +0200 Subject: [PATCH 25/33] Fix bug in Move pp --- common/src/main/java/ctbrec/recorder/postprocessing/Move.java | 3 ++- .../ctbrec/recorder/postprocessing/MoveSingleFileTest.java | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/common/src/main/java/ctbrec/recorder/postprocessing/Move.java b/common/src/main/java/ctbrec/recorder/postprocessing/Move.java index 5079385c..894faee3 100644 --- a/common/src/main/java/ctbrec/recorder/postprocessing/Move.java +++ b/common/src/main/java/ctbrec/recorder/postprocessing/Move.java @@ -29,6 +29,7 @@ public class Move extends AbstractPlaceholderAwarePostProcessor { String pathTemplate = getConfig().getOrDefault(PATH_TEMPLATE, DEFAULT); String path = fillInPlaceHolders(pathTemplate, rec, config); File src = rec.getPostProcessedFile(); + boolean isFile = src.isFile(); File target = new File(path, src.getName()); if (Objects.equals(src, target)) { return; @@ -42,7 +43,7 @@ public class Move extends AbstractPlaceholderAwarePostProcessor { } rec.getAssociatedFiles().remove(src.getCanonicalPath()); rec.getAssociatedFiles().add(target.getCanonicalPath()); - if (src.isFile()) { + if (isFile) { deleteEmptyParents(src.getParentFile()); } else { deleteEmptyParents(src); diff --git a/common/src/test/java/ctbrec/recorder/postprocessing/MoveSingleFileTest.java b/common/src/test/java/ctbrec/recorder/postprocessing/MoveSingleFileTest.java index e11cde09..b2199b80 100644 --- a/common/src/test/java/ctbrec/recorder/postprocessing/MoveSingleFileTest.java +++ b/common/src/test/java/ctbrec/recorder/postprocessing/MoveSingleFileTest.java @@ -25,6 +25,7 @@ public class MoveSingleFileTest extends AbstractPpTest { rec.setAbsoluteFile(original); rec.setStartDate(now); rec.setSingleFile(true); + Move pp = new Move(); pp.getConfig().put(Move.PATH_TEMPLATE, new File(baseDir.toFile(), Move.DEFAULT).getAbsolutePath()); pp.postprocess(rec, recordingManager, config); From 6fa6c63c85639cf1f72e65c5127b86a1c4880f60 Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Sat, 26 Sep 2020 16:07:11 +0200 Subject: [PATCH 26/33] Add test for DeleteTooShort pp --- .../postprocessing/DeleteTooShortTest.java | 98 +++++++++++++++++++ 1 file changed, 98 insertions(+) create mode 100644 common/src/test/java/ctbrec/recorder/postprocessing/DeleteTooShortTest.java diff --git a/common/src/test/java/ctbrec/recorder/postprocessing/DeleteTooShortTest.java b/common/src/test/java/ctbrec/recorder/postprocessing/DeleteTooShortTest.java new file mode 100644 index 00000000..6c3a5e08 --- /dev/null +++ b/common/src/test/java/ctbrec/recorder/postprocessing/DeleteTooShortTest.java @@ -0,0 +1,98 @@ +package ctbrec.recorder.postprocessing; + +import static org.junit.Assert.*; +import static org.mockito.Mockito.*; + +import java.io.File; +import java.io.IOException; +import java.time.Duration; +import java.util.Collections; + +import org.junit.Test; + +import ctbrec.Config; +import ctbrec.Recording; +import ctbrec.recorder.RecordingManager; +import ctbrec.recorder.download.Download; + +public class DeleteTooShortTest extends AbstractPpTest { + + @Test + public void tooShortSingleFileRecShouldBeDeleted() throws IOException, InterruptedException { + testProcess(original); + } + + @Test + public void tooShortDirectoryRecShouldBeDeleted() throws IOException, InterruptedException { + testProcess(originalDir); + } + + private void testProcess(File original) throws IOException { + Recording rec = createRec(original); + Config config = mockConfig(); + RecordingManager recordingManager = new RecordingManager(config, Collections.emptyList()); + recordingManager.add(rec); + + assertEquals(1, recordingManager.getAll().size()); + + DeleteTooShort pp = new DeleteTooShort(); + pp.getConfig().put(DeleteTooShort.MIN_LEN_IN_SECS, "10"); + pp.postprocess(rec, recordingManager, config); + + assertFalse(rec.getAbsoluteFile().exists()); + assertFalse(original.exists()); + assertEquals(0, recordingManager.getAll().size()); + } + + @Test + public void testGetName() { + assertEquals("delete too short", new DeleteTooShort().getName()); + } + + @Test + public void testDisabledWithSingleFile() throws IOException, InterruptedException { + Recording rec = createRec(original); + Config config = mockConfig(); + RecordingManager recordingManager = new RecordingManager(config, Collections.emptyList()); + recordingManager.add(rec); + assertEquals(1, recordingManager.getAll().size()); + + DeleteTooShort pp = new DeleteTooShort(); + pp.getConfig().put(DeleteTooShort.MIN_LEN_IN_SECS, "0"); + pp.postprocess(rec, recordingManager, config); + + assertTrue(rec.getAbsoluteFile().exists()); + assertTrue(original.exists()); + assertEquals(1, recordingManager.getAll().size()); + } + + @Test + public void longEnoughVideoShouldStay() throws IOException, InterruptedException { + Recording rec = createRec(original); + Config config = mockConfig(); + RecordingManager recordingManager = new RecordingManager(config, Collections.emptyList()); + recordingManager.add(rec); + assertEquals(1, recordingManager.getAll().size()); + + DeleteTooShort pp = new DeleteTooShort(); + pp.getConfig().put(DeleteTooShort.MIN_LEN_IN_SECS, "1"); + pp.postprocess(rec, recordingManager, config); + + assertTrue(rec.getAbsoluteFile().exists()); + assertTrue(original.exists()); + assertEquals(1, recordingManager.getAll().size()); + } + + private Recording createRec(File original) { + Download download = mock(Download.class); + when(download.getLength()).thenReturn(Duration.ofSeconds(5)); + Recording rec = new Recording(); + rec.setModel(mockModel()); + rec.setAbsoluteFile(original); + rec.setPostProcessedFile(original); + rec.setStartDate(now); + rec.setSingleFile(true); + rec.setDownload(download); + return rec; + } +} From 4aca8613901f7b6a2bf03025740120b0fb09656b Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Sat, 26 Sep 2020 16:32:31 +0200 Subject: [PATCH 27/33] Remove unused settings --- common/src/main/java/ctbrec/Settings.java | 2 -- common/src/main/java/ctbrec/recorder/NextGenLocalRecorder.java | 3 --- server/src/main/java/ctbrec/recorder/server/ConfigServlet.java | 1 - 3 files changed, 6 deletions(-) diff --git a/common/src/main/java/ctbrec/Settings.java b/common/src/main/java/ctbrec/Settings.java index ba7feac0..a4599529 100644 --- a/common/src/main/java/ctbrec/Settings.java +++ b/common/src/main/java/ctbrec/Settings.java @@ -84,7 +84,6 @@ public class Settings { public String mfcModelsTableSortType = ""; public String mfcPassword = ""; public String mfcUsername = ""; - public int minimumLengthInSeconds = 0; public long minimumSpaceLeftInBytes = 0; public Map modelNotes = new HashMap<>(); public List models = new ArrayList<>(); @@ -112,7 +111,6 @@ public class Settings { public String recordingsSortColumn = ""; public String recordingsSortType = ""; public boolean recordSingleFile = false; - public boolean removeRecordingAfterPostProcessing = false; public boolean requireAuthentication = false; public String servletContext = ""; public boolean showPlayerStarting = false; diff --git a/common/src/main/java/ctbrec/recorder/NextGenLocalRecorder.java b/common/src/main/java/ctbrec/recorder/NextGenLocalRecorder.java index 3623b5da..136e9e16 100644 --- a/common/src/main/java/ctbrec/recorder/NextGenLocalRecorder.java +++ b/common/src/main/java/ctbrec/recorder/NextGenLocalRecorder.java @@ -169,9 +169,6 @@ public class NextGenLocalRecorder implements Recorder { setRecordingStatus(recording, State.FINISHED); recordingManager.saveRecording(recording); LOG.info("Post-processing finished for {}", recording.getModel().getName()); - if (config.getSettings().removeRecordingAfterPostProcessing) { - recordingManager.remove(recording); - } } catch (Exception e) { if (e instanceof InterruptedException) { // NOSONAR Thread.currentThread().interrupt(); diff --git a/server/src/main/java/ctbrec/recorder/server/ConfigServlet.java b/server/src/main/java/ctbrec/recorder/server/ConfigServlet.java index 7d0e73e8..fe8f51bb 100644 --- a/server/src/main/java/ctbrec/recorder/server/ConfigServlet.java +++ b/server/src/main/java/ctbrec/recorder/server/ConfigServlet.java @@ -57,7 +57,6 @@ public class ConfigServlet extends AbstractCtbrecServlet { addParameter("generatePlaylist", "Generate Playlist", DataType.BOOLEAN, settings.generatePlaylist, json); addParameter("minimumResolution", "Minimum Resolution", DataType.INTEGER, settings.minimumResolution, json); addParameter("maximumResolution", "Maximum Resolution", DataType.INTEGER, settings.maximumResolution, json); - addParameter("minimumLengthInSeconds", "Minimum Length (secs)", DataType.INTEGER, settings.minimumLengthInSeconds, json); addParameter("minimumSpaceLeftInBytes", "Leave Space On Device (GiB)", DataType.LONG, settings.minimumSpaceLeftInBytes, json); addParameter("onlineCheckIntervalInSecs", "Online Check Interval (secs)", DataType.INTEGER, settings.onlineCheckIntervalInSecs, json); addParameter("postProcessing", "Post-Processing", DataType.STRING, settings.postProcessing, json); From b8ffdb32cec931c8009a6eeddc33cdede6febbf4 Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Sat, 26 Sep 2020 17:04:17 +0200 Subject: [PATCH 28/33] Change config mock, so that the config dir is in the temp test dir --- .../test/java/ctbrec/recorder/postprocessing/AbstractPpTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/common/src/test/java/ctbrec/recorder/postprocessing/AbstractPpTest.java b/common/src/test/java/ctbrec/recorder/postprocessing/AbstractPpTest.java index 2b19e7f7..0d9e65e7 100644 --- a/common/src/test/java/ctbrec/recorder/postprocessing/AbstractPpTest.java +++ b/common/src/test/java/ctbrec/recorder/postprocessing/AbstractPpTest.java @@ -63,6 +63,7 @@ public abstract class AbstractPpTest { Config config = mock(Config.class); when(config.getSettings()).thenReturn(mockSettings()); when(config.getModelNotes(any())).thenReturn("tag, foo, bar"); + when(config.getConfigDir()).thenReturn(new File(baseDir.toFile(), "config")); configStatic = mockStatic(Config.class); configStatic.when(Config::getInstance).thenReturn(config); return config; From 93deeb3e52d99167082275df7e1d3d73165ab03f Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Sat, 26 Sep 2020 17:22:42 +0200 Subject: [PATCH 29/33] Remove unused postProcessing setting --- client/pom.xml | 2 +- common/pom.xml | 2 +- common/src/main/java/ctbrec/Settings.java | 1 - .../recorder/download/AbstractDownload.java | 48 ------------------- .../recorder/download/dash/DashDownload.java | 1 - .../recorder/download/hls/FFmpegDownload.java | 6 --- .../recorder/download/hls/HlsDownload.java | 1 - .../download/hls/MergedFfmpegHlsDownload.java | 6 --- master/pom.xml | 2 +- server/pom.xml | 2 +- .../ctbrec/recorder/server/ConfigServlet.java | 1 - 11 files changed, 4 insertions(+), 68 deletions(-) diff --git a/client/pom.xml b/client/pom.xml index 2bfba9df..5f7f04d4 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -8,7 +8,7 @@ ctbrec master - 3.9.0 + 3.10.0 ../master diff --git a/common/pom.xml b/common/pom.xml index 6713bf45..10e5b090 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -8,7 +8,7 @@ ctbrec master - 3.9.0 + 3.10.0 ../master diff --git a/common/src/main/java/ctbrec/Settings.java b/common/src/main/java/ctbrec/Settings.java index a4599529..d95a50eb 100644 --- a/common/src/main/java/ctbrec/Settings.java +++ b/common/src/main/java/ctbrec/Settings.java @@ -92,7 +92,6 @@ public class Settings { public boolean onlineCheckSkipsPausedModels = false; public int overviewUpdateIntervalInSecs = 10; public String password = ""; // chaturbate password TODO maybe rename this onetime - public String postProcessing = ""; public int postProcessingThreads = 2; public List postProcessors = new ArrayList<>(); public String proxyHost; diff --git a/common/src/main/java/ctbrec/recorder/download/AbstractDownload.java b/common/src/main/java/ctbrec/recorder/download/AbstractDownload.java index 3e522da3..c1f1a939 100644 --- a/common/src/main/java/ctbrec/recorder/download/AbstractDownload.java +++ b/common/src/main/java/ctbrec/recorder/download/AbstractDownload.java @@ -1,59 +1,11 @@ package ctbrec.recorder.download; -import java.io.File; -import java.io.IOException; import java.time.Instant; -import java.util.Arrays; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import ctbrec.Config; -import ctbrec.OS; -import ctbrec.Recording; -import ctbrec.io.StreamRedirectThread; public abstract class AbstractDownload implements Download { - private static final Logger LOG = LoggerFactory.getLogger(AbstractDownload.class); - protected Instant startTime; - protected void runPostProcessingScript(Recording recording) throws IOException, InterruptedException { - String postProcessing = Config.getInstance().getSettings().postProcessing; - if (postProcessing != null && !postProcessing.isEmpty()) { - File target = recording.getAbsoluteFile(); - Runtime rt = Runtime.getRuntime(); - String[] args = new String[] { - postProcessing, - target.getParentFile().getAbsolutePath(), - target.getAbsolutePath(), - getModel().getName(), - getModel().getSite().getName(), - Long.toString(recording.getStartDate().getEpochSecond()) - }; - if(LOG.isDebugEnabled()) { - LOG.debug("Running {}", Arrays.toString(args)); - } - Process process = rt.exec(args, OS.getEnvironment()); - // TODO maybe write these to a separate log file, e.g. recname.ts.pp.log - Thread std = new Thread(new StreamRedirectThread(process.getInputStream(), System.out)); - std.setName("Process stdout pipe"); - std.setDaemon(true); - std.start(); - Thread err = new Thread(new StreamRedirectThread(process.getErrorStream(), System.err)); - err.setName("Process stderr pipe"); - err.setDaemon(true); - err.start(); - - int exitCode = process.waitFor(); - LOG.debug("Process finished with exit code {}", exitCode); - if (exitCode != 0) { - throw new ProcessExitedUncleanException("Post-Processing finished with exit code " + exitCode); - } - } - } - @Override public Instant getStartTime() { return startTime; diff --git a/common/src/main/java/ctbrec/recorder/download/dash/DashDownload.java b/common/src/main/java/ctbrec/recorder/download/dash/DashDownload.java index d6203b2a..e3f1a170 100644 --- a/common/src/main/java/ctbrec/recorder/download/dash/DashDownload.java +++ b/common/src/main/java/ctbrec/recorder/download/dash/DashDownload.java @@ -398,7 +398,6 @@ public class DashDownload extends AbstractDownload { new FfmpegMuxer(dir, file); targetFile = file; recording.setPath(path.substring(0, path.length() - 5)); - runPostProcessingScript(recording); } catch (Exception e) { throw new PostProcessingException(e); } diff --git a/common/src/main/java/ctbrec/recorder/download/hls/FFmpegDownload.java b/common/src/main/java/ctbrec/recorder/download/hls/FFmpegDownload.java index 2e25a9f9..52ed297c 100644 --- a/common/src/main/java/ctbrec/recorder/download/hls/FFmpegDownload.java +++ b/common/src/main/java/ctbrec/recorder/download/hls/FFmpegDownload.java @@ -125,12 +125,6 @@ public class FFmpegDownload extends AbstractHlsDownload { @Override public void postprocess(Recording recording) { - Thread.currentThread().setName("PP " + model.getName()); - try { - runPostProcessingScript(recording); - } catch (Exception e) { - throw new PostProcessingException(e); - } } @Override diff --git a/common/src/main/java/ctbrec/recorder/download/hls/HlsDownload.java b/common/src/main/java/ctbrec/recorder/download/hls/HlsDownload.java index b7f8435e..1d2f034b 100644 --- a/common/src/main/java/ctbrec/recorder/download/hls/HlsDownload.java +++ b/common/src/main/java/ctbrec/recorder/download/hls/HlsDownload.java @@ -215,7 +215,6 @@ public class HlsDownload extends AbstractHlsDownload { try { generatePlaylist(recording); recording.setStatusWithEvent(State.POST_PROCESSING); - runPostProcessingScript(recording); } catch (Exception e) { throw new PostProcessingException(e); } diff --git a/common/src/main/java/ctbrec/recorder/download/hls/MergedFfmpegHlsDownload.java b/common/src/main/java/ctbrec/recorder/download/hls/MergedFfmpegHlsDownload.java index 2721efb6..609866b5 100644 --- a/common/src/main/java/ctbrec/recorder/download/hls/MergedFfmpegHlsDownload.java +++ b/common/src/main/java/ctbrec/recorder/download/hls/MergedFfmpegHlsDownload.java @@ -489,12 +489,6 @@ public class MergedFfmpegHlsDownload extends AbstractHlsDownload { @Override public void postprocess(Recording recording) { - Thread.currentThread().setName("PP " + model.getName()); - try { - runPostProcessingScript(recording); - } catch (Exception e) { - throw new PostProcessingException(e); - } } public void downloadFinishedRecording(String segmentPlaylistUri, File target, ProgressListener progressListener, long sizeInBytes) throws Exception { diff --git a/master/pom.xml b/master/pom.xml index c81fb52f..9eb04530 100644 --- a/master/pom.xml +++ b/master/pom.xml @@ -6,7 +6,7 @@ ctbrec master pom - 3.9.0 + 3.10.0 ../common diff --git a/server/pom.xml b/server/pom.xml index 9dd10055..d7413299 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -8,7 +8,7 @@ ctbrec master - 3.9.0 + 3.10.0 ../master diff --git a/server/src/main/java/ctbrec/recorder/server/ConfigServlet.java b/server/src/main/java/ctbrec/recorder/server/ConfigServlet.java index fe8f51bb..36d9e13c 100644 --- a/server/src/main/java/ctbrec/recorder/server/ConfigServlet.java +++ b/server/src/main/java/ctbrec/recorder/server/ConfigServlet.java @@ -59,7 +59,6 @@ public class ConfigServlet extends AbstractCtbrecServlet { addParameter("maximumResolution", "Maximum Resolution", DataType.INTEGER, settings.maximumResolution, json); addParameter("minimumSpaceLeftInBytes", "Leave Space On Device (GiB)", DataType.LONG, settings.minimumSpaceLeftInBytes, json); addParameter("onlineCheckIntervalInSecs", "Online Check Interval (secs)", DataType.INTEGER, settings.onlineCheckIntervalInSecs, json); - addParameter("postProcessing", "Post-Processing", DataType.STRING, settings.postProcessing, json); addParameter("postProcessingThreads", "Post-Processing Threads", DataType.INTEGER, settings.postProcessingThreads, json); addParameter("recordingsDir", "Recordings Directory", DataType.STRING, settings.recordingsDir, json); addParameter("recordSingleFile", "Record Single File", DataType.BOOLEAN, settings.recordSingleFile, json); From f575958fe922e21ae068e50bee338d362c5ce7d5 Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Sat, 26 Sep 2020 18:10:16 +0200 Subject: [PATCH 30/33] Fix equals and hashcode equals and hashcode used the path field, which is empty for old recordings. Fixed by using getAbsoluteFile() instead --- common/src/main/java/ctbrec/Recording.java | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/common/src/main/java/ctbrec/Recording.java b/common/src/main/java/ctbrec/Recording.java index 12559935..b95a10a3 100644 --- a/common/src/main/java/ctbrec/Recording.java +++ b/common/src/main/java/ctbrec/Recording.java @@ -220,7 +220,7 @@ public class Recording implements Serializable { int result = 1; result = prime * result + ((getStartDate() == null) ? 0 : (int) (getStartDate().toEpochMilli() ^ (getStartDate().toEpochMilli() >>> 32))); result = prime * result + ((model == null) ? 0 : model.hashCode()); - result = prime * result + ((path == null) ? 0 : path.hashCode()); + result = prime * result + ((getAbsoluteFile() == null) ? 0 : getAbsoluteFile().hashCode()); return result; } @@ -241,11 +241,7 @@ public class Recording implements Serializable { } else if (!getModel().equals(other.getModel())) { return false; } - if (path == null) { - if (other.path != null) { - return false; - } - } else if (!path.equals(other.path)) { + if (!getAbsoluteFile().equals(other.getAbsoluteFile())) { return false; } if (getStartDate() == null) { From 9ed4ded25847f64e47bb3221d38a2973c1a1fb20 Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Sat, 26 Sep 2020 19:12:16 +0200 Subject: [PATCH 31/33] Disable post-processing steps in remote mode --- client/src/main/java/ctbrec/ui/settings/SettingsTab.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/client/src/main/java/ctbrec/ui/settings/SettingsTab.java b/client/src/main/java/ctbrec/ui/settings/SettingsTab.java index 3f7fdd00..812f1b33 100644 --- a/client/src/main/java/ctbrec/ui/settings/SettingsTab.java +++ b/client/src/main/java/ctbrec/ui/settings/SettingsTab.java @@ -98,6 +98,7 @@ public class SettingsTab extends Tab implements TabSelectionListener { private ExclusiveSelectionProperty recordLocal; private SimpleIntegerProperty postProcessingThreads; private IgnoreList ignoreList; + private PostProcessingStepPanel postProcessingStepPanel; public SettingsTab(List sites, Recorder recorder) { this.sites = sites; @@ -148,6 +149,7 @@ public class SettingsTab extends Tab implements TabSelectionListener { } private void createGui() { + postProcessingStepPanel = new PostProcessingStepPanel(config); ignoreList = new IgnoreList(sites); List siteCategories = new ArrayList<>(); for (Site site : sites) { @@ -201,7 +203,7 @@ public class SettingsTab extends Tab implements TabSelectionListener { Category.of("Post-Processing", Group.of("Post-Processing", Setting.of("Threads", postProcessingThreads), - Setting.of("Steps", new PostProcessingStepPanel(config)) + Setting.of("Steps", postProcessingStepPanel) ) ), Category.of("Events & Actions", new ActionSettingsPanel(recorder)), @@ -238,6 +240,8 @@ public class SettingsTab extends Tab implements TabSelectionListener { prefs.getSetting("removeRecordingAfterPostProcessing").ifPresent(s -> bindEnabledProperty(s, recordLocal.not())); prefs.getSetting("minimumLengthInSeconds").ifPresent(s -> bindEnabledProperty(s, recordLocal.not())); prefs.getSetting("concurrentRecordings").ifPresent(s -> bindEnabledProperty(s, recordLocal.not())); + prefs.getSetting("concurrentRecordings").ifPresent(s -> bindEnabledProperty(s, recordLocal.not())); + postProcessingStepPanel.disableProperty().bind(recordLocal.not()); } private void bindEnabledProperty(Setting s, BooleanExpression bindTo) { From 5fa72eaaa0d932e01496a908d9eb05d4a16930c0 Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Sat, 26 Sep 2020 19:52:09 +0200 Subject: [PATCH 32/33] Add setting to define the download file name --- .../java/ctbrec/ui/settings/SettingsTab.java | 4 ++++ .../ctbrec/ui/tabs/DownloadPostprocessor.java | 21 +++++++++++++++++++ .../java/ctbrec/ui/tabs/RecordingsTab.java | 12 +++++------ common/src/main/java/ctbrec/Settings.java | 1 + .../java/ctbrec/recorder/RemoteRecorder.java | 13 ++++++++++++ ...AbstractPlaceholderAwarePostProcessor.java | 5 +++-- 6 files changed, 48 insertions(+), 8 deletions(-) create mode 100644 client/src/main/java/ctbrec/ui/tabs/DownloadPostprocessor.java diff --git a/client/src/main/java/ctbrec/ui/settings/SettingsTab.java b/client/src/main/java/ctbrec/ui/settings/SettingsTab.java index 812f1b33..02cb7e2e 100644 --- a/client/src/main/java/ctbrec/ui/settings/SettingsTab.java +++ b/client/src/main/java/ctbrec/ui/settings/SettingsTab.java @@ -93,6 +93,7 @@ public class SettingsTab extends Tab implements TabSelectionListener { private SimpleStringProperty server; private SimpleIntegerProperty port; private SimpleStringProperty path; + private SimpleStringProperty downloadFilename; private SimpleBooleanProperty requireAuthentication; private SimpleBooleanProperty transportLayerSecurity; private ExclusiveSelectionProperty recordLocal; @@ -140,6 +141,7 @@ public class SettingsTab extends Tab implements TabSelectionListener { server = new SimpleStringProperty(null, "httpServer", settings.httpServer); port = new SimpleIntegerProperty(null, "httpPort", settings.httpPort); path = new SimpleStringProperty(null, "servletContext", settings.servletContext); + downloadFilename = new SimpleStringProperty(null, "downloadFilename", settings.downloadFilename); requireAuthentication = new SimpleBooleanProperty(null, "requireAuthentication", settings.requireAuthentication); requireAuthentication.addListener(this::requireAuthenticationChanged); transportLayerSecurity = new SimpleBooleanProperty(null, "transportLayerSecurity", settings.transportLayerSecurity); @@ -196,6 +198,7 @@ public class SettingsTab extends Tab implements TabSelectionListener { Setting.of("Server", server), Setting.of("Port", port), Setting.of("Path", path, "Leave empty, if you didn't change the servletContext in the server config"), + Setting.of("Download Filename", downloadFilename, "File name pattern for downloads"), Setting.of("Require authentication", requireAuthentication), Setting.of("Use Secure Communication (TLS)", transportLayerSecurity) ) @@ -241,6 +244,7 @@ public class SettingsTab extends Tab implements TabSelectionListener { prefs.getSetting("minimumLengthInSeconds").ifPresent(s -> bindEnabledProperty(s, recordLocal.not())); prefs.getSetting("concurrentRecordings").ifPresent(s -> bindEnabledProperty(s, recordLocal.not())); prefs.getSetting("concurrentRecordings").ifPresent(s -> bindEnabledProperty(s, recordLocal.not())); + prefs.getSetting("downloadFilename").ifPresent(s -> bindEnabledProperty(s, recordLocal)); postProcessingStepPanel.disableProperty().bind(recordLocal.not()); } diff --git a/client/src/main/java/ctbrec/ui/tabs/DownloadPostprocessor.java b/client/src/main/java/ctbrec/ui/tabs/DownloadPostprocessor.java new file mode 100644 index 00000000..3db557a0 --- /dev/null +++ b/client/src/main/java/ctbrec/ui/tabs/DownloadPostprocessor.java @@ -0,0 +1,21 @@ +package ctbrec.ui.tabs; + +import java.io.IOException; + +import ctbrec.Config; +import ctbrec.Recording; +import ctbrec.recorder.RecordingManager; +import ctbrec.recorder.postprocessing.AbstractPlaceholderAwarePostProcessor; + +public class DownloadPostprocessor extends AbstractPlaceholderAwarePostProcessor { + + @Override + public String getName() { + return "download renamer"; + } + + @Override + public void postprocess(Recording rec, RecordingManager recordingManager, Config config) throws IOException, InterruptedException { + // nothing really to do in here, we just inherit from AbstractPlaceholderAwarePostProcessor to use fillInPlaceHolders + } +} diff --git a/client/src/main/java/ctbrec/ui/tabs/RecordingsTab.java b/client/src/main/java/ctbrec/ui/tabs/RecordingsTab.java index 1583606e..d905f303 100644 --- a/client/src/main/java/ctbrec/ui/tabs/RecordingsTab.java +++ b/client/src/main/java/ctbrec/ui/tabs/RecordingsTab.java @@ -581,15 +581,15 @@ public class RecordingsTab extends Tab implements TabSelectionListener { String filename = proposeTargetFilename(recording); FileChooser chooser = new FileChooser(); chooser.setInitialFileName(filename); - if(config.getSettings().lastDownloadDir != null && !config.getSettings().lastDownloadDir.equals("")) { + if (config.getSettings().lastDownloadDir != null && !config.getSettings().lastDownloadDir.equals("")) { File dir = new File(config.getSettings().lastDownloadDir); - while(!dir.exists()) { + while (!dir.exists()) { dir = dir.getParentFile(); } chooser.setInitialDirectory(dir); } File target = chooser.showSaveDialog(null); - if(target != null) { + if (target != null) { config.getSettings().lastDownloadDir = target.getParent(); startDownloadThread(target, recording); recording.setStatus(DOWNLOADING); @@ -598,12 +598,12 @@ public class RecordingsTab extends Tab implements TabSelectionListener { } private String proposeTargetFilename(Recording recording) { - String path = recording.getAbsoluteFile().getAbsolutePath().substring(1); if(recording.isSingleFile()) { - return new File(path).getName(); + return recording.getAbsoluteFile().getName(); } else { + String downloadFilename = config.getSettings().downloadFilename; String fileSuffix = config.getSettings().ffmpegFileSuffix; - String filename = path.replace("/", "-").replace(".mp4", "") + '.' + fileSuffix; + String filename = new DownloadPostprocessor().fillInPlaceHolders(downloadFilename, recording, config) + '.' + fileSuffix; return filename; } } diff --git a/common/src/main/java/ctbrec/Settings.java b/common/src/main/java/ctbrec/Settings.java index d95a50eb..2fc2f41c 100644 --- a/common/src/main/java/ctbrec/Settings.java +++ b/common/src/main/java/ctbrec/Settings.java @@ -48,6 +48,7 @@ public class Settings { public int concurrentRecordings = 0; public boolean determineResolution = false; public List disabledSites = new ArrayList<>(); + public String downloadFilename = "${modelSanitizedName}-${localDateTime}"; public List eventHandlers = new ArrayList<>(); public String fc2livePassword = ""; public String fc2liveUsername = ""; diff --git a/common/src/main/java/ctbrec/recorder/RemoteRecorder.java b/common/src/main/java/ctbrec/recorder/RemoteRecorder.java index 321bf46c..4b473772 100644 --- a/common/src/main/java/ctbrec/recorder/RemoteRecorder.java +++ b/common/src/main/java/ctbrec/recorder/RemoteRecorder.java @@ -350,6 +350,19 @@ public class RemoteRecorder implements Recorder { } recordings = newRecordings; + + // assign a site to the model + for (Site site : sites) { + for (Recording recording : recordings) { + Model m = recording.getModel(); + if (m.getSite() == null) { + if (site.isSiteForModel(m)) { + m.setSite(site); + continue; + } + } + } + } } else { LOG.error(SERVER_RETURNED_ERROR, resp.status, resp.msg); } diff --git a/common/src/main/java/ctbrec/recorder/postprocessing/AbstractPlaceholderAwarePostProcessor.java b/common/src/main/java/ctbrec/recorder/postprocessing/AbstractPlaceholderAwarePostProcessor.java index 5007d97f..b83a1790 100644 --- a/common/src/main/java/ctbrec/recorder/postprocessing/AbstractPlaceholderAwarePostProcessor.java +++ b/common/src/main/java/ctbrec/recorder/postprocessing/AbstractPlaceholderAwarePostProcessor.java @@ -11,6 +11,7 @@ import java.util.regex.Pattern; import ctbrec.Config; import ctbrec.Recording; +import ctbrec.sites.Site; public abstract class AbstractPlaceholderAwarePostProcessor extends AbstractPostProcessor { @@ -36,7 +37,7 @@ public abstract class AbstractPlaceholderAwarePostProcessor extends AbstractPost .replace("${modelName}", ofNullable(rec.getModel().getName()).orElse("modelName")) .replace("${modelDisplayName}", ofNullable(rec.getModel().getDisplayName()).orElse("displayName")) .replace("${modelSanitizedName}", ofNullable(rec.getModel().getSanitizedNamed()).orElse("sanitizedName")) - .replace("${siteName}", rec.getModel().getSite().getName()) + .replace("${siteName}", ofNullable(rec.getModel().getSite()).map(Site::getName).orElse("site")) .replace("${siteSanitizedName}", getSanitizedSiteName(rec)) .replace("${fileSuffix}", getFileSuffix(rec)) .replace("${epochSecond}", Long.toString(rec.getStartDate().getEpochSecond())) @@ -91,7 +92,7 @@ public abstract class AbstractPlaceholderAwarePostProcessor extends AbstractPost } private CharSequence getSanitizedSiteName(Recording rec) { - return rec.getModel().getSite().getName().replace(' ', '_').replace('\\', '_').replace('/', '_'); + return ofNullable(rec.getModel().getSite()).map(Site::getName).orElse("").replace(' ', '_').replace('\\', '_').replace('/', '_'); } } From 43cf0a0bc1361511f3060b1eec5e687b9bc6f6ed Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Sat, 26 Sep 2020 20:18:17 +0200 Subject: [PATCH 33/33] Add placeholder for recording notes --- common/src/main/java/ctbrec/StringUtil.java | 10 ++++++++++ .../AbstractPlaceholderAwarePostProcessor.java | 10 ++++++++-- .../AbstractPlaceholderAwarePostProcessorTest.java | 2 +- 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/common/src/main/java/ctbrec/StringUtil.java b/common/src/main/java/ctbrec/StringUtil.java index 589e296b..236fcca4 100644 --- a/common/src/main/java/ctbrec/StringUtil.java +++ b/common/src/main/java/ctbrec/StringUtil.java @@ -60,4 +60,14 @@ public class StringUtil { } return hex; } + + // @formatter:off + public static String sanitize(String input) { + return input + .replace(' ', '_') + .replace('\\', '_') + .replace('/', '_') + .replace('\'', '_') + .replace('"', '_'); + } // @formatter:on } diff --git a/common/src/main/java/ctbrec/recorder/postprocessing/AbstractPlaceholderAwarePostProcessor.java b/common/src/main/java/ctbrec/recorder/postprocessing/AbstractPlaceholderAwarePostProcessor.java index b83a1790..3cf306e5 100644 --- a/common/src/main/java/ctbrec/recorder/postprocessing/AbstractPlaceholderAwarePostProcessor.java +++ b/common/src/main/java/ctbrec/recorder/postprocessing/AbstractPlaceholderAwarePostProcessor.java @@ -1,5 +1,6 @@ package ctbrec.recorder.postprocessing; +import static ctbrec.StringUtil.*; import static java.util.Optional.*; import java.time.ZoneId; @@ -26,6 +27,7 @@ public abstract class AbstractPlaceholderAwarePostProcessor extends AbstractPost "${epochSecond}", "${fileSuffix}", "${modelNotes}", + "${recordingNotes}", "${recordingsDir}", "${absolutePath}", "${absoluteParentPath}" @@ -41,7 +43,8 @@ public abstract class AbstractPlaceholderAwarePostProcessor extends AbstractPost .replace("${siteSanitizedName}", getSanitizedSiteName(rec)) .replace("${fileSuffix}", getFileSuffix(rec)) .replace("${epochSecond}", Long.toString(rec.getStartDate().getEpochSecond())) - .replace("${modelNotes}", config.getModelNotes(rec.getModel())) + .replace("${modelNotes}", sanitize(config.getModelNotes(rec.getModel()))) + .replace("${recordingNotes}", getSanitizedRecordingNotes(rec)) .replace("${recordingsDir}", config.getSettings().recordingsDir) .replace("${absolutePath}", rec.getPostProcessedFile().getAbsolutePath()) .replace("${absoluteParentPath}", rec.getPostProcessedFile().getParentFile().getAbsolutePath()) @@ -92,7 +95,10 @@ public abstract class AbstractPlaceholderAwarePostProcessor extends AbstractPost } private CharSequence getSanitizedSiteName(Recording rec) { - return ofNullable(rec.getModel().getSite()).map(Site::getName).orElse("").replace(' ', '_').replace('\\', '_').replace('/', '_'); + return sanitize(ofNullable(rec.getModel().getSite()).map(Site::getName).orElse("")); } + private CharSequence getSanitizedRecordingNotes(Recording rec) { + return sanitize(ofNullable(rec.getNote()).orElse("")); + } } diff --git a/common/src/test/java/ctbrec/recorder/postprocessing/AbstractPlaceholderAwarePostProcessorTest.java b/common/src/test/java/ctbrec/recorder/postprocessing/AbstractPlaceholderAwarePostProcessorTest.java index 649c6ec8..04e4e13d 100644 --- a/common/src/test/java/ctbrec/recorder/postprocessing/AbstractPlaceholderAwarePostProcessorTest.java +++ b/common/src/test/java/ctbrec/recorder/postprocessing/AbstractPlaceholderAwarePostProcessorTest.java @@ -120,6 +120,6 @@ public class AbstractPlaceholderAwarePostProcessorTest extends AbstractPpTest { @Test public void testModelNotesReplacement() { String input = "asdf_${modelNotes}_asdf"; - assertEquals("asdf_tag, foo, bar_asdf", placeHolderAwarePp.fillInPlaceHolders(input, rec, config)); + assertEquals("asdf_tag,_foo,_bar_asdf", placeHolderAwarePp.fillInPlaceHolders(input, rec, config)); } }