Added auto-merging for local mode

In local mode files can be automatically merged after a recording is
finished. This can be configured in the settings.

src/main/java/ctbrec/Recording.java

@@ -1,5 +1,6 @@
 package ctbrec;
 
+import java.io.File;
 import java.text.ParseException;
 import java.text.SimpleDateFormat;
 import java.time.Instant;
@@ -116,4 +117,23 @@ public class Recording {
             return false;
         return true;
     }
+
+    public static File mergedFileFromDirectory(File recDir) {
+        String date = recDir.getName();
+        String model = recDir.getParentFile().getName();
+        String filename = model + "-" + date + ".ts";
+        File mergedFile = new File(recDir, filename);
+        return mergedFile;
+    }
+
+    public static boolean isMergedRecording(File recDir) {
+        File mergedFile = mergedFileFromDirectory(recDir);
+        return mergedFile.exists();
+    }
+
+    public static boolean isMergedRecording(Recording recording) {
+        String recordingsDir = Config.getInstance().getSettings().recordingsDir;
+        File recDir = new File(recordingsDir, recording.getPath());
+        return isMergedRecording(recDir);
+    }
 }

src/main/java/ctbrec/Settings.java

@@ -15,6 +15,8 @@ public class Settings {
     public String password = "";
     public String lastDownloadDir = "";
     public List<Model> models = new ArrayList<Model>();
+    public boolean automerge = false;
+    public boolean automergeKeepSegments = false;
     public boolean determineResolution = false;
     public boolean requireAuthentication = false;
     public byte[] key = null;

src/main/java/ctbrec/recorder/LocalRecorder.java

@@ -30,10 +30,10 @@ import ctbrec.Config;
 import ctbrec.HttpClient;
 import ctbrec.Model;
 import ctbrec.Recording;
+import ctbrec.Recording.STATUS;
+import ctbrec.recorder.PlaylistGenerator.InvalidPlaylistException;
 import ctbrec.recorder.download.Download;
 import ctbrec.recorder.download.HlsDownload;
-import ctbrec.recorder.server.PlaylistGenerator;
-import ctbrec.recorder.server.PlaylistGenerator.InvalidPlaylistException;
 
 public class LocalRecorder implements Recorder {
 
@@ -43,12 +43,14 @@ public class LocalRecorder implements Recorder {
     private Lock lock = new ReentrantLock();
     private Map<Model, Download> recordingProcesses = Collections.synchronizedMap(new HashMap<>());
     private Map<File, PlaylistGenerator> playlistGenerators = new HashMap<>();
+    private Map<File, SegmentMerger> segmentMergers = new HashMap<>();
     private Config config;
     private ProcessMonitor processMonitor;
     private OnlineMonitor onlineMonitor;
     private PlaylistGeneratorTrigger playlistGenTrigger;
     private HttpClient client = HttpClient.getInstance();
     private volatile boolean recording = true;
+    private List<File> deleteInProgress = Collections.synchronizedList(new ArrayList<>());
 
     public LocalRecorder(Config config) {
         this.config = config;
@@ -238,7 +240,7 @@ public class LocalRecorder implements Recorder {
                             LOG.debug("Recording terminated for model {}", m.getName());
                             iterator.remove();
                             restart.add(m);
-                            generatePlaylist(d.getDirectory());
+                            finishRecording(d.getDirectory());
                         }
                     }
                     for (Model m : restart) {
@@ -257,34 +259,73 @@ public class LocalRecorder implements Recorder {
             }
             LOG.debug(getName() + " terminated");
         }
-
     }
 
-    private void generatePlaylist(File recDir) {
+    private void finishRecording(File directory) {
         Thread t = new Thread() {
             @Override
             public void run() {
-                PlaylistGenerator playlistGenerator = new PlaylistGenerator();
+                boolean local = Config.getInstance().getSettings().localRecording;
-                playlistGenerators.put(recDir, playlistGenerator);
+                boolean automerge = Config.getInstance().getSettings().automerge;
-                try {
+                generatePlaylist(directory);
-                    playlistGenerator.generate(recDir);
+                if(local && automerge) {
-                    playlistGenerator.validate(recDir);
+                    File mergedFile = merge(directory);
-                } catch (IOException | ParseException | PlaylistException e) {
+                    if(mergedFile != null && mergedFile.exists() && mergedFile.length() > 0) {
-                    LOG.error("Couldn't generate playlist file", e);
+                        LOG.debug("Merged file {}", mergedFile.getAbsolutePath());
-                } catch (InvalidPlaylistException e) {
+                        if (!Config.getInstance().getSettings().automergeKeepSegments) {
-                    LOG.error("Playlist is invalid", e);
+                            try {
-                    File playlist = new File(recDir, "playlist.m3u8");
+                                LOG.debug("Deleting directory {}", directory);
-                    playlist.delete();
+                                delete(directory, mergedFile);
-                } finally {
+                            } catch (IOException e) {
-                    playlistGenerators.remove(recDir);
+                                LOG.error("Couldn't delete directory {}", directory, e);
+                            }
+                        }
+                    } else {
+                        LOG.error("Merged file not found {}", mergedFile);
+                    }
                 }
             }
         };
         t.setDaemon(true);
-        t.setName("Playlist Generator " + recDir.toString());
+        t.setName("Postprocessing" + directory.toString());
         t.start();
     }
 
+
+    private File merge(File recDir) {
+        SegmentMerger segmentMerger = new SegmentMerger();
+        segmentMergers.put(recDir, segmentMerger);
+        try {
+            File mergedFile = Recording.mergedFileFromDirectory(recDir);
+            segmentMerger.merge(recDir, mergedFile);
+            return mergedFile;
+        } catch (IOException e) {
+            LOG.error("Couldn't generate playlist file", e);
+        } catch (ParseException | PlaylistException | InvalidPlaylistException e) {
+            LOG.error("Playlist is invalid", e);
+        } finally {
+            segmentMergers.remove(recDir);
+        }
+        return null;
+    }
+
+    private void generatePlaylist(File recDir) {
+        PlaylistGenerator playlistGenerator = new PlaylistGenerator();
+        playlistGenerators.put(recDir, playlistGenerator);
+        try {
+            playlistGenerator.generate(recDir);
+            playlistGenerator.validate(recDir);
+        } catch (IOException | ParseException | PlaylistException e) {
+            LOG.error("Couldn't generate playlist file", e);
+        } catch (InvalidPlaylistException e) {
+            LOG.error("Playlist is invalid", e);
+            File playlist = new File(recDir, "playlist.m3u8");
+            playlist.delete();
+        } finally {
+            playlistGenerators.remove(recDir);
+        }
+    }
+
     private class OnlineMonitor extends Thread {
         private volatile boolean running = false;
 
@@ -354,8 +395,11 @@ public class LocalRecorder implements Recorder {
                                 }
                             }
                             if(!recordingProcessFound) {
-                                // finished recording without playlist -> generate it
+                                if(deleteInProgress.contains(recDir)) {
-                                generatePlaylist(recDir);
+                                    LOG.debug("{} is being deleted. Not going to generate a playlist", recDir);
+                                } else {
+                                    finishRecording(recDir);
+                                }
                             }
                         }
                     }
@@ -406,12 +450,23 @@ public class LocalRecorder implements Recorder {
                         if(recording.hasPlaylist()) {
                             recording.setStatus(FINISHED);
                         } else {
-                            PlaylistGenerator playlistGenerator = playlistGenerators.get(rec);
+                            // this might be a merged recording
-                            if(playlistGenerator != null) {
+                            if(Recording.isMergedRecording(rec)) {
-                                recording.setStatus(GENERATING_PLAYLIST);
+                                recording.setStatus(FINISHED);
-                                recording.setProgress(playlistGenerator.getProgress());
                             } else {
-                                recording.setStatus(RECORDING);
+                                PlaylistGenerator playlistGenerator = playlistGenerators.get(rec);
+                                if(playlistGenerator != null) {
+                                    recording.setStatus(GENERATING_PLAYLIST);
+                                    recording.setProgress(playlistGenerator.getProgress());
+                                } else {
+                                    SegmentMerger merger = segmentMergers.get(rec);
+                                    if(merger != null) {
+                                        recording.setStatus(STATUS.MERGING);
+                                        recording.setProgress(merger.getProgress());
+                                    } else {
+                                        recording.setStatus(RECORDING);
+                                    }
+                                }
                             }
                         }
                         recordings.add(recording);
@@ -437,31 +492,72 @@ public class LocalRecorder implements Recorder {
     public void delete(Recording recording) throws IOException {
         File recordingsDir = new File(config.getSettings().recordingsDir);
         File directory = new File(recordingsDir, recording.getPath());
+        delete(directory, new File[] {});
+    }
+
+    private void delete(File directory, File...excludes) throws IOException {
         if(!directory.exists()) {
             throw new IOException("Recording does not exist");
         }
-        File[] files = directory.listFiles();
-        boolean deletedAllFiles = true;
-        for (File file : files) {
-            try {
-                Files.delete(file.toPath());
-            } catch (Exception e) {
-                deletedAllFiles = false;
-                LOG.debug("Couldn't delete {}", file, e);
-            }
-        }
 
-        if(deletedAllFiles) {
+        try {
-            boolean deleted = directory.delete();
+            deleteInProgress.add(directory);
-            if(deleted) {
+            File[] files = directory.listFiles();
-                if(directory.getParentFile().list().length == 0) {
+            boolean deletedAllFiles = true;
-                    directory.getParentFile().delete();
+            for (File file : files) {
+                boolean skip = false;
+                for (File exclude : excludes) {
+                    if(file.equals(exclude)) {
+                        skip = true;
+                        break;
+                    }
+                }
+                if(skip) {
+                    continue;
+                }
+
+                try {
+                    Files.delete(file.toPath());
+                } catch (Exception e) {
+                    deletedAllFiles = false;
+                    LOG.debug("Couldn't delete {}", file, e);
+                }
+
+                try {
+                    Thread.sleep(50);
+                } catch (InterruptedException e) {
+                    // TODO Auto-generated catch block
+                    e.printStackTrace();
+                }
+            }
+
+            if(deletedAllFiles) {
+                if(directory.list().length == 0) {
+                    boolean deleted = directory.delete();
+                    if(deleted) {
+                        if(directory.getParentFile().list().length == 0) {
+                            directory.getParentFile().delete();
+                        }
+                    } else {
+                        throw new IOException("Couldn't delete " + directory);
+                    }
                 }
             } else {
-                throw new IOException("Couldn't delete " + directory);
+                throw new IOException("Couldn't delete all files in " + directory);
             }
-        } else {
+        } finally {
-            throw new IOException("Couldn't delete all files in " + directory);
+            deleteInProgress.remove(directory);
+        }
+    }
+
+    @Override
+    public void merge(Recording rec, boolean keepSegments) throws IOException {
+        File recordingsDir = new File(config.getSettings().recordingsDir);
+        File directory = new File(recordingsDir, rec.getPath());
+        merge(directory);
+        if(!keepSegments) {
+            File mergedFile = Recording.mergedFileFromDirectory(directory);
+            delete(directory, mergedFile);
         }
     }
 }

src/main/java/ctbrec/recorder/PlaylistGenerator.java

@@ -1,4 +1,4 @@
-package ctbrec.recorder.server;
+package ctbrec.recorder;
 
 import java.io.File;
 import java.io.FileInputStream;

src/main/java/ctbrec/recorder/ProgressListener.java

@@ -1,4 +1,4 @@
-package ctbrec.recorder.server;
+package ctbrec.recorder;
 
 @FunctionalInterface
 public interface ProgressListener {

src/main/java/ctbrec/recorder/Recorder.java

@@ -23,6 +23,8 @@ public interface Recorder {
 
     public List<Recording> getRecordings() throws IOException, InvalidKeyException, NoSuchAlgorithmException, IllegalStateException;
 
+    public void merge(Recording recording, boolean keepSegments) throws IOException;
+
     public void delete(Recording recording) throws IOException, InvalidKeyException, NoSuchAlgorithmException, IllegalStateException;
 
     public void shutdown();

src/main/java/ctbrec/recorder/RemoteRecorder.java

@@ -233,4 +233,9 @@ public class RemoteRecorder implements Recorder {
             //throw new IOException("Couldn't delete recording: " + response.code() + " " + json);
         }
     }
+
+    @Override
+    public void merge(Recording recording, boolean keepSegments) throws IOException {
+        throw new RuntimeException("Merging not available for remote recorder");
+    }
 }

src/main/java/ctbrec/recorder/SegmentMerger.java

@@ -0,0 +1,51 @@
+package ctbrec.recorder;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.List;
+
+import com.iheartradio.m3u8.Encoding;
+import com.iheartradio.m3u8.Format;
+import com.iheartradio.m3u8.ParseException;
+import com.iheartradio.m3u8.PlaylistException;
+import com.iheartradio.m3u8.PlaylistParser;
+import com.iheartradio.m3u8.data.MediaPlaylist;
+import com.iheartradio.m3u8.data.Playlist;
+import com.iheartradio.m3u8.data.TrackData;
+
+
+public class SegmentMerger {
+    private int lastPercentage;
+
+    public void merge(File recDir, File targetFile) throws IOException, ParseException, PlaylistException {
+        if (targetFile.exists()) {
+            return;
+        }
+
+        File playlistFile = new File(recDir, "playlist.m3u8");
+        try (FileInputStream fin = new FileInputStream(playlistFile); FileOutputStream fos = new FileOutputStream(targetFile)) {
+            PlaylistParser parser = new PlaylistParser(fin, Format.EXT_M3U, Encoding.UTF_8);
+            Playlist playlist = parser.parse();
+            MediaPlaylist mediaPlaylist = playlist.getMediaPlaylist();
+            List<TrackData> tracks = mediaPlaylist.getTracks();
+            for (int i = 0; i < tracks.size(); i++) {
+                TrackData trackData = tracks.get(i);
+                File segment = new File(recDir, trackData.getUri());
+                try (FileInputStream segmentStream = new FileInputStream(segment)) {
+                    int length = -1;
+                    byte[] b = new byte[1024 * 1024];
+                    while ((length = segmentStream.read(b)) >= 0) {
+                        fos.write(b, 0, length);
+                    }
+                    lastPercentage = (int) (i * 100.0 / tracks.size());
+                }
+            }
+        }
+    }
+
+    public int getProgress() {
+        return lastPercentage;
+    }
+}

src/main/java/ctbrec/ui/Launcher.java

@@ -56,11 +56,12 @@ public class Launcher extends Application {
             new Thread() {
                 @Override
                 public void run() {
-                    try {
+                    // TODO enable this again
-                        client.login();
+                    //                    try {
-                    } catch (IOException e1) {
+                    //                        client.login();
-                        LOG.warn("Initial login failed" , e1);
+                    //                    } catch (IOException e1) {
-                    }
+                    //                        LOG.warn("Initial login failed" , e1);
+                    //                    }
                 };
             }.start();
         }

src/main/java/ctbrec/ui/Player.java

@@ -71,7 +71,12 @@ public class Player {
             try {
                 if (Config.getInstance().getSettings().localRecording && rec != null) {
                     File dir = new File(Config.getInstance().getSettings().recordingsDir, rec.getPath());
-                    File file = new File(dir, "playlist.m3u8");
+                    File file = null;
+                    if(Recording.isMergedRecording(rec)) {
+                        file = Recording.mergedFileFromDirectory(dir);
+                    } else {
+                        file = new File(dir, "playlist.m3u8");
+                    }
                     playerProcess = rt.exec(Config.getInstance().getSettings().mediaPlayer + " " + file, OS.getEnvironment(), dir);
                 } else {
                     playerProcess = rt.exec(Config.getInstance().getSettings().mediaPlayer + " " + url);

src/main/java/ctbrec/ui/RecordingsTab.java

@@ -4,7 +4,6 @@ import static javafx.scene.control.ButtonType.NO;
 import static javafx.scene.control.ButtonType.YES;
 
 import java.io.File;
-import java.io.FileInputStream;
 import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
 import java.io.IOException;
@@ -46,6 +45,7 @@ import javafx.scene.Cursor;
 import javafx.scene.control.Alert.AlertType;
 import javafx.scene.control.ButtonType;
 import javafx.scene.control.ContextMenu;
+import javafx.scene.control.Menu;
 import javafx.scene.control.MenuItem;
 import javafx.scene.control.ScrollPane;
 import javafx.scene.control.Tab;
@@ -250,23 +250,36 @@ public class RecordingsTab extends Tab implements TabSelectionListener {
             contextMenu.getItems().add(downloadRecording);
         }
 
-        MenuItem mergeRecording = new MenuItem("Merge segments");
+        Menu mergeRecording = new Menu("Merge segments");
-        mergeRecording.setOnAction((e) -> {
+        MenuItem mergeKeep = new MenuItem("… and keep segments");
+        mergeKeep.setOnAction((e) -> {
             try {
-                merge(recording);
+                merge(recording, true);
             } catch (IOException e1) {
                 showErrorDialog("Error while merging recording", "The playlist does not exist", e1);
                 LOG.error("Error while merging recording", e);
             }
         });
+        MenuItem mergeDelete = new MenuItem("… and delete segments");
+        mergeDelete.setOnAction((e) -> {
+            try {
+                merge(recording, false);
+            } catch (IOException e1) {
+                showErrorDialog("Error while merging recording", "The playlist does not exist", e1);
+                LOG.error("Error while merging recording", e);
+            }
+        });
+        mergeRecording.getItems().addAll(mergeKeep, mergeDelete);
         if (Config.getInstance().getSettings().localRecording && recording.getStatus() == STATUS.FINISHED) {
-            contextMenu.getItems().add(mergeRecording);
+            if(!Recording.isMergedRecording(recording)) {
+                contextMenu.getItems().add(mergeRecording);
+            }
         }
 
         return contextMenu;
     }
 
-    private void merge(Recording recording) throws IOException {
+    private void merge(Recording recording, boolean keepSegments) throws IOException {
         File recDir = new File (Config.getInstance().getSettings().recordingsDir, recording.getPath());
         File playlistFile = new File(recDir, "playlist.m3u8");
         if(!playlistFile.exists()) {
@@ -282,36 +295,11 @@ public class RecordingsTab extends Tab implements TabSelectionListener {
         Thread t = new Thread() {
             @Override
             public void run() {
-                try(
+                try {
-                        FileInputStream fin = new FileInputStream(playlistFile);
+                    recorder.merge(recording, keepSegments);
-                        FileOutputStream fos = new FileOutputStream(targetFile))
-                {
-                    PlaylistParser parser = new PlaylistParser(fin, Format.EXT_M3U, Encoding.UTF_8);
-                    Playlist playlist = parser.parse();
-                    MediaPlaylist mediaPlaylist = playlist.getMediaPlaylist();
-                    List<TrackData> tracks = mediaPlaylist.getTracks();
-                    for (int i = 0; i < tracks.size(); i++) {
-                        TrackData trackData = tracks.get(i);
-                        File segment = new File(recDir, trackData.getUri());
-                        try(FileInputStream segmentStream = new FileInputStream(segment)) {
-                            int length = -1;
-                            byte[] b = new byte[1024 * 1024];
-                            while( (length = segmentStream.read(b)) >= 0 ) {
-                                fos.write(b, 0, length);
-                            }
-                            int progress = (int)(i * 100.0 / tracks.size());
-                            Platform.runLater(() -> {
-                                recording.setStatus(STATUS.MERGING);
-                                recording.setProgress(progress);
-                            });
-                        }
-                    }
                 } catch (IOException e) {
                     showErrorDialog("Error while merging segments", "The merged file could not be created", e);
                     LOG.error("Error while merging segments", e);
-                } catch (ParseException | PlaylistException e) {
-                    showErrorDialog("Error while merging recording", "Couldn't read playlist", e);
-                    LOG.error("Error while merging recording", e);
                 } finally {
                     Platform.runLater(() -> {
                         recording.setStatus(STATUS.FINISHED);
@@ -323,7 +311,6 @@ public class RecordingsTab extends Tab implements TabSelectionListener {
         t.setDaemon(true);
         t.setName("Segment Merger " + recording.getPath());
         t.start();
-
     }
 
     private void download(Recording recording) throws IOException, ParseException, PlaylistException {

src/main/java/ctbrec/ui/SettingsTab.java

@@ -45,8 +45,11 @@ public class SettingsTab extends Tab {
     private TextField username;
     private TextField server;
     private TextField port;
+    private static final int CHECKBOX_MARGIN = 6;
     private CheckBox loadResolution;
-    private CheckBox secureCommunication;
+    private CheckBox secureCommunication = new CheckBox();
+    private CheckBox automerge = new CheckBox();
+    private CheckBox automergeKeepSegments = new CheckBox();
     private PasswordField password;
     private RadioButton recordLocal;
     private RadioButton recordRemote;
@@ -107,7 +110,8 @@ public class SettingsTab extends Tab {
         GridPane.setColumnSpan(password, 2);
         layout.add(password, 1, row);
 
-        layout.add(new Label("Display stream resolution in overview"), 0, ++row);
+        Label l = new Label("Display stream resolution in overview");
+        layout.add(l, 0, ++row);
         loadResolution = new CheckBox();
         loadResolution.setSelected(Config.getInstance().getSettings().determineResolution);
         loadResolution.setOnAction((e) -> {
@@ -116,9 +120,22 @@ public class SettingsTab extends Tab {
                 ThumbOverviewTab.queue.clear();
             }
         });
+        GridPane.setMargin(l, new Insets(CHECKBOX_MARGIN, 0, 0, 0));
+        GridPane.setMargin(loadResolution, new Insets(CHECKBOX_MARGIN, 0, 0, 0));
         layout.add(loadResolution, 1, row);
 
-        layout.add(new Label(), 0, ++row);
+        l = new Label("Auto-merge recordings");
+        layout.add(l, 0, ++row);
+        automerge.setSelected(Config.getInstance().getSettings().automerge);
+        automerge.setOnAction((e) -> Config.getInstance().getSettings().automerge = automerge.isSelected());
+        GridPane.setMargin(l, new Insets(CHECKBOX_MARGIN, 0, 0, 0));
+        GridPane.setMargin(automerge, new Insets(CHECKBOX_MARGIN, 0, 0, 0));
+        layout.add(automerge, 1, row);
+
+        automergeKeepSegments.setText("Keep segments");
+        automergeKeepSegments.setOnAction((e) -> Config.getInstance().getSettings().automergeKeepSegments = automergeKeepSegments.isSelected());
+        GridPane.setMargin(automergeKeepSegments, new Insets(CHECKBOX_MARGIN, 0, 30, 0));
+        layout.add(automergeKeepSegments, 1, ++row);
 
         layout.add(new Label("Record Location"), 0, ++row);
         recordLocation = new ToggleGroup();
@@ -132,9 +149,7 @@ public class SettingsTab extends Tab {
         layout.add(recordRemote, 2, row);
         recordLocation.selectedToggleProperty().addListener((e) -> {
             Config.getInstance().getSettings().localRecording = recordLocal.isSelected();
-            server.setDisable(recordLocal.isSelected());
+            setRecordingMode(recordLocal.isSelected());
-            port.setDisable(recordLocal.isSelected());
-
             Alert restart = new AutosizeAlert(AlertType.INFORMATION);
             restart.setTitle("Restart required");
             restart.setHeaderText("Restart required");
@@ -173,8 +188,8 @@ public class SettingsTab extends Tab {
         GridPane.setColumnSpan(port, 2);
         layout.add(port, 1, row);
 
-        layout.add(new Label("Require authentication"), 0, ++row);
+        l = new Label("Require authentication");
-        secureCommunication = new CheckBox();
+        layout.add(l, 0, ++row);
         secureCommunication.setSelected(Config.getInstance().getSettings().requireAuthentication);
         secureCommunication.setOnAction((e) -> {
             Config.getInstance().getSettings().requireAuthentication = secureCommunication.isSelected();
@@ -196,11 +211,19 @@ public class SettingsTab extends Tab {
                 keyDialog.show();
             }
         });
+        GridPane.setMargin(l, new Insets(CHECKBOX_MARGIN, 0, 0, 0));
+        GridPane.setMargin(secureCommunication, new Insets(CHECKBOX_MARGIN, 0, 0, 0));
         layout.add(secureCommunication, 1, row);
 
+        setRecordingMode(recordLocal.isSelected());
+    }
 
-        server.setDisable(recordLocal.isSelected());
+    private void setRecordingMode(boolean local) {
-        port.setDisable(recordLocal.isSelected());
+        server.setDisable(local);
+        port.setDisable(local);
+        secureCommunication.setDisable(local);
+        automerge.setDisable(!local);
+        automergeKeepSegments.setDisable(!local);
     }
 
     private ChangeListener<? super Boolean> createRecordingsDirectoryFocusListener() {