Fix recording downloads from server to client

client/src/main/java/ctbrec/ui/FileDownload.java

@@ -0,0 +1,54 @@
+package ctbrec.ui;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URL;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import ctbrec.io.HttpClient;
+import ctbrec.recorder.ProgressListener;
+import okhttp3.Request;
+import okhttp3.Response;
+
+public class FileDownload {
+
+    private static final Logger LOG = LoggerFactory.getLogger(FileDownload.class);
+
+    private HttpClient httpClient;
+    private ProgressListener downloadListener;
+
+    public FileDownload(HttpClient httpClient, ProgressListener downloadListener) {
+        this.httpClient = httpClient;
+        this.downloadListener = downloadListener;
+    }
+
+    public void start(URL url, File target) throws IOException {
+        LOG.trace("Downloading file {} to {}", url, target);
+        Request request = new Request.Builder().url(url).addHeader("connection", "keep-alive").build();
+        Response response = httpClient.execute(request);
+        long fileSize = Long.parseLong(response.header("Content-Length", String.valueOf(Long.MAX_VALUE)));
+        InputStream in = null;
+        try (FileOutputStream fos = new FileOutputStream(target)) {
+            in = response.body().byteStream();
+            byte[] b = new byte[1024 * 100];
+            long totalBytesRead = 0;
+            int length = -1;
+            while ((length = in.read(b)) >= 0) {
+                fos.write(b, 0, length);
+                totalBytesRead += length;
+                int progress = (int)(totalBytesRead * 100d / fileSize);
+                downloadListener.update(progress);
+            }
+        } finally {
+            if (in != null) {
+                in.close();
+            }
+            response.close();
+        }
+    }
+
+}

client/src/main/java/ctbrec/ui/RecordingsTab.java

@@ -4,6 +4,7 @@ import static ctbrec.Recording.State.*;
 import static javafx.scene.control.ButtonType.*;
 
 import java.io.File;
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.net.URL;
 import java.nio.file.NoSuchFileException;
@@ -32,6 +33,7 @@ import ctbrec.Config;
 import ctbrec.Recording;
 import ctbrec.Recording.State;
 import ctbrec.StringUtil;
+import ctbrec.recorder.ProgressListener;
 import ctbrec.recorder.Recorder;
 import ctbrec.recorder.download.hls.MergedHlsDownload;
 import ctbrec.sites.Site;
@@ -406,21 +408,14 @@ public class RecordingsTab extends Tab implements TabSelectionListener {
         }
 
         MenuItem downloadRecording = new MenuItem("Download");
-        downloadRecording.setOnAction(e -> {
+        downloadRecording.setOnAction(e -> download(first));
-            try {
-                download(first);
-            } catch (IOException e1) {
-                showErrorDialog(ERROR_WHILE_DOWNLOADING_RECORDING, "The recording could not be downloaded", e1);
-                LOG.error(ERROR_WHILE_DOWNLOADING_RECORDING, e1);
-            }
-        });
         if (!Config.getInstance().getSettings().localRecording && first.getStatus() == State.FINISHED) {
             contextMenu.getItems().add(downloadRecording);
         }
 
         MenuItem rerunPostProcessing = new MenuItem("Rerun Post-Processing");
         rerunPostProcessing.setOnAction(e -> triggerPostProcessing(first));
-        if (first.getStatus() == State.FINISHED || first.getStatus() == State.WAITING) {
+        if ((first.getStatus() == State.FINISHED || first.getStatus() == State.WAITING) && !first.isSegmented()) {
             contextMenu.getItems().add(rerunPostProcessing);
         }
 
@@ -451,8 +446,9 @@ public class RecordingsTab extends Tab implements TabSelectionListener {
         }).start();
     }
 
-    private void download(Recording recording) throws IOException {
+    private void download(Recording recording) {
-        String filename = recording.getPath().substring(1).replace("/", "-") + ".ts";
+        LOG.debug("Path {}", recording.getPath());
+        String filename = proposeTargetFilename(recording);
         FileChooser chooser = new FileChooser();
         chooser.setInitialFileName(filename);
         if(config.getSettings().lastDownloadDir != null && !config.getSettings().lastDownloadDir.equals("")) {
@@ -465,36 +461,45 @@ public class RecordingsTab extends Tab implements TabSelectionListener {
         File target = chooser.showSaveDialog(null);
         if(target != null) {
             config.getSettings().lastDownloadDir = target.getParent();
-            String hlsBase = config.getServerUrl() + "/hls";
+            startDownloadThread(target, recording);
-            URL url = new URL(hlsBase + recording.getPath() + "/playlist.m3u8");
-            LOG.info("Downloading {}", recording.getPath());
-            startDownloadThread(url, target, recording);
             recording.setStatus(State.DOWNLOADING);
             recording.setProgress(0);
         }
     }
 
-    private void startDownloadThread(URL url, File target, Recording recording) {
+    private String proposeTargetFilename(Recording recording) {
+        String path = recording.getPath().substring(1);
+        if(recording.isSegmented()) {
+            String filename = path.replace("/", "-");
+            if(!filename.endsWith(".mp4")) {
+                filename += ".mp4";
+            }
+            return filename;
+        } else {
+            return new File(path).getName();
+        }
+    }
+
+    private void startDownloadThread(File target, Recording recording) {
         Thread t = new Thread(() -> {
             try {
-                MergedHlsDownload download = new MergedHlsDownload(CamrecApplication.httpClient);
+                String hlsBase = config.getServerUrl() + "/hls";
-                LOG.info("Downloading {}", url);
+                if (recording.isSegmented()) {
-                //                download.start(url.toString(), target, progress -> Platform.runLater(() -> {
+                    URL url = new URL(hlsBase + recording.getPath() + "/playlist.m3u8");
-                //                    if (progress == 100) {
+                    MergedHlsDownload download = new MergedHlsDownload(CamrecApplication.httpClient);
-                //                        recording.setStatus(FINISHED);
+                    LOG.info("Downloading {}", url);
-                //                        recording.setProgress(-1);
+                    download.downloadFinishedRecording(url.toString(), target, createDownloadListener(recording));
-                //                        LOG.debug("Download finished for recording {}", recording.getPath());
+                } else {
-                //                    } else {
+                    URL url = new URL(hlsBase + recording.getPath());
-                //                        recording.setStatus(DOWNLOADING);
+                    FileDownload download = new FileDownload(CamrecApplication.httpClient, createDownloadListener(recording));
-                //                        recording.setProgress(progress);
+                    download.start(url, target);
-                //                    }
+                }
-                //                }));
+            } catch (FileNotFoundException e) {
-                //            } catch (FileNotFoundException e) {
+                showErrorDialog(ERROR_WHILE_DOWNLOADING_RECORDING, "The target file couldn't be created", e);
-                //                showErrorDialog(ERROR_WHILE_DOWNLOADING_RECORDING, "The target file couldn't be created", e);
+                LOG.error(ERROR_WHILE_DOWNLOADING_RECORDING, e);
-                //                LOG.error(ERROR_WHILE_DOWNLOADING_RECORDING, e);
+            } catch (Exception e) {
-                //            } catch (IOException e) {
+                showErrorDialog(ERROR_WHILE_DOWNLOADING_RECORDING, "The recording could not be downloaded", e);
-                //                showErrorDialog(ERROR_WHILE_DOWNLOADING_RECORDING, "The recording could not be downloaded", e);
+                LOG.error(ERROR_WHILE_DOWNLOADING_RECORDING, e);
-                //                LOG.error(ERROR_WHILE_DOWNLOADING_RECORDING, e);
             } finally {
                 Platform.runLater(() -> {
                     recording.setStatus(FINISHED);
@@ -507,6 +512,19 @@ public class RecordingsTab extends Tab implements TabSelectionListener {
         t.start();
     }
 
+    private ProgressListener createDownloadListener(Recording recording) {
+        return progress -> Platform.runLater(() -> {
+            if (progress == 100) {
+                recording.setStatus(FINISHED);
+                recording.setProgress(-1);
+                LOG.debug("Download finished for recording {}", recording.getPath());
+            } else {
+                recording.setStatus(DOWNLOADING);
+                recording.setProgress(progress);
+            }
+        });
+    }
+
     private void showErrorDialog(final String title, final String msg, final Exception e) {
         Platform.runLater(() -> {
             AutosizeAlert autosizeAlert = new AutosizeAlert(AlertType.ERROR, getTabPane().getScene());
@@ -531,7 +549,7 @@ public class RecordingsTab extends Tab implements TabSelectionListener {
             }.start();
         } else {
             String hlsBase = Config.getInstance().getServerUrl() + "/hls";
-            url = hlsBase + recording.getPath() + (recording.getPath().endsWith(".mp4") ? "" : "/playlist.m3u8");
+            url = hlsBase + recording.getPath() + (recording.isSegmented() ? "/playlist.m3u8" : "");
             new Thread() {
                 @Override
                 public void run() {

common/src/main/java/ctbrec/Recording.java

@@ -6,6 +6,7 @@ import java.time.Instant;
 import java.time.LocalDateTime;
 import java.time.ZoneId;
 import java.time.format.DateTimeFormatter;
+import java.util.Optional;
 
 import ctbrec.event.EventBusHolder;
 import ctbrec.event.RecordingStateChangedEvent;
@@ -216,4 +217,8 @@ public class Recording {
     public void refresh() {
         sizeInByte = getSize();
     }
+
+    public boolean isSegmented() {
+        return !Optional.ofNullable(getPath()).orElse("").endsWith(".mp4");
+    }
 }

common/src/main/java/ctbrec/recorder/download/hls/MergedHlsDownload.java

@@ -1,17 +1,29 @@
 package ctbrec.recorder.download.hls;
 
 import java.io.File;
+import java.io.FileOutputStream;
 import java.io.IOException;
+import java.io.InputStream;
+import java.net.URL;
 import java.nio.file.Files;
+import java.security.InvalidKeyException;
+import java.security.NoSuchAlgorithmException;
 import java.util.Arrays;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.iheartradio.m3u8.ParseException;
+import com.iheartradio.m3u8.PlaylistException;
+
 import ctbrec.Config;
+import ctbrec.Hmac;
 import ctbrec.OS;
 import ctbrec.io.HttpClient;
 import ctbrec.io.StreamRedirectThread;
+import ctbrec.recorder.ProgressListener;
+import okhttp3.Request;
+import okhttp3.Response;
 
 public class MergedHlsDownload extends HlsDownload {
     private static final Logger LOG = LoggerFactory.getLogger(MergedHlsDownload.class);
@@ -28,8 +40,18 @@ public class MergedHlsDownload extends HlsDownload {
         if (!playlist.exists()) {
             super.generatePlaylist(recording);
         }
-
+        File targetFile = new File(dir, "0merged.mp4");
         try {
+            postprocess(playlist, targetFile);
+            recording.setPath(recording.getPath() + '/' + "0merged.mp4"); // TODO set the actual name
+        } catch (PostProcessingException e) {
+            LOG.error("An error occurred during post-processing", e);
+        }
+    }
+
+    private void postprocess(File playlist, File target) throws PostProcessingException {
+        try {
+            File dir = playlist.getParentFile();
             // @formatter:off
             String[] cmdline = OS.getFFmpegCommand(
                     "-i", playlist.getAbsolutePath(),
@@ -37,7 +59,7 @@ public class MergedHlsDownload extends HlsDownload {
                     "-c:a", "copy",
                     "-movflags", "faststart",
                     "-f", "mp4",
-                    new File(dir, "0merged.mp4").getAbsolutePath()
+                    target.getAbsolutePath()
                     );
             // @formatter:on
             LOG.debug("Command line: {}", Arrays.toString(cmdline));
@@ -46,12 +68,13 @@ public class MergedHlsDownload extends HlsDownload {
             new Thread(new StreamRedirectThread(ffmpeg.getErrorStream(), System.err)).start(); // NOSONAR
             int exitCode = ffmpeg.waitFor();
             if (exitCode == 0) {
-                recording.setPath(recording.getPath() + '/' + "0merged.mp4");
                 Files.delete(playlist.toPath());
                 File[] segments = dir.listFiles((directory, filename) -> filename.endsWith(".ts"));
                 for (File segment : segments) {
                     Files.delete(segment.toPath());
                 }
+            } else {
+                throw new PostProcessingException("FFmpeg exit code was " + exitCode);
             }
         } catch (InterruptedException e) {
             Thread.currentThread().interrupt();
@@ -60,4 +83,57 @@ public class MergedHlsDownload extends HlsDownload {
             LOG.error("Couldn't execute FFMPEG", e);
         }
     }
+
+    public void downloadFinishedRecording(String segmentPlaylistUri, File target, ProgressListener progressListener)
+            throws IOException, ParseException, PlaylistException, InvalidKeyException, NoSuchAlgorithmException, PostProcessingException {
+        if (Config.getInstance().getSettings().requireAuthentication) {
+            URL u = new URL(segmentPlaylistUri);
+            String path = u.getPath();
+            byte[] key = Config.getInstance().getSettings().key;
+            if (!Config.getInstance().getContextPath().isEmpty()) {
+                path = path.substring(Config.getInstance().getContextPath().length());
+            }
+            String hmac = Hmac.calculate(path, key);
+            segmentPlaylistUri = segmentPlaylistUri + "?hmac=" + hmac;
+        }
+
+        File tempDir = new File(target.getParentFile(), "ctbrec-download-tmp-" + target.getName());
+        Files.createDirectories(tempDir.toPath());
+
+        downloadFile(segmentPlaylistUri, tempDir);
+        SegmentPlaylist segmentPlaylist = getNextSegments(segmentPlaylistUri);
+        int fileCounter = 0;
+        for (String segmentUrl : segmentPlaylist.segments) {
+            downloadFile(segmentUrl, tempDir);
+            fileCounter++;
+            int total = segmentPlaylist.segments.size();
+            int progress = (int) (fileCounter / (double) total * 100);
+            progressListener.update(progress);
+        }
+
+        File downloadedPlaylist = new File(tempDir, "playlist.m3u8");
+        postprocess(downloadedPlaylist, target);
+        Files.delete(tempDir.toPath());
+    }
+
+    private void downloadFile(String fileUri, File tempDir) throws IOException {
+        LOG.trace("Downloading file {} to {}", fileUri, tempDir);
+        Request request = new Request.Builder().url(fileUri).addHeader("connection", "keep-alive").build();
+        Response response = client.execute(request);
+        InputStream in = null;
+        File file = new File(request.url().encodedPath());
+        try (FileOutputStream fos = new FileOutputStream(new File(tempDir, file.getName()))) {
+            in = response.body().byteStream();
+            byte[] b = new byte[1024 * 100];
+            int length = -1;
+            while ((length = in.read(b)) >= 0) {
+                fos.write(b, 0, length);
+            }
+        } finally {
+            if (in != null) {
+                in.close();
+            }
+            response.close();
+        }
+    }
 }

common/src/main/java/ctbrec/recorder/download/hls/PostProcessingException.java

@@ -0,0 +1,9 @@
+package ctbrec.recorder.download.hls;
+
+public class PostProcessingException extends Exception {
+
+    public PostProcessingException(String msg) {
+        super(msg);
+    }
+
+}