Improve error handling in downloads

common/src/main/java/ctbrec/recorder/download/dash/DashDownload.java

@@ -81,6 +81,7 @@ public class DashDownload extends AbstractDownload {
                 .header("Connection", "keep-alive")
                 .build(); // @formatter:on
         LOG.trace("Loading manifest {}", url);
+        // TODO try 10 times
         try (Response response = httpClient.execute(request)) {
             if (response.isSuccessful()) {
                 return response.body().string();
@@ -224,6 +225,7 @@ public class DashDownload extends AbstractDownload {
     @Override
     public void start() throws IOException {
         try {
+            Thread.currentThread().setName("Download " + model.getName());
             running = true;
             splitRecStartTime = ZonedDateTime.now();
             JAXBContext jc = JAXBContext.newInstance(MPDtype.class.getPackage().getName());
@@ -237,6 +239,16 @@ public class DashDownload extends AbstractDownload {
                     break;
                 }
             }
+        } catch(HttpException e) {
+            if(e.getResponseCode() == 404) {
+                LOG.debug("Manifest not found (404). Model {} probably went offline", model);
+                waitSomeTime(10_000);
+            } else if(e.getResponseCode() == 403) {
+                LOG.debug("Manifest access forbidden (403). Model {} probably went private or offline", model);
+                waitSomeTime(10_000);
+            } else {
+                throw e;
+            }
         } catch (Exception e) {
             LOG.error("Error while downloading dash stream", e);
         } finally {
@@ -349,6 +361,7 @@ public class DashDownload extends AbstractDownload {
     @Override
     public void postprocess(Recording recording) {
         try {
+            Thread.currentThread().setName("PP " + model.getName());
             recording.setStatus(POST_PROCESSING);
             String path = recording.getPath();
             File dir = new File(Config.getInstance().getSettings().recordingsDir, path);

common/src/main/java/ctbrec/recorder/download/hls/AbstractHlsDownload.java

@@ -38,6 +38,7 @@ import ctbrec.Recording.State;
 import ctbrec.UnknownModel;
 import ctbrec.io.HttpClient;
 import ctbrec.io.HttpException;
+import ctbrec.recorder.PlaylistGenerator.InvalidPlaylistException;
 import ctbrec.recorder.download.AbstractDownload;
 import ctbrec.recorder.download.StreamSource;
 import okhttp3.Request;
@@ -69,7 +70,7 @@ public abstract class AbstractHlsDownload extends AbstractDownload {
         };
     }
 
-    protected SegmentPlaylist getNextSegments(String segmentsURL) throws IOException, ParseException, PlaylistException {
+    protected SegmentPlaylist getNextSegments(String segmentsURL) throws Exception {
         URL segmentsUrl = new URL(segmentsURL);
         Request request = new Request.Builder()
                 .url(segmentsUrl)
@@ -80,46 +81,59 @@ public abstract class AbstractHlsDownload extends AbstractDownload {
                 .header("Referer", model.getSite().getBaseUrl())
                 .header("Connection", "keep-alive")
                 .build();
-        try(Response response = client.execute(request)) {
+        int tries = 1;
-            if(response.isSuccessful()) {
+        Exception lastException = null;
-                String body = response.body().string();
+        for (int i = 0; i <= 10; i++) {
-                if(!body.contains("#EXTINF")) {
+            try (Response response = client.execute(request)) {
-                    // no segments, empty playlist
+                if (response.isSuccessful()) {
-                    return new SegmentPlaylist(segmentsURL);
+                    String body = response.body().string();
-                }
+                    if (!body.contains("#EXTINF")) {
-
+                        // no segments, empty playlist
-                InputStream inputStream = new ByteArrayInputStream(body.getBytes("utf-8"));
+                        return new SegmentPlaylist(segmentsURL);
-                PlaylistParser parser = new PlaylistParser(inputStream, Format.EXT_M3U, Encoding.UTF_8, ParsingMode.LENIENT);
-                Playlist playlist = parser.parse();
-                if(playlist.hasMediaPlaylist()) {
-                    MediaPlaylist mediaPlaylist = playlist.getMediaPlaylist();
-                    SegmentPlaylist lsp = new SegmentPlaylist(segmentsURL);
-                    lsp.seq = mediaPlaylist.getMediaSequenceNumber();
-                    lsp.targetDuration = mediaPlaylist.getTargetDuration();
-                    List<TrackData> tracks = mediaPlaylist.getTracks();
-                    for (TrackData trackData : tracks) {
-                        String uri = trackData.getUri();
-                        if(!uri.startsWith("http")) {
-                            String tmpurl = segmentsUrl.toString();
-                            tmpurl = tmpurl.substring(0, tmpurl.lastIndexOf('/') + 1);
-                            uri = tmpurl + uri;
-                        }
-                        lsp.totalDuration += trackData.getTrackInfo().duration;
-                        lsp.lastSegDuration = trackData.getTrackInfo().duration;
-                        lsp.segments.add(uri);
-                        if(trackData.hasEncryptionData()) {
-                            lsp.encrypted = true;
-                            EncryptionData data = trackData.getEncryptionData();
-                            lsp.encryptionKeyUrl = data.getUri();
-                            lsp.encryptionMethod = data.getMethod().getValue();
-                        }
                     }
-                    return lsp;
+
+                    InputStream inputStream = new ByteArrayInputStream(body.getBytes("utf-8"));
+                    PlaylistParser parser = new PlaylistParser(inputStream, Format.EXT_M3U, Encoding.UTF_8, ParsingMode.LENIENT);
+                    Playlist playlist = parser.parse();
+                    if (playlist.hasMediaPlaylist()) {
+                        MediaPlaylist mediaPlaylist = playlist.getMediaPlaylist();
+                        SegmentPlaylist lsp = new SegmentPlaylist(segmentsURL);
+                        lsp.seq = mediaPlaylist.getMediaSequenceNumber();
+                        lsp.targetDuration = mediaPlaylist.getTargetDuration();
+                        List<TrackData> tracks = mediaPlaylist.getTracks();
+                        for (TrackData trackData : tracks) {
+                            String uri = trackData.getUri();
+                            if (!uri.startsWith("http")) {
+                                String tmpurl = segmentsUrl.toString();
+                                tmpurl = tmpurl.substring(0, tmpurl.lastIndexOf('/') + 1);
+                                uri = tmpurl + uri;
+                            }
+                            lsp.totalDuration += trackData.getTrackInfo().duration;
+                            lsp.lastSegDuration = trackData.getTrackInfo().duration;
+                            lsp.segments.add(uri);
+                            if (trackData.hasEncryptionData()) {
+                                lsp.encrypted = true;
+                                EncryptionData data = trackData.getEncryptionData();
+                                lsp.encryptionKeyUrl = data.getUri();
+                                lsp.encryptionMethod = data.getMethod().getValue();
+                            }
+                        }
+                        return lsp;
+                    }
+                    throw new InvalidPlaylistException("Playlist has no media playlist");
+                } else {
+                    throw new HttpException(response.code(), response.message());
                 }
-                return null;
+            } catch (Exception e) {
-            } else {
+                LOG.debug("Couldn't download HLS playlist (try {}) {} - {}", tries, segmentsURL, e.getMessage());
-                throw new HttpException(response.code(), response.message());
+                lastException = e;
             }
+            waitSomeTime(100 * tries);
+        }
+        if (lastException != null) {
+            throw lastException;
+        } else {
+            throw new IOException("Couldn't download HLS playlist");
         }
     }
 
@@ -183,7 +197,21 @@ public abstract class AbstractHlsDownload extends AbstractDownload {
         return model;
     }
 
-
+    /**
+     * Causes the current thread to sleep for a short amount of time.
+     * This is used to slow down retries, if something is wrong with the playlist.
+     * E.g. HTTP 403 or 404
+     */
+    protected void waitSomeTime(long waitForMillis) {
+        try {
+            Thread.sleep(waitForMillis);
+        } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
+            if(running) {
+                LOG.error("Couldn't sleep. This might mess up the download!");
+            }
+        }
+    }
 
     public static class SegmentPlaylist {
         public String url;

common/src/main/java/ctbrec/recorder/download/hls/HlsDownload.java

@@ -78,30 +78,32 @@ public class HlsDownload extends AbstractHlsDownload {
     public void start() throws IOException {
         try {
             running = true;
+            Thread.currentThread().setName("Download " + model.getName());
             splitRecStartTime = ZonedDateTime.now();
 
-            if(!model.isOnline()) {
+            if (!model.isOnline()) {
-                throw new IOException(model.getName() +"'s room is not public");
+                throw new IOException(model.getName() + "'s room is not public");
             }
 
             String segments = getSegmentPlaylistUrl(model);
-            if(segments != null) {
+            if (segments != null) {
                 if (!downloadDir.toFile().exists()) {
                     Files.createDirectories(downloadDir);
                 }
                 int lastSegmentNumber = 0;
                 int nextSegmentNumber = 0;
                 int waitFactor = 1;
-                while(running) {
+                while (running) {
                     SegmentPlaylist playlist = getNextSegments(segments);
                     emptyPlaylistCheck(playlist);
-                    if(nextSegmentNumber > 0 && playlist.seq > nextSegmentNumber) {
+                    if (nextSegmentNumber > 0 && playlist.seq > nextSegmentNumber) {
                         waitFactor *= 2;
-                        LOG.warn("Missed segments {} < {} in download for {} - setting wait factor to 1/{}", nextSegmentNumber, playlist.seq, model, waitFactor);
+                        LOG.warn("Missed segments {} < {} in download for {} - setting wait factor to 1/{}", nextSegmentNumber, playlist.seq, model,
+                                waitFactor);
                     }
                     int skip = nextSegmentNumber - playlist.seq;
                     for (String segment : playlist.segments) {
-                        if(skip > 0) {
+                        if (skip > 0) {
                             skip--;
                         } else {
                             URL segmentUrl = new URL(segment);
@@ -118,7 +120,7 @@ public class HlsDownload extends AbstractHlsDownload {
                     }
 
                     long waitForMillis = 0;
-                    if(lastSegmentNumber == playlist.seq) {
+                    if (lastSegmentNumber == playlist.seq) {
                         // playlist didn't change -> wait for at least half the target duration
                         waitForMillis = (long) playlist.targetDuration * 1000 / waitFactor;
                         LOG.trace("Playlist didn't change... waiting for {}ms", waitForMillis);
@@ -133,31 +135,31 @@ public class HlsDownload extends AbstractHlsDownload {
                     // this if check makes sure, that we don't decrease nextSegment. for some reason
                     // streamate playlists sometimes jump back. e.g. max sequence = 79 -> 80 -> 79
                     lastSegmentNumber = playlist.seq;
-                    if(lastSegmentNumber + playlist.segments.size() > nextSegmentNumber) {
+                    if (lastSegmentNumber + playlist.segments.size() > nextSegmentNumber) {
                         nextSegmentNumber = lastSegmentNumber + playlist.segments.size();
                     }
                 }
             } else {
                 throw new IOException("Couldn't determine segments uri");
             }
-        } catch(ParseException e) {
+        } catch (ParseException e) {
             throw new IOException("Couldn't parse HLS playlist:\n" + e.getInput(), e);
-        } catch(PlaylistException e) {
+        } catch (PlaylistException e) {
             throw new IOException("Couldn't parse HLS playlist", e);
-        } catch(EOFException e) {
+        } catch (EOFException e) {
             // end of playlist reached
             LOG.debug("Reached end of playlist for model {}", model);
-        } catch(HttpException e) {
+        } catch (HttpException e) {
-            if(e.getResponseCode() == 404) {
+            if (e.getResponseCode() == 404) {
                 LOG.debug("Playlist not found (404). Model {} probably went offline", model);
                 waitSomeTime(10_000);
-            } else if(e.getResponseCode() == 403) {
+            } else if (e.getResponseCode() == 403) {
                 LOG.debug("Playlist access forbidden (403). Model {} probably went private or offline", model);
                 waitSomeTime(10_000);
             } else {
                 throw e;
             }
-        } catch(Exception e) {
+        } catch (Exception e) {
             throw new IOException("Couldn't download segment", e);
         } finally {
             downloadThreadPool.shutdown();
@@ -176,6 +178,7 @@ public class HlsDownload extends AbstractHlsDownload {
 
     @Override
     public void postprocess(Recording recording) {
+        Thread.currentThread().setName("PP " + model.getName());
         recording.setStatusWithEvent(State.GENERATING_PLAYLIST);
         generatePlaylist(recording);
         recording.setStatusWithEvent(State.POST_PROCESSING);
@@ -188,7 +191,7 @@ public class HlsDownload extends AbstractHlsDownload {
 
     protected File generatePlaylist(Recording recording) {
         File recDir = recording.getAbsoluteFile();
-        if(!config.getSettings().generatePlaylist) {
+        if (!config.getSettings().generatePlaylist) {
             return null;
         }
 
@@ -197,7 +200,7 @@ public class HlsDownload extends AbstractHlsDownload {
 
         try {
             File playlist = playlistGenerator.generate(recDir);
-            if(playlist != null) {
+            if (playlist != null) {
                 playlistGenerator.validate(recDir);
             }
             recording.setProgress(-1);
@@ -205,7 +208,7 @@ public class HlsDownload extends AbstractHlsDownload {
         } catch (IOException | ParseException e) {
             LOG.error("Couldn't generate playlist file", e);
         } catch (PlaylistException e) {
-            if(e.getErrors().isEmpty()) {
+            if (e.getErrors().isEmpty()) {
                 LOG.error("Couldn't generate playlist file", e);
             } else {
                 LOG.error("Playlist contains errors");
@@ -217,7 +220,7 @@ public class HlsDownload extends AbstractHlsDownload {
             LOG.error("Playlist is invalid and will be deleted", e);
             File playlist = new File(recDir, "playlist.m3u8");
             try {
-                Files.delete(playlist.toPath());
+                Files.deleteIfExists(playlist.toPath());
             } catch (IOException e1) {
                 LOG.error("Couldn't delete playlist {}", playlist, e1);
             }
@@ -227,10 +230,10 @@ public class HlsDownload extends AbstractHlsDownload {
     }
 
     private boolean splitRecording() {
-        if(config.getSettings().splitRecordings > 0) {
+        if (config.getSettings().splitRecordings > 0) {
             Duration recordingDuration = Duration.between(splitRecStartTime, ZonedDateTime.now());
             long seconds = recordingDuration.getSeconds();
-            if(seconds >= config.getSettings().splitRecordings) {
+            if (seconds >= config.getSettings().splitRecordings) {
                 internalStop();
                 return true;
             }
@@ -275,36 +278,41 @@ public class HlsDownload extends AbstractHlsDownload {
         @Override
         public Boolean call() throws Exception {
             LOG.trace("Downloading segment to {}", file);
-            int maxTries = 3;
+            int maxTries = 5;
             for (int i = 1; i <= maxTries; i++) {
                 Request request = new Request.Builder().url(url).addHeader("connection", "keep-alive").build();
-                Response response = client.execute(request);
+                try (Response response = client.execute(request)) {
-                InputStream in = null;
+                    if (response.isSuccessful()) {
-                try (FileOutputStream fos = new FileOutputStream(file.toFile())) {
+                        InputStream in = null;
-                    in = response.body().byteStream();
+                        try (FileOutputStream fos = new FileOutputStream(file.toFile())) {
-                    if(playlist.encrypted) {
+                            in = response.body().byteStream();
-                        in = new Crypto(playlist.encryptionKeyUrl, client).wrap(in);
+                            if (playlist.encrypted) {
-                    }
+                                in = new Crypto(playlist.encryptionKeyUrl, client).wrap(in);
-                    byte[] b = new byte[1024 * 100];
+                            }
-                    int length = -1;
+                            byte[] b = new byte[1024 * 100];
-                    while( (length = in.read(b)) >= 0 ) {
+                            int length = -1;
-                        fos.write(b, 0, length);
+                            while ((length = in.read(b)) >= 0) {
-                    }
+                                fos.write(b, 0, length);
-                    return true;
+                            }
-                } catch(FileNotFoundException e) {
+                            return true;
-                    LOG.debug("Segment does not exist {}", url.getFile());
+                        } catch (FileNotFoundException e) {
-                    break;
+                            LOG.debug("Segment does not exist {}", url.getFile());
-                } catch(Exception e) {
+                            break;
-                    if (i == maxTries) {
+                        } catch (Exception e) {
-                        LOG.warn("Error while downloading segment. Segment {} finally failed", file.toFile().getName());
+                            if (i == maxTries) {
+                                LOG.warn("Error while downloading segment. Segment {} finally failed", file.toFile().getName());
+                            } else {
+                                LOG.warn("Error while downloading segment on try {}", i);
+                            }
+                        }
                     } else {
-                        LOG.warn("Error while downloading segment on try {}", i);
+                        // wait a bit and retry
+                        try {
+                            Thread.sleep(50 * i);
+                        } catch (InterruptedException e) {
+                            Thread.currentThread().interrupt();
+                        }
                     }
-                } finally {
-                    if(in != null) {
-                        in.close();
-                    }
-                    response.close();
                 }
             }
             return false;
@@ -339,7 +347,7 @@ public class HlsDownload extends AbstractHlsDownload {
 
     private double getPlaylistLength(File playlist) throws IOException, ParseException, PlaylistException {
         if (playlist.exists()) {
-            try(FileInputStream fin = new FileInputStream(playlist)) {
+            try (FileInputStream fin = new FileInputStream(playlist)) {
                 PlaylistParser playlistParser = new PlaylistParser(fin, Format.EXT_M3U, Encoding.UTF_8, ParsingMode.LENIENT);
                 Playlist m3u = playlistParser.parse();
                 MediaPlaylist mediaPlaylist = m3u.getMediaPlaylist();
@@ -353,20 +361,4 @@ public class HlsDownload extends AbstractHlsDownload {
             throw new FileNotFoundException(playlist.getAbsolutePath() + " does not exist");
         }
     }
-
-    /**
-     * Causes the current thread to sleep for a short amount of time.
-     * This is used to slow down retries, if something is wrong with the playlist.
-     * E.g. HTTP 403 or 404
-     */
-    private void waitSomeTime(long waitForMillis) {
-        try {
-            Thread.sleep(waitForMillis);
-        } catch (InterruptedException e) {
-            Thread.currentThread().interrupt();
-            if(running) {
-                LOG.error("Couldn't sleep. This might mess up the download!");
-            }
-        }
-    }
 }

common/src/main/java/ctbrec/recorder/download/hls/MergedHlsDownload.java

@@ -6,8 +6,6 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.net.URL;
 import java.nio.file.Files;
-import java.security.InvalidKeyException;
-import java.security.NoSuchAlgorithmException;
 import java.time.Duration;
 import java.time.Instant;
 import java.util.Arrays;
@@ -18,9 +16,6 @@ import org.jcodec.containers.mp4.boxes.MovieBox;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.iheartradio.m3u8.ParseException;
-import com.iheartradio.m3u8.PlaylistException;
-
 import ctbrec.Config;
 import ctbrec.Hmac;
 import ctbrec.Model;
@@ -56,6 +51,7 @@ public class MergedHlsDownload extends HlsDownload {
 
     @Override
     public void postprocess(ctbrec.Recording recording) {
+        Thread.currentThread().setName("PP " + model.getName());
         try {
             File playlist = super.generatePlaylist(recording);
             Objects.requireNonNull(playlist, "Generated playlist is null");
@@ -110,7 +106,7 @@ public class MergedHlsDownload extends HlsDownload {
     }
 
     public void downloadFinishedRecording(String segmentPlaylistUri, File target, ProgressListener progressListener)
-            throws IOException, ParseException, PlaylistException, InvalidKeyException, NoSuchAlgorithmException {
+            throws Exception {
         if (Config.getInstance().getSettings().requireAuthentication) {
             URL u = new URL(segmentPlaylistUri);
             String path = u.getPath();

common/src/main/java/ctbrec/sites/jasmin/LiveJasminHlsDownload.java

@@ -17,7 +17,7 @@ import ctbrec.recorder.download.hls.HlsDownload;
 
 public class LiveJasminHlsDownload extends HlsDownload {
 
-    private static final transient Logger LOG = LoggerFactory.getLogger(LiveJasminHlsDownload.class);
+    private static final Logger LOG = LoggerFactory.getLogger(LiveJasminHlsDownload.class);
     private long lastMasterPlaylistUpdate = 0;
     private String segmentUrl;
 
@@ -26,7 +26,7 @@ public class LiveJasminHlsDownload extends HlsDownload {
     }
 
     @Override
-    protected SegmentPlaylist getNextSegments(String segments) throws IOException, ParseException, PlaylistException {
+    protected SegmentPlaylist getNextSegments(String segments) throws Exception {
         if(this.segmentUrl == null) {
             this.segmentUrl = segments;
         }

common/src/main/java/ctbrec/sites/jasmin/LiveJasminMergedHlsDownload.java

@@ -17,7 +17,7 @@ import ctbrec.recorder.download.hls.MergedHlsDownload;
 
 public class LiveJasminMergedHlsDownload extends MergedHlsDownload {
 
-    private static final transient Logger LOG = LoggerFactory.getLogger(LiveJasminMergedHlsDownload.class);
+    private static final Logger LOG = LoggerFactory.getLogger(LiveJasminMergedHlsDownload.class);
     private long lastMasterPlaylistUpdate = 0;
     private String segmentUrl;
 
@@ -26,7 +26,7 @@ public class LiveJasminMergedHlsDownload extends MergedHlsDownload {
     }
 
     @Override
-    protected SegmentPlaylist getNextSegments(String segments) throws IOException, ParseException, PlaylistException {
+    protected SegmentPlaylist getNextSegments(String segments) throws Exception {
         if(this.segmentUrl == null) {
             this.segmentUrl = segments;
         }