Improve error handling in downloads

common/src/main/java/ctbrec/recorder/download/dash/DashDownload.java

@@ -81,6 +81,7 @@ public class DashDownload extends AbstractDownload {
                 .header("Connection", "keep-alive")
                 .build(); // @formatter:on
         LOG.trace("Loading manifest {}", url);
+        // TODO try 10 times
         try (Response response = httpClient.execute(request)) {
             if (response.isSuccessful()) {
                 return response.body().string();
@@ -224,6 +225,7 @@ public class DashDownload extends AbstractDownload {
     @Override
     public void start() throws IOException {
         try {
+            Thread.currentThread().setName("Download " + model.getName());
             running = true;
             splitRecStartTime = ZonedDateTime.now();
             JAXBContext jc = JAXBContext.newInstance(MPDtype.class.getPackage().getName());
@@ -237,6 +239,16 @@ public class DashDownload extends AbstractDownload {
                     break;
                 }
             }
+        } catch(HttpException e) {
+            if(e.getResponseCode() == 404) {
+                LOG.debug("Manifest not found (404). Model {} probably went offline", model);
+                waitSomeTime(10_000);
+            } else if(e.getResponseCode() == 403) {
+                LOG.debug("Manifest access forbidden (403). Model {} probably went private or offline", model);
+                waitSomeTime(10_000);
+            } else {
+                throw e;
+            }
         } catch (Exception e) {
             LOG.error("Error while downloading dash stream", e);
         } finally {
@@ -349,6 +361,7 @@ public class DashDownload extends AbstractDownload {
     @Override
     public void postprocess(Recording recording) {
         try {
+            Thread.currentThread().setName("PP " + model.getName());
             recording.setStatus(POST_PROCESSING);
             String path = recording.getPath();
             File dir = new File(Config.getInstance().getSettings().recordingsDir, path);

common/src/main/java/ctbrec/recorder/download/hls/AbstractHlsDownload.java

@@ -38,6 +38,7 @@ import ctbrec.Recording.State;
 import ctbrec.UnknownModel;
 import ctbrec.io.HttpClient;
 import ctbrec.io.HttpException;
+import ctbrec.recorder.PlaylistGenerator.InvalidPlaylistException;
 import ctbrec.recorder.download.AbstractDownload;
 import ctbrec.recorder.download.StreamSource;
 import okhttp3.Request;
@@ -69,7 +70,7 @@ public abstract class AbstractHlsDownload extends AbstractDownload {
         };
     }
 
-    protected SegmentPlaylist getNextSegments(String segmentsURL) throws IOException, ParseException, PlaylistException {
+    protected SegmentPlaylist getNextSegments(String segmentsURL) throws Exception {
         URL segmentsUrl = new URL(segmentsURL);
         Request request = new Request.Builder()
                 .url(segmentsUrl)
@@ -80,6 +81,9 @@ public abstract class AbstractHlsDownload extends AbstractDownload {
                 .header("Referer", model.getSite().getBaseUrl())
                 .header("Connection", "keep-alive")
                 .build();
+        int tries = 1;
+        Exception lastException = null;
+        for (int i = 0; i <= 10; i++) {
             try (Response response = client.execute(request)) {
                 if (response.isSuccessful()) {
                     String body = response.body().string();
@@ -116,10 +120,20 @@ public abstract class AbstractHlsDownload extends AbstractDownload {
                         }
                         return lsp;
                     }
-                return null;
+                    throw new InvalidPlaylistException("Playlist has no media playlist");
                 } else {
                     throw new HttpException(response.code(), response.message());
                 }
+            } catch (Exception e) {
+                LOG.debug("Couldn't download HLS playlist (try {}) {} - {}", tries, segmentsURL, e.getMessage());
+                lastException = e;
+            }
+            waitSomeTime(100 * tries);
+        }
+        if (lastException != null) {
+            throw lastException;
+        } else {
+            throw new IOException("Couldn't download HLS playlist");
         }
     }
 
@@ -183,7 +197,21 @@ public abstract class AbstractHlsDownload extends AbstractDownload {
         return model;
     }
 
-
+    /**
+     * Causes the current thread to sleep for a short amount of time.
+     * This is used to slow down retries, if something is wrong with the playlist.
+     * E.g. HTTP 403 or 404
+     */
+    protected void waitSomeTime(long waitForMillis) {
+        try {
+            Thread.sleep(waitForMillis);
+        } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
+            if(running) {
+                LOG.error("Couldn't sleep. This might mess up the download!");
+            }
+        }
+    }
 
     public static class SegmentPlaylist {
         public String url;

common/src/main/java/ctbrec/recorder/download/hls/HlsDownload.java

@@ -78,6 +78,7 @@ public class HlsDownload extends AbstractHlsDownload {
     public void start() throws IOException {
         try {
             running = true;
+            Thread.currentThread().setName("Download " + model.getName());
             splitRecStartTime = ZonedDateTime.now();
 
             if (!model.isOnline()) {
@@ -97,7 +98,8 @@ public class HlsDownload extends AbstractHlsDownload {
                     emptyPlaylistCheck(playlist);
                     if (nextSegmentNumber > 0 && playlist.seq > nextSegmentNumber) {
                         waitFactor *= 2;
-                        LOG.warn("Missed segments {} < {} in download for {} - setting wait factor to 1/{}", nextSegmentNumber, playlist.seq, model, waitFactor);
+                        LOG.warn("Missed segments {} < {} in download for {} - setting wait factor to 1/{}", nextSegmentNumber, playlist.seq, model,
+                                waitFactor);
                     }
                     int skip = nextSegmentNumber - playlist.seq;
                     for (String segment : playlist.segments) {
@@ -176,6 +178,7 @@ public class HlsDownload extends AbstractHlsDownload {
 
     @Override
     public void postprocess(Recording recording) {
+        Thread.currentThread().setName("PP " + model.getName());
         recording.setStatusWithEvent(State.GENERATING_PLAYLIST);
         generatePlaylist(recording);
         recording.setStatusWithEvent(State.POST_PROCESSING);
@@ -217,7 +220,7 @@ public class HlsDownload extends AbstractHlsDownload {
             LOG.error("Playlist is invalid and will be deleted", e);
             File playlist = new File(recDir, "playlist.m3u8");
             try {
-                Files.delete(playlist.toPath());
+                Files.deleteIfExists(playlist.toPath());
             } catch (IOException e1) {
                 LOG.error("Couldn't delete playlist {}", playlist, e1);
             }
@@ -275,10 +278,11 @@ public class HlsDownload extends AbstractHlsDownload {
         @Override
         public Boolean call() throws Exception {
             LOG.trace("Downloading segment to {}", file);
-            int maxTries = 3;
+            int maxTries = 5;
             for (int i = 1; i <= maxTries; i++) {
                 Request request = new Request.Builder().url(url).addHeader("connection", "keep-alive").build();
-                Response response = client.execute(request);
+                try (Response response = client.execute(request)) {
+                    if (response.isSuccessful()) {
                         InputStream in = null;
                         try (FileOutputStream fos = new FileOutputStream(file.toFile())) {
                             in = response.body().byteStream();
@@ -300,11 +304,15 @@ public class HlsDownload extends AbstractHlsDownload {
                             } else {
                                 LOG.warn("Error while downloading segment on try {}", i);
                             }
-                } finally {
-                    if(in != null) {
-                        in.close();
                         }
-                    response.close();
+                    } else {
+                        // wait a bit and retry
+                        try {
+                            Thread.sleep(50 * i);
+                        } catch (InterruptedException e) {
+                            Thread.currentThread().interrupt();
+                        }
+                    }
                 }
             }
             return false;
@@ -353,20 +361,4 @@ public class HlsDownload extends AbstractHlsDownload {
             throw new FileNotFoundException(playlist.getAbsolutePath() + " does not exist");
         }
     }
-
-    /**
-     * Causes the current thread to sleep for a short amount of time.
-     * This is used to slow down retries, if something is wrong with the playlist.
-     * E.g. HTTP 403 or 404
-     */
-    private void waitSomeTime(long waitForMillis) {
-        try {
-            Thread.sleep(waitForMillis);
-        } catch (InterruptedException e) {
-            Thread.currentThread().interrupt();
-            if(running) {
-                LOG.error("Couldn't sleep. This might mess up the download!");
-            }
-        }
-    }
 }

common/src/main/java/ctbrec/recorder/download/hls/MergedHlsDownload.java

@@ -6,8 +6,6 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.net.URL;
 import java.nio.file.Files;
-import java.security.InvalidKeyException;
-import java.security.NoSuchAlgorithmException;
 import java.time.Duration;
 import java.time.Instant;
 import java.util.Arrays;
@@ -18,9 +16,6 @@ import org.jcodec.containers.mp4.boxes.MovieBox;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.iheartradio.m3u8.ParseException;
-import com.iheartradio.m3u8.PlaylistException;
-
 import ctbrec.Config;
 import ctbrec.Hmac;
 import ctbrec.Model;
@@ -56,6 +51,7 @@ public class MergedHlsDownload extends HlsDownload {
 
     @Override
     public void postprocess(ctbrec.Recording recording) {
+        Thread.currentThread().setName("PP " + model.getName());
         try {
             File playlist = super.generatePlaylist(recording);
             Objects.requireNonNull(playlist, "Generated playlist is null");
@@ -110,7 +106,7 @@ public class MergedHlsDownload extends HlsDownload {
     }
 
     public void downloadFinishedRecording(String segmentPlaylistUri, File target, ProgressListener progressListener)
-            throws IOException, ParseException, PlaylistException, InvalidKeyException, NoSuchAlgorithmException {
+            throws Exception {
         if (Config.getInstance().getSettings().requireAuthentication) {
             URL u = new URL(segmentPlaylistUri);
             String path = u.getPath();

common/src/main/java/ctbrec/sites/jasmin/LiveJasminHlsDownload.java

@@ -17,7 +17,7 @@ import ctbrec.recorder.download.hls.HlsDownload;
 
 public class LiveJasminHlsDownload extends HlsDownload {
 
-    private static final transient Logger LOG = LoggerFactory.getLogger(LiveJasminHlsDownload.class);
+    private static final Logger LOG = LoggerFactory.getLogger(LiveJasminHlsDownload.class);
     private long lastMasterPlaylistUpdate = 0;
     private String segmentUrl;
 
@@ -26,7 +26,7 @@ public class LiveJasminHlsDownload extends HlsDownload {
     }
 
     @Override
-    protected SegmentPlaylist getNextSegments(String segments) throws IOException, ParseException, PlaylistException {
+    protected SegmentPlaylist getNextSegments(String segments) throws Exception {
         if(this.segmentUrl == null) {
             this.segmentUrl = segments;
         }

common/src/main/java/ctbrec/sites/jasmin/LiveJasminMergedHlsDownload.java

@@ -17,7 +17,7 @@ import ctbrec.recorder.download.hls.MergedHlsDownload;
 
 public class LiveJasminMergedHlsDownload extends MergedHlsDownload {
 
-    private static final transient Logger LOG = LoggerFactory.getLogger(LiveJasminMergedHlsDownload.class);
+    private static final Logger LOG = LoggerFactory.getLogger(LiveJasminMergedHlsDownload.class);
     private long lastMasterPlaylistUpdate = 0;
     private String segmentUrl;
 
@@ -26,7 +26,7 @@ public class LiveJasminMergedHlsDownload extends MergedHlsDownload {
     }
 
     @Override
-    protected SegmentPlaylist getNextSegments(String segments) throws IOException, ParseException, PlaylistException {
+    protected SegmentPlaylist getNextSegments(String segments) throws Exception {
         if(this.segmentUrl == null) {
             this.segmentUrl = segments;
         }