Stop recording, if a certain amount of segment errors occurred

common/src/main/java/ctbrec/Settings.java

@@ -164,6 +164,8 @@ public class Settings {
     public long recordUntilDefaultDurationInMinutes = 24 * 60L;
     public boolean removeRecordingAfterPostProcessing = false;
     public boolean requireAuthentication = false;
+    public int segmentErrorMeasurePeriodInSecs = 20;
+    public int segmentErrorThresholdToStopRecording = 5;
     public String servletContext = "";
     public boolean showGridLinesInTables = true;
     public boolean showPlayerStarting = false;

common/src/main/java/ctbrec/recorder/download/hls/AbstractHlsDownload.java

@@ -70,6 +70,7 @@ public abstract class AbstractHlsDownload extends AbstractDownload {
     private int consecutivePlaylistTimeouts = 0;
     private int consecutivePlaylistErrors = 0;
     private Instant lastSegmentDownload = Instant.MIN;
+    private final List<Instant> segmentErrorTimestamps = new LinkedList<>();
     private int selectedResolution = UNKNOWN;
 
     private final List<RecordingEvent> recordingEvents = new LinkedList<>();
@@ -164,14 +165,38 @@ public abstract class AbstractHlsDownload extends AbstractDownload {
                     segmentDownloadFinished(future.get());
                 } catch (InterruptedException e) {
                     Thread.currentThread().interrupt();
-                    LOG.error("Error in segmentDownloadFinished", e);
+                    LOG.error("Thread interrupted during segment download", e);
                 } catch (ExecutionException e) {
-                    LOG.error("Error in segmentDownloadFinished", e);
+                    // Something went wrong during the segment download.
+                    // At this point we have taken care of that, but we take note of the error and if a certain threshold of errors is exceeded in a certain
+                    // amount of time, we stop the download and hope for the best to get routed to a better server.
+                    stopRecordingOnHighSegmentErrorCount();
                 }
             }
         });
     }
 
+    private void stopRecordingOnHighSegmentErrorCount() {
+        segmentErrorTimestamps.add(Instant.now());
+        int errorsInMeasurePeriod = 0;
+        int measurePeriodInSecs = config.getSettings().segmentErrorMeasurePeriodInSecs;
+        Instant measureStart = Instant.now().minusSeconds(measurePeriodInSecs);
+        for (Iterator<Instant> iterator = segmentErrorTimestamps.iterator(); iterator.hasNext(); ) {
+            Instant timestamp = iterator.next();
+            if (timestamp.isAfter(measureStart)) {
+                errorsInMeasurePeriod++;
+            } else {
+                // too old, can be removed
+                iterator.remove();
+            }
+        }
+        LOG.debug("Segment errors in last {} secs for {}: {}", measurePeriodInSecs, getModel(), errorsInMeasurePeriod);
+        if (errorsInMeasurePeriod > config.getSettings().segmentErrorThresholdToStopRecording) {
+            LOG.info("Too many ({}) segment errors for {} - stopping recording", errorsInMeasurePeriod, getModel());
+            stop();
+        }
+    }
+
     protected abstract void execute(SegmentDownload segmentDownload);
 
     protected void handleHttpException(HttpException e) {

common/src/main/java/ctbrec/recorder/download/hls/HlsDownload.java

@@ -128,7 +128,7 @@ public class HlsDownload extends AbstractHlsDownload {
         while ((future = segmentDownloads.peek()) != null && !Thread.currentThread().isInterrupted()) {
             try {
                 if (running && future.isDone()) {
-                    segmentDownloads.poll(); // future is done remove from queue
+                    segmentDownloads.poll();// future is done remove from queue
                     SegmentDownload segmentDownload = future.get();
                     segments.add(toTrack(segmentDownload.getSegment()));
                 } else {
@@ -138,7 +138,7 @@ public class HlsDownload extends AbstractHlsDownload {
             } catch (InterruptedException e) {
                 Thread.currentThread().interrupt();
             } catch (Exception e) {
-                LOG.error("Segment download failed for model {}", model, e);
+                LOG.info("Segment download failed for model {} - skipping adding segment to playlist", model);
             }
         }
     }

common/src/main/java/ctbrec/recorder/download/hls/SegmentDownload.java

@@ -52,7 +52,7 @@ public class SegmentDownload implements Callable<SegmentDownload> {
     }
 
     @Override
-    public SegmentDownload call() {
+    public SegmentDownload call() throws InvalidAlgorithmParameterException, NoSuchPaddingException, NoSuchAlgorithmException, IOException, InvalidKeyException {
         for (int tries = 1; tries <= 3 && !Thread.currentThread().isInterrupted(); tries++) { // NOSONAR
             Request request = createRequest();
             try (Response response = client.execute(request)) {
@@ -66,6 +66,7 @@ public class SegmentDownload implements Callable<SegmentDownload> {
             } catch (Exception e) {
                 if (tries == 3) {
                     LOG.warn("Error while downloading segment for {}. Segment {} finally failed: {}", model, url.getFile(), e.getMessage());
+                    throw e;
                 } else {
                     LOG.debug("Error while downloading segment {} for {} on try {} - {}", url.getFile(), model, tries, e.getMessage());
                 }