diff --git a/common/src/main/java/ctbrec/Settings.java b/common/src/main/java/ctbrec/Settings.java index c92c0ae4..a58ad1c7 100644 --- a/common/src/main/java/ctbrec/Settings.java +++ b/common/src/main/java/ctbrec/Settings.java @@ -164,6 +164,8 @@ public class Settings { public long recordUntilDefaultDurationInMinutes = 24 * 60L; public boolean removeRecordingAfterPostProcessing = false; public boolean requireAuthentication = false; + public int segmentErrorMeasurePeriodInSecs = 20; + public int segmentErrorThresholdToStopRecording = 5; public String servletContext = ""; public boolean showGridLinesInTables = true; public boolean showPlayerStarting = false; diff --git a/common/src/main/java/ctbrec/recorder/download/hls/AbstractHlsDownload.java b/common/src/main/java/ctbrec/recorder/download/hls/AbstractHlsDownload.java index 607cdca3..a48dd9de 100644 --- a/common/src/main/java/ctbrec/recorder/download/hls/AbstractHlsDownload.java +++ b/common/src/main/java/ctbrec/recorder/download/hls/AbstractHlsDownload.java @@ -70,6 +70,7 @@ public abstract class AbstractHlsDownload extends AbstractDownload { private int consecutivePlaylistTimeouts = 0; private int consecutivePlaylistErrors = 0; private Instant lastSegmentDownload = Instant.MIN; + private final List segmentErrorTimestamps = new LinkedList<>(); private int selectedResolution = UNKNOWN; private final List recordingEvents = new LinkedList<>(); @@ -164,14 +165,38 @@ public abstract class AbstractHlsDownload extends AbstractDownload { segmentDownloadFinished(future.get()); } catch (InterruptedException e) { Thread.currentThread().interrupt(); - LOG.error("Error in segmentDownloadFinished", e); + LOG.error("Thread interrupted during segment download", e); } catch (ExecutionException e) { - LOG.error("Error in segmentDownloadFinished", e); + // Something went wrong during the segment download. + // At this point we have taken care of that, but we take note of the error and if a certain threshold of errors is exceeded in a certain + // amount of time, we stop the download and hope for the best to get routed to a better server. + stopRecordingOnHighSegmentErrorCount(); } } }); } + private void stopRecordingOnHighSegmentErrorCount() { + segmentErrorTimestamps.add(Instant.now()); + int errorsInMeasurePeriod = 0; + int measurePeriodInSecs = config.getSettings().segmentErrorMeasurePeriodInSecs; + Instant measureStart = Instant.now().minusSeconds(measurePeriodInSecs); + for (Iterator iterator = segmentErrorTimestamps.iterator(); iterator.hasNext(); ) { + Instant timestamp = iterator.next(); + if (timestamp.isAfter(measureStart)) { + errorsInMeasurePeriod++; + } else { + // too old, can be removed + iterator.remove(); + } + } + LOG.debug("Segment errors in last {} secs for {}: {}", measurePeriodInSecs, getModel(), errorsInMeasurePeriod); + if (errorsInMeasurePeriod > config.getSettings().segmentErrorThresholdToStopRecording) { + LOG.info("Too many ({}) segment errors for {} - stopping recording", errorsInMeasurePeriod, getModel()); + stop(); + } + } + protected abstract void execute(SegmentDownload segmentDownload); protected void handleHttpException(HttpException e) { diff --git a/common/src/main/java/ctbrec/recorder/download/hls/HlsDownload.java b/common/src/main/java/ctbrec/recorder/download/hls/HlsDownload.java index 0168b162..ffdf44ff 100644 --- a/common/src/main/java/ctbrec/recorder/download/hls/HlsDownload.java +++ b/common/src/main/java/ctbrec/recorder/download/hls/HlsDownload.java @@ -128,7 +128,7 @@ public class HlsDownload extends AbstractHlsDownload { while ((future = segmentDownloads.peek()) != null && !Thread.currentThread().isInterrupted()) { try { if (running && future.isDone()) { - segmentDownloads.poll(); // future is done remove from queue + segmentDownloads.poll();// future is done remove from queue SegmentDownload segmentDownload = future.get(); segments.add(toTrack(segmentDownload.getSegment())); } else { @@ -138,7 +138,7 @@ public class HlsDownload extends AbstractHlsDownload { } catch (InterruptedException e) { Thread.currentThread().interrupt(); } catch (Exception e) { - LOG.error("Segment download failed for model {}", model, e); + LOG.info("Segment download failed for model {} - skipping adding segment to playlist", model); } } } diff --git a/common/src/main/java/ctbrec/recorder/download/hls/SegmentDownload.java b/common/src/main/java/ctbrec/recorder/download/hls/SegmentDownload.java index c2fc8497..bfb04a16 100644 --- a/common/src/main/java/ctbrec/recorder/download/hls/SegmentDownload.java +++ b/common/src/main/java/ctbrec/recorder/download/hls/SegmentDownload.java @@ -52,7 +52,7 @@ public class SegmentDownload implements Callable { } @Override - public SegmentDownload call() { + public SegmentDownload call() throws InvalidAlgorithmParameterException, NoSuchPaddingException, NoSuchAlgorithmException, IOException, InvalidKeyException { for (int tries = 1; tries <= 3 && !Thread.currentThread().isInterrupted(); tries++) { // NOSONAR Request request = createRequest(); try (Response response = client.execute(request)) { @@ -66,6 +66,7 @@ public class SegmentDownload implements Callable { } catch (Exception e) { if (tries == 3) { LOG.warn("Error while downloading segment for {}. Segment {} finally failed: {}", model, url.getFile(), e.getMessage()); + throw e; } else { LOG.debug("Error while downloading segment {} for {} on try {} - {}", url.getFile(), model, tries, e.getMessage()); }