davebluke
/
ctbrec
forked from j62/ctbrec
1
0
Fork 0
Code Pull Requests Activity

Stop recording, if a certain amount of segment errors occurred

This commit is contained in:
0xb00bface 2021-12-21 18:45:24 +01:00
parent 3c145f4924
commit 3ad7b6aca6
4 changed files with 33 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
common/src/main/java/ctbrec/Settings.java
View File

@ -164,6 +164,8 @@ public class Settings {
public long recordUntilDefaultDurationInMinutes = 24 * 60L; public long recordUntilDefaultDurationInMinutes = 24 * 60L;
public boolean removeRecordingAfterPostProcessing = false; public boolean removeRecordingAfterPostProcessing = false;
public boolean requireAuthentication = false; public boolean requireAuthentication = false;
public int segmentErrorMeasurePeriodInSecs = 20;
public int segmentErrorThresholdToStopRecording = 5;
public String servletContext = ""; public String servletContext = "";
public boolean showGridLinesInTables = true; public boolean showGridLinesInTables = true;
public boolean showPlayerStarting = false; public boolean showPlayerStarting = false;

29
common/src/main/java/ctbrec/recorder/download/hls/AbstractHlsDownload.java
View File

@ -70,6 +70,7 @@ public abstract class AbstractHlsDownload extends AbstractDownload {
private int consecutivePlaylistTimeouts = 0; private int consecutivePlaylistTimeouts = 0;
private int consecutivePlaylistErrors = 0; private int consecutivePlaylistErrors = 0;
private Instant lastSegmentDownload = Instant.MIN; private Instant lastSegmentDownload = Instant.MIN;
private final List<Instant> segmentErrorTimestamps = new LinkedList<>();
private int selectedResolution = UNKNOWN; private int selectedResolution = UNKNOWN;
private final List<RecordingEvent> recordingEvents = new LinkedList<>(); private final List<RecordingEvent> recordingEvents = new LinkedList<>();
@ -164,14 +165,38 @@ public abstract class AbstractHlsDownload extends AbstractDownload {
segmentDownloadFinished(future.get()); segmentDownloadFinished(future.get());
} catch (InterruptedException e) { } catch (InterruptedException e) {
Thread.currentThread().interrupt(); Thread.currentThread().interrupt();
LOG.error("Error in segmentDownloadFinished", e); LOG.error("Thread interrupted during segment download", e);
} catch (ExecutionException e) { } catch (ExecutionException e) {
LOG.error("Error in segmentDownloadFinished", e); // Something went wrong during the segment download.
// At this point we have taken care of that, but we take note of the error and if a certain threshold of errors is exceeded in a certain
// amount of time, we stop the download and hope for the best to get routed to a better server.
stopRecordingOnHighSegmentErrorCount();
} }
} }
}); });
} }
private void stopRecordingOnHighSegmentErrorCount() {
segmentErrorTimestamps.add(Instant.now());
int errorsInMeasurePeriod = 0;
int measurePeriodInSecs = config.getSettings().segmentErrorMeasurePeriodInSecs;
Instant measureStart = Instant.now().minusSeconds(measurePeriodInSecs);
for (Iterator<Instant> iterator = segmentErrorTimestamps.iterator(); iterator.hasNext(); ) {
Instant timestamp = iterator.next();
if (timestamp.isAfter(measureStart)) {
errorsInMeasurePeriod++;
} else {
// too old, can be removed
iterator.remove();
}
}
LOG.debug("Segment errors in last {} secs for {}: {}", measurePeriodInSecs, getModel(), errorsInMeasurePeriod);
if (errorsInMeasurePeriod > config.getSettings().segmentErrorThresholdToStopRecording) {
LOG.info("Too many ({}) segment errors for {} - stopping recording", errorsInMeasurePeriod, getModel());
stop();
}
}
protected abstract void execute(SegmentDownload segmentDownload); protected abstract void execute(SegmentDownload segmentDownload);
protected void handleHttpException(HttpException e) { protected void handleHttpException(HttpException e) {

2
common/src/main/java/ctbrec/recorder/download/hls/HlsDownload.java
View File

@ -138,7 +138,7 @@ public class HlsDownload extends AbstractHlsDownload {
} catch (InterruptedException e) { } catch (InterruptedException e) {
Thread.currentThread().interrupt(); Thread.currentThread().interrupt();
} catch (Exception e) { } catch (Exception e) {
LOG.error("Segment download failed for model {}", model, e); LOG.info("Segment download failed for model {} - skipping adding segment to playlist", model);
} }
} }
} }

3
common/src/main/java/ctbrec/recorder/download/hls/SegmentDownload.java
View File

@ -52,7 +52,7 @@ public class SegmentDownload implements Callable<SegmentDownload> {
} }
@Override @Override
public SegmentDownload call() { public SegmentDownload call() throws InvalidAlgorithmParameterException, NoSuchPaddingException, NoSuchAlgorithmException, IOException, InvalidKeyException {
for (int tries = 1; tries <= 3 && !Thread.currentThread().isInterrupted(); tries++) { // NOSONAR for (int tries = 1; tries <= 3 && !Thread.currentThread().isInterrupted(); tries++) { // NOSONAR
Request request = createRequest(); Request request = createRequest();
try (Response response = client.execute(request)) { try (Response response = client.execute(request)) {
@ -66,6 +66,7 @@ public class SegmentDownload implements Callable<SegmentDownload> {
} catch (Exception e) { } catch (Exception e) {
if (tries == 3) { if (tries == 3) {
LOG.warn("Error while downloading segment for {}. Segment {} finally failed: {}", model, url.getFile(), e.getMessage()); LOG.warn("Error while downloading segment for {}. Segment {} finally failed: {}", model, url.getFile(), e.getMessage());
throw e;
} else { } else {
LOG.debug("Error while downloading segment {} for {} on try {} - {}", url.getFile(), model, tries, e.getMessage()); LOG.debug("Error while downloading segment {} for {} on try {} - {}", url.getFile(), model, tries, e.getMessage());
} }