j62
/
ctbrec
2
2
Fork 2
Code Issues 9 Pull Requests Packages Projects Releases 3 Wiki Activity

Improve error handling in downloads

This commit is contained in:
0xboobface 2019-12-26 21:41:54 +01:00
parent c774a29421
commit c2901284ef
6 changed files with 143 additions and 114 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
common/src/main/java/ctbrec/recorder/download/dash/DashDownload.java
View File

@ -81,6 +81,7 @@ public class DashDownload extends AbstractDownload {
.header("Connection", "keep-alive")
.build(); // @formatter:on
LOG.trace("Loading manifest {}", url);
// TODO try 10 times
try (Response response = httpClient.execute(request)) {
if (response.isSuccessful()) {
return response.body().string();
@ -224,6 +225,7 @@ public class DashDownload extends AbstractDownload {
@Override
public void start() throws IOException {
try {
Thread.currentThread().setName("Download " + model.getName());
running = true;
splitRecStartTime = ZonedDateTime.now();
JAXBContext jc = JAXBContext.newInstance(MPDtype.class.getPackage().getName());
@ -237,6 +239,16 @@ public class DashDownload extends AbstractDownload {
break;
}
}
} catch(HttpException e) {
if(e.getResponseCode() == 404) {
LOG.debug("Manifest not found (404). Model {} probably went offline", model);
waitSomeTime(10_000);
} else if(e.getResponseCode() == 403) {
LOG.debug("Manifest access forbidden (403). Model {} probably went private or offline", model);
waitSomeTime(10_000);
} else {
throw e;
}
} catch (Exception e) {
LOG.error("Error while downloading dash stream", e);
} finally {
@ -349,6 +361,7 @@ public class DashDownload extends AbstractDownload {
@Override
public void postprocess(Recording recording) {
try {
Thread.currentThread().setName("PP " + model.getName());
recording.setStatus(POST_PROCESSING);
String path = recording.getPath();
File dir = new File(Config.getInstance().getSettings().recordingsDir, path);

34
common/src/main/java/ctbrec/recorder/download/hls/AbstractHlsDownload.java
View File

@ -38,6 +38,7 @@ import ctbrec.Recording.State;
import ctbrec.UnknownModel;
import ctbrec.io.HttpClient;
import ctbrec.io.HttpException;
import ctbrec.recorder.PlaylistGenerator.InvalidPlaylistException;
import ctbrec.recorder.download.AbstractDownload;
import ctbrec.recorder.download.StreamSource;
import okhttp3.Request;
@ -69,7 +70,7 @@ public abstract class AbstractHlsDownload extends AbstractDownload {
};
}
protected SegmentPlaylist getNextSegments(String segmentsURL) throws IOException, ParseException, PlaylistException {
protected SegmentPlaylist getNextSegments(String segmentsURL) throws Exception {
URL segmentsUrl = new URL(segmentsURL);
Request request = new Request.Builder()
.url(segmentsUrl)
@ -80,6 +81,9 @@ public abstract class AbstractHlsDownload extends AbstractDownload {
.header("Referer", model.getSite().getBaseUrl())
.header("Connection", "keep-alive")
.build();
int tries = 1;
Exception lastException = null;
for (int i = 0; i <= 10; i++) {
try (Response response = client.execute(request)) {
if (response.isSuccessful()) {
String body = response.body().string();
@ -116,10 +120,20 @@ public abstract class AbstractHlsDownload extends AbstractDownload {
}
return lsp;
}
return null;
throw new InvalidPlaylistException("Playlist has no media playlist");
} else {
throw new HttpException(response.code(), response.message());
}
} catch (Exception e) {
LOG.debug("Couldn't download HLS playlist (try {}) {} - {}", tries, segmentsURL, e.getMessage());
lastException = e;
}
waitSomeTime(100 * tries);
}
if (lastException != null) {
throw lastException;
} else {
throw new IOException("Couldn't download HLS playlist");
}
}
@ -183,7 +197,21 @@ public abstract class AbstractHlsDownload extends AbstractDownload {
return model;
}
/**
* Causes the current thread to sleep for a short amount of time.
* This is used to slow down retries, if something is wrong with the playlist.
* E.g. HTTP 403 or 404
*/
protected void waitSomeTime(long waitForMillis) {
try {
Thread.sleep(waitForMillis);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
if(running) {
LOG.error("Couldn't sleep. This might mess up the download!");
}
}
}
public static class SegmentPlaylist {
public String url;

40
common/src/main/java/ctbrec/recorder/download/hls/HlsDownload.java
View File

@ -78,6 +78,7 @@ public class HlsDownload extends AbstractHlsDownload {
public void start() throws IOException {
try {
running = true;
Thread.currentThread().setName("Download " + model.getName());
splitRecStartTime = ZonedDateTime.now();
if (!model.isOnline()) {
@ -97,7 +98,8 @@ public class HlsDownload extends AbstractHlsDownload {
emptyPlaylistCheck(playlist);
if (nextSegmentNumber > 0 && playlist.seq > nextSegmentNumber) {
waitFactor *= 2;
LOG.warn("Missed segments {} < {} in download for {} - setting wait factor to 1/{}", nextSegmentNumber, playlist.seq, model, waitFactor);
LOG.warn("Missed segments {} < {} in download for {} - setting wait factor to 1/{}", nextSegmentNumber, playlist.seq, model,
waitFactor);
}
int skip = nextSegmentNumber - playlist.seq;
for (String segment : playlist.segments) {
@ -176,6 +178,7 @@ public class HlsDownload extends AbstractHlsDownload {
@Override
public void postprocess(Recording recording) {
Thread.currentThread().setName("PP " + model.getName());
recording.setStatusWithEvent(State.GENERATING_PLAYLIST);
generatePlaylist(recording);
recording.setStatusWithEvent(State.POST_PROCESSING);
@ -217,7 +220,7 @@ public class HlsDownload extends AbstractHlsDownload {
LOG.error("Playlist is invalid and will be deleted", e);
File playlist = new File(recDir, "playlist.m3u8");
try {
Files.delete(playlist.toPath());
Files.deleteIfExists(playlist.toPath());
} catch (IOException e1) {
LOG.error("Couldn't delete playlist {}", playlist, e1);
}
@ -275,10 +278,11 @@ public class HlsDownload extends AbstractHlsDownload {
@Override
public Boolean call() throws Exception {
LOG.trace("Downloading segment to {}", file);
int maxTries = 3;
int maxTries = 5;
for (int i = 1; i <= maxTries; i++) {
Request request = new Request.Builder().url(url).addHeader("connection", "keep-alive").build();
Response response = client.execute(request);
try (Response response = client.execute(request)) {
if (response.isSuccessful()) {
InputStream in = null;
try (FileOutputStream fos = new FileOutputStream(file.toFile())) {
in = response.body().byteStream();
@ -300,11 +304,15 @@ public class HlsDownload extends AbstractHlsDownload {
} else {
LOG.warn("Error while downloading segment on try {}", i);
}
} finally {
if(in != null) {
in.close();
}
response.close();
} else {
// wait a bit and retry
try {
Thread.sleep(50 * i);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
}
}
}
return false;
@ -353,20 +361,4 @@ public class HlsDownload extends AbstractHlsDownload {
throw new FileNotFoundException(playlist.getAbsolutePath() + " does not exist");
}
}
/**
* Causes the current thread to sleep for a short amount of time.
* This is used to slow down retries, if something is wrong with the playlist.
* E.g. HTTP 403 or 404
*/
private void waitSomeTime(long waitForMillis) {
try {
Thread.sleep(waitForMillis);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
if(running) {
LOG.error("Couldn't sleep. This might mess up the download!");
}
}
}
}

8
common/src/main/java/ctbrec/recorder/download/hls/MergedHlsDownload.java
View File

@ -6,8 +6,6 @@ import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.nio.file.Files;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import java.time.Duration;
import java.time.Instant;
import java.util.Arrays;
@ -18,9 +16,6 @@ import org.jcodec.containers.mp4.boxes.MovieBox;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.iheartradio.m3u8.ParseException;
import com.iheartradio.m3u8.PlaylistException;
import ctbrec.Config;
import ctbrec.Hmac;
import ctbrec.Model;
@ -56,6 +51,7 @@ public class MergedHlsDownload extends HlsDownload {
@Override
public void postprocess(ctbrec.Recording recording) {
Thread.currentThread().setName("PP " + model.getName());
try {
File playlist = super.generatePlaylist(recording);
Objects.requireNonNull(playlist, "Generated playlist is null");
@ -110,7 +106,7 @@ public class MergedHlsDownload extends HlsDownload {
}
public void downloadFinishedRecording(String segmentPlaylistUri, File target, ProgressListener progressListener)
throws IOException, ParseException, PlaylistException, InvalidKeyException, NoSuchAlgorithmException {
throws Exception {
if (Config.getInstance().getSettings().requireAuthentication) {
URL u = new URL(segmentPlaylistUri);
String path = u.getPath();

4
common/src/main/java/ctbrec/sites/jasmin/LiveJasminHlsDownload.java
View File

@ -17,7 +17,7 @@ import ctbrec.recorder.download.hls.HlsDownload;
public class LiveJasminHlsDownload extends HlsDownload {
private static final transient Logger LOG = LoggerFactory.getLogger(LiveJasminHlsDownload.class);
private static final Logger LOG = LoggerFactory.getLogger(LiveJasminHlsDownload.class);
private long lastMasterPlaylistUpdate = 0;
private String segmentUrl;
@ -26,7 +26,7 @@ public class LiveJasminHlsDownload extends HlsDownload {
}
@Override
protected SegmentPlaylist getNextSegments(String segments) throws IOException, ParseException, PlaylistException {
protected SegmentPlaylist getNextSegments(String segments) throws Exception {
if(this.segmentUrl == null) {
this.segmentUrl = segments;
}

4
common/src/main/java/ctbrec/sites/jasmin/LiveJasminMergedHlsDownload.java
View File

@ -17,7 +17,7 @@ import ctbrec.recorder.download.hls.MergedHlsDownload;
public class LiveJasminMergedHlsDownload extends MergedHlsDownload {
private static final transient Logger LOG = LoggerFactory.getLogger(LiveJasminMergedHlsDownload.class);
private static final Logger LOG = LoggerFactory.getLogger(LiveJasminMergedHlsDownload.class);
private long lastMasterPlaylistUpdate = 0;
private String segmentUrl;
@ -26,7 +26,7 @@ public class LiveJasminMergedHlsDownload extends MergedHlsDownload {
}
@Override
protected SegmentPlaylist getNextSegments(String segments) throws IOException, ParseException, PlaylistException {
protected SegmentPlaylist getNextSegments(String segments) throws Exception {
if(this.segmentUrl == null) {
this.segmentUrl = segments;
}