ctbrecuser
/
ctbrec
forked from j62/ctbrec
1
0
Fork 0
Code Pull Requests Activity

Improve error handling in downloads

This commit is contained in:
0xboobface 2019-12-26 21:41:54 +01:00
parent c774a29421
commit c2901284ef
6 changed files with 143 additions and 114 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
common/src/main/java/ctbrec/recorder/download/dash/DashDownload.java
View File

@ -81,6 +81,7 @@ public class DashDownload extends AbstractDownload {
.header("Connection", "keep-alive")
.build(); // @formatter:on
LOG.trace("Loading manifest {}", url);
// TODO try 10 times
try (Response response = httpClient.execute(request)) {
if (response.isSuccessful()) {
return response.body().string();
@ -224,6 +225,7 @@ public class DashDownload extends AbstractDownload {
@Override
public void start() throws IOException {
try {
Thread.currentThread().setName("Download " + model.getName());
running = true;
splitRecStartTime = ZonedDateTime.now();
JAXBContext jc = JAXBContext.newInstance(MPDtype.class.getPackage().getName());
@ -237,6 +239,16 @@ public class DashDownload extends AbstractDownload {
break;
}
}
} catch(HttpException e) {
if(e.getResponseCode() == 404) {
LOG.debug("Manifest not found (404). Model {} probably went offline", model);
waitSomeTime(10_000);
} else if(e.getResponseCode() == 403) {
LOG.debug("Manifest access forbidden (403). Model {} probably went private or offline", model);
waitSomeTime(10_000);
} else {
throw e;
}
} catch (Exception e) {
LOG.error("Error while downloading dash stream", e);
} finally {
@ -349,6 +361,7 @@ public class DashDownload extends AbstractDownload {
@Override
public void postprocess(Recording recording) {
try {
Thread.currentThread().setName("PP " + model.getName());
recording.setStatus(POST_PROCESSING);
String path = recording.getPath();
File dir = new File(Config.getInstance().getSettings().recordingsDir, path);

106
common/src/main/java/ctbrec/recorder/download/hls/AbstractHlsDownload.java
View File

@ -38,6 +38,7 @@ import ctbrec.Recording.State;
import ctbrec.UnknownModel;
import ctbrec.io.HttpClient;
import ctbrec.io.HttpException;
import ctbrec.recorder.PlaylistGenerator.InvalidPlaylistException;
import ctbrec.recorder.download.AbstractDownload;
import ctbrec.recorder.download.StreamSource;
import okhttp3.Request;
@ -69,7 +70,7 @@ public abstract class AbstractHlsDownload extends AbstractDownload {
};
}
protected SegmentPlaylist getNextSegments(String segmentsURL) throws IOException, ParseException, PlaylistException {
protected SegmentPlaylist getNextSegments(String segmentsURL) throws Exception {
URL segmentsUrl = new URL(segmentsURL);
Request request = new Request.Builder()
.url(segmentsUrl)
@ -80,46 +81,59 @@ public abstract class AbstractHlsDownload extends AbstractDownload {
.header("Referer", model.getSite().getBaseUrl())
.header("Connection", "keep-alive")
.build();
try(Response response = client.execute(request)) {
if(response.isSuccessful()) {
String body = response.body().string();
if(!body.contains("#EXTINF")) {
// no segments, empty playlist
return new SegmentPlaylist(segmentsURL);
}
InputStream inputStream = new ByteArrayInputStream(body.getBytes("utf-8"));
PlaylistParser parser = new PlaylistParser(inputStream, Format.EXT_M3U, Encoding.UTF_8, ParsingMode.LENIENT);
Playlist playlist = parser.parse();
if(playlist.hasMediaPlaylist()) {
MediaPlaylist mediaPlaylist = playlist.getMediaPlaylist();
SegmentPlaylist lsp = new SegmentPlaylist(segmentsURL);
lsp.seq = mediaPlaylist.getMediaSequenceNumber();
lsp.targetDuration = mediaPlaylist.getTargetDuration();
List<TrackData> tracks = mediaPlaylist.getTracks();
for (TrackData trackData : tracks) {
String uri = trackData.getUri();
if(!uri.startsWith("http")) {
String tmpurl = segmentsUrl.toString();
tmpurl = tmpurl.substring(0, tmpurl.lastIndexOf('/') + 1);
uri = tmpurl + uri;
}
lsp.totalDuration += trackData.getTrackInfo().duration;
lsp.lastSegDuration = trackData.getTrackInfo().duration;
lsp.segments.add(uri);
if(trackData.hasEncryptionData()) {
lsp.encrypted = true;
EncryptionData data = trackData.getEncryptionData();
lsp.encryptionKeyUrl = data.getUri();
lsp.encryptionMethod = data.getMethod().getValue();
}
int tries = 1;
Exception lastException = null;
for (int i = 0; i <= 10; i++) {
try (Response response = client.execute(request)) {
if (response.isSuccessful()) {
String body = response.body().string();
if (!body.contains("#EXTINF")) {
// no segments, empty playlist
return new SegmentPlaylist(segmentsURL);
}
return lsp;
InputStream inputStream = new ByteArrayInputStream(body.getBytes("utf-8"));
PlaylistParser parser = new PlaylistParser(inputStream, Format.EXT_M3U, Encoding.UTF_8, ParsingMode.LENIENT);
Playlist playlist = parser.parse();
if (playlist.hasMediaPlaylist()) {
MediaPlaylist mediaPlaylist = playlist.getMediaPlaylist();
SegmentPlaylist lsp = new SegmentPlaylist(segmentsURL);
lsp.seq = mediaPlaylist.getMediaSequenceNumber();
lsp.targetDuration = mediaPlaylist.getTargetDuration();
List<TrackData> tracks = mediaPlaylist.getTracks();
for (TrackData trackData : tracks) {
String uri = trackData.getUri();
if (!uri.startsWith("http")) {
String tmpurl = segmentsUrl.toString();
tmpurl = tmpurl.substring(0, tmpurl.lastIndexOf('/') + 1);
uri = tmpurl + uri;
}
lsp.totalDuration += trackData.getTrackInfo().duration;
lsp.lastSegDuration = trackData.getTrackInfo().duration;
lsp.segments.add(uri);
if (trackData.hasEncryptionData()) {
lsp.encrypted = true;
EncryptionData data = trackData.getEncryptionData();
lsp.encryptionKeyUrl = data.getUri();
lsp.encryptionMethod = data.getMethod().getValue();
}
}
return lsp;
}
throw new InvalidPlaylistException("Playlist has no media playlist");
} else {
throw new HttpException(response.code(), response.message());
}
return null;
} else {
throw new HttpException(response.code(), response.message());
} catch (Exception e) {
LOG.debug("Couldn't download HLS playlist (try {}) {} - {}", tries, segmentsURL, e.getMessage());
lastException = e;
}
waitSomeTime(100 * tries);
}
if (lastException != null) {
throw lastException;
} else {
throw new IOException("Couldn't download HLS playlist");
}
}
@ -183,7 +197,21 @@ public abstract class AbstractHlsDownload extends AbstractDownload {
return model;
}
/**
* Causes the current thread to sleep for a short amount of time.
* This is used to slow down retries, if something is wrong with the playlist.
* E.g. HTTP 403 or 404
*/
protected void waitSomeTime(long waitForMillis) {
try {
Thread.sleep(waitForMillis);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
if(running) {
LOG.error("Couldn't sleep. This might mess up the download!");
}
}
}
public static class SegmentPlaylist {
public String url;

122
common/src/main/java/ctbrec/recorder/download/hls/HlsDownload.java
View File

@ -78,30 +78,32 @@ public class HlsDownload extends AbstractHlsDownload {
public void start() throws IOException {
try {
running = true;
Thread.currentThread().setName("Download " + model.getName());
splitRecStartTime = ZonedDateTime.now();
if(!model.isOnline()) {
throw new IOException(model.getName() +"'s room is not public");
if (!model.isOnline()) {
throw new IOException(model.getName() + "'s room is not public");
}
String segments = getSegmentPlaylistUrl(model);
if(segments != null) {
if (segments != null) {
if (!downloadDir.toFile().exists()) {
Files.createDirectories(downloadDir);
}
int lastSegmentNumber = 0;
int nextSegmentNumber = 0;
int waitFactor = 1;
while(running) {
while (running) {
SegmentPlaylist playlist = getNextSegments(segments);
emptyPlaylistCheck(playlist);
if(nextSegmentNumber > 0 && playlist.seq > nextSegmentNumber) {
if (nextSegmentNumber > 0 && playlist.seq > nextSegmentNumber) {
waitFactor *= 2;
LOG.warn("Missed segments {} < {} in download for {} - setting wait factor to 1/{}", nextSegmentNumber, playlist.seq, model, waitFactor);
LOG.warn("Missed segments {} < {} in download for {} - setting wait factor to 1/{}", nextSegmentNumber, playlist.seq, model,
waitFactor);
}
int skip = nextSegmentNumber - playlist.seq;
for (String segment : playlist.segments) {
if(skip > 0) {
if (skip > 0) {
skip--;
} else {
URL segmentUrl = new URL(segment);
@ -118,7 +120,7 @@ public class HlsDownload extends AbstractHlsDownload {
}
long waitForMillis = 0;
if(lastSegmentNumber == playlist.seq) {
if (lastSegmentNumber == playlist.seq) {
// playlist didn't change -> wait for at least half the target duration
waitForMillis = (long) playlist.targetDuration * 1000 / waitFactor;
LOG.trace("Playlist didn't change... waiting for {}ms", waitForMillis);
@ -133,31 +135,31 @@ public class HlsDownload extends AbstractHlsDownload {
// this if check makes sure, that we don't decrease nextSegment. for some reason
// streamate playlists sometimes jump back. e.g. max sequence = 79 -> 80 -> 79
lastSegmentNumber = playlist.seq;
if(lastSegmentNumber + playlist.segments.size() > nextSegmentNumber) {
if (lastSegmentNumber + playlist.segments.size() > nextSegmentNumber) {
nextSegmentNumber = lastSegmentNumber + playlist.segments.size();
}
}
} else {
throw new IOException("Couldn't determine segments uri");
}
} catch(ParseException e) {
} catch (ParseException e) {
throw new IOException("Couldn't parse HLS playlist:\n" + e.getInput(), e);
} catch(PlaylistException e) {
} catch (PlaylistException e) {
throw new IOException("Couldn't parse HLS playlist", e);
} catch(EOFException e) {
} catch (EOFException e) {
// end of playlist reached
LOG.debug("Reached end of playlist for model {}", model);
} catch(HttpException e) {
if(e.getResponseCode() == 404) {
} catch (HttpException e) {
if (e.getResponseCode() == 404) {
LOG.debug("Playlist not found (404). Model {} probably went offline", model);
waitSomeTime(10_000);
} else if(e.getResponseCode() == 403) {
} else if (e.getResponseCode() == 403) {
LOG.debug("Playlist access forbidden (403). Model {} probably went private or offline", model);
waitSomeTime(10_000);
} else {
throw e;
}
} catch(Exception e) {
} catch (Exception e) {
throw new IOException("Couldn't download segment", e);
} finally {
downloadThreadPool.shutdown();
@ -176,6 +178,7 @@ public class HlsDownload extends AbstractHlsDownload {
@Override
public void postprocess(Recording recording) {
Thread.currentThread().setName("PP " + model.getName());
recording.setStatusWithEvent(State.GENERATING_PLAYLIST);
generatePlaylist(recording);
recording.setStatusWithEvent(State.POST_PROCESSING);
@ -188,7 +191,7 @@ public class HlsDownload extends AbstractHlsDownload {
protected File generatePlaylist(Recording recording) {
File recDir = recording.getAbsoluteFile();
if(!config.getSettings().generatePlaylist) {
if (!config.getSettings().generatePlaylist) {
return null;
}
@ -197,7 +200,7 @@ public class HlsDownload extends AbstractHlsDownload {
try {
File playlist = playlistGenerator.generate(recDir);
if(playlist != null) {
if (playlist != null) {
playlistGenerator.validate(recDir);
}
recording.setProgress(-1);
@ -205,7 +208,7 @@ public class HlsDownload extends AbstractHlsDownload {
} catch (IOException | ParseException e) {
LOG.error("Couldn't generate playlist file", e);
} catch (PlaylistException e) {
if(e.getErrors().isEmpty()) {
if (e.getErrors().isEmpty()) {
LOG.error("Couldn't generate playlist file", e);
} else {
LOG.error("Playlist contains errors");
@ -217,7 +220,7 @@ public class HlsDownload extends AbstractHlsDownload {
LOG.error("Playlist is invalid and will be deleted", e);
File playlist = new File(recDir, "playlist.m3u8");
try {
Files.delete(playlist.toPath());
Files.deleteIfExists(playlist.toPath());
} catch (IOException e1) {
LOG.error("Couldn't delete playlist {}", playlist, e1);
}
@ -227,10 +230,10 @@ public class HlsDownload extends AbstractHlsDownload {
}
private boolean splitRecording() {
if(config.getSettings().splitRecordings > 0) {
if (config.getSettings().splitRecordings > 0) {
Duration recordingDuration = Duration.between(splitRecStartTime, ZonedDateTime.now());
long seconds = recordingDuration.getSeconds();
if(seconds >= config.getSettings().splitRecordings) {
if (seconds >= config.getSettings().splitRecordings) {
internalStop();
return true;
}
@ -275,36 +278,41 @@ public class HlsDownload extends AbstractHlsDownload {
@Override
public Boolean call() throws Exception {
LOG.trace("Downloading segment to {}", file);
int maxTries = 3;
int maxTries = 5;
for (int i = 1; i <= maxTries; i++) {
Request request = new Request.Builder().url(url).addHeader("connection", "keep-alive").build();
Response response = client.execute(request);
InputStream in = null;
try (FileOutputStream fos = new FileOutputStream(file.toFile())) {
in = response.body().byteStream();
if(playlist.encrypted) {
in = new Crypto(playlist.encryptionKeyUrl, client).wrap(in);
}
byte[] b = new byte[1024 * 100];
int length = -1;
while( (length = in.read(b)) >= 0 ) {
fos.write(b, 0, length);
}
return true;
} catch(FileNotFoundException e) {
LOG.debug("Segment does not exist {}", url.getFile());
break;
} catch(Exception e) {
if (i == maxTries) {
LOG.warn("Error while downloading segment. Segment {} finally failed", file.toFile().getName());
try (Response response = client.execute(request)) {
if (response.isSuccessful()) {
InputStream in = null;
try (FileOutputStream fos = new FileOutputStream(file.toFile())) {
in = response.body().byteStream();
if (playlist.encrypted) {
in = new Crypto(playlist.encryptionKeyUrl, client).wrap(in);
}
byte[] b = new byte[1024 * 100];
int length = -1;
while ((length = in.read(b)) >= 0) {
fos.write(b, 0, length);
}
return true;
} catch (FileNotFoundException e) {
LOG.debug("Segment does not exist {}", url.getFile());
break;
} catch (Exception e) {
if (i == maxTries) {
LOG.warn("Error while downloading segment. Segment {} finally failed", file.toFile().getName());
} else {
LOG.warn("Error while downloading segment on try {}", i);
}
}
} else {
LOG.warn("Error while downloading segment on try {}", i);
// wait a bit and retry
try {
Thread.sleep(50 * i);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
}
} finally {
if(in != null) {
in.close();
}
response.close();
}
}
return false;
@ -339,7 +347,7 @@ public class HlsDownload extends AbstractHlsDownload {
private double getPlaylistLength(File playlist) throws IOException, ParseException, PlaylistException {
if (playlist.exists()) {
try(FileInputStream fin = new FileInputStream(playlist)) {
try (FileInputStream fin = new FileInputStream(playlist)) {
PlaylistParser playlistParser = new PlaylistParser(fin, Format.EXT_M3U, Encoding.UTF_8, ParsingMode.LENIENT);
Playlist m3u = playlistParser.parse();
MediaPlaylist mediaPlaylist = m3u.getMediaPlaylist();
@ -353,20 +361,4 @@ public class HlsDownload extends AbstractHlsDownload {
throw new FileNotFoundException(playlist.getAbsolutePath() + " does not exist");
}
}
/**
* Causes the current thread to sleep for a short amount of time.
* This is used to slow down retries, if something is wrong with the playlist.
* E.g. HTTP 403 or 404
*/
private void waitSomeTime(long waitForMillis) {
try {
Thread.sleep(waitForMillis);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
if(running) {
LOG.error("Couldn't sleep. This might mess up the download!");
}
}
}
}

8
common/src/main/java/ctbrec/recorder/download/hls/MergedHlsDownload.java
View File

@ -6,8 +6,6 @@ import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.nio.file.Files;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import java.time.Duration;
import java.time.Instant;
import java.util.Arrays;
@ -18,9 +16,6 @@ import org.jcodec.containers.mp4.boxes.MovieBox;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.iheartradio.m3u8.ParseException;
import com.iheartradio.m3u8.PlaylistException;
import ctbrec.Config;
import ctbrec.Hmac;
import ctbrec.Model;
@ -56,6 +51,7 @@ public class MergedHlsDownload extends HlsDownload {
@Override
public void postprocess(ctbrec.Recording recording) {
Thread.currentThread().setName("PP " + model.getName());
try {
File playlist = super.generatePlaylist(recording);
Objects.requireNonNull(playlist, "Generated playlist is null");
@ -110,7 +106,7 @@ public class MergedHlsDownload extends HlsDownload {
}
public void downloadFinishedRecording(String segmentPlaylistUri, File target, ProgressListener progressListener)
throws IOException, ParseException, PlaylistException, InvalidKeyException, NoSuchAlgorithmException {
throws Exception {
if (Config.getInstance().getSettings().requireAuthentication) {
URL u = new URL(segmentPlaylistUri);
String path = u.getPath();

4
common/src/main/java/ctbrec/sites/jasmin/LiveJasminHlsDownload.java
View File

@ -17,7 +17,7 @@ import ctbrec.recorder.download.hls.HlsDownload;
public class LiveJasminHlsDownload extends HlsDownload {
private static final transient Logger LOG = LoggerFactory.getLogger(LiveJasminHlsDownload.class);
private static final Logger LOG = LoggerFactory.getLogger(LiveJasminHlsDownload.class);
private long lastMasterPlaylistUpdate = 0;
private String segmentUrl;
@ -26,7 +26,7 @@ public class LiveJasminHlsDownload extends HlsDownload {
}
@Override
protected SegmentPlaylist getNextSegments(String segments) throws IOException, ParseException, PlaylistException {
protected SegmentPlaylist getNextSegments(String segments) throws Exception {
if(this.segmentUrl == null) {
this.segmentUrl = segments;
}

4
common/src/main/java/ctbrec/sites/jasmin/LiveJasminMergedHlsDownload.java
View File

@ -17,7 +17,7 @@ import ctbrec.recorder.download.hls.MergedHlsDownload;
public class LiveJasminMergedHlsDownload extends MergedHlsDownload {
private static final transient Logger LOG = LoggerFactory.getLogger(LiveJasminMergedHlsDownload.class);
private static final Logger LOG = LoggerFactory.getLogger(LiveJasminMergedHlsDownload.class);
private long lastMasterPlaylistUpdate = 0;
private String segmentUrl;
@ -26,7 +26,7 @@ public class LiveJasminMergedHlsDownload extends MergedHlsDownload {
}
@Override
protected SegmentPlaylist getNextSegments(String segments) throws IOException, ParseException, PlaylistException {
protected SegmentPlaylist getNextSegments(String segments) throws Exception {
if(this.segmentUrl == null) {
this.segmentUrl = segments;
}