diff --git a/common/src/main/java/ctbrec/io/HttpConstants.java b/common/src/main/java/ctbrec/io/HttpConstants.java new file mode 100644 index 00000000..72436eb5 --- /dev/null +++ b/common/src/main/java/ctbrec/io/HttpConstants.java @@ -0,0 +1,17 @@ +package ctbrec.io; + +public class HttpConstants { + + public static final String ACCEPT = "Accept"; + public static final String ACCEPT_LANGUAGE = "Accept-Language"; + public static final String CONNECTION = "connection"; + public static final String KEEP_ALIVE = "keep-alive"; + public static final String MIMETYPE_APPLICATION_JSON = "application/json"; + public static final String REFERER = "Referer"; + public static final String USER_AGENT = "User-Agent"; + public static final String XML_HTTP_REQUEST = "XMLHttpRequest"; + public static final String X_CSRF_TOKEN = "X-CSRF-Token"; + public static final String X_REQUESTED_WITH = "X-Requested-With"; + + private HttpConstants() {} +} diff --git a/common/src/main/java/ctbrec/recorder/download/hls/AbstractHlsDownload.java b/common/src/main/java/ctbrec/recorder/download/hls/AbstractHlsDownload.java index 9ed1c495..4f056a43 100644 --- a/common/src/main/java/ctbrec/recorder/download/hls/AbstractHlsDownload.java +++ b/common/src/main/java/ctbrec/recorder/download/hls/AbstractHlsDownload.java @@ -8,7 +8,6 @@ import java.util.ArrayList; import java.util.Collections; import java.util.Iterator; import java.util.List; -import java.util.concurrent.BlockingQueue; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.LinkedBlockingQueue; @@ -52,7 +51,7 @@ public abstract class AbstractHlsDownload extends AbstractDownload { protected HttpClient client; protected volatile boolean running = false; protected Model model = new UnknownModel(); - protected BlockingQueue downloadQueue = new LinkedBlockingQueue<>(50); + protected LinkedBlockingQueue downloadQueue = new LinkedBlockingQueue<>(50); protected ExecutorService downloadThreadPool = new ThreadPoolExecutor(5, 5, 2, TimeUnit.MINUTES, downloadQueue, createThreadFactory()); protected State state = State.UNKNOWN; private int playlistEmptyCount = 0; @@ -124,7 +123,7 @@ public abstract class AbstractHlsDownload extends AbstractDownload { throw new HttpException(response.code(), response.message()); } } catch (Exception e) { - LOG.debug("Couldn't download HLS playlist (try {}) {} - {}", tries, segmentsURL, e.getMessage()); + LOG.debug("Couldn't download HLS playlist (try {}) {} - {}", tries, e.getMessage(), segmentsURL); lastException = e; } waitSomeTime(100 * tries); diff --git a/common/src/main/java/ctbrec/recorder/download/hls/HlsDownload.java b/common/src/main/java/ctbrec/recorder/download/hls/HlsDownload.java index 2a8af237..e7c53b7e 100644 --- a/common/src/main/java/ctbrec/recorder/download/hls/HlsDownload.java +++ b/common/src/main/java/ctbrec/recorder/download/hls/HlsDownload.java @@ -1,5 +1,7 @@ package ctbrec.recorder.download.hls; +import static ctbrec.io.HttpConstants.*; + import java.io.EOFException; import java.io.File; import java.io.FileInputStream; @@ -19,6 +21,7 @@ import java.time.ZoneId; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.util.concurrent.Callable; +import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.TimeUnit; import java.util.regex.Pattern; @@ -109,7 +112,20 @@ public class HlsDownload extends AbstractHlsDownload { URL segmentUrl = new URL(segment); String prefix = nf.format(segmentCounter++); SegmentDownload segmentDownload = new SegmentDownload(playlist, segmentUrl, downloadDir, client, prefix); - downloadThreadPool.submit(segmentDownload); + try { + downloadThreadPool.submit(segmentDownload); + if (downloadQueue.remainingCapacity() < 10) { + LOG.debug("space left in queue {}", downloadQueue.remainingCapacity()); + // if the queue is running full, we might be struggling with timeouts + // let's check, if the model is still online + if (!model.isOnline(true)) { + downloadQueue.clear(); + internalStop(); + } + } + } catch(RejectedExecutionException e) { + LOG.warn("Download queue is full ({}). Skipping segment {} {}", downloadQueue.size(), prefix, segmentUrl); + } } } @@ -278,40 +294,34 @@ public class HlsDownload extends AbstractHlsDownload { @Override public Boolean call() throws Exception { LOG.trace("Downloading segment to {}", file); - int maxTries = 5; - for (int i = 1; i <= maxTries; i++) { - Request request = new Request.Builder().url(url).addHeader("connection", "keep-alive").build(); - try (Response response = client.execute(request)) { + for (int tries = 1; tries <= 3; tries++) { + Request request = new Request.Builder() + .url(url) + .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) + .header(CONNECTION, KEEP_ALIVE) + .build(); + InputStream in = null; + try (Response response = client.execute(request); FileOutputStream fos = new FileOutputStream(file.toFile())) { if (response.isSuccessful()) { - InputStream in = null; - try (FileOutputStream fos = new FileOutputStream(file.toFile())) { - in = response.body().byteStream(); - if (playlist.encrypted) { - in = new Crypto(playlist.encryptionKeyUrl, client).wrap(in); - } - byte[] b = new byte[1024 * 100]; - int length = -1; - while ((length = in.read(b)) >= 0) { - fos.write(b, 0, length); - } - return true; - } catch (FileNotFoundException e) { - LOG.debug("Segment does not exist {}", url.getFile()); - break; - } catch (Exception e) { - if (i == maxTries) { - LOG.warn("Error while downloading segment. Segment {} finally failed", file.toFile().getName()); - } else { - LOG.warn("Error while downloading segment on try {}", i); - } + in = response.body().byteStream(); + if(playlist.encrypted) { + in = new Crypto(playlist.encryptionKeyUrl, client).wrap(in); } + byte[] b = new byte[1024 * 100]; + int length = -1; + while( (length = in.read(b)) >= 0 ) { + fos.write(b, 0, length); + } + return true; + } + } catch(FileNotFoundException e) { + LOG.debug("Segment does not exist {}", url.getFile()); + break; + } catch(Exception e) { + if (tries == 3) { + LOG.warn("Error while downloading segment. Segment {} finally failed: {}", file.toFile().getName(), e.getMessage()); } else { - // wait a bit and retry - try { - Thread.sleep(50 * i); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } + LOG.debug("Error while downloading segment on try {} - {}", tries, e.getMessage()); } } }