ctbrecuser
/
ctbrec
forked from j62/ctbrec
1
0
Fork 0
Code Pull Requests Activity

Introduce new HttpConstants class for header constants etc

This commit is contained in:
0xboobface 2019-12-27 14:27:10 +01:00
parent 4f852bd5f3
commit b91bfdc353
3 changed files with 61 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
common/src/main/java/ctbrec/io/HttpConstants.java Normal file
View File

@ -0,0 +1,17 @@
package ctbrec.io;
public class HttpConstants {
public static final String ACCEPT = "Accept";
public static final String ACCEPT_LANGUAGE = "Accept-Language";
public static final String CONNECTION = "connection";
public static final String KEEP_ALIVE = "keep-alive";
public static final String MIMETYPE_APPLICATION_JSON = "application/json";
public static final String REFERER = "Referer";
public static final String USER_AGENT = "User-Agent";
public static final String XML_HTTP_REQUEST = "XMLHttpRequest";
public static final String X_CSRF_TOKEN = "X-CSRF-Token";
public static final String X_REQUESTED_WITH = "X-Requested-With";
private HttpConstants() {}
}

5
common/src/main/java/ctbrec/recorder/download/hls/AbstractHlsDownload.java
View File

@ -8,7 +8,6 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.LinkedBlockingQueue;
@ -52,7 +51,7 @@ public abstract class AbstractHlsDownload extends AbstractDownload {
protected HttpClient client;
protected volatile boolean running = false;
protected Model model = new UnknownModel();
protected BlockingQueue<Runnable> downloadQueue = new LinkedBlockingQueue<>(50);
protected LinkedBlockingQueue<Runnable> downloadQueue = new LinkedBlockingQueue<>(50);
protected ExecutorService downloadThreadPool = new ThreadPoolExecutor(5, 5, 2, TimeUnit.MINUTES, downloadQueue, createThreadFactory());
protected State state = State.UNKNOWN;
private int playlistEmptyCount = 0;
@ -124,7 +123,7 @@ public abstract class AbstractHlsDownload extends AbstractDownload {
throw new HttpException(response.code(), response.message());
}
} catch (Exception e) {
LOG.debug("Couldn't download HLS playlist (try {}) {} - {}", tries, segmentsURL, e.getMessage());
LOG.debug("Couldn't download HLS playlist (try {}) {} - {}", tries, e.getMessage(), segmentsURL);
lastException = e;
}
waitSomeTime(100 * tries);

74
common/src/main/java/ctbrec/recorder/download/hls/HlsDownload.java
View File

@ -1,5 +1,7 @@
package ctbrec.recorder.download.hls;
import static ctbrec.io.HttpConstants.*;
import java.io.EOFException;
import java.io.File;
import java.io.FileInputStream;
@ -19,6 +21,7 @@ import java.time.ZoneId;
import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
import java.util.concurrent.Callable;
import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.regex.Pattern;
@ -109,7 +112,20 @@ public class HlsDownload extends AbstractHlsDownload {
URL segmentUrl = new URL(segment);
String prefix = nf.format(segmentCounter++);
SegmentDownload segmentDownload = new SegmentDownload(playlist, segmentUrl, downloadDir, client, prefix);
downloadThreadPool.submit(segmentDownload);
try {
downloadThreadPool.submit(segmentDownload);
if (downloadQueue.remainingCapacity() < 10) {
LOG.debug("space left in queue {}", downloadQueue.remainingCapacity());
// if the queue is running full, we might be struggling with timeouts
// let's check, if the model is still online
if (!model.isOnline(true)) {
downloadQueue.clear();
internalStop();
}
}
} catch(RejectedExecutionException e) {
LOG.warn("Download queue is full ({}). Skipping segment {} {}", downloadQueue.size(), prefix, segmentUrl);
}
}
}
@ -278,40 +294,34 @@ public class HlsDownload extends AbstractHlsDownload {
@Override
public Boolean call() throws Exception {
LOG.trace("Downloading segment to {}", file);
int maxTries = 5;
for (int i = 1; i <= maxTries; i++) {
Request request = new Request.Builder().url(url).addHeader("connection", "keep-alive").build();
try (Response response = client.execute(request)) {
for (int tries = 1; tries <= 3; tries++) {
Request request = new Request.Builder()
.url(url)
.header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
.header(CONNECTION, KEEP_ALIVE)
.build();
InputStream in = null;
try (Response response = client.execute(request); FileOutputStream fos = new FileOutputStream(file.toFile())) {
if (response.isSuccessful()) {
InputStream in = null;
try (FileOutputStream fos = new FileOutputStream(file.toFile())) {
in = response.body().byteStream();
if (playlist.encrypted) {
in = new Crypto(playlist.encryptionKeyUrl, client).wrap(in);
}
byte[] b = new byte[1024 * 100];
int length = -1;
while ((length = in.read(b)) >= 0) {
fos.write(b, 0, length);
}
return true;
} catch (FileNotFoundException e) {
LOG.debug("Segment does not exist {}", url.getFile());
break;
} catch (Exception e) {
if (i == maxTries) {
LOG.warn("Error while downloading segment. Segment {} finally failed", file.toFile().getName());
} else {
LOG.warn("Error while downloading segment on try {}", i);
}
in = response.body().byteStream();
if(playlist.encrypted) {
in = new Crypto(playlist.encryptionKeyUrl, client).wrap(in);
}
byte[] b = new byte[1024 * 100];
int length = -1;
while( (length = in.read(b)) >= 0 ) {
fos.write(b, 0, length);
}
return true;
}
} catch(FileNotFoundException e) {
LOG.debug("Segment does not exist {}", url.getFile());
break;
} catch(Exception e) {
if (tries == 3) {
LOG.warn("Error while downloading segment. Segment {} finally failed: {}", file.toFile().getName(), e.getMessage());
} else {
// wait a bit and retry
try {
Thread.sleep(50 * i);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
LOG.debug("Error while downloading segment on try {} - {}", tries, e.getMessage());
}
}
}