jafea7
/
ctbrec-5.3.2-experimental
mirror of https://github.com/Jafea7/ctbrec-5.3.2-original.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add HttpHeaderFactory mechanism 

Each model can provide its own factory to provide HTTP headers for the
different download requests (playlist, segment playlist, segments)
This commit is contained in:
0xboobface 2020-06-12 18:24:34 +02:00
parent 06029a1331
commit 43d2676e11
9 changed files with 123 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
client/src/main/java/ctbrec/ui/JavaFxModel.java
View File

@ -14,6 +14,7 @@ import com.squareup.moshi.JsonWriter;
import ctbrec.Model;
import ctbrec.recorder.download.Download;
import ctbrec.recorder.download.HttpHeaderFactory;
import ctbrec.recorder.download.StreamSource;
import ctbrec.sites.Site;
import javafx.beans.property.BooleanProperty;
@ -280,4 +281,9 @@ public class JavaFxModel implements Model {
public Download createDownload() {
return delegate.createDownload();
}
@Override
public HttpHeaderFactory getHttpHeaderFactory() {
return delegate.getHttpHeaderFactory();
}
}

12
common/src/main/java/ctbrec/AbstractModel.java
View File

@ -3,6 +3,7 @@ package ctbrec;
import java.io.IOException;
import java.time.Instant;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.ExecutionException;
@ -11,6 +12,8 @@ import com.squareup.moshi.JsonReader;
import com.squareup.moshi.JsonWriter;
import ctbrec.recorder.download.Download;
import ctbrec.recorder.download.HttpHeaderFactory;
import ctbrec.recorder.download.HttpHeaderFactoryImpl;
import ctbrec.recorder.download.hls.HlsDownload;
import ctbrec.recorder.download.hls.MergedFfmpegHlsDownload;
import ctbrec.sites.Site;
@ -236,4 +239,13 @@ public abstract class AbstractModel implements Model {
return new MergedFfmpegHlsDownload(getSite().getHttpClient());
}
}
@Override
public HttpHeaderFactory getHttpHeaderFactory() {
HttpHeaderFactoryImpl fac = new HttpHeaderFactoryImpl();
fac.setMasterPlaylistHeaders(new HashMap<>());
fac.setSegmentPlaylistHeaders(new HashMap<>());
fac.setSegmentHeaders(new HashMap<>());
return fac;
}
}

3
common/src/main/java/ctbrec/Model.java
View File

@ -14,6 +14,7 @@ import com.squareup.moshi.JsonReader;
import com.squareup.moshi.JsonWriter;
import ctbrec.recorder.download.Download;
import ctbrec.recorder.download.HttpHeaderFactory;
import ctbrec.recorder.download.StreamSource;
import ctbrec.sites.Site;
@ -125,4 +126,6 @@ public interface Model extends Comparable<Model>, Serializable {
public int getPriority();
public HttpHeaderFactory getHttpHeaderFactory();
}

10
common/src/main/java/ctbrec/recorder/download/HttpHeaderFactory.java Normal file
View File

@ -0,0 +1,10 @@
package ctbrec.recorder.download;
import java.util.Map;
public interface HttpHeaderFactory {
Map<String, String> createMasterPlaylistHeaders();
Map<String, String> createSegmentPlaylistHeaders();
Map<String, String> createSegmentHeaders();
}

37
common/src/main/java/ctbrec/recorder/download/HttpHeaderFactoryImpl.java Normal file
View File

@ -0,0 +1,37 @@
package ctbrec.recorder.download;
import java.util.Map;
public class HttpHeaderFactoryImpl implements HttpHeaderFactory {
private Map<String, String> masterPlaylistHeaders;
private Map<String, String> segmentPlaylistHeaders;
private Map<String, String> segmentHeaders;
@Override
public Map<String, String> createMasterPlaylistHeaders() {
return masterPlaylistHeaders;
}
@Override
public Map<String, String> createSegmentPlaylistHeaders() {
return segmentPlaylistHeaders;
}
@Override
public Map<String, String> createSegmentHeaders() {
return segmentHeaders;
}
public void setMasterPlaylistHeaders(Map<String, String> masterPlaylistHeaders) {
this.masterPlaylistHeaders = masterPlaylistHeaders;
}
public void setSegmentPlaylistHeaders(Map<String, String> segmentPlaylistHeaders) {
this.segmentPlaylistHeaders = segmentPlaylistHeaders;
}
public void setSegmentHeaders(Map<String, String> segmentHeaders) {
this.segmentHeaders = segmentHeaders;
}
}

31
common/src/main/java/ctbrec/recorder/download/hls/AbstractHlsDownload.java
View File

@ -9,9 +9,12 @@ import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Optional;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
@ -45,9 +48,11 @@ import ctbrec.io.HttpClient;
import ctbrec.io.HttpException;
import ctbrec.recorder.PlaylistGenerator.InvalidPlaylistException;
import ctbrec.recorder.download.AbstractDownload;
import ctbrec.recorder.download.HttpHeaderFactory;
import ctbrec.recorder.download.StreamSource;
import ctbrec.sites.Site;
import okhttp3.Request;
import okhttp3.Request.Builder;
import okhttp3.Response;
public abstract class AbstractHlsDownload extends AbstractDownload {
@ -78,15 +83,10 @@ public abstract class AbstractHlsDownload extends AbstractDownload {
protected SegmentPlaylist getNextSegments(String segmentsURL) throws IOException, ParseException, PlaylistException {
URL segmentsUrl = new URL(segmentsURL);
Request request = new Request.Builder()
.url(segmentsUrl)
.header(ACCEPT, "*/*")
.header(ACCEPT_LANGUAGE, Locale.ENGLISH.getLanguage())
.header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
.header(CONNECTION, KEEP_ALIVE)
.header(ORIGIN, Optional.ofNullable(model).map(Model::getSite).map(Site::getBaseUrl).orElse(""))
.header(REFERER, Optional.ofNullable(model).map(Model::getSite).map(Site::getBaseUrl).orElse(""))
.build();
Builder builder = new Request.Builder()
.url(segmentsUrl);
addHeaders(builder, Optional.ofNullable(model).map(Model::getHttpHeaderFactory).map(HttpHeaderFactory::createSegmentPlaylistHeaders).orElse(new HashMap<>()));
Request request = builder.build();
try (Response response = client.execute(request)) {
if (response.isSuccessful()) {
@ -134,6 +134,19 @@ public abstract class AbstractHlsDownload extends AbstractDownload {
}
protected void addHeaders(Builder builder, Map<String, String> headers) {
headers.putIfAbsent(ACCEPT, "*/*");
headers.putIfAbsent(ACCEPT_LANGUAGE, Locale.ENGLISH.getLanguage());
headers.putIfAbsent(USER_AGENT, Config.getInstance().getSettings().httpUserAgent);
headers.putIfAbsent(CONNECTION, KEEP_ALIVE);
headers.computeIfAbsent(ORIGIN, k -> Optional.ofNullable(model).map(Model::getSite).map(Site::getBaseUrl).orElse(null));
headers.computeIfAbsent(REFERER, k -> Optional.ofNullable(model).map(Model::getSite).map(Site::getBaseUrl).orElse(null));
for (Entry<String, String> header : headers.entrySet()) {
builder.header(header.getKey(), header.getValue());
}
}
protected String getSegmentPlaylistUrl(Model model) throws IOException, ExecutionException, ParseException, PlaylistException, JAXBException {
LOG.debug("{} stream idx: {}", model.getName(), model.getStreamUrlIndex());
List<StreamSource> streamSources = model.getStreamSources();

14
common/src/main/java/ctbrec/recorder/download/hls/HlsDownload.java
View File

@ -1,7 +1,5 @@
package ctbrec.recorder.download.hls;
import static ctbrec.io.HttpConstants.*;
import java.io.EOFException;
import java.io.File;
import java.io.FileInputStream;
@ -21,6 +19,8 @@ import java.time.Instant;
import java.time.ZoneId;
import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
import java.util.HashMap;
import java.util.Optional;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.RejectedExecutionException;
@ -49,7 +49,9 @@ import ctbrec.io.BandwidthMeter;
import ctbrec.io.HttpClient;
import ctbrec.io.HttpException;
import ctbrec.recorder.PlaylistGenerator;
import ctbrec.recorder.download.HttpHeaderFactory;
import okhttp3.Request;
import okhttp3.Request.Builder;
import okhttp3.Response;
public class HlsDownload extends AbstractHlsDownload {
@ -293,11 +295,9 @@ public class HlsDownload extends AbstractHlsDownload {
public Boolean call() throws Exception {
LOG.trace("Downloading segment {} to {}", url, file);
for (int tries = 1; tries <= 3 && !Thread.currentThread().isInterrupted(); tries++) {
Request request = new Request.Builder()
.url(url)
.header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
.header(CONNECTION, KEEP_ALIVE)
.build();
Builder builder = new Request.Builder().url(url);
addHeaders(builder, Optional.ofNullable(model).map(Model::getHttpHeaderFactory).map(HttpHeaderFactory::createSegmentHeaders).orElse(new HashMap<>()));
Request request = builder.build();
InputStream in = null;
try (Response response = client.execute(request); FileOutputStream fos = new FileOutputStream(file.toFile())) {
if (response.isSuccessful()) {

12
common/src/main/java/ctbrec/recorder/download/hls/MergedFfmpegHlsDownload.java
View File

@ -1,7 +1,5 @@
package ctbrec.recorder.download.hls;
import static ctbrec.io.HttpConstants.*;
import java.io.EOFException;
import java.io.File;
import java.io.FileOutputStream;
@ -14,6 +12,7 @@ import java.time.Duration;
import java.time.Instant;
import java.time.ZonedDateTime;
import java.util.Arrays;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.Optional;
import java.util.Queue;
@ -39,8 +38,10 @@ import ctbrec.io.HttpClient;
import ctbrec.io.HttpException;
import ctbrec.io.StreamRedirectThread;
import ctbrec.recorder.ProgressListener;
import ctbrec.recorder.download.HttpHeaderFactory;
import ctbrec.recorder.download.ProcessExitedUncleanException;
import okhttp3.Request;
import okhttp3.Request.Builder;
import okhttp3.Response;
public class MergedFfmpegHlsDownload extends AbstractHlsDownload {
@ -413,10 +414,9 @@ public class MergedFfmpegHlsDownload extends AbstractHlsDownload {
LOG.trace("Downloading segment {}", url.getFile());
int maxTries = 3;
for (int i = 1; i <= maxTries && running; i++) {
Request request = new Request.Builder().url(url)
.header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
.header(CONNECTION, KEEP_ALIVE)
.build();
Builder builder = new Request.Builder().url(url);
addHeaders(builder, Optional.ofNullable(model).map(Model::getHttpHeaderFactory).map(HttpHeaderFactory::createSegmentHeaders).orElse(new HashMap<>()));
Request request = builder.build();
try (Response response = client.execute(request)) {
if (response.isSuccessful()) {
byte[] segment = response.body().bytes();

20
common/src/main/java/ctbrec/sites/mfc/MyFreeCamsModel.java
View File

@ -8,8 +8,10 @@ import java.io.UnsupportedEncodingException;
import java.net.URLDecoder;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.ExecutionException;
@ -29,6 +31,8 @@ import ctbrec.Config;
import ctbrec.io.HtmlParser;
import ctbrec.io.HttpException;
import ctbrec.recorder.download.Download;
import ctbrec.recorder.download.HttpHeaderFactory;
import ctbrec.recorder.download.HttpHeaderFactoryImpl;
import ctbrec.recorder.download.StreamSource;
import okhttp3.FormBody;
import okhttp3.Request;
@ -345,4 +349,20 @@ public class MyFreeCamsModel extends AbstractModel {
// return new MyFreeCamsWebrtcDownload(uid, streamUrl, ((MyFreeCams)site).getClient());
// }
}
@Override
public HttpHeaderFactory getHttpHeaderFactory() {
HttpHeaderFactoryImpl fac = new HttpHeaderFactoryImpl();
Map<String, String> headers = new HashMap<>();
headers.put(ACCEPT, "*/*");
headers.put(ACCEPT_LANGUAGE, Locale.ENGLISH.getLanguage());
headers.put(CONNECTION, KEEP_ALIVE);
headers.put(ORIGIN, getSite().getBaseUrl());
headers.put(REFERER, getSite().getBaseUrl());
headers.put(USER_AGENT, Config.getInstance().getSettings().httpUserAgent);
fac.setMasterPlaylistHeaders(headers);
fac.setSegmentPlaylistHeaders(headers);
fac.setSegmentHeaders(headers);
return fac;
}
}