From 43d2676e11b35ec919f2463c4d6d778803a49e8e Mon Sep 17 00:00:00 2001 From: 0xboobface <0xboobface@gmail.com> Date: Fri, 12 Jun 2020 18:24:34 +0200 Subject: [PATCH] Add HttpHeaderFactory mechanism Each model can provide its own factory to provide HTTP headers for the different download requests (playlist, segment playlist, segments) --- .../src/main/java/ctbrec/ui/JavaFxModel.java | 6 +++ .../src/main/java/ctbrec/AbstractModel.java | 12 ++++++ common/src/main/java/ctbrec/Model.java | 3 ++ .../recorder/download/HttpHeaderFactory.java | 10 +++++ .../download/HttpHeaderFactoryImpl.java | 37 +++++++++++++++++++ .../download/hls/AbstractHlsDownload.java | 31 +++++++++++----- .../recorder/download/hls/HlsDownload.java | 14 +++---- .../download/hls/MergedFfmpegHlsDownload.java | 12 +++--- .../ctbrec/sites/mfc/MyFreeCamsModel.java | 20 ++++++++++ 9 files changed, 123 insertions(+), 22 deletions(-) create mode 100644 common/src/main/java/ctbrec/recorder/download/HttpHeaderFactory.java create mode 100644 common/src/main/java/ctbrec/recorder/download/HttpHeaderFactoryImpl.java diff --git a/client/src/main/java/ctbrec/ui/JavaFxModel.java b/client/src/main/java/ctbrec/ui/JavaFxModel.java index cedfedbc..bc404953 100644 --- a/client/src/main/java/ctbrec/ui/JavaFxModel.java +++ b/client/src/main/java/ctbrec/ui/JavaFxModel.java @@ -14,6 +14,7 @@ import com.squareup.moshi.JsonWriter; import ctbrec.Model; import ctbrec.recorder.download.Download; +import ctbrec.recorder.download.HttpHeaderFactory; import ctbrec.recorder.download.StreamSource; import ctbrec.sites.Site; import javafx.beans.property.BooleanProperty; @@ -280,4 +281,9 @@ public class JavaFxModel implements Model { public Download createDownload() { return delegate.createDownload(); } + + @Override + public HttpHeaderFactory getHttpHeaderFactory() { + return delegate.getHttpHeaderFactory(); + } } diff --git a/common/src/main/java/ctbrec/AbstractModel.java b/common/src/main/java/ctbrec/AbstractModel.java index 14c066ad..a9d48b3a 100644 --- a/common/src/main/java/ctbrec/AbstractModel.java +++ b/common/src/main/java/ctbrec/AbstractModel.java @@ -3,6 +3,7 @@ package ctbrec; import java.io.IOException; import java.time.Instant; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Optional; import java.util.concurrent.ExecutionException; @@ -11,6 +12,8 @@ import com.squareup.moshi.JsonReader; import com.squareup.moshi.JsonWriter; import ctbrec.recorder.download.Download; +import ctbrec.recorder.download.HttpHeaderFactory; +import ctbrec.recorder.download.HttpHeaderFactoryImpl; import ctbrec.recorder.download.hls.HlsDownload; import ctbrec.recorder.download.hls.MergedFfmpegHlsDownload; import ctbrec.sites.Site; @@ -236,4 +239,13 @@ public abstract class AbstractModel implements Model { return new MergedFfmpegHlsDownload(getSite().getHttpClient()); } } + + @Override + public HttpHeaderFactory getHttpHeaderFactory() { + HttpHeaderFactoryImpl fac = new HttpHeaderFactoryImpl(); + fac.setMasterPlaylistHeaders(new HashMap<>()); + fac.setSegmentPlaylistHeaders(new HashMap<>()); + fac.setSegmentHeaders(new HashMap<>()); + return fac; + } } diff --git a/common/src/main/java/ctbrec/Model.java b/common/src/main/java/ctbrec/Model.java index 7d0133e6..6fb12ff5 100644 --- a/common/src/main/java/ctbrec/Model.java +++ b/common/src/main/java/ctbrec/Model.java @@ -14,6 +14,7 @@ import com.squareup.moshi.JsonReader; import com.squareup.moshi.JsonWriter; import ctbrec.recorder.download.Download; +import ctbrec.recorder.download.HttpHeaderFactory; import ctbrec.recorder.download.StreamSource; import ctbrec.sites.Site; @@ -125,4 +126,6 @@ public interface Model extends Comparable, Serializable { public int getPriority(); + public HttpHeaderFactory getHttpHeaderFactory(); + } \ No newline at end of file diff --git a/common/src/main/java/ctbrec/recorder/download/HttpHeaderFactory.java b/common/src/main/java/ctbrec/recorder/download/HttpHeaderFactory.java new file mode 100644 index 00000000..041b0032 --- /dev/null +++ b/common/src/main/java/ctbrec/recorder/download/HttpHeaderFactory.java @@ -0,0 +1,10 @@ +package ctbrec.recorder.download; + +import java.util.Map; + +public interface HttpHeaderFactory { + + Map createMasterPlaylistHeaders(); + Map createSegmentPlaylistHeaders(); + Map createSegmentHeaders(); +} diff --git a/common/src/main/java/ctbrec/recorder/download/HttpHeaderFactoryImpl.java b/common/src/main/java/ctbrec/recorder/download/HttpHeaderFactoryImpl.java new file mode 100644 index 00000000..34c9f87e --- /dev/null +++ b/common/src/main/java/ctbrec/recorder/download/HttpHeaderFactoryImpl.java @@ -0,0 +1,37 @@ +package ctbrec.recorder.download; + +import java.util.Map; + +public class HttpHeaderFactoryImpl implements HttpHeaderFactory { + + private Map masterPlaylistHeaders; + private Map segmentPlaylistHeaders; + private Map segmentHeaders; + + @Override + public Map createMasterPlaylistHeaders() { + return masterPlaylistHeaders; + } + + @Override + public Map createSegmentPlaylistHeaders() { + return segmentPlaylistHeaders; + } + + @Override + public Map createSegmentHeaders() { + return segmentHeaders; + } + + public void setMasterPlaylistHeaders(Map masterPlaylistHeaders) { + this.masterPlaylistHeaders = masterPlaylistHeaders; + } + + public void setSegmentPlaylistHeaders(Map segmentPlaylistHeaders) { + this.segmentPlaylistHeaders = segmentPlaylistHeaders; + } + + public void setSegmentHeaders(Map segmentHeaders) { + this.segmentHeaders = segmentHeaders; + } +} diff --git a/common/src/main/java/ctbrec/recorder/download/hls/AbstractHlsDownload.java b/common/src/main/java/ctbrec/recorder/download/hls/AbstractHlsDownload.java index ce885e78..0c5e8a64 100644 --- a/common/src/main/java/ctbrec/recorder/download/hls/AbstractHlsDownload.java +++ b/common/src/main/java/ctbrec/recorder/download/hls/AbstractHlsDownload.java @@ -9,9 +9,12 @@ import java.net.URL; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Locale; +import java.util.Map; +import java.util.Map.Entry; import java.util.Optional; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; @@ -45,9 +48,11 @@ import ctbrec.io.HttpClient; import ctbrec.io.HttpException; import ctbrec.recorder.PlaylistGenerator.InvalidPlaylistException; import ctbrec.recorder.download.AbstractDownload; +import ctbrec.recorder.download.HttpHeaderFactory; import ctbrec.recorder.download.StreamSource; import ctbrec.sites.Site; import okhttp3.Request; +import okhttp3.Request.Builder; import okhttp3.Response; public abstract class AbstractHlsDownload extends AbstractDownload { @@ -78,15 +83,10 @@ public abstract class AbstractHlsDownload extends AbstractDownload { protected SegmentPlaylist getNextSegments(String segmentsURL) throws IOException, ParseException, PlaylistException { URL segmentsUrl = new URL(segmentsURL); - Request request = new Request.Builder() - .url(segmentsUrl) - .header(ACCEPT, "*/*") - .header(ACCEPT_LANGUAGE, Locale.ENGLISH.getLanguage()) - .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) - .header(CONNECTION, KEEP_ALIVE) - .header(ORIGIN, Optional.ofNullable(model).map(Model::getSite).map(Site::getBaseUrl).orElse("")) - .header(REFERER, Optional.ofNullable(model).map(Model::getSite).map(Site::getBaseUrl).orElse("")) - .build(); + Builder builder = new Request.Builder() + .url(segmentsUrl); + addHeaders(builder, Optional.ofNullable(model).map(Model::getHttpHeaderFactory).map(HttpHeaderFactory::createSegmentPlaylistHeaders).orElse(new HashMap<>())); + Request request = builder.build(); try (Response response = client.execute(request)) { if (response.isSuccessful()) { @@ -134,6 +134,19 @@ public abstract class AbstractHlsDownload extends AbstractDownload { } + protected void addHeaders(Builder builder, Map headers) { + headers.putIfAbsent(ACCEPT, "*/*"); + headers.putIfAbsent(ACCEPT_LANGUAGE, Locale.ENGLISH.getLanguage()); + headers.putIfAbsent(USER_AGENT, Config.getInstance().getSettings().httpUserAgent); + headers.putIfAbsent(CONNECTION, KEEP_ALIVE); + headers.computeIfAbsent(ORIGIN, k -> Optional.ofNullable(model).map(Model::getSite).map(Site::getBaseUrl).orElse(null)); + headers.computeIfAbsent(REFERER, k -> Optional.ofNullable(model).map(Model::getSite).map(Site::getBaseUrl).orElse(null)); + + for (Entry header : headers.entrySet()) { + builder.header(header.getKey(), header.getValue()); + } + } + protected String getSegmentPlaylistUrl(Model model) throws IOException, ExecutionException, ParseException, PlaylistException, JAXBException { LOG.debug("{} stream idx: {}", model.getName(), model.getStreamUrlIndex()); List streamSources = model.getStreamSources(); diff --git a/common/src/main/java/ctbrec/recorder/download/hls/HlsDownload.java b/common/src/main/java/ctbrec/recorder/download/hls/HlsDownload.java index 4360e878..ee756947 100644 --- a/common/src/main/java/ctbrec/recorder/download/hls/HlsDownload.java +++ b/common/src/main/java/ctbrec/recorder/download/hls/HlsDownload.java @@ -1,7 +1,5 @@ package ctbrec.recorder.download.hls; -import static ctbrec.io.HttpConstants.*; - import java.io.EOFException; import java.io.File; import java.io.FileInputStream; @@ -21,6 +19,8 @@ import java.time.Instant; import java.time.ZoneId; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; +import java.util.HashMap; +import java.util.Optional; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; import java.util.concurrent.RejectedExecutionException; @@ -49,7 +49,9 @@ import ctbrec.io.BandwidthMeter; import ctbrec.io.HttpClient; import ctbrec.io.HttpException; import ctbrec.recorder.PlaylistGenerator; +import ctbrec.recorder.download.HttpHeaderFactory; import okhttp3.Request; +import okhttp3.Request.Builder; import okhttp3.Response; public class HlsDownload extends AbstractHlsDownload { @@ -293,11 +295,9 @@ public class HlsDownload extends AbstractHlsDownload { public Boolean call() throws Exception { LOG.trace("Downloading segment {} to {}", url, file); for (int tries = 1; tries <= 3 && !Thread.currentThread().isInterrupted(); tries++) { - Request request = new Request.Builder() - .url(url) - .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) - .header(CONNECTION, KEEP_ALIVE) - .build(); + Builder builder = new Request.Builder().url(url); + addHeaders(builder, Optional.ofNullable(model).map(Model::getHttpHeaderFactory).map(HttpHeaderFactory::createSegmentHeaders).orElse(new HashMap<>())); + Request request = builder.build(); InputStream in = null; try (Response response = client.execute(request); FileOutputStream fos = new FileOutputStream(file.toFile())) { if (response.isSuccessful()) { diff --git a/common/src/main/java/ctbrec/recorder/download/hls/MergedFfmpegHlsDownload.java b/common/src/main/java/ctbrec/recorder/download/hls/MergedFfmpegHlsDownload.java index fc71c099..27e1c242 100644 --- a/common/src/main/java/ctbrec/recorder/download/hls/MergedFfmpegHlsDownload.java +++ b/common/src/main/java/ctbrec/recorder/download/hls/MergedFfmpegHlsDownload.java @@ -1,7 +1,5 @@ package ctbrec.recorder.download.hls; -import static ctbrec.io.HttpConstants.*; - import java.io.EOFException; import java.io.File; import java.io.FileOutputStream; @@ -14,6 +12,7 @@ import java.time.Duration; import java.time.Instant; import java.time.ZonedDateTime; import java.util.Arrays; +import java.util.HashMap; import java.util.LinkedList; import java.util.Optional; import java.util.Queue; @@ -39,8 +38,10 @@ import ctbrec.io.HttpClient; import ctbrec.io.HttpException; import ctbrec.io.StreamRedirectThread; import ctbrec.recorder.ProgressListener; +import ctbrec.recorder.download.HttpHeaderFactory; import ctbrec.recorder.download.ProcessExitedUncleanException; import okhttp3.Request; +import okhttp3.Request.Builder; import okhttp3.Response; public class MergedFfmpegHlsDownload extends AbstractHlsDownload { @@ -413,10 +414,9 @@ public class MergedFfmpegHlsDownload extends AbstractHlsDownload { LOG.trace("Downloading segment {}", url.getFile()); int maxTries = 3; for (int i = 1; i <= maxTries && running; i++) { - Request request = new Request.Builder().url(url) - .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) - .header(CONNECTION, KEEP_ALIVE) - .build(); + Builder builder = new Request.Builder().url(url); + addHeaders(builder, Optional.ofNullable(model).map(Model::getHttpHeaderFactory).map(HttpHeaderFactory::createSegmentHeaders).orElse(new HashMap<>())); + Request request = builder.build(); try (Response response = client.execute(request)) { if (response.isSuccessful()) { byte[] segment = response.body().bytes(); diff --git a/common/src/main/java/ctbrec/sites/mfc/MyFreeCamsModel.java b/common/src/main/java/ctbrec/sites/mfc/MyFreeCamsModel.java index 0ff45d69..bc664492 100644 --- a/common/src/main/java/ctbrec/sites/mfc/MyFreeCamsModel.java +++ b/common/src/main/java/ctbrec/sites/mfc/MyFreeCamsModel.java @@ -8,8 +8,10 @@ import java.io.UnsupportedEncodingException; import java.net.URLDecoder; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Locale; +import java.util.Map; import java.util.Objects; import java.util.concurrent.ExecutionException; @@ -29,6 +31,8 @@ import ctbrec.Config; import ctbrec.io.HtmlParser; import ctbrec.io.HttpException; import ctbrec.recorder.download.Download; +import ctbrec.recorder.download.HttpHeaderFactory; +import ctbrec.recorder.download.HttpHeaderFactoryImpl; import ctbrec.recorder.download.StreamSource; import okhttp3.FormBody; import okhttp3.Request; @@ -345,4 +349,20 @@ public class MyFreeCamsModel extends AbstractModel { // return new MyFreeCamsWebrtcDownload(uid, streamUrl, ((MyFreeCams)site).getClient()); // } } + + @Override + public HttpHeaderFactory getHttpHeaderFactory() { + HttpHeaderFactoryImpl fac = new HttpHeaderFactoryImpl(); + Map headers = new HashMap<>(); + headers.put(ACCEPT, "*/*"); + headers.put(ACCEPT_LANGUAGE, Locale.ENGLISH.getLanguage()); + headers.put(CONNECTION, KEEP_ALIVE); + headers.put(ORIGIN, getSite().getBaseUrl()); + headers.put(REFERER, getSite().getBaseUrl()); + headers.put(USER_AGENT, Config.getInstance().getSettings().httpUserAgent); + fac.setMasterPlaylistHeaders(headers); + fac.setSegmentPlaylistHeaders(headers); + fac.setSegmentHeaders(headers); + return fac; + } }