From abffa14f8df2164916d956970bb15523e0c9584e Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Tue, 12 Oct 2021 19:58:33 +0200 Subject: [PATCH] Fix AmateurTV downloads. They switched from HLS segments to a MP4 stream --- .../sites/amateurtv/AmateurTvDownload.java | 170 ++++++++++++++++++ .../sites/amateurtv/AmateurTvModel.java | 90 ++++------ 2 files changed, 207 insertions(+), 53 deletions(-) create mode 100644 common/src/main/java/ctbrec/sites/amateurtv/AmateurTvDownload.java diff --git a/common/src/main/java/ctbrec/sites/amateurtv/AmateurTvDownload.java b/common/src/main/java/ctbrec/sites/amateurtv/AmateurTvDownload.java new file mode 100644 index 00000000..74c601d0 --- /dev/null +++ b/common/src/main/java/ctbrec/sites/amateurtv/AmateurTvDownload.java @@ -0,0 +1,170 @@ +package ctbrec.sites.amateurtv; + +import ctbrec.Config; +import ctbrec.Model; +import ctbrec.Recording; +import ctbrec.io.BandwidthMeter; +import ctbrec.io.HttpClient; +import ctbrec.io.HttpException; +import ctbrec.recorder.download.AbstractDownload; +import ctbrec.recorder.download.Download; +import ctbrec.recorder.download.StreamSource; +import okhttp3.Request; +import okhttp3.Response; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.time.Duration; +import java.time.Instant; +import java.util.Objects; +import java.util.concurrent.ExecutorService; +import java.util.regex.Pattern; + +import static ctbrec.io.HttpConstants.*; + +public class AmateurTvDownload extends AbstractDownload { + + private static final Logger LOG = LoggerFactory.getLogger(AmateurTvDownload.class); + private static final int MAX_SECONDS_WITHOUT_TRANSFER = 20; + + private final HttpClient httpClient; + private FileOutputStream fout; + private Instant timeOfLastTransfer = Instant.MAX; + + private volatile boolean running; + private volatile boolean started; + + private File targetFile; + + public AmateurTvDownload(HttpClient httpClient) { + this.httpClient = httpClient; + } + + @Override + public void init(Config config, Model model, Instant startTime, ExecutorService executorService) throws IOException { + this.config = config; + this.model = model; + this.startTime = startTime; + this.downloadExecutor = executorService; + splittingStrategy = initSplittingStrategy(config.getSettings()); + targetFile = config.getFileForRecording(model, "mp4", startTime); + timeOfLastTransfer = Instant.now(); + } + + @Override + public void stop() { + running = false; + } + + @Override + public void finalizeDownload() { + if (fout != null) { + try { + LOG.debug("Closing recording file {}", targetFile); + fout.close(); + } catch (IOException e) { + LOG.error("Error while closing recording file {}", targetFile, e); + } + } + } + + @Override + public boolean isRunning() { + return running; + } + + @Override + public void postprocess(Recording recording) { + // nothing to do + } + + @Override + public File getTarget() { + return targetFile; + } + + @Override + public String getPath(Model model) { + String absolutePath = targetFile.getAbsolutePath(); + String recordingsDir = Config.getInstance().getSettings().recordingsDir; + String relativePath = absolutePath.replaceFirst(Pattern.quote(recordingsDir), ""); + return relativePath; + } + + @Override + public boolean isSingleFile() { + return true; + } + + @Override + public long getSizeInByte() { + return getTarget().length(); + } + + @Override + public Download call() throws Exception { + if (!started) { + started = true; + startDownload(); + } + + if (splittingStrategy.splitNecessary(this)) { + stop(); + rescheduleTime = Instant.now(); + } else { + rescheduleTime = Instant.now().plusSeconds(5); + } + if (!model.isOnline(true)) { + stop(); + } + if (Duration.between(timeOfLastTransfer, Instant.now()).getSeconds() > MAX_SECONDS_WITHOUT_TRANSFER) { + LOG.info("No video data received for {} seconds. Stopping recording for model {}", MAX_SECONDS_WITHOUT_TRANSFER, model); + stop(); + } + return this; + } + + private void startDownload() { + downloadExecutor.submit(() -> { + running = true; + try { + StreamSource src = model.getStreamSources().get(0); + LOG.debug("Loading video from {}", src.mediaPlaylistUrl); + Request request = new Request.Builder() + .url(src.mediaPlaylistUrl) + .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) + .header(ACCEPT, "*/*") + .header(ACCEPT_LANGUAGE, "en") + .header(ORIGIN, AmateurTv.baseUrl) + .build(); + try (Response resp = httpClient.execute(request)) { + if (resp.isSuccessful()) { + LOG.debug("Recording video stream to {}", targetFile); + Files.createDirectories(targetFile.getParentFile().toPath()); + fout = new FileOutputStream(targetFile); + + InputStream in = Objects.requireNonNull(resp.body()).byteStream(); + byte[] b = new byte[1024]; + int len; + while (running && !Thread.currentThread().isInterrupted() && (len = in.read(b)) >= 0) { + fout.write(b, 0, len); + timeOfLastTransfer = Instant.now(); + BandwidthMeter.add(len); + } + } else { + throw new HttpException(resp.code(), resp.message()); + } + } + } catch (Exception e) { + LOG.error("Error while downloading MP4", e); + } + running = false; + }); + } + +} diff --git a/common/src/main/java/ctbrec/sites/amateurtv/AmateurTvModel.java b/common/src/main/java/ctbrec/sites/amateurtv/AmateurTvModel.java index 661e5372..a2b1360c 100644 --- a/common/src/main/java/ctbrec/sites/amateurtv/AmateurTvModel.java +++ b/common/src/main/java/ctbrec/sites/amateurtv/AmateurTvModel.java @@ -1,8 +1,23 @@ package ctbrec.sites.amateurtv; -import static ctbrec.Model.State.*; -import static ctbrec.io.HttpConstants.*; +import com.iheartradio.m3u8.*; +import com.iheartradio.m3u8.data.MediaPlaylist; +import com.iheartradio.m3u8.data.Playlist; +import ctbrec.AbstractModel; +import ctbrec.Config; +import ctbrec.io.HttpClient; +import ctbrec.io.HttpException; +import ctbrec.recorder.download.Download; +import ctbrec.recorder.download.StreamSource; +import okhttp3.FormBody; +import okhttp3.Request; +import okhttp3.RequestBody; +import okhttp3.Response; +import org.json.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import javax.xml.bind.JAXBException; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; @@ -11,33 +26,8 @@ import java.util.Locale; import java.util.Objects; import java.util.concurrent.ExecutionException; -import javax.xml.bind.JAXBException; - -import org.json.JSONObject; -import org.jsoup.nodes.Element; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.iheartradio.m3u8.Encoding; -import com.iheartradio.m3u8.Format; -import com.iheartradio.m3u8.ParseException; -import com.iheartradio.m3u8.ParsingMode; -import com.iheartradio.m3u8.PlaylistException; -import com.iheartradio.m3u8.PlaylistParser; -import com.iheartradio.m3u8.data.MasterPlaylist; -import com.iheartradio.m3u8.data.Playlist; -import com.iheartradio.m3u8.data.PlaylistData; -import com.iheartradio.m3u8.data.StreamInfo; - -import ctbrec.AbstractModel; -import ctbrec.Config; -import ctbrec.io.HttpClient; -import ctbrec.io.HttpException; -import ctbrec.recorder.download.StreamSource; -import okhttp3.FormBody; -import okhttp3.Request; -import okhttp3.RequestBody; -import okhttp3.Response; +import static ctbrec.Model.State.*; +import static ctbrec.io.HttpConstants.*; public class AmateurTvModel extends AbstractModel { @@ -79,28 +69,17 @@ public class AmateurTvModel extends AbstractModel { Request req = new Request.Builder().url(streamUrl).build(); try (Response response = site.getHttpClient().execute(req)) { if (response.isSuccessful()) { - InputStream inputStream = response.body().byteStream(); + InputStream inputStream = Objects.requireNonNull(response.body()).byteStream(); PlaylistParser parser = new PlaylistParser(inputStream, Format.EXT_M3U, Encoding.UTF_8, ParsingMode.LENIENT); Playlist playlist = parser.parse(); - MasterPlaylist master = playlist.getMasterPlaylist(); - for (PlaylistData playlistData : master.getPlaylists()) { - StreamSource streamsource = new StreamSource(); - Element img = new Element("img"); - img.setBaseUri(streamUrl); - img.attr("src", playlistData.getUri()); - streamsource.mediaPlaylistUrl = img.absUrl("src"); - if (playlistData.hasStreamInfo()) { - StreamInfo info = playlistData.getStreamInfo(); - streamsource.bandwidth = info.getBandwidth(); - streamsource.width = info.hasResolution() ? info.getResolution().width : 0; - streamsource.height = info.hasResolution() ? info.getResolution().height : 0; - } else { - streamsource.bandwidth = 0; - streamsource.width = 0; - streamsource.height = 0; - } - streamSources.add(streamsource); - } + MediaPlaylist media = playlist.getMediaPlaylist(); + String baseUrl = streamUrl.substring(0, streamUrl.lastIndexOf('/') + 1); + String vodUri = baseUrl + media.getTracks().get(0).getUri(); + StreamSource streamsource = new StreamSource(); + streamsource.mediaPlaylistUrl = vodUri; + streamsource.width = 0; + streamsource.height = 0; + streamSources.add(streamsource); } else { throw new HttpException(response.code(), response.message()); } @@ -111,7 +90,7 @@ public class AmateurTvModel extends AbstractModel { private String getStreamUrl() throws IOException { JSONObject json = getModelInfo(); JSONObject videoTech = json.getJSONObject("videoTechnologies"); - return videoTech.getString("hlsV2"); + return videoTech.getString("fmp4-hls"); } @Override @@ -127,7 +106,7 @@ public class AmateurTvModel extends AbstractModel { @Override public int[] getStreamResolution(boolean failFast) throws ExecutionException { try { - return new int[] { getStreamSources().get(0).width, getStreamSources().get(0).height }; + return new int[]{getStreamSources().get(0).width, getStreamSources().get(0).height}; } catch (Exception e) { throw new ExecutionException(e); } @@ -146,7 +125,7 @@ public class AmateurTvModel extends AbstractModel { } private boolean followUnfollow(String url) throws IOException { - if(!getSite().login()) { + if (!getSite().login()) { throw new IOException("Not logged in"); } @@ -165,7 +144,7 @@ public class AmateurTvModel extends AbstractModel { .build(); try (Response resp = site.getHttpClient().execute(req)) { if (resp.isSuccessful()) { - String msg = resp.body().string(); + String msg = Objects.requireNonNull(resp.body()).string(); JSONObject json = new JSONObject(msg); if (Objects.equals(json.getString("result"), "OK")) { LOG.debug("Follow/Unfollow -> {}", msg); @@ -193,4 +172,9 @@ public class AmateurTvModel extends AbstractModel { return json; } } + + @Override + public Download createDownload() { + return new AmateurTvDownload(getSite().getHttpClient()); + } }