From 0f5a05bfd7530d77329edd91faa83e56ce707e92 Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Sun, 24 Jan 2021 20:12:52 +0100 Subject: [PATCH] Make Showup downloads work in v4 Also fix the Followed tab --- .../showup/ShowupFollowedUpdateService.java | 5 +- .../ui/sites/showup/ShowupTabProvider.java | 4 +- .../download/hls/AbstractHlsDownload.java | 2 +- .../download/hls/MergedFfmpegHlsDownload.java | 14 +- .../sites/showup/ShowupMergedDownload.java | 127 ++++++++++++++++++ .../java/ctbrec/sites/showup/ShowupModel.java | 9 ++ 6 files changed, 150 insertions(+), 11 deletions(-) diff --git a/client/src/main/java/ctbrec/ui/sites/showup/ShowupFollowedUpdateService.java b/client/src/main/java/ctbrec/ui/sites/showup/ShowupFollowedUpdateService.java index e3d6e4d0..7aac6a75 100644 --- a/client/src/main/java/ctbrec/ui/sites/showup/ShowupFollowedUpdateService.java +++ b/client/src/main/java/ctbrec/ui/sites/showup/ShowupFollowedUpdateService.java @@ -53,7 +53,7 @@ public class ShowupFollowedUpdateService extends PaginatedScheduledService { if (json.optString("status").equalsIgnoreCase("success")) { Map onlineModels = parseOnlineModels(json); return parseFavorites(json).stream() - .filter(m -> onlineModels.containsKey(m.getName()) == showOnline) + .filter(m -> onlineModels.containsKey(m.getUid()) == showOnline) .collect(Collectors.toList()); } else { throw new UnexpectedResponseException("Request was not successful: " + body); @@ -71,9 +71,10 @@ public class ShowupFollowedUpdateService extends PaginatedScheduledService { JSONObject m = list.getJSONObject(i); ShowupModel model = new ShowupModel(); model.setSite(site); + model.setUid(m.optString("fav_uid")); model.setName(m.optString("username")); model.setUrl(site.getBaseUrl() + '/' + model.getName()); - + favorites.add(model); } return favorites; } diff --git a/client/src/main/java/ctbrec/ui/sites/showup/ShowupTabProvider.java b/client/src/main/java/ctbrec/ui/sites/showup/ShowupTabProvider.java index 8e9e7165..a412636e 100644 --- a/client/src/main/java/ctbrec/ui/sites/showup/ShowupTabProvider.java +++ b/client/src/main/java/ctbrec/ui/sites/showup/ShowupTabProvider.java @@ -23,7 +23,9 @@ public class ShowupTabProvider extends TabProvider { tabs.add(createTab("Women", "female")); tabs.add(createTab("Men", "male")); tabs.add(createTab("All", "all")); - tabs.add(new ShowupFollowedTab("Favorites", site)); + ShowupFollowedTab showupFollowedTab = new ShowupFollowedTab("Favorites", site); + showupFollowedTab.setRecorder(site.getRecorder()); + tabs.add(showupFollowedTab); return tabs; } diff --git a/common/src/main/java/ctbrec/recorder/download/hls/AbstractHlsDownload.java b/common/src/main/java/ctbrec/recorder/download/hls/AbstractHlsDownload.java index acb4b55d..6a02177c 100644 --- a/common/src/main/java/ctbrec/recorder/download/hls/AbstractHlsDownload.java +++ b/common/src/main/java/ctbrec/recorder/download/hls/AbstractHlsDownload.java @@ -164,7 +164,7 @@ public abstract class AbstractHlsDownload extends AbstractDownload { }); } - private void handleHttpException(HttpException e) throws IOException { + protected void handleHttpException(HttpException e) throws IOException { if (e.getResponseCode() == 404) { checkIfModelIsStillOnline("Playlist not found (404). Model {} probably went offline. Model state: {}"); } else if (e.getResponseCode() == 403) { diff --git a/common/src/main/java/ctbrec/recorder/download/hls/MergedFfmpegHlsDownload.java b/common/src/main/java/ctbrec/recorder/download/hls/MergedFfmpegHlsDownload.java index 01b72d17..9eb892a0 100644 --- a/common/src/main/java/ctbrec/recorder/download/hls/MergedFfmpegHlsDownload.java +++ b/common/src/main/java/ctbrec/recorder/download/hls/MergedFfmpegHlsDownload.java @@ -37,12 +37,12 @@ public class MergedFfmpegHlsDownload extends AbstractHlsDownload { private static final Logger LOG = LoggerFactory.getLogger(MergedFfmpegHlsDownload.class); - private File targetFile; - private transient FFmpeg ffmpeg; - private transient Process ffmpegProcess; - private transient OutputStream ffmpegStdIn; - private transient BlockingQueue> queue = new LinkedBlockingQueue<>(); - private transient Lock ffmpegStreamLock = new ReentrantLock(); + protected File targetFile; + protected transient FFmpeg ffmpeg; + protected transient Process ffmpegProcess; + protected transient OutputStream ffmpegStdIn; + protected transient BlockingQueue> queue = new LinkedBlockingQueue<>(); + protected transient Lock ffmpegStreamLock = new ReentrantLock(); public MergedFfmpegHlsDownload(HttpClient client) { super(client); @@ -80,7 +80,7 @@ public class MergedFfmpegHlsDownload extends AbstractHlsDownload { return this; } - private void streamSegmentDataToFfmpeg() { + protected void streamSegmentDataToFfmpeg() { downloadExecutor.submit(() -> { ffmpegStreamLock.lock(); try { diff --git a/common/src/main/java/ctbrec/sites/showup/ShowupMergedDownload.java b/common/src/main/java/ctbrec/sites/showup/ShowupMergedDownload.java index 205ee2bf..b726d081 100644 --- a/common/src/main/java/ctbrec/sites/showup/ShowupMergedDownload.java +++ b/common/src/main/java/ctbrec/sites/showup/ShowupMergedDownload.java @@ -1,12 +1,139 @@ package ctbrec.sites.showup; +import static ctbrec.io.HttpConstants.*; + +import java.io.IOException; +import java.io.InputStream; +import java.time.Instant; +import java.util.concurrent.ExecutorService; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.iheartradio.m3u8.ParseException; +import com.iheartradio.m3u8.PlaylistException; + +import ctbrec.Config; +import ctbrec.Model; +import ctbrec.io.BandwidthMeter; import ctbrec.io.HttpClient; +import ctbrec.io.HttpException; +import ctbrec.recorder.download.ProcessExitedUncleanException; +import ctbrec.recorder.download.hls.AbstractHlsDownload; import ctbrec.recorder.download.hls.MergedFfmpegHlsDownload; +import ctbrec.recorder.download.hls.SegmentPlaylist; +import okhttp3.Request; +import okhttp3.Response; public class ShowupMergedDownload extends MergedFfmpegHlsDownload { + private static final transient Logger LOG = LoggerFactory.getLogger(ShowupMergedDownload.class); + + private transient Response response; + private transient InputStream in; + private transient byte[] buffer = new byte[10240]; + public ShowupMergedDownload(HttpClient client) { super(client); } + @Override + public void init(Config config, Model model, Instant startTime, ExecutorService executorService) throws IOException { + super.init(config, model, startTime, executorService); + try { + if (segmentPlaylistUrl == null) { + segmentPlaylistUrl = getSegmentPlaylistUrl(model); + } + startDownload(segmentPlaylistUrl); + } catch (Exception e) { + LOG.error("Error starting the stream", e); + stop(); + } + } + + protected void startDownload(String segmentPlaylistUri) throws IOException, ParseException, PlaylistException { + try { + SegmentPlaylist lsp = getNextSegments(segmentPlaylistUri); + emptyPlaylistCheck(lsp); + + String segment = lsp.segments.get(0); + Request request = new Request.Builder().url(segment) + .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) + .header(CONNECTION, KEEP_ALIVE) + .build(); + response = client.execute(request); + if (response.isSuccessful()) { + in = response.body().byteStream(); + } else { + throw new HttpException(response.code(), response.message()); + } + } catch (HttpException e) { + handleHttpException(e); + } catch (Exception e) { + LOG.info("Unexpected error while downloading {}", model, e); + stop(); + } + } + + @Override + public AbstractHlsDownload call() throws Exception { + if (!running) { + return this; + } + + try { + if (!ffmpegProcess.isAlive()) { + running = false; + int exitValue = ffmpegProcess.exitValue(); + ffmpeg.shutdown(exitValue); + } + } catch (ProcessExitedUncleanException e) { + LOG.error("FFmpeg exited unclean", e); + } + + streamSegmentDataToFfmpeg(); + rescheduleTime = Instant.now().plusSeconds(1); + return this; + } + + @Override + protected void streamSegmentDataToFfmpeg() { + downloadExecutor.submit(() -> { + if (!ffmpegStreamLock.tryLock()) { + return; + } + try { + int length = -1; + boolean keepGoing = true; + while ((length = in.read(buffer)) > 0 && keepGoing) { + BandwidthMeter.add(length); + ffmpegStdIn.write(buffer, 0, length); + keepGoing = running && !Thread.interrupted() && model.isOnline(true); + splittingStrategy.splitNecessary(this); + } + if (length == -1) { + LOG.info("End of stream reached for {}", model); + stop(); + } + LOG.debug("loop finished"); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } catch (Exception e) { + LOG.error("Error while streaming data to FFmpeg", e); + } finally { + ffmpegStreamLock.unlock(); + } + }); + } + + @Override + public void finalizeDownload() { + super.finalizeDownload(); + try { + in.close(); + response.close(); + } catch (IOException e) { + LOG.error("Error while finalizing download", e); + } + } } diff --git a/common/src/main/java/ctbrec/sites/showup/ShowupModel.java b/common/src/main/java/ctbrec/sites/showup/ShowupModel.java index e29f0d0b..8c92b6e7 100644 --- a/common/src/main/java/ctbrec/sites/showup/ShowupModel.java +++ b/common/src/main/java/ctbrec/sites/showup/ShowupModel.java @@ -28,6 +28,7 @@ import ctbrec.recorder.download.StreamSource; public class ShowupModel extends AbstractModel { + private String uid; private String streamId; private String streamTranscoderAddr; private int[] resolution = new int[2]; @@ -128,6 +129,14 @@ public class ShowupModel extends AbstractModel { this.streamTranscoderAddr = streamTranscoderAddr; } + public String getUid() { + return uid; + } + + public void setUid(String uid) { + this.uid = uid; + } + @Override public Download createDownload() { if (Config.isServerMode() && !Config.getInstance().getSettings().recordSingleFile) {