From dc8a4d419043c5cb26f3c1dafa871a1b6f900fe3 Mon Sep 17 00:00:00 2001 From: 0xboobface <0xboobface@gmail.com> Date: Thu, 13 Dec 2018 17:09:40 +0100 Subject: [PATCH 01/44] Don't add model to models or update it, if uid is not set --- common/src/main/java/ctbrec/sites/mfc/MyFreeCamsClient.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/common/src/main/java/ctbrec/sites/mfc/MyFreeCamsClient.java b/common/src/main/java/ctbrec/sites/mfc/MyFreeCamsClient.java index 9f5c26d6..50ac1bbc 100644 --- a/common/src/main/java/ctbrec/sites/mfc/MyFreeCamsClient.java +++ b/common/src/main/java/ctbrec/sites/mfc/MyFreeCamsClient.java @@ -420,6 +420,11 @@ public class MyFreeCamsClient { return; } + // uid not set, we can't identify this model + if(state.getUid() == null || state.getUid() <= 0) { + return; + } + MyFreeCamsModel model = models.getIfPresent(state.getUid()); if(model == null) { model = mfc.createModel(state.getNm()); From 560e73c1dd85c20413097be40316a6abcfbab8ac Mon Sep 17 00:00:00 2001 From: 0xboobface <0xboobface@gmail.com> Date: Thu, 13 Dec 2018 20:51:14 +0100 Subject: [PATCH 02/44] Reduce log level for unused message types --- common/src/main/java/ctbrec/sites/mfc/MyFreeCamsClient.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common/src/main/java/ctbrec/sites/mfc/MyFreeCamsClient.java b/common/src/main/java/ctbrec/sites/mfc/MyFreeCamsClient.java index 50ac1bbc..cc4a3e06 100644 --- a/common/src/main/java/ctbrec/sites/mfc/MyFreeCamsClient.java +++ b/common/src/main/java/ctbrec/sites/mfc/MyFreeCamsClient.java @@ -277,7 +277,7 @@ public class MyFreeCamsClient { case ROOMDATA: LOG.debug("ROOMDATA: {}", message); case UEOPT: - LOG.debug("UEOPT: {}", message); + LOG.trace("UEOPT: {}", message); break; case SLAVEVSHARE: // LOG.debug("SLAVEVSHARE {}", message); @@ -295,7 +295,7 @@ public class MyFreeCamsClient { } break; default: - LOG.debug("Unknown message {}", message); + LOG.trace("Unknown message {}", message); break; } } From ceb7c07aa8b115b395a4abaf91dd2cc34d65b7a4 Mon Sep 17 00:00:00 2001 From: 0xboobface <0xboobface@gmail.com> Date: Thu, 13 Dec 2018 20:54:10 +0100 Subject: [PATCH 03/44] Add setting for minimum recording length If a recording is shorter than x seconds, it gets deleted --- .../java/ctbrec/ui/settings/SettingsTab.java | 21 ++++++ common/src/main/java/ctbrec/MpegUtil.java | 73 +++++++++++++++++++ common/src/main/java/ctbrec/Settings.java | 1 + .../java/ctbrec/recorder/LocalRecorder.java | 73 +++++++++++++++++++ .../ctbrec/recorder/PlaylistGenerator.java | 55 +------------- 5 files changed, 171 insertions(+), 52 deletions(-) create mode 100644 common/src/main/java/ctbrec/MpegUtil.java diff --git a/client/src/main/java/ctbrec/ui/settings/SettingsTab.java b/client/src/main/java/ctbrec/ui/settings/SettingsTab.java index d9630ff9..c06733b2 100644 --- a/client/src/main/java/ctbrec/ui/settings/SettingsTab.java +++ b/client/src/main/java/ctbrec/ui/settings/SettingsTab.java @@ -58,6 +58,7 @@ public class SettingsTab extends Tab implements TabSelectionListener { private TextField port; private TextField onlineCheckIntervalInSecs; private TextField leaveSpaceOnDevice; + private TextField minimumLengthInSecs; private CheckBox loadResolution; private CheckBox secureCommunication = new CheckBox(); private CheckBox chooseStreamQuality = new CheckBox(); @@ -360,6 +361,26 @@ public class SettingsTab extends Tab implements TabSelectionListener { GridPane.setMargin(leaveSpaceOnDevice, new Insets(0, 0, 0, CHECKBOX_MARGIN)); layout.add(leaveSpaceOnDevice, 1, row++); + tt = new Tooltip("Delete recordings, which are shorter than x seconds. 0 to disable."); + l = new Label("Delete recordings shorter than (secs)"); + l.setTooltip(tt); + layout.add(l, 0, row); + int minimumLengthInSeconds = Config.getInstance().getSettings().minimumLengthInSeconds; + minimumLengthInSecs = new TextField(Integer.toString(minimumLengthInSeconds)); + minimumLengthInSecs.setTooltip(tt); + minimumLengthInSecs.textProperty().addListener((observable, oldValue, newValue) -> { + if (!newValue.matches("\\d*")) { + minimumLengthInSecs.setText(newValue.replaceAll("[^\\d]", "")); + } + if(!minimumLengthInSecs.getText().isEmpty()) { + int minimumLength = Integer.parseInt(minimumLengthInSecs.getText()); + Config.getInstance().getSettings().minimumLengthInSeconds = minimumLength; + saveConfig(); + } + }); + GridPane.setMargin(minimumLengthInSecs, new Insets(0, 0, 0, CHECKBOX_MARGIN)); + layout.add(minimumLengthInSecs, 1, row++); + TitledPane locations = new TitledPane("Recorder", layout); locations.setCollapsible(false); return locations; diff --git a/common/src/main/java/ctbrec/MpegUtil.java b/common/src/main/java/ctbrec/MpegUtil.java new file mode 100644 index 00000000..4ef37a53 --- /dev/null +++ b/common/src/main/java/ctbrec/MpegUtil.java @@ -0,0 +1,73 @@ +package ctbrec; + +import java.io.File; +import java.io.IOException; +import java.nio.channels.ReadableByteChannel; +import java.time.Duration; +import java.util.Set; + +import org.jcodec.common.Demuxer; +import org.jcodec.common.DemuxerTrack; +import org.jcodec.common.TrackType; +import org.jcodec.common.Tuple; +import org.jcodec.common.Tuple._2; +import org.jcodec.common.io.FileChannelWrapper; +import org.jcodec.common.io.NIOUtils; +import org.jcodec.common.model.Packet; +import org.jcodec.containers.mps.MPSDemuxer; +import org.jcodec.containers.mps.MTSDemuxer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class MpegUtil { + private static final transient Logger LOG = LoggerFactory.getLogger(MpegUtil.class); + + public static void main(String[] args) throws IOException { + readFile(new File("../../test-recs/ff.ts")); + } + + public static void readFile(File file) throws IOException { + System.out.println(file.getCanonicalPath()); + double duration = MpegUtil.getFileDuration(file); + System.out.println(Duration.ofSeconds((long) duration)); + } + + public static double getFileDuration(File file) throws IOException { + try(FileChannelWrapper ch = NIOUtils.readableChannel(file)) { + _2 m2tsDemuxer = createM2TSDemuxer(ch, TrackType.VIDEO); + Demuxer demuxer = m2tsDemuxer.v1; + DemuxerTrack videoDemux = demuxer.getTracks().get(0); + Packet videoFrame = null; + double totalDuration = 0; + while( (videoFrame = videoDemux.nextFrame()) != null) { + totalDuration += videoFrame.getDurationD(); + } + return totalDuration; + } + } + + public static _2 createM2TSDemuxer(FileChannelWrapper ch, TrackType targetTrack) throws IOException { + MTSDemuxer mts = new MTSDemuxer(ch); + Set programs = mts.getPrograms(); + if (programs.size() == 0) { + LOG.error("The MPEG TS stream contains no programs"); + return null; + } + Tuple._2 found = null; + for (Integer pid : programs) { + ReadableByteChannel program = mts.getProgram(pid); + if (found != null) { + program.close(); + continue; + } + MPSDemuxer demuxer = new MPSDemuxer(program); + if (targetTrack == TrackType.AUDIO && demuxer.getAudioTracks().size() > 0 + || targetTrack == TrackType.VIDEO && demuxer.getVideoTracks().size() > 0) { + found = org.jcodec.common.Tuple._2(pid, (Demuxer) demuxer); + } else { + program.close(); + } + } + return found; + } +} diff --git a/common/src/main/java/ctbrec/Settings.java b/common/src/main/java/ctbrec/Settings.java index 384ea432..043b55af 100644 --- a/common/src/main/java/ctbrec/Settings.java +++ b/common/src/main/java/ctbrec/Settings.java @@ -41,6 +41,7 @@ public class Settings { public String recordingsDir = System.getProperty("user.home") + File.separator + "ctbrec"; public DirectoryStructure recordingsDirStructure = DirectoryStructure.FLAT; public long minimumSpaceLeftInBytes = 0; + public int minimumLengthInSeconds = 0; public String mediaPlayer = "/usr/bin/mpv"; public String postProcessing = ""; public String username = ""; // chaturbate username TODO maybe rename this onetime diff --git a/common/src/main/java/ctbrec/recorder/LocalRecorder.java b/common/src/main/java/ctbrec/recorder/LocalRecorder.java index 0732220b..6e16c809 100644 --- a/common/src/main/java/ctbrec/recorder/LocalRecorder.java +++ b/common/src/main/java/ctbrec/recorder/LocalRecorder.java @@ -4,6 +4,8 @@ import static ctbrec.Recording.State.*; import static ctbrec.event.Event.Type.*; import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; import java.io.FilenameFilter; import java.io.IOException; import java.nio.file.FileStore; @@ -11,6 +13,7 @@ import java.nio.file.Files; import java.security.InvalidKeyException; import java.security.NoSuchAlgorithmException; import java.text.SimpleDateFormat; +import java.time.Duration; import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; @@ -34,11 +37,19 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.common.eventbus.Subscribe; +import com.iheartradio.m3u8.Encoding; +import com.iheartradio.m3u8.Format; import com.iheartradio.m3u8.ParseException; +import com.iheartradio.m3u8.ParsingMode; import com.iheartradio.m3u8.PlaylistException; +import com.iheartradio.m3u8.PlaylistParser; +import com.iheartradio.m3u8.data.MediaPlaylist; +import com.iheartradio.m3u8.data.Playlist; +import com.iheartradio.m3u8.data.TrackData; import ctbrec.Config; import ctbrec.Model; +import ctbrec.MpegUtil; import ctbrec.OS; import ctbrec.Recording; import ctbrec.Recording.State; @@ -740,9 +751,71 @@ public class LocalRecorder implements Recorder { fireRecordingStateChanged(download.getTarget(), GENERATING_PLAYLIST, download.getModel(), download.getStartTime()); generatePlaylist(download.getTarget()); } + boolean deleted = deleteIfTooShort(download); + if(deleted) { + // recording was too short. stop here and don't do post-processing + return; + } fireRecordingStateChanged(download.getTarget(), POST_PROCESSING, download.getModel(), download.getStartTime()); postprocess(download); fireRecordingStateChanged(download.getTarget(), FINISHED, download.getModel(), download.getStartTime()); }; } + + + // TODO maybe get file size and bitrate and check, if the values are plausible + // we could also compare the length with the time elapsed since starting the recording + private boolean deleteIfTooShort(Download download) { + long minimumLengthInSeconds = Config.getInstance().getSettings().minimumLengthInSeconds; + if(minimumLengthInSeconds <= 0) { + return false; + } + + try { + LOG.debug("Determining video length for {}", download.getTarget()); + File target = download.getTarget(); + double duration = 0; + if(target.isDirectory()) { + File playlist = new File(target, "playlist.m3u8"); + duration = getPlaylistLength(playlist); + } else { + duration = MpegUtil.getFileDuration(target); + } + Duration minLength = Duration.ofSeconds(minimumLengthInSeconds); + Duration videoLength = Duration.ofSeconds((long) duration); + LOG.debug("Recording started at:{}. Video length is {}", download.getStartTime(), videoLength); + if(videoLength.minus(minLength).isNegative()) { + LOG.debug("Video too short {} {}", videoLength, download.getTarget()); + LOG.debug("Deleting {}", target); + if(target.isDirectory()) { + deleteDirectory(target); + deleteEmptyParents(target); + } else { + Files.delete(target.toPath()); + deleteEmptyParents(target.getParentFile()); + } + return true; + } else { + return false; + } + } catch (Exception e) { + LOG.error("Couldn't check video length", e); + return false; + } + } + + private double getPlaylistLength(File playlist) throws IOException, ParseException, PlaylistException { + if(playlist.exists()) { + PlaylistParser playlistParser = new PlaylistParser(new FileInputStream(playlist), Format.EXT_M3U, Encoding.UTF_8, ParsingMode.LENIENT); + Playlist m3u = playlistParser.parse(); + MediaPlaylist mediaPlaylist = m3u.getMediaPlaylist(); + double length = 0; + for (TrackData trackData : mediaPlaylist.getTracks()) { + length += trackData.getTrackInfo().duration; + } + return length; + } else { + throw new FileNotFoundException(playlist.getAbsolutePath() + " does not exist"); + } + } } diff --git a/common/src/main/java/ctbrec/recorder/PlaylistGenerator.java b/common/src/main/java/ctbrec/recorder/PlaylistGenerator.java index 2fec113c..a4180765 100644 --- a/common/src/main/java/ctbrec/recorder/PlaylistGenerator.java +++ b/common/src/main/java/ctbrec/recorder/PlaylistGenerator.java @@ -6,24 +6,12 @@ import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.FilenameFilter; import java.io.IOException; -import java.nio.channels.ReadableByteChannel; import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; -import org.jcodec.common.Demuxer; -import org.jcodec.common.DemuxerTrack; -import org.jcodec.common.TrackType; -import org.jcodec.common.Tuple; -import org.jcodec.common.Tuple._2; -import org.jcodec.common.io.FileChannelWrapper; -import org.jcodec.common.io.NIOUtils; -import org.jcodec.common.model.Packet; -import org.jcodec.containers.mps.MPSDemuxer; -import org.jcodec.containers.mps.MTSDemuxer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -40,6 +28,8 @@ import com.iheartradio.m3u8.data.PlaylistType; import com.iheartradio.m3u8.data.TrackData; import com.iheartradio.m3u8.data.TrackInfo; +import ctbrec.MpegUtil; + public class PlaylistGenerator { private static final transient Logger LOG = LoggerFactory.getLogger(PlaylistGenerator.class); @@ -72,7 +62,7 @@ public class PlaylistGenerator { try { track.add(new TrackData.Builder() .withUri(file.getName()) - .withTrackInfo(new TrackInfo((float) getFileDuration(file), file.getName())) + .withTrackInfo(new TrackInfo((float) MpegUtil.getFileDuration(file), file.getName())) .build()); } catch(Exception e) { LOG.warn("Couldn't determine duration for {}. Skipping this file.", file.getName()); @@ -141,45 +131,6 @@ public class PlaylistGenerator { return targetDuration; } - private double getFileDuration(File file) throws IOException { - try(FileChannelWrapper ch = NIOUtils.readableChannel(file)) { - _2 m2tsDemuxer = createM2TSDemuxer(ch, TrackType.VIDEO); - Demuxer demuxer = m2tsDemuxer.v1; - DemuxerTrack videoDemux = demuxer.getTracks().get(0); - Packet videoFrame = null; - double totalDuration = 0; - while( (videoFrame = videoDemux.nextFrame()) != null) { - totalDuration += videoFrame.getDurationD(); - } - return totalDuration; - } - } - - public static _2 createM2TSDemuxer(FileChannelWrapper ch, TrackType targetTrack) throws IOException { - MTSDemuxer mts = new MTSDemuxer(ch); - Set programs = mts.getPrograms(); - if (programs.size() == 0) { - LOG.error("The MPEG TS stream contains no programs"); - return null; - } - Tuple._2 found = null; - for (Integer pid : programs) { - ReadableByteChannel program = mts.getProgram(pid); - if (found != null) { - program.close(); - continue; - } - MPSDemuxer demuxer = new MPSDemuxer(program); - if (targetTrack == TrackType.AUDIO && demuxer.getAudioTracks().size() > 0 - || targetTrack == TrackType.VIDEO && demuxer.getVideoTracks().size() > 0) { - found = org.jcodec.common.Tuple._2(pid, (Demuxer) demuxer); - } else { - program.close(); - } - } - return found; - } - public void addProgressListener(ProgressListener l) { listeners.add(l); } From 150af23d1475df9ebb1b0bac4e253d22496bdf11 Mon Sep 17 00:00:00 2001 From: 0xboobface <0xboobface@gmail.com> Date: Thu, 13 Dec 2018 20:54:26 +0100 Subject: [PATCH 04/44] Fix log messages --- common/src/main/java/ctbrec/event/ExecuteProgram.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common/src/main/java/ctbrec/event/ExecuteProgram.java b/common/src/main/java/ctbrec/event/ExecuteProgram.java index 28cb4807..5bb2b321 100644 --- a/common/src/main/java/ctbrec/event/ExecuteProgram.java +++ b/common/src/main/java/ctbrec/event/ExecuteProgram.java @@ -48,9 +48,9 @@ public class ExecuteProgram extends Action { err.start(); process.waitFor(); - LOG.debug("executing {} finished", executable); + LOG.debug("Executing {} finished", executable); } catch (Exception e) { - LOG.error("Error while processing {}", e); + LOG.error("Error while executing {}", executable, e); } } From 52cdf8d60127fd47c05b3afe3412cd20e04287e1 Mon Sep 17 00:00:00 2001 From: 0xboobface <0xboobface@gmail.com> Date: Thu, 13 Dec 2018 23:48:16 +0100 Subject: [PATCH 05/44] Add classes and first code for Streamate --- .../java/ctbrec/ui/CamrecApplication.java | 2 + .../main/java/ctbrec/ui/SiteUiFactory.java | 8 + .../ui/sites/streamate/StreamateSiteUi.java | 33 +++ .../sites/streamate/StreamateTabProvider.java | 62 +++++ .../streamate/StreamateUpdateService.java | 95 +++++++ .../ctbrec/ui/sites/streamate/girls.sml | 18 ++ .../main/java/ctbrec/io/XmlParserUtils.java | 117 ++++++++ .../ctbrec/sites/streamate/Streamate.java | 193 ++++++++++++++ .../sites/streamate/StreamateHttpClient.java | 75 ++++++ .../sites/streamate/StreamateModel.java | 251 ++++++++++++++++++ 10 files changed, 854 insertions(+) create mode 100644 client/src/main/java/ctbrec/ui/sites/streamate/StreamateSiteUi.java create mode 100644 client/src/main/java/ctbrec/ui/sites/streamate/StreamateTabProvider.java create mode 100644 client/src/main/java/ctbrec/ui/sites/streamate/StreamateUpdateService.java create mode 100644 client/src/main/resources/ctbrec/ui/sites/streamate/girls.sml create mode 100644 common/src/main/java/ctbrec/io/XmlParserUtils.java create mode 100644 common/src/main/java/ctbrec/sites/streamate/Streamate.java create mode 100644 common/src/main/java/ctbrec/sites/streamate/StreamateHttpClient.java create mode 100644 common/src/main/java/ctbrec/sites/streamate/StreamateModel.java diff --git a/client/src/main/java/ctbrec/ui/CamrecApplication.java b/client/src/main/java/ctbrec/ui/CamrecApplication.java index d698dbdb..d91a9d5e 100644 --- a/client/src/main/java/ctbrec/ui/CamrecApplication.java +++ b/client/src/main/java/ctbrec/ui/CamrecApplication.java @@ -38,6 +38,7 @@ import ctbrec.sites.cam4.Cam4; import ctbrec.sites.camsoda.Camsoda; import ctbrec.sites.chaturbate.Chaturbate; import ctbrec.sites.mfc.MyFreeCams; +import ctbrec.sites.streamate.Streamate; import ctbrec.ui.settings.SettingsTab; import javafx.application.Application; import javafx.application.HostServices; @@ -76,6 +77,7 @@ public class CamrecApplication extends Application { sites.add(new Camsoda()); sites.add(new Chaturbate()); sites.add(new MyFreeCams()); + sites.add(new Streamate()); loadConfig(); registerAlertSystem(); createHttpClient(); diff --git a/client/src/main/java/ctbrec/ui/SiteUiFactory.java b/client/src/main/java/ctbrec/ui/SiteUiFactory.java index 8ef694d1..7475868c 100644 --- a/client/src/main/java/ctbrec/ui/SiteUiFactory.java +++ b/client/src/main/java/ctbrec/ui/SiteUiFactory.java @@ -6,11 +6,13 @@ import ctbrec.sites.cam4.Cam4; import ctbrec.sites.camsoda.Camsoda; import ctbrec.sites.chaturbate.Chaturbate; import ctbrec.sites.mfc.MyFreeCams; +import ctbrec.sites.streamate.Streamate; import ctbrec.ui.sites.bonga.BongaCamsSiteUi; import ctbrec.ui.sites.cam4.Cam4SiteUi; import ctbrec.ui.sites.camsoda.CamsodaSiteUi; import ctbrec.ui.sites.chaturbate.ChaturbateSiteUi; import ctbrec.ui.sites.myfreecams.MyFreeCamsSiteUi; +import ctbrec.ui.sites.streamate.StreamateSiteUi; public class SiteUiFactory { @@ -19,6 +21,7 @@ public class SiteUiFactory { private static CamsodaSiteUi camsodaSiteUi; private static ChaturbateSiteUi ctbSiteUi; private static MyFreeCamsSiteUi mfcSiteUi; + private static StreamateSiteUi streamateSiteUi; public static synchronized SiteUI getUi(Site site) { if (site instanceof BongaCams) { @@ -46,6 +49,11 @@ public class SiteUiFactory { mfcSiteUi = new MyFreeCamsSiteUi((MyFreeCams) site); } return mfcSiteUi; + } else if (site instanceof Streamate) { + if (streamateSiteUi == null) { + streamateSiteUi = new StreamateSiteUi((Streamate) site); + } + return streamateSiteUi; } throw new RuntimeException("Unknown site " + site.getName()); } diff --git a/client/src/main/java/ctbrec/ui/sites/streamate/StreamateSiteUi.java b/client/src/main/java/ctbrec/ui/sites/streamate/StreamateSiteUi.java new file mode 100644 index 00000000..8d31d020 --- /dev/null +++ b/client/src/main/java/ctbrec/ui/sites/streamate/StreamateSiteUi.java @@ -0,0 +1,33 @@ +package ctbrec.ui.sites.streamate; + +import java.io.IOException; + +import ctbrec.sites.ConfigUI; +import ctbrec.sites.streamate.Streamate; +import ctbrec.ui.SiteUI; +import ctbrec.ui.TabProvider; + +public class StreamateSiteUi implements SiteUI { + + private StreamateTabProvider tabProvider; + + public StreamateSiteUi(Streamate streamate) { + tabProvider = new StreamateTabProvider(streamate); + } + + @Override + public TabProvider getTabProvider() { + return tabProvider; + } + + @Override + public ConfigUI getConfigUI() { + return null; + } + + @Override + public boolean login() throws IOException { + return false; + } + +} diff --git a/client/src/main/java/ctbrec/ui/sites/streamate/StreamateTabProvider.java b/client/src/main/java/ctbrec/ui/sites/streamate/StreamateTabProvider.java new file mode 100644 index 00000000..5c74b825 --- /dev/null +++ b/client/src/main/java/ctbrec/ui/sites/streamate/StreamateTabProvider.java @@ -0,0 +1,62 @@ +package ctbrec.ui.sites.streamate; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import ctbrec.recorder.Recorder; +import ctbrec.sites.streamate.Streamate; +import ctbrec.ui.TabProvider; +import ctbrec.ui.ThumbOverviewTab; +import javafx.scene.Scene; +import javafx.scene.control.Tab; + +public class StreamateTabProvider extends TabProvider { + private static final transient Logger LOG = LoggerFactory.getLogger(StreamateTabProvider.class); + private Streamate streamate; + private Recorder recorder; + + public StreamateTabProvider(Streamate streamate) { + this.streamate = streamate; + this.recorder = streamate.getRecorder(); + } + + @Override + public List getTabs(Scene scene) { + List tabs = new ArrayList<>(); + try { + tabs.add(createTab("Girls", "/ctbrec/ui/sites/streamate/girls.sml")); + } catch (IOException e) { + LOG.error("Couldn't create streamate tab", e); + } + return tabs; + } + + @Override + public Tab getFollowedTab() { + return null; + } + + private Tab createTab(String title, String queryFile) throws IOException { + StreamateUpdateService updateService = new StreamateUpdateService(loadQuery(queryFile), streamate); + ThumbOverviewTab tab = new ThumbOverviewTab(title, updateService, streamate); + tab.setRecorder(recorder); + return tab; + } + + private String loadQuery(String file) throws IOException { + InputStream is = getClass().getResourceAsStream(file); + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + byte[] b = new byte[1024]; + int len = -1; + while( (len = is.read(b)) >= 0) { + bos.write(b, 0, len); + } + return new String(bos.toByteArray(), "utf-8"); + } +} diff --git a/client/src/main/java/ctbrec/ui/sites/streamate/StreamateUpdateService.java b/client/src/main/java/ctbrec/ui/sites/streamate/StreamateUpdateService.java new file mode 100644 index 00000000..1c594e93 --- /dev/null +++ b/client/src/main/java/ctbrec/ui/sites/streamate/StreamateUpdateService.java @@ -0,0 +1,95 @@ +package ctbrec.ui.sites.streamate; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.xpath.XPathExpressionException; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.w3c.dom.Document; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; +import org.xml.sax.SAXException; + +import ctbrec.Config; +import ctbrec.Model; +import ctbrec.io.XmlParserUtils; +import ctbrec.sites.streamate.Streamate; +import ctbrec.sites.streamate.StreamateModel; +import ctbrec.ui.PaginatedScheduledService; +import javafx.concurrent.Task; +import okhttp3.MediaType; +import okhttp3.Request; +import okhttp3.RequestBody; +import okhttp3.Response; + +public class StreamateUpdateService extends PaginatedScheduledService { + + private static final transient Logger LOG = LoggerFactory.getLogger(StreamateUpdateService.class); + + private static final String URL = "http://affiliate.streamate.com/SMLive/SMLResult.xml"; + private Streamate streamate; + private String query; + + public StreamateUpdateService(String query, Streamate streamate) { + this.query = query; + this.streamate = streamate; + } + + @Override + protected Task> createTask() { + return new Task>() { + @Override + public List call() throws IOException, SAXException, ParserConfigurationException, XPathExpressionException { + LOG.debug("Fetching page {}", URL); + String q = query + .replace("{maxresults}", "50") + .replace("{pagenum}", Integer.toString(page)); + //LOG.debug("Query:\n{}", q); + RequestBody body = RequestBody.create(MediaType.parse("text/xml"), q); + Request request = new Request.Builder() + .url(URL) + .addHeader("User-Agent", Config.getInstance().getSettings().httpUserAgent) + .addHeader("Accept", "text/xml, */*") + .addHeader("Accept-Language", "en") + .addHeader("Referer", streamate.getBaseUrl()) + .post(body) + .build(); + Response response = streamate.getHttpClient().execute(request); + if (response.isSuccessful()) { + List models = new ArrayList<>(); + String content = response.body().string(); + LOG.debug(content); + ByteArrayInputStream in = new ByteArrayInputStream(content.getBytes("utf-8")); + Document doc = DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(in); + NodeList performers = doc.getElementsByTagName("Performer"); + for (int i = 0; i < performers.getLength(); i++) { + Node performer = performers.item(i); + String name = performer.getAttributes().getNamedItem("Name").getNodeValue(); + String id = performer.getAttributes().getNamedItem("Id").getNodeValue(); + String GoldShow = performer.getAttributes().getNamedItem("GoldShow").getNodeValue(); + String PreGoldShow = performer.getAttributes().getNamedItem("PreGoldShow").getNodeValue(); + String PartyChat = performer.getAttributes().getNamedItem("PartyChat").getNodeValue(); + StreamateModel model = (StreamateModel) streamate.createModel(name); + model.setId(id); + models.add(model); + Node pic = XmlParserUtils.getNodeWithXpath(performer, "Media/Pic/Full"); + String previewUrl = "https:" + pic.getAttributes().getNamedItem("Src").getNodeValue(); + model.setPreview(previewUrl); + LOG.debug("Name {} - {}{}{}", name, PartyChat, PreGoldShow, GoldShow); + } + return models; + } else { + int code = response.code(); + response.close(); + throw new IOException("HTTP status " + code); + } + } + }; + } +} diff --git a/client/src/main/resources/ctbrec/ui/sites/streamate/girls.sml b/client/src/main/resources/ctbrec/ui/sites/streamate/girls.sml new file mode 100644 index 00000000..f7841ae4 --- /dev/null +++ b/client/src/main/resources/ctbrec/ui/sites/streamate/girls.sml @@ -0,0 +1,18 @@ + + + + + + + + biopic, staticbiopic + + + + live,recorded + + + + \ No newline at end of file diff --git a/common/src/main/java/ctbrec/io/XmlParserUtils.java b/common/src/main/java/ctbrec/io/XmlParserUtils.java new file mode 100644 index 00000000..a1ac9cf6 --- /dev/null +++ b/common/src/main/java/ctbrec/io/XmlParserUtils.java @@ -0,0 +1,117 @@ +package ctbrec.io; + +import java.io.IOException; +import java.io.StringReader; +import java.util.List; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.xpath.XPath; +import javax.xml.xpath.XPathConstants; +import javax.xml.xpath.XPathExpressionException; +import javax.xml.xpath.XPathFactory; + +import org.w3c.dom.Document; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; +import org.xml.sax.InputSource; +import org.xml.sax.SAXException; + +public class XmlParserUtils { + + public static Document parse(String xml) throws ParserConfigurationException, SAXException, IOException { + DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + DocumentBuilder builder = factory.newDocumentBuilder(); + return builder.parse(new InputSource(new StringReader(xml))); + } + + public static Node getFirstElementByTagName(Document doc, String tagName) { + NodeList list = doc.getElementsByTagName(tagName); + if (list.getLength() > 0) { + return list.item(0); + } else { + return null; + } + } + + public static String getTextContent(Document doc, String tagName) { + Node node = getFirstElementByTagName(doc, tagName); + if (node != null) { + return node.getTextContent(); + } else { + return null; + } + } + + public static String getTextContent(Node parent, String tagName) { + Node node = findChildWithTagName(parent, tagName); + if (node != null) { + return node.getTextContent(); + } else { + return null; + } + } + + public static Node findChildWithTagName(Node parent, String tagName) { + if (parent == null) { + return null; + } + + NodeList childs = parent.getChildNodes(); + for (int i = 0; i < childs.getLength(); i++) { + Node child = childs.item(i); + if (child.getNodeName().equals(tagName)) { + return child; + } else if (child.hasChildNodes()) { + Node result = findChildWithTagName(child, tagName); + if (result != null) { + return result; + } + } + } + + return null; + } + + public static void getElementsByTagName(Node parent, String tagName, List result) { + if (parent == null) { + return; + } + + NodeList childs = parent.getChildNodes(); + for (int i = 0; i < childs.getLength(); i++) { + Node child = childs.item(i); + if (child.getNodeName().equals(tagName)) { + result.add(child); + } else if (child.hasChildNodes()) { + getElementsByTagName(child, tagName, result); + } + } + } + + public static String getStringWithXpath(String xml, String xpath) throws XPathExpressionException { + XPath xp = XPathFactory.newInstance().newXPath(); + return xp.evaluate(xpath, new InputSource(new StringReader(xml))); + } + + public static String getStringWithXpath(Node node, String xpath) throws XPathExpressionException { + XPath xp = XPathFactory.newInstance().newXPath(); + return xp.evaluate(xpath, node); + } + + public static Node getNodeWithXpath(String xml, String xpath) throws XPathExpressionException { + XPath xp = XPathFactory.newInstance().newXPath(); + return (Node) xp.evaluate(xpath, new InputSource(new StringReader(xml)), XPathConstants.NODE); + } + + public static Node getNodeWithXpath(Node node, String xpath) throws XPathExpressionException { + XPath xp = XPathFactory.newInstance().newXPath(); + return (Node) xp.evaluate(xpath, node, XPathConstants.NODE); + } + + public static NodeList getNodeListWithXpath(String xml, String xpath) throws XPathExpressionException { + XPath xp = XPathFactory.newInstance().newXPath(); + return (NodeList) xp.evaluate(xpath, new InputSource(new StringReader(xml)), XPathConstants.NODESET); + } +} diff --git a/common/src/main/java/ctbrec/sites/streamate/Streamate.java b/common/src/main/java/ctbrec/sites/streamate/Streamate.java new file mode 100644 index 00000000..e0da236b --- /dev/null +++ b/common/src/main/java/ctbrec/sites/streamate/Streamate.java @@ -0,0 +1,193 @@ +package ctbrec.sites.streamate; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import ctbrec.Model; +import ctbrec.io.HttpClient; +import ctbrec.sites.AbstractSite; + +public class Streamate extends AbstractSite { + + private static final transient Logger LOG = LoggerFactory.getLogger(Streamate.class); + + public static final String BASE_URL = "https://www.streamate.com"; + + private StreamateHttpClient httpClient; + + @Override + public String getName() { + return "Streamate"; + } + + @Override + public String getBaseUrl() { + return BASE_URL; + } + + @Override + public String getAffiliateLink() { + return BASE_URL + "/landing/click/?AFNO=2-11330.2"; + } + + @Override + public Model createModel(String name) { + StreamateModel model = new StreamateModel(); + model.setName(name); + model.setUrl(BASE_URL + "/cam/" + name); + model.setDescription(""); + model.setSite(this); + return model; + } + + @Override + public Integer getTokenBalance() throws IOException { + // int userId = ((StreamateHttpClient)getHttpClient()).getUserId(); + // String url = Streamate.BASE_URL + "/tools/amf.php"; + // RequestBody body = new FormBody.Builder() + // .add("method", "ping") + // .add("args[]", Integer.toString(userId)) + // .build(); + // Request request = new Request.Builder() + // .url(url) + // .addHeader("User-Agent", Config.getInstance().getSettings().httpUserAgent) + // .addHeader("Accept", "application/json, text/javascript, */*") + // .addHeader("Accept-Language", "en") + // .addHeader("Referer", Streamate.BASE_URL) + // .addHeader("X-Requested-With", "XMLHttpRequest") + // .post(body) + // .build(); + // try(Response response = getHttpClient().execute(request)) { + // if(response.isSuccessful()) { + // JSONObject json = new JSONObject(response.body().string()); + // if(json.optString("status").equals("online")) { + // JSONObject userData = json.getJSONObject("userData"); + // return userData.getInt("balance"); + // } else { + // throw new IOException("Request was not successful: " + json.toString(2)); + // } + // } else { + // throw new HttpException(response.code(), response.message()); + // } + // } + return 0; + } + + @Override + public String getBuyTokensLink() { + return getAffiliateLink(); + } + + @Override + public synchronized boolean login() throws IOException { + return credentialsAvailable() && getHttpClient().login(); + } + + @Override + public HttpClient getHttpClient() { + if(httpClient == null) { + httpClient = new StreamateHttpClient(); + } + return httpClient; + } + + @Override + public void init() throws IOException { + } + + @Override + public void shutdown() { + if(httpClient != null) { + httpClient.shutdown(); + } + } + + @Override + public boolean supportsTips() { + return false; + } + + @Override + public boolean supportsFollow() { + return false; + } + + @Override + public boolean supportsSearch() { + return false; + } + + @Override + public boolean searchRequiresLogin() { + return true; + } + + @Override + public List search(String q) throws IOException, InterruptedException { + // String url = BASE_URL + "/tools/listing_v3.php?offset=0&model_search[display_name][text]=" + URLEncoder.encode(q, "utf-8"); + // Request req = new Request.Builder() + // .url(url) + // .addHeader("User-Agent", Config.getInstance().getSettings().httpUserAgent) + // .addHeader("Accept", "application/json, text/javascript, */*") + // .addHeader("Accept-Language", "en") + // .addHeader("Referer", Streamate.BASE_URL) + // .addHeader("X-Requested-With", "XMLHttpRequest") + // .build(); + // try(Response response = getHttpClient().execute(req)) { + // if(response.isSuccessful()) { + // String body = response.body().string(); + // JSONObject json = new JSONObject(body); + // if(json.optString("status").equals("success")) { + // List models = new ArrayList<>(); + // JSONArray results = json.getJSONArray("models"); + // for (int i = 0; i < results.length(); i++) { + // JSONObject result = results.getJSONObject(i); + // Model model = createModel(result.getString("username")); + // String thumb = result.getString("thumb_image"); + // if(thumb != null) { + // model.setPreview("https:" + thumb); + // } + // if(result.has("display_name")) { + // model.setDisplayName(result.getString("display_name")); + // } + // models.add(model); + // } + // return models; + // } else { + // LOG.warn("Search result: " + json.toString(2)); + // return Collections.emptyList(); + // } + // } else { + // throw new HttpException(response.code(), response.message()); + // } + // } + return Collections.emptyList(); + } + + @Override + public boolean isSiteForModel(Model m) { + return m instanceof StreamateModel; + } + + @Override + public boolean credentialsAvailable() { + return false; + } + + @Override + public Model createModelFromUrl(String url) { + Matcher m = Pattern.compile("https?://.*?streamate.com/cam/([^/]*?)/?").matcher(url); + if(m.matches()) { + String modelName = m.group(1); + return createModel(modelName); + } else { + return super.createModelFromUrl(url); + } + } +} diff --git a/common/src/main/java/ctbrec/sites/streamate/StreamateHttpClient.java b/common/src/main/java/ctbrec/sites/streamate/StreamateHttpClient.java new file mode 100644 index 00000000..6772eadd --- /dev/null +++ b/common/src/main/java/ctbrec/sites/streamate/StreamateHttpClient.java @@ -0,0 +1,75 @@ +package ctbrec.sites.streamate; + +import java.io.IOException; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import ctbrec.io.HttpClient; + +public class StreamateHttpClient extends HttpClient { + + private static final transient Logger LOG = LoggerFactory.getLogger(StreamateHttpClient.class); + + public StreamateHttpClient() { + super("streamate"); + } + + @Override + public synchronized boolean login() throws IOException { + if(loggedIn) { + return true; + } + + boolean cookiesWorked = checkLoginSuccess(); + if(cookiesWorked) { + loggedIn = true; + LOG.debug("Logged in with cookies"); + return true; + } + + return false; + } + + /** + * Check, if the login worked + * @throws IOException + */ + public boolean checkLoginSuccess() throws IOException { + return false; + // String modelName = getAnyModelName(); + // // we request the roomData of a random model, because it contains + // // user data, if the user is logged in, which we can use to verify, that the login worked + // String url = Streamate.BASE_URL + "/tools/amf.php"; + // RequestBody body = new FormBody.Builder() + // .add("method", "getRoomData") + // .add("args[]", modelName) + // .add("args[]", "false") + // //.add("method", "ping") // TODO alternative request, but + // //.add("args[]", ) // where to get the userId + // .build(); + // Request request = new Request.Builder() + // .url(url) + // .addHeader("User-Agent", Config.getInstance().getSettings().httpUserAgent) + // .addHeader("Accept", "application/json, text/javascript, */*") + // .addHeader("Accept-Language", "en") + // .addHeader("Referer", Streamate.BASE_URL) + // .addHeader("X-Requested-With", "XMLHttpRequest") + // .post(body) + // .build(); + // try(Response response = execute(request)) { + // if(response.isSuccessful()) { + // JSONObject json = new JSONObject(response.body().string()); + // if(json.optString("status").equals("success")) { + // JSONObject userData = json.getJSONObject("userData"); + // userId = userData.optInt("userId"); + // return userId > 0; + // } else { + // throw new IOException("Request was not successful: " + json.toString(2)); + // } + // } else { + // throw new HttpException(response.code(), response.message()); + // } + // } + } +} diff --git a/common/src/main/java/ctbrec/sites/streamate/StreamateModel.java b/common/src/main/java/ctbrec/sites/streamate/StreamateModel.java new file mode 100644 index 00000000..20c80be9 --- /dev/null +++ b/common/src/main/java/ctbrec/sites/streamate/StreamateModel.java @@ -0,0 +1,251 @@ +package ctbrec.sites.streamate; + +import static ctbrec.Model.State.*; + +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.ExecutionException; + +import org.json.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.iheartradio.m3u8.Encoding; +import com.iheartradio.m3u8.Format; +import com.iheartradio.m3u8.ParseException; +import com.iheartradio.m3u8.ParsingMode; +import com.iheartradio.m3u8.PlaylistException; +import com.iheartradio.m3u8.PlaylistParser; +import com.iheartradio.m3u8.data.MasterPlaylist; +import com.iheartradio.m3u8.data.Playlist; +import com.iheartradio.m3u8.data.PlaylistData; +import com.iheartradio.m3u8.data.StreamInfo; + +import ctbrec.AbstractModel; +import ctbrec.Config; +import ctbrec.io.HttpException; +import ctbrec.recorder.download.StreamSource; +import okhttp3.FormBody; +import okhttp3.Request; +import okhttp3.RequestBody; +import okhttp3.Response; + +public class StreamateModel extends AbstractModel { + + private static final transient Logger LOG = LoggerFactory.getLogger(StreamateModel.class); + + private boolean online = false; + private List streamSources = new ArrayList<>(); + private int[] resolution; + private String id; + + @Override + public boolean isOnline(boolean ignoreCache) throws IOException, ExecutionException, InterruptedException { + if(ignoreCache) { + String url = getStreamUrl(); + Request req = new Request.Builder().url(url).build(); + try(Response resp = site.getHttpClient().execute(req)) { + online = resp.isSuccessful(); + } + } + return online; + } + + private JSONObject getRoomData() throws IOException { + String url = Streamate.BASE_URL + "/tools/amf.php"; + RequestBody body = new FormBody.Builder() + .add("method", "getRoomData") + .add("args[]", getName()) + .add("args[]", "false") + .build(); + Request request = new Request.Builder() + .url(url) + .addHeader("User-Agent", Config.getInstance().getSettings().httpUserAgent) + .addHeader("Accept", "application/json, text/javascript, */*") + .addHeader("Accept-Language", "en") + .addHeader("Referer", Streamate.BASE_URL) + .addHeader("X-Requested-With", "XMLHttpRequest") + .post(body) + .build(); + try(Response response = site.getHttpClient().execute(request)) { + if(response.isSuccessful()) { + JSONObject json = new JSONObject(response.body().string()); + return json; + } else { + throw new IOException(response.code() + " " + response.message()); + } + } + } + + public void setOnline(boolean online) { + this.online = online; + } + + @Override + public State getOnlineState(boolean failFast) throws IOException, ExecutionException { + if(failFast) { + return onlineState; + } else { + if(onlineState == UNKNOWN) { + return online ? ONLINE : OFFLINE; + } + return onlineState; + } + } + + @Override + public void setOnlineState(State onlineState) { + this.onlineState = onlineState; + } + + @Override + public List getStreamSources() throws IOException, ExecutionException, ParseException, PlaylistException { + String streamUrl = getStreamUrl(); + if (streamUrl == null) { + return Collections.emptyList(); + } + Request req = new Request.Builder().url(streamUrl).build(); + try(Response response = site.getHttpClient().execute(req)) { + if(response.isSuccessful()) { + InputStream inputStream = response.body().byteStream(); + PlaylistParser parser = new PlaylistParser(inputStream, Format.EXT_M3U, Encoding.UTF_8, ParsingMode.LENIENT); + Playlist playlist = parser.parse(); + MasterPlaylist master = playlist.getMasterPlaylist(); + streamSources.clear(); + for (PlaylistData playlistData : master.getPlaylists()) { + StreamSource streamsource = new StreamSource(); + streamsource.mediaPlaylistUrl = streamUrl.replace("playlist.m3u8", playlistData.getUri()); + if (playlistData.hasStreamInfo()) { + StreamInfo info = playlistData.getStreamInfo(); + streamsource.bandwidth = info.getBandwidth(); + streamsource.width = info.hasResolution() ? info.getResolution().width : 0; + streamsource.height = info.hasResolution() ? info.getResolution().height : 0; + } else { + streamsource.bandwidth = 0; + streamsource.width = 0; + streamsource.height = 0; + } + streamSources.add(streamsource); + } + } else { + throw new HttpException(response.code(), response.message()); + } + } + return streamSources; + } + + private String getStreamUrl() throws IOException { + String url = "https://hybridclient.naiadsystems.com/api/v1/config/?sabasic=&sakey=&sk=www.streamate.com&userid=0&version=6.3.16&ajax=1&name=" + getName(); + Request req = new Request.Builder() + .addHeader("User-Agent", Config.getInstance().getSettings().httpUserAgent) + .addHeader("Accept", "application/json, text/javascript, */*") + .addHeader("Accept-Language", "en") + .addHeader("Referer", Streamate.BASE_URL + '/' + getName()) + .addHeader("X-Requested-With", "XMLHttpRequest") + .url(url) + .build(); + try(Response response = site.getHttpClient().execute(req)) { + if(response.isSuccessful()) { + JSONObject json = new JSONObject(response.body().string()); + JSONObject performer = json.getJSONObject("performer"); + id = performer.getString("id"); + JSONObject stream = json.getJSONObject("stream"); + String sserver = stream.getString("serverId"); + String streamId = stream.getString("streamId"); + String wsHost = stream.getString("nodeHost"); + LOG.debug(json.toString(2)); + + String wsUrl = wsHost + "/socket.io/?" + + "performerid=" + id + + "&sserver=" + sserver + + "&streamid=" + streamId + + "&sakey=&sessiontype=preview&perfdiscountid=0&minduration=0&goldshowid=0&version=7&referrer=hybrid.client.6.3.16/avchat.swf&usertype=false&lang=en&EIO=3&transport=websocket"; + } else { + throw new IOException(response.code() + ' ' + response.message()); + } + } + return ""; + } + + @Override + public void invalidateCacheEntries() { + resolution = null; + } + + @Override + public void receiveTip(int tokens) throws IOException { + // String url = Streamate.BASE_URL + "/chat-ajax-amf-service?" + System.currentTimeMillis(); + // int userId = ((StreamateHttpClient)site.getHttpClient()).getUserId(); + // RequestBody body = new FormBody.Builder() + // .add("method", "tipModel") + // .add("args[]", getName()) + // .add("args[]", Integer.toString(tokens)) + // .add("args[]", Integer.toString(userId)) + // .add("args[3]", "") + // .build(); + // Request request = new Request.Builder() + // .url(url) + // .addHeader("User-Agent", Config.getInstance().getSettings().httpUserAgent) + // .addHeader("Accept", "application/json, text/javascript, */*") + // .addHeader("Accept-Language", "en") + // .addHeader("Referer", Streamate.BASE_URL + '/' + getName()) + // .addHeader("X-Requested-With", "XMLHttpRequest") + // .post(body) + // .build(); + // try(Response response = site.getHttpClient().execute(request)) { + // if(response.isSuccessful()) { + // JSONObject json = new JSONObject(response.body().string()); + // if(!json.optString("status").equals("success")) { + // LOG.error("Sending tip failed {}", json.toString(2)); + // throw new IOException("Sending tip failed"); + // } + // } else { + // throw new IOException(response.code() + ' ' + response.message()); + // } + // } + } + + @Override + public int[] getStreamResolution(boolean failFast) throws ExecutionException { + if(resolution == null) { + if(failFast) { + return new int[2]; + } + try { + if(!isOnline()) { + return new int[2]; + } + List streamSources = getStreamSources(); + Collections.sort(streamSources); + StreamSource best = streamSources.get(streamSources.size()-1); + resolution = new int[] {best.width, best.height}; + } catch (ExecutionException | IOException | ParseException | PlaylistException | InterruptedException e) { + LOG.warn("Couldn't determine stream resolution for {} - {}", getName(), e.getMessage()); + } + return resolution; + } else { + return resolution; + } + } + + @Override + public boolean follow() throws IOException { + return false; + } + + @Override + public boolean unfollow() throws IOException { + return false; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } +} From 64c60eaeaabec6a94db32ddd2d68563c8330618d Mon Sep 17 00:00:00 2001 From: 0xboobface <0xboobface@gmail.com> Date: Fri, 14 Dec 2018 02:39:07 +0100 Subject: [PATCH 06/44] Add determination of stream url and stream sources --- .../streamate/StreamateUpdateService.java | 3 +- .../src/main/java/ctbrec/io/HttpClient.java | 7 + .../sites/streamate/StreamateModel.java | 130 ++++++++++++------ .../streamate/StreamateWebsocketClient.java | 74 ++++++++++ 4 files changed, 167 insertions(+), 47 deletions(-) create mode 100644 common/src/main/java/ctbrec/sites/streamate/StreamateWebsocketClient.java diff --git a/client/src/main/java/ctbrec/ui/sites/streamate/StreamateUpdateService.java b/client/src/main/java/ctbrec/ui/sites/streamate/StreamateUpdateService.java index 1c594e93..7f9816c6 100644 --- a/client/src/main/java/ctbrec/ui/sites/streamate/StreamateUpdateService.java +++ b/client/src/main/java/ctbrec/ui/sites/streamate/StreamateUpdateService.java @@ -64,7 +64,6 @@ public class StreamateUpdateService extends PaginatedScheduledService { if (response.isSuccessful()) { List models = new ArrayList<>(); String content = response.body().string(); - LOG.debug(content); ByteArrayInputStream in = new ByteArrayInputStream(content.getBytes("utf-8")); Document doc = DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(in); NodeList performers = doc.getElementsByTagName("Performer"); @@ -81,7 +80,7 @@ public class StreamateUpdateService extends PaginatedScheduledService { Node pic = XmlParserUtils.getNodeWithXpath(performer, "Media/Pic/Full"); String previewUrl = "https:" + pic.getAttributes().getNamedItem("Src").getNodeValue(); model.setPreview(previewUrl); - LOG.debug("Name {} - {}{}{}", name, PartyChat, PreGoldShow, GoldShow); + //LOG.debug("Name {} - {}{}{}", name, PartyChat, PreGoldShow, GoldShow); } return models; } else { diff --git a/common/src/main/java/ctbrec/io/HttpClient.java b/common/src/main/java/ctbrec/io/HttpClient.java index 9dc81010..5b2d8d9c 100644 --- a/common/src/main/java/ctbrec/io/HttpClient.java +++ b/common/src/main/java/ctbrec/io/HttpClient.java @@ -29,6 +29,8 @@ import okhttp3.OkHttpClient.Builder; import okhttp3.Request; import okhttp3.Response; import okhttp3.Route; +import okhttp3.WebSocket; +import okhttp3.WebSocketListener; public abstract class HttpClient { private static final transient Logger LOG = LoggerFactory.getLogger(HttpClient.class); @@ -219,4 +221,9 @@ public abstract class HttpClient { getCookieJar().clear(); loggedIn = false; } + + public WebSocket newWebSocket(String url, WebSocketListener l) { + Request request = new Request.Builder().url(url).build(); + return client.newWebSocket(request, l); + } } diff --git a/common/src/main/java/ctbrec/sites/streamate/StreamateModel.java b/common/src/main/java/ctbrec/sites/streamate/StreamateModel.java index 20c80be9..cecf91da 100644 --- a/common/src/main/java/ctbrec/sites/streamate/StreamateModel.java +++ b/common/src/main/java/ctbrec/sites/streamate/StreamateModel.java @@ -4,11 +4,13 @@ import static ctbrec.Model.State.*; import java.io.IOException; import java.io.InputStream; +import java.net.URLEncoder; import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.concurrent.ExecutionException; +import org.json.JSONArray; import org.json.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -28,9 +30,7 @@ import ctbrec.AbstractModel; import ctbrec.Config; import ctbrec.io.HttpException; import ctbrec.recorder.download.StreamSource; -import okhttp3.FormBody; import okhttp3.Request; -import okhttp3.RequestBody; import okhttp3.Response; public class StreamateModel extends AbstractModel { @@ -54,32 +54,6 @@ public class StreamateModel extends AbstractModel { return online; } - private JSONObject getRoomData() throws IOException { - String url = Streamate.BASE_URL + "/tools/amf.php"; - RequestBody body = new FormBody.Builder() - .add("method", "getRoomData") - .add("args[]", getName()) - .add("args[]", "false") - .build(); - Request request = new Request.Builder() - .url(url) - .addHeader("User-Agent", Config.getInstance().getSettings().httpUserAgent) - .addHeader("Accept", "application/json, text/javascript, */*") - .addHeader("Accept-Language", "en") - .addHeader("Referer", Streamate.BASE_URL) - .addHeader("X-Requested-With", "XMLHttpRequest") - .post(body) - .build(); - try(Response response = site.getHttpClient().execute(request)) { - if(response.isSuccessful()) { - JSONObject json = new JSONObject(response.body().string()); - return json; - } else { - throw new IOException(response.code() + " " + response.message()); - } - } - } - public void setOnline(boolean online) { this.online = online; } @@ -107,6 +81,7 @@ public class StreamateModel extends AbstractModel { if (streamUrl == null) { return Collections.emptyList(); } + LOG.debug(streamUrl); Request req = new Request.Builder().url(streamUrl).build(); try(Response response = site.getHttpClient().execute(req)) { if(response.isSuccessful()) { @@ -117,7 +92,7 @@ public class StreamateModel extends AbstractModel { streamSources.clear(); for (PlaylistData playlistData : master.getPlaylists()) { StreamSource streamsource = new StreamSource(); - streamsource.mediaPlaylistUrl = streamUrl.replace("playlist.m3u8", playlistData.getUri()); + streamsource.mediaPlaylistUrl = playlistData.getUri(); if (playlistData.hasStreamInfo()) { StreamInfo info = playlistData.getStreamInfo(); streamsource.bandwidth = info.getBandwidth(); @@ -138,6 +113,85 @@ public class StreamateModel extends AbstractModel { } private String getStreamUrl() throws IOException { + JSONObject json = getRoomInfo(); + JSONObject performer = json.getJSONObject("performer"); + id = Long.toString(performer.getLong("id")); + JSONObject stream = json.getJSONObject("stream"); + String sserver = stream.getString("serverId"); + String streamId = stream.getString("streamId"); + String wsHost = stream.getString("nodeHost"); + JSONObject liveservices = json.getJSONObject("liveservices"); + String streamHost = liveservices.getString("host").replace("wss", "https"); + + String roomId; + try { + roomId = getRoomId(wsHost, sserver, streamId); + LOG.debug("room id: {}", roomId); + } catch (InterruptedException e) { + throw new IOException("Couldn't get room id", e); + } + + String streamFormatUrl = getStreamFormatUrl(streamHost, roomId); + return getMasterPlaylistUrl(streamFormatUrl); + } + + private String getMasterPlaylistUrl(String url) throws IOException { + LOG.debug(url); + Request req = new Request.Builder() + .addHeader("User-Agent", Config.getInstance().getSettings().httpUserAgent) + .addHeader("Accept", "*/*") + .addHeader("Accept-Language", "en") + .addHeader("Referer", Streamate.BASE_URL + '/' + getName()) + .addHeader("X-Requested-With", "XMLHttpRequest") + .url(url) + .build(); + try(Response response = site.getHttpClient().execute(req)) { + if(response.isSuccessful()) { + JSONObject json = new JSONObject(response.body().string()); + JSONObject formats = json.getJSONObject("formats"); + JSONObject hls = formats.getJSONObject("mp4-hls"); + return hls.getString("manifest"); + } else { + throw new HttpException(response.code(), response.message()); + } + } + } + + private String getStreamFormatUrl(String streamHost, String roomId) throws IOException { + String url = streamHost + "/videourl?payload=" + + URLEncoder.encode("{\"puserid\":" + id + ",\"roomid\":\"" + roomId + "\",\"showtype\":1,\"nginx\":1}", "utf-8"); + LOG.debug(url); + Request req = new Request.Builder() + .addHeader("User-Agent", Config.getInstance().getSettings().httpUserAgent) + .addHeader("Accept", "*/*") + .addHeader("Accept-Language", "en") + .addHeader("Referer", Streamate.BASE_URL + '/' + getName()) + .addHeader("X-Requested-With", "XMLHttpRequest") + .url(url) + .build(); + try(Response response = site.getHttpClient().execute(req)) { + if(response.isSuccessful()) { + JSONArray streamConfig = new JSONArray(response.body().string()); + JSONObject obj = streamConfig.getJSONObject(0); + return obj.getString("url"); + } else { + throw new HttpException(response.code(), response.message()); + } + } + } + + private String getRoomId(String wsHost, String sserver, String streamId) throws InterruptedException { + String wsUrl = wsHost + "/socket.io/?" + + "performerid=" + id + + "&sserver=" + sserver + + "&streamid=" + streamId + + "&sakey=&sessiontype=preview&perfdiscountid=0&minduration=0&goldshowid=0&version=7&referrer=hybrid.client.6.3.16/avchat.swf&usertype=false&lang=en&EIO=3&transport=websocket"; + + StreamateWebsocketClient wsClient = new StreamateWebsocketClient(wsUrl, site.getHttpClient()); + return wsClient.getRoomId(); + } + + private JSONObject getRoomInfo() throws IOException { String url = "https://hybridclient.naiadsystems.com/api/v1/config/?sabasic=&sakey=&sk=www.streamate.com&userid=0&version=6.3.16&ajax=1&name=" + getName(); Request req = new Request.Builder() .addHeader("User-Agent", Config.getInstance().getSettings().httpUserAgent) @@ -149,25 +203,11 @@ public class StreamateModel extends AbstractModel { .build(); try(Response response = site.getHttpClient().execute(req)) { if(response.isSuccessful()) { - JSONObject json = new JSONObject(response.body().string()); - JSONObject performer = json.getJSONObject("performer"); - id = performer.getString("id"); - JSONObject stream = json.getJSONObject("stream"); - String sserver = stream.getString("serverId"); - String streamId = stream.getString("streamId"); - String wsHost = stream.getString("nodeHost"); - LOG.debug(json.toString(2)); - - String wsUrl = wsHost + "/socket.io/?" - + "performerid=" + id - + "&sserver=" + sserver - + "&streamid=" + streamId - + "&sakey=&sessiontype=preview&perfdiscountid=0&minduration=0&goldshowid=0&version=7&referrer=hybrid.client.6.3.16/avchat.swf&usertype=false&lang=en&EIO=3&transport=websocket"; + return new JSONObject(response.body().string()); } else { - throw new IOException(response.code() + ' ' + response.message()); + throw new HttpException(response.code(), response.message()); } } - return ""; } @Override diff --git a/common/src/main/java/ctbrec/sites/streamate/StreamateWebsocketClient.java b/common/src/main/java/ctbrec/sites/streamate/StreamateWebsocketClient.java new file mode 100644 index 00000000..d13cde56 --- /dev/null +++ b/common/src/main/java/ctbrec/sites/streamate/StreamateWebsocketClient.java @@ -0,0 +1,74 @@ +package ctbrec.sites.streamate; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import ctbrec.io.HttpClient; +import okhttp3.Response; +import okhttp3.WebSocket; +import okhttp3.WebSocketListener; +import okio.ByteString; + +public class StreamateWebsocketClient { + + private static final transient Logger LOG = LoggerFactory.getLogger(StreamateWebsocketClient.class); + private String url; + private HttpClient client; + + public StreamateWebsocketClient(String url, HttpClient client) { + this.url = url; + this.client = client; + } + + String roomId = ""; + public String getRoomId() throws InterruptedException { + LOG.debug("Connecting to {}", url); + Object monitor = new Object(); + client.newWebSocket(url, new WebSocketListener() { + @Override + public void onOpen(WebSocket webSocket, Response response) { + response.close(); + } + + @Override + public void onMessage(WebSocket webSocket, String text) { + if(text.contains("NaiadAuthorized")) { + Matcher m = Pattern.compile("\"roomid\":\"(.*?)\"").matcher(text); + if(m.find()) { + roomId = m.group(1); + webSocket.close(1000, ""); + } + } + } + + @Override + public void onMessage(WebSocket webSocket, ByteString bytes) { + LOG.debug("ws btxt {}", bytes.toString()); + } + + @Override + public void onClosed(WebSocket webSocket, int code, String reason) { + synchronized (monitor) { + monitor.notify(); + } + } + + @Override + public void onFailure(WebSocket webSocket, Throwable t, Response response) { + LOG.debug("ws failure", t); + response.close(); + synchronized (monitor) { + monitor.notify(); + } + } + }); + synchronized (monitor) { + monitor.wait(); + } + return roomId; + } +} + From 70f4fa930f810dbd390cd207457fbd31b8de1353 Mon Sep 17 00:00:00 2001 From: 0xboobface <0xboobface@gmail.com> Date: Fri, 14 Dec 2018 13:42:20 +0100 Subject: [PATCH 07/44] Implement search for Streamate --- .../ctbrec/sites/streamate/Streamate.java | 156 +++++++++--------- .../sites/streamate/StreamateHttpClient.java | 11 ++ .../sites/streamate/StreamateModel.java | 9 +- 3 files changed, 96 insertions(+), 80 deletions(-) diff --git a/common/src/main/java/ctbrec/sites/streamate/Streamate.java b/common/src/main/java/ctbrec/sites/streamate/Streamate.java index e0da236b..12cf63c7 100644 --- a/common/src/main/java/ctbrec/sites/streamate/Streamate.java +++ b/common/src/main/java/ctbrec/sites/streamate/Streamate.java @@ -1,17 +1,25 @@ package ctbrec.sites.streamate; import java.io.IOException; +import java.net.URLEncoder; +import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.json.JSONArray; +import org.json.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import ctbrec.Config; import ctbrec.Model; import ctbrec.io.HttpClient; +import ctbrec.io.HttpException; import ctbrec.sites.AbstractSite; +import okhttp3.Request; +import okhttp3.Response; public class Streamate extends AbstractSite { @@ -33,7 +41,8 @@ public class Streamate extends AbstractSite { @Override public String getAffiliateLink() { - return BASE_URL + "/landing/click/?AFNO=2-11330.2"; + return BASE_URL + "/landing/click/?AFNO=2-11329.1"; + // return BASE_URL + "/landing/click/?AFNO=2-11330.2"; } @Override @@ -48,34 +57,34 @@ public class Streamate extends AbstractSite { @Override public Integer getTokenBalance() throws IOException { - // int userId = ((StreamateHttpClient)getHttpClient()).getUserId(); - // String url = Streamate.BASE_URL + "/tools/amf.php"; - // RequestBody body = new FormBody.Builder() - // .add("method", "ping") - // .add("args[]", Integer.toString(userId)) - // .build(); - // Request request = new Request.Builder() - // .url(url) - // .addHeader("User-Agent", Config.getInstance().getSettings().httpUserAgent) - // .addHeader("Accept", "application/json, text/javascript, */*") - // .addHeader("Accept-Language", "en") - // .addHeader("Referer", Streamate.BASE_URL) - // .addHeader("X-Requested-With", "XMLHttpRequest") - // .post(body) - // .build(); - // try(Response response = getHttpClient().execute(request)) { - // if(response.isSuccessful()) { - // JSONObject json = new JSONObject(response.body().string()); - // if(json.optString("status").equals("online")) { - // JSONObject userData = json.getJSONObject("userData"); - // return userData.getInt("balance"); - // } else { - // throw new IOException("Request was not successful: " + json.toString(2)); - // } - // } else { - // throw new HttpException(response.code(), response.message()); - // } - // } + // int userId = ((StreamateHttpClient)getHttpClient()).getUserId(); + // String url = Streamate.BASE_URL + "/tools/amf.php"; + // RequestBody body = new FormBody.Builder() + // .add("method", "ping") + // .add("args[]", Integer.toString(userId)) + // .build(); + // Request request = new Request.Builder() + // .url(url) + // .addHeader("User-Agent", Config.getInstance().getSettings().httpUserAgent) + // .addHeader("Accept", "application/json, text/javascript, */*") + // .addHeader("Accept-Language", "en") + // .addHeader("Referer", Streamate.BASE_URL) + // .addHeader("X-Requested-With", "XMLHttpRequest") + // .post(body) + // .build(); + // try(Response response = getHttpClient().execute(request)) { + // if(response.isSuccessful()) { + // JSONObject json = new JSONObject(response.body().string()); + // if(json.optString("status").equals("online")) { + // JSONObject userData = json.getJSONObject("userData"); + // return userData.getInt("balance"); + // } else { + // throw new IOException("Request was not successful: " + json.toString(2)); + // } + // } else { + // throw new HttpException(response.code(), response.message()); + // } + // } return 0; } @@ -91,7 +100,7 @@ public class Streamate extends AbstractSite { @Override public HttpClient getHttpClient() { - if(httpClient == null) { + if (httpClient == null) { httpClient = new StreamateHttpClient(); } return httpClient; @@ -103,7 +112,7 @@ public class Streamate extends AbstractSite { @Override public void shutdown() { - if(httpClient != null) { + if (httpClient != null) { httpClient.shutdown(); } } @@ -120,54 +129,51 @@ public class Streamate extends AbstractSite { @Override public boolean supportsSearch() { - return false; - } - - @Override - public boolean searchRequiresLogin() { return true; } + @Override + public boolean searchRequiresLogin() { + return false; + } + @Override public List search(String q) throws IOException, InterruptedException { - // String url = BASE_URL + "/tools/listing_v3.php?offset=0&model_search[display_name][text]=" + URLEncoder.encode(q, "utf-8"); - // Request req = new Request.Builder() - // .url(url) - // .addHeader("User-Agent", Config.getInstance().getSettings().httpUserAgent) - // .addHeader("Accept", "application/json, text/javascript, */*") - // .addHeader("Accept-Language", "en") - // .addHeader("Referer", Streamate.BASE_URL) - // .addHeader("X-Requested-With", "XMLHttpRequest") - // .build(); - // try(Response response = getHttpClient().execute(req)) { - // if(response.isSuccessful()) { - // String body = response.body().string(); - // JSONObject json = new JSONObject(body); - // if(json.optString("status").equals("success")) { - // List models = new ArrayList<>(); - // JSONArray results = json.getJSONArray("models"); - // for (int i = 0; i < results.length(); i++) { - // JSONObject result = results.getJSONObject(i); - // Model model = createModel(result.getString("username")); - // String thumb = result.getString("thumb_image"); - // if(thumb != null) { - // model.setPreview("https:" + thumb); - // } - // if(result.has("display_name")) { - // model.setDisplayName(result.getString("display_name")); - // } - // models.add(model); - // } - // return models; - // } else { - // LOG.warn("Search result: " + json.toString(2)); - // return Collections.emptyList(); - // } - // } else { - // throw new HttpException(response.code(), response.message()); - // } - // } - return Collections.emptyList(); + String url = BASE_URL + "/api/search/autocomplete?exact=false&skin_search_kids=0&results_per_page=10&query=" + URLEncoder.encode(q, "utf-8"); + Request req = new Request.Builder().url(url) + .addHeader("User-Agent", Config.getInstance().getSettings().httpUserAgent) + .addHeader("Accept", "application/json, text/javascript, */*") + .addHeader("Accept-Language", "en") + .addHeader("Referer", Streamate.BASE_URL) + .addHeader("X-Requested-With", "XMLHttpRequest").build(); + try (Response response = getHttpClient().execute(req)) { + if (response.isSuccessful()) { + String body = response.body().string(); + JSONObject json = new JSONObject(body); + if (json.optString("status").equals("SM_OK")) { + List models = new ArrayList<>(); + JSONObject results = json.getJSONObject("results"); + JSONArray nickname = results.getJSONArray("nickname"); + for (int i = 0; i < nickname.length(); i++) { + JSONObject result = nickname.getJSONObject(i); + StreamateModel model = (StreamateModel) createModel(result.getString("nickname")); + model.setId(result.getString("performerId")); + String thumb = result.getString("thumbnail"); + if (thumb != null) { + model.setPreview(thumb); + } + model.setOnline(result.optString("liveStatus").equals("live")); + models.add(model); + } + return models; + } else { + LOG.warn("Search result: " + json.toString(2)); + return Collections.emptyList(); + } + } else { + throw new HttpException(response.code(), response.message()); + } + } } @Override @@ -183,7 +189,7 @@ public class Streamate extends AbstractSite { @Override public Model createModelFromUrl(String url) { Matcher m = Pattern.compile("https?://.*?streamate.com/cam/([^/]*?)/?").matcher(url); - if(m.matches()) { + if (m.matches()) { String modelName = m.group(1); return createModel(modelName); } else { diff --git a/common/src/main/java/ctbrec/sites/streamate/StreamateHttpClient.java b/common/src/main/java/ctbrec/sites/streamate/StreamateHttpClient.java index 6772eadd..aa64b61f 100644 --- a/common/src/main/java/ctbrec/sites/streamate/StreamateHttpClient.java +++ b/common/src/main/java/ctbrec/sites/streamate/StreamateHttpClient.java @@ -1,11 +1,14 @@ package ctbrec.sites.streamate; import java.io.IOException; +import java.util.Collections; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import ctbrec.io.HttpClient; +import okhttp3.Cookie; +import okhttp3.HttpUrl; public class StreamateHttpClient extends HttpClient { @@ -13,6 +16,14 @@ public class StreamateHttpClient extends HttpClient { public StreamateHttpClient() { super("streamate"); + + // this cookie is needed for the search + Cookie searchCookie = new Cookie.Builder() + .domain("streamate.com") + .name("Xld_rct") + .value("1") + .build(); + getCookieJar().saveFromResponse(HttpUrl.parse(Streamate.BASE_URL), Collections.singletonList(searchCookie)); } @Override diff --git a/common/src/main/java/ctbrec/sites/streamate/StreamateModel.java b/common/src/main/java/ctbrec/sites/streamate/StreamateModel.java index cecf91da..425009d7 100644 --- a/common/src/main/java/ctbrec/sites/streamate/StreamateModel.java +++ b/common/src/main/java/ctbrec/sites/streamate/StreamateModel.java @@ -45,11 +45,10 @@ public class StreamateModel extends AbstractModel { @Override public boolean isOnline(boolean ignoreCache) throws IOException, ExecutionException, InterruptedException { if(ignoreCache) { - String url = getStreamUrl(); - Request req = new Request.Builder().url(url).build(); - try(Response resp = site.getHttpClient().execute(req)) { - online = resp.isSuccessful(); - } + JSONObject roomInfo = getRoomInfo(); + JSONObject stream = roomInfo.getJSONObject("stream"); + String serverId = stream.optString("serverId"); + online = !serverId.equals("0"); } return online; } From 461e65ed84c4aac989b8776c3543bdfb9b4c0ad6 Mon Sep 17 00:00:00 2001 From: 0xboobface <0xboobface@gmail.com> Date: Fri, 14 Dec 2018 14:58:12 +0100 Subject: [PATCH 08/44] Switch to much simpler JSON api --- .../sites/streamate/StreamateTabProvider.java | 27 ++- .../streamate/StreamateUpdateService.java | 77 ++++---- .../ctbrec/ui/sites/streamate/girls.sml | 18 -- .../sites/streamate/StreamateModel.java | 168 ++++-------------- 4 files changed, 79 insertions(+), 211 deletions(-) delete mode 100644 client/src/main/resources/ctbrec/ui/sites/streamate/girls.sml diff --git a/client/src/main/java/ctbrec/ui/sites/streamate/StreamateTabProvider.java b/client/src/main/java/ctbrec/ui/sites/streamate/StreamateTabProvider.java index 5c74b825..137854a5 100644 --- a/client/src/main/java/ctbrec/ui/sites/streamate/StreamateTabProvider.java +++ b/client/src/main/java/ctbrec/ui/sites/streamate/StreamateTabProvider.java @@ -1,8 +1,6 @@ package ctbrec.ui.sites.streamate; -import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.InputStream; import java.util.ArrayList; import java.util.List; @@ -30,7 +28,15 @@ public class StreamateTabProvider extends TabProvider { public List getTabs(Scene scene) { List tabs = new ArrayList<>(); try { - tabs.add(createTab("Girls", "/ctbrec/ui/sites/streamate/girls.sml")); + tabs.add(createTab("Girls", Streamate.BASE_URL + "/api/search/list?domain=streamate.com&index=availperf&filters=gender:f")); + tabs.add(createTab("Guys", Streamate.BASE_URL + "/api/search/list?domain=streamate.com&index=availperf&filters=gender:m")); + tabs.add(createTab("Couples", Streamate.BASE_URL + "/api/search/list?domain=streamate.com&index=availperf&filters=gender:mf")); + tabs.add(createTab("Lesbian", Streamate.BASE_URL + "/api/search/list?domain=streamate.com&index=availperf&filters=gender:ff")); + tabs.add(createTab("Gay", Streamate.BASE_URL + "/api/search/list?domain=streamate.com&index=availperf&filters=gender:mm")); + tabs.add(createTab("Groups", Streamate.BASE_URL + "/api/search/list?domain=streamate.com&index=availperf&filters=gender:g")); + tabs.add(createTab("Trans female", Streamate.BASE_URL + "/api/search/list?domain=streamate.com&index=availperf&filters=gender:tm2f")); + tabs.add(createTab("Trans male", Streamate.BASE_URL + "/api/search/list?domain=streamate.com&index=availperf&filters=gender:tf2m")); + tabs.add(createTab("New", Streamate.BASE_URL + "/api/search/list?domain=streamate.com&index=availperf&filters=new:true")); } catch (IOException e) { LOG.error("Couldn't create streamate tab", e); } @@ -42,21 +48,10 @@ public class StreamateTabProvider extends TabProvider { return null; } - private Tab createTab(String title, String queryFile) throws IOException { - StreamateUpdateService updateService = new StreamateUpdateService(loadQuery(queryFile), streamate); + private Tab createTab(String title, String url) throws IOException { + StreamateUpdateService updateService = new StreamateUpdateService(streamate, url); ThumbOverviewTab tab = new ThumbOverviewTab(title, updateService, streamate); tab.setRecorder(recorder); return tab; } - - private String loadQuery(String file) throws IOException { - InputStream is = getClass().getResourceAsStream(file); - ByteArrayOutputStream bos = new ByteArrayOutputStream(); - byte[] b = new byte[1024]; - int len = -1; - while( (len = is.read(b)) >= 0) { - bos.write(b, 0, len); - } - return new String(bos.toByteArray(), "utf-8"); - } } diff --git a/client/src/main/java/ctbrec/ui/sites/streamate/StreamateUpdateService.java b/client/src/main/java/ctbrec/ui/sites/streamate/StreamateUpdateService.java index 7f9816c6..0703e662 100644 --- a/client/src/main/java/ctbrec/ui/sites/streamate/StreamateUpdateService.java +++ b/client/src/main/java/ctbrec/ui/sites/streamate/StreamateUpdateService.java @@ -1,44 +1,39 @@ package ctbrec.ui.sites.streamate; -import java.io.ByteArrayInputStream; import java.io.IOException; import java.util.ArrayList; import java.util.List; -import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; import javax.xml.xpath.XPathExpressionException; +import org.json.JSONArray; +import org.json.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.w3c.dom.Document; -import org.w3c.dom.Node; -import org.w3c.dom.NodeList; import org.xml.sax.SAXException; import ctbrec.Config; import ctbrec.Model; -import ctbrec.io.XmlParserUtils; +import ctbrec.io.HttpException; import ctbrec.sites.streamate.Streamate; import ctbrec.sites.streamate.StreamateModel; import ctbrec.ui.PaginatedScheduledService; import javafx.concurrent.Task; -import okhttp3.MediaType; import okhttp3.Request; -import okhttp3.RequestBody; import okhttp3.Response; public class StreamateUpdateService extends PaginatedScheduledService { private static final transient Logger LOG = LoggerFactory.getLogger(StreamateUpdateService.class); - private static final String URL = "http://affiliate.streamate.com/SMLive/SMLResult.xml"; + private static final int MODELS_PER_PAGE = 48; private Streamate streamate; - private String query; + private String url; - public StreamateUpdateService(String query, Streamate streamate) { - this.query = query; + public StreamateUpdateService(Streamate streamate, String url) { this.streamate = streamate; + this.url = url; } @Override @@ -46,47 +41,35 @@ public class StreamateUpdateService extends PaginatedScheduledService { return new Task>() { @Override public List call() throws IOException, SAXException, ParserConfigurationException, XPathExpressionException { - LOG.debug("Fetching page {}", URL); - String q = query - .replace("{maxresults}", "50") - .replace("{pagenum}", Integer.toString(page)); - //LOG.debug("Query:\n{}", q); - RequestBody body = RequestBody.create(MediaType.parse("text/xml"), q); + int from = (page - 1) * MODELS_PER_PAGE; + String _url = url + "&from=" + from + "&size=" + MODELS_PER_PAGE; + LOG.debug("Fetching page {}", _url); Request request = new Request.Builder() - .url(URL) + .url(_url) .addHeader("User-Agent", Config.getInstance().getSettings().httpUserAgent) - .addHeader("Accept", "text/xml, */*") + .addHeader("Accept", "application/json, */*") .addHeader("Accept-Language", "en") .addHeader("Referer", streamate.getBaseUrl()) - .post(body) .build(); - Response response = streamate.getHttpClient().execute(request); - if (response.isSuccessful()) { - List models = new ArrayList<>(); - String content = response.body().string(); - ByteArrayInputStream in = new ByteArrayInputStream(content.getBytes("utf-8")); - Document doc = DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(in); - NodeList performers = doc.getElementsByTagName("Performer"); - for (int i = 0; i < performers.getLength(); i++) { - Node performer = performers.item(i); - String name = performer.getAttributes().getNamedItem("Name").getNodeValue(); - String id = performer.getAttributes().getNamedItem("Id").getNodeValue(); - String GoldShow = performer.getAttributes().getNamedItem("GoldShow").getNodeValue(); - String PreGoldShow = performer.getAttributes().getNamedItem("PreGoldShow").getNodeValue(); - String PartyChat = performer.getAttributes().getNamedItem("PartyChat").getNodeValue(); - StreamateModel model = (StreamateModel) streamate.createModel(name); - model.setId(id); - models.add(model); - Node pic = XmlParserUtils.getNodeWithXpath(performer, "Media/Pic/Full"); - String previewUrl = "https:" + pic.getAttributes().getNamedItem("Src").getNodeValue(); - model.setPreview(previewUrl); - //LOG.debug("Name {} - {}{}{}", name, PartyChat, PreGoldShow, GoldShow); + try(Response response = streamate.getHttpClient().execute(request)) { + if (response.isSuccessful()) { + List models = new ArrayList<>(); + String content = response.body().string(); + JSONObject json = new JSONObject(content); + JSONArray performers = json.getJSONArray("performers"); + for (int i = 0; i < performers.length(); i++) { + JSONObject p = performers.getJSONObject(i); + String nickname = p.getString("nickname"); + StreamateModel model = (StreamateModel) streamate.createModel(nickname); + model.setId(Long.toString(p.getLong("id"))); + model.setPreview(p.getString("thumbnail")); + model.setOnline(p.optBoolean("online")); + models.add(model); + } + return models; + } else { + throw new HttpException(response.code(), response.message()); } - return models; - } else { - int code = response.code(); - response.close(); - throw new IOException("HTTP status " + code); } } }; diff --git a/client/src/main/resources/ctbrec/ui/sites/streamate/girls.sml b/client/src/main/resources/ctbrec/ui/sites/streamate/girls.sml deleted file mode 100644 index f7841ae4..00000000 --- a/client/src/main/resources/ctbrec/ui/sites/streamate/girls.sml +++ /dev/null @@ -1,18 +0,0 @@ - - - - - - - - biopic, staticbiopic - - - - live,recorded - - - - \ No newline at end of file diff --git a/common/src/main/java/ctbrec/sites/streamate/StreamateModel.java b/common/src/main/java/ctbrec/sites/streamate/StreamateModel.java index 425009d7..66e58376 100644 --- a/common/src/main/java/ctbrec/sites/streamate/StreamateModel.java +++ b/common/src/main/java/ctbrec/sites/streamate/StreamateModel.java @@ -3,8 +3,6 @@ package ctbrec.sites.streamate; import static ctbrec.Model.State.*; import java.io.IOException; -import java.io.InputStream; -import java.net.URLEncoder; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -15,16 +13,8 @@ import org.json.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.iheartradio.m3u8.Encoding; -import com.iheartradio.m3u8.Format; import com.iheartradio.m3u8.ParseException; -import com.iheartradio.m3u8.ParsingMode; import com.iheartradio.m3u8.PlaylistException; -import com.iheartradio.m3u8.PlaylistParser; -import com.iheartradio.m3u8.data.MasterPlaylist; -import com.iheartradio.m3u8.data.Playlist; -import com.iheartradio.m3u8.data.PlaylistData; -import com.iheartradio.m3u8.data.StreamInfo; import ctbrec.AbstractModel; import ctbrec.Config; @@ -45,10 +35,17 @@ public class StreamateModel extends AbstractModel { @Override public boolean isOnline(boolean ignoreCache) throws IOException, ExecutionException, InterruptedException { if(ignoreCache) { - JSONObject roomInfo = getRoomInfo(); - JSONObject stream = roomInfo.getJSONObject("stream"); - String serverId = stream.optString("serverId"); - online = !serverId.equals("0"); + String url = "https://sea1c-ls.naiadsystems.com/sea1c-edge-ls/80/live/s:" + getName() + ".json"; + Request req = new Request.Builder().url(url) + .addHeader("User-Agent", Config.getInstance().getSettings().httpUserAgent) + .addHeader("Accept", "*/*") + .addHeader("Accept-Language", "en") + .addHeader("Referer", Streamate.BASE_URL + '/' + getName()) + .addHeader("X-Requested-With", "XMLHttpRequest") + .build(); + try(Response response = site.getHttpClient().execute(req)) { + online = response.isSuccessful(); + } } return online; } @@ -76,137 +73,48 @@ public class StreamateModel extends AbstractModel { @Override public List getStreamSources() throws IOException, ExecutionException, ParseException, PlaylistException { - String streamUrl = getStreamUrl(); - if (streamUrl == null) { - return Collections.emptyList(); - } - LOG.debug(streamUrl); - Request req = new Request.Builder().url(streamUrl).build(); - try(Response response = site.getHttpClient().execute(req)) { - if(response.isSuccessful()) { - InputStream inputStream = response.body().byteStream(); - PlaylistParser parser = new PlaylistParser(inputStream, Format.EXT_M3U, Encoding.UTF_8, ParsingMode.LENIENT); - Playlist playlist = parser.parse(); - MasterPlaylist master = playlist.getMasterPlaylist(); - streamSources.clear(); - for (PlaylistData playlistData : master.getPlaylists()) { - StreamSource streamsource = new StreamSource(); - streamsource.mediaPlaylistUrl = playlistData.getUri(); - if (playlistData.hasStreamInfo()) { - StreamInfo info = playlistData.getStreamInfo(); - streamsource.bandwidth = info.getBandwidth(); - streamsource.width = info.hasResolution() ? info.getResolution().width : 0; - streamsource.height = info.hasResolution() ? info.getResolution().height : 0; - } else { - streamsource.bandwidth = 0; - streamsource.width = 0; - streamsource.height = 0; - } - streamSources.add(streamsource); - } - } else { - throw new HttpException(response.code(), response.message()); - } - } - return streamSources; - } - - private String getStreamUrl() throws IOException { - JSONObject json = getRoomInfo(); - JSONObject performer = json.getJSONObject("performer"); - id = Long.toString(performer.getLong("id")); - JSONObject stream = json.getJSONObject("stream"); - String sserver = stream.getString("serverId"); - String streamId = stream.getString("streamId"); - String wsHost = stream.getString("nodeHost"); - JSONObject liveservices = json.getJSONObject("liveservices"); - String streamHost = liveservices.getString("host").replace("wss", "https"); - - String roomId; - try { - roomId = getRoomId(wsHost, sserver, streamId); - LOG.debug("room id: {}", roomId); - } catch (InterruptedException e) { - throw new IOException("Couldn't get room id", e); - } - - String streamFormatUrl = getStreamFormatUrl(streamHost, roomId); - return getMasterPlaylistUrl(streamFormatUrl); - } - - private String getMasterPlaylistUrl(String url) throws IOException { - LOG.debug(url); - Request req = new Request.Builder() + String url = "https://sea1c-ls.naiadsystems.com/sea1c-edge-ls/80/live/s:" + getName() + ".json"; + Request req = new Request.Builder().url(url) .addHeader("User-Agent", Config.getInstance().getSettings().httpUserAgent) .addHeader("Accept", "*/*") .addHeader("Accept-Language", "en") .addHeader("Referer", Streamate.BASE_URL + '/' + getName()) .addHeader("X-Requested-With", "XMLHttpRequest") - .url(url) .build(); try(Response response = site.getHttpClient().execute(req)) { if(response.isSuccessful()) { JSONObject json = new JSONObject(response.body().string()); JSONObject formats = json.getJSONObject("formats"); + JSONObject ws = formats.getJSONObject("mp4-ws"); JSONObject hls = formats.getJSONObject("mp4-hls"); - return hls.getString("manifest"); - } else { - throw new HttpException(response.code(), response.message()); - } - } - } - - private String getStreamFormatUrl(String streamHost, String roomId) throws IOException { - String url = streamHost + "/videourl?payload=" - + URLEncoder.encode("{\"puserid\":" + id + ",\"roomid\":\"" + roomId + "\",\"showtype\":1,\"nginx\":1}", "utf-8"); - LOG.debug(url); - Request req = new Request.Builder() - .addHeader("User-Agent", Config.getInstance().getSettings().httpUserAgent) - .addHeader("Accept", "*/*") - .addHeader("Accept-Language", "en") - .addHeader("Referer", Streamate.BASE_URL + '/' + getName()) - .addHeader("X-Requested-With", "XMLHttpRequest") - .url(url) - .build(); - try(Response response = site.getHttpClient().execute(req)) { - if(response.isSuccessful()) { - JSONArray streamConfig = new JSONArray(response.body().string()); - JSONObject obj = streamConfig.getJSONObject(0); - return obj.getString("url"); - } else { - throw new HttpException(response.code(), response.message()); - } - } - } - - private String getRoomId(String wsHost, String sserver, String streamId) throws InterruptedException { - String wsUrl = wsHost + "/socket.io/?" - + "performerid=" + id - + "&sserver=" + sserver - + "&streamid=" + streamId - + "&sakey=&sessiontype=preview&perfdiscountid=0&minduration=0&goldshowid=0&version=7&referrer=hybrid.client.6.3.16/avchat.swf&usertype=false&lang=en&EIO=3&transport=websocket"; - - StreamateWebsocketClient wsClient = new StreamateWebsocketClient(wsUrl, site.getHttpClient()); - return wsClient.getRoomId(); - } - - private JSONObject getRoomInfo() throws IOException { - String url = "https://hybridclient.naiadsystems.com/api/v1/config/?sabasic=&sakey=&sk=www.streamate.com&userid=0&version=6.3.16&ajax=1&name=" + getName(); - Request req = new Request.Builder() - .addHeader("User-Agent", Config.getInstance().getSettings().httpUserAgent) - .addHeader("Accept", "application/json, text/javascript, */*") - .addHeader("Accept-Language", "en") - .addHeader("Referer", Streamate.BASE_URL + '/' + getName()) - .addHeader("X-Requested-With", "XMLHttpRequest") - .url(url) - .build(); - try(Response response = site.getHttpClient().execute(req)) { - if(response.isSuccessful()) { - return new JSONObject(response.body().string()); + + // add encodings + JSONArray encodings = hls.getJSONArray("encodings"); + streamSources.clear(); + for (int i = 0; i < encodings.length(); i++) { + JSONObject encoding = encodings.getJSONObject(i); + StreamSource src = new StreamSource(); + src.mediaPlaylistUrl = encoding.getString("location"); + src.width = encoding.optInt("videoWidth"); + src.height = encoding.optInt("videoHeight"); + src.bandwidth = (encoding.optInt("videoKbps") + encoding.optInt("audioKbps")) * 1024; + streamSources.add(src); + } + + // add raw source stream + JSONObject origin = hls.getJSONObject("origin"); + StreamSource src = new StreamSource(); + src.mediaPlaylistUrl = origin.getString("location"); + origin = ws.getJSONObject("origin"); // switch to web socket origin, because it has width, height and bitrates + src.width = origin.optInt("videoWidth"); + src.height = origin.optInt("videoHeight"); + src.bandwidth = (origin.optInt("videoKbps") + origin.optInt("audioKbps")) * 1024; + streamSources.add(src); } else { throw new HttpException(response.code(), response.message()); } } + return streamSources; } @Override From 6b52906811529f6d55340b69e98369ed7a8a412a Mon Sep 17 00:00:00 2001 From: 0xboobface <0xboobface@gmail.com> Date: Fri, 14 Dec 2018 15:51:15 +0100 Subject: [PATCH 09/44] Add configuration ui for the credentials --- .../ui/sites/streamate/StreamateConfigUI.java | 86 +++++++++++++++++++ .../ui/sites/streamate/StreamateSiteUi.java | 4 +- .../streamate/StreamateUpdateService.java | 9 ++ common/src/main/java/ctbrec/Settings.java | 6 +- 4 files changed, 102 insertions(+), 3 deletions(-) create mode 100644 client/src/main/java/ctbrec/ui/sites/streamate/StreamateConfigUI.java diff --git a/client/src/main/java/ctbrec/ui/sites/streamate/StreamateConfigUI.java b/client/src/main/java/ctbrec/ui/sites/streamate/StreamateConfigUI.java new file mode 100644 index 00000000..d338cab1 --- /dev/null +++ b/client/src/main/java/ctbrec/ui/sites/streamate/StreamateConfigUI.java @@ -0,0 +1,86 @@ +package ctbrec.ui.sites.streamate; + +import ctbrec.Config; +import ctbrec.Settings; +import ctbrec.sites.streamate.Streamate; +import ctbrec.ui.DesktopIntegration; +import ctbrec.ui.settings.SettingsTab; +import ctbrec.ui.sites.AbstractConfigUI; +import javafx.geometry.Insets; +import javafx.scene.Parent; +import javafx.scene.control.Button; +import javafx.scene.control.CheckBox; +import javafx.scene.control.Label; +import javafx.scene.control.PasswordField; +import javafx.scene.control.TextField; +import javafx.scene.layout.GridPane; +import javafx.scene.layout.Priority; + +public class StreamateConfigUI extends AbstractConfigUI { + private Streamate streamate; + + public StreamateConfigUI(Streamate streamate) { + this.streamate = streamate; + } + + @Override + public Parent createConfigPanel() { + GridPane layout = SettingsTab.createGridLayout(); + Settings settings = Config.getInstance().getSettings(); + + int row = 0; + Label l = new Label("Active"); + layout.add(l, 0, row); + CheckBox enabled = new CheckBox(); + enabled.setSelected(!settings.disabledSites.contains(streamate.getName())); + enabled.setOnAction((e) -> { + if(enabled.isSelected()) { + settings.disabledSites.remove(streamate.getName()); + } else { + settings.disabledSites.add(streamate.getName()); + } + save(); + }); + GridPane.setMargin(enabled, new Insets(0, 0, 0, SettingsTab.CHECKBOX_MARGIN)); + layout.add(enabled, 1, row++); + + layout.add(new Label("Streamate User"), 0, row); + TextField username = new TextField(settings.streamateUsername); + username.textProperty().addListener((ob, o, n) -> { + if(!n.equals(Config.getInstance().getSettings().streamateUsername)) { + Config.getInstance().getSettings().streamateUsername = username.getText(); + streamate.getHttpClient().logout(); + save(); + } + }); + GridPane.setFillWidth(username, true); + GridPane.setHgrow(username, Priority.ALWAYS); + GridPane.setColumnSpan(username, 2); + layout.add(username, 1, row++); + + layout.add(new Label("Streamate Password"), 0, row); + PasswordField password = new PasswordField(); + password.setText(settings.streamatePassword); + password.textProperty().addListener((ob, o, n) -> { + if(!n.equals(Config.getInstance().getSettings().streamatePassword)) { + Config.getInstance().getSettings().streamatePassword = password.getText(); + streamate.getHttpClient().logout(); + save(); + } + }); + GridPane.setFillWidth(password, true); + GridPane.setHgrow(password, Priority.ALWAYS); + GridPane.setColumnSpan(password, 2); + layout.add(password, 1, row++); + + Button createAccount = new Button("Create new Account"); + createAccount.setOnAction((e) -> DesktopIntegration.open(streamate.getAffiliateLink())); + layout.add(createAccount, 1, row++); + GridPane.setColumnSpan(createAccount, 2); + GridPane.setMargin(username, new Insets(0, 0, 0, SettingsTab.CHECKBOX_MARGIN)); + GridPane.setMargin(password, new Insets(0, 0, 0, SettingsTab.CHECKBOX_MARGIN)); + GridPane.setMargin(createAccount, new Insets(0, 0, 0, SettingsTab.CHECKBOX_MARGIN)); + return layout; + } + +} diff --git a/client/src/main/java/ctbrec/ui/sites/streamate/StreamateSiteUi.java b/client/src/main/java/ctbrec/ui/sites/streamate/StreamateSiteUi.java index 8d31d020..bd29af2b 100644 --- a/client/src/main/java/ctbrec/ui/sites/streamate/StreamateSiteUi.java +++ b/client/src/main/java/ctbrec/ui/sites/streamate/StreamateSiteUi.java @@ -10,9 +10,11 @@ import ctbrec.ui.TabProvider; public class StreamateSiteUi implements SiteUI { private StreamateTabProvider tabProvider; + private StreamateConfigUI configUi; public StreamateSiteUi(Streamate streamate) { tabProvider = new StreamateTabProvider(streamate); + configUi = new StreamateConfigUI(streamate); } @Override @@ -22,7 +24,7 @@ public class StreamateSiteUi implements SiteUI { @Override public ConfigUI getConfigUI() { - return null; + return configUi; } @Override diff --git a/client/src/main/java/ctbrec/ui/sites/streamate/StreamateUpdateService.java b/client/src/main/java/ctbrec/ui/sites/streamate/StreamateUpdateService.java index 0703e662..37ea045c 100644 --- a/client/src/main/java/ctbrec/ui/sites/streamate/StreamateUpdateService.java +++ b/client/src/main/java/ctbrec/ui/sites/streamate/StreamateUpdateService.java @@ -64,6 +64,15 @@ public class StreamateUpdateService extends PaginatedScheduledService { model.setId(Long.toString(p.getLong("id"))); model.setPreview(p.getString("thumbnail")); model.setOnline(p.optBoolean("online")); + // TODO figure out, what all the states mean + // liveState {…} + // exclusiveShow false + // goldShow true + // onBreak false + // partyChat true + // preGoldShow true + // privateChat false + // specialShow false models.add(model); } return models; diff --git a/common/src/main/java/ctbrec/Settings.java b/common/src/main/java/ctbrec/Settings.java index 043b55af..168f2e25 100644 --- a/common/src/main/java/ctbrec/Settings.java +++ b/common/src/main/java/ctbrec/Settings.java @@ -60,8 +60,10 @@ public class Settings { public boolean mfcIgnoreUpscaled = false; public String camsodaUsername = ""; public String camsodaPassword = ""; - public String cam4Username; - public String cam4Password; + public String cam4Username = ""; + public String cam4Password = ""; + public String streamateUsername = ""; + public String streamatePassword = ""; public String lastDownloadDir = ""; public List models = new ArrayList<>(); From c7e07b4b261cf0e793a99d39d77755539880c95d Mon Sep 17 00:00:00 2001 From: 0xboobface <0xboobface@gmail.com> Date: Fri, 14 Dec 2018 17:36:24 +0100 Subject: [PATCH 10/44] Implement login and favorites tab --- .../streamate/StreamateFollowedService.java | 94 +++++++++++++++++++ .../sites/streamate/StreamateFollowedTab.java | 77 +++++++++++++++ .../ui/sites/streamate/StreamateSiteUi.java | 4 +- .../sites/streamate/StreamateTabProvider.java | 7 +- .../ctbrec/sites/streamate/Streamate.java | 5 +- .../sites/streamate/StreamateHttpClient.java | 51 +++++++++- 6 files changed, 234 insertions(+), 4 deletions(-) create mode 100644 client/src/main/java/ctbrec/ui/sites/streamate/StreamateFollowedService.java create mode 100644 client/src/main/java/ctbrec/ui/sites/streamate/StreamateFollowedTab.java diff --git a/client/src/main/java/ctbrec/ui/sites/streamate/StreamateFollowedService.java b/client/src/main/java/ctbrec/ui/sites/streamate/StreamateFollowedService.java new file mode 100644 index 00000000..6aab7d5b --- /dev/null +++ b/client/src/main/java/ctbrec/ui/sites/streamate/StreamateFollowedService.java @@ -0,0 +1,94 @@ +package ctbrec.ui.sites.streamate; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.xpath.XPathExpressionException; + +import org.json.JSONArray; +import org.json.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.xml.sax.SAXException; + +import ctbrec.Config; +import ctbrec.Model; +import ctbrec.io.HttpException; +import ctbrec.sites.streamate.Streamate; +import ctbrec.sites.streamate.StreamateHttpClient; +import ctbrec.sites.streamate.StreamateModel; +import ctbrec.ui.PaginatedScheduledService; +import javafx.concurrent.Task; +import okhttp3.Request; +import okhttp3.Response; + +public class StreamateFollowedService extends PaginatedScheduledService { + + private static final transient Logger LOG = LoggerFactory.getLogger(StreamateFollowedService.class); + + private static final int MODELS_PER_PAGE = 48; + private Streamate streamate; + private StreamateHttpClient httpClient; + private String url; + private boolean showOnline = true; + + public StreamateFollowedService(Streamate streamate) { + this.streamate = streamate; + this.httpClient = (StreamateHttpClient) streamate.getHttpClient(); + this.url = streamate.getBaseUrl() + "/api/search/v1/favorites?host=streamate.com&domain=streamate.com"; + } + + @Override + protected Task> createTask() { + return new Task>() { + @Override + public List call() throws IOException, SAXException, ParserConfigurationException, XPathExpressionException { + httpClient.login(); + String saKey = httpClient.getSaKey(); + String userId = httpClient.getUserId(); + String _url = url + "&page_number=" + page + "&results_per_page=" + MODELS_PER_PAGE + "&sakey=" + saKey + "&userid=" + userId; + LOG.debug("Fetching page {}", _url); + Request request = new Request.Builder() + .url(_url) + .addHeader("User-Agent", Config.getInstance().getSettings().httpUserAgent) + .addHeader("Accept", "application/json, */*") + .addHeader("Accept-Language", "en") + .addHeader("Referer", streamate.getBaseUrl()) + .build(); + try(Response response = streamate.getHttpClient().execute(request)) { + if (response.isSuccessful()) { + List models = new ArrayList<>(); + String content = response.body().string(); + JSONObject json = new JSONObject(content); + if(json.optString("status").equals("SM_OK")) { + JSONArray performers = json.getJSONArray("Results"); + for (int i = 0; i < performers.length(); i++) { + JSONObject p = performers.getJSONObject(i); + String nickname = p.getString("Nickname"); + StreamateModel model = (StreamateModel) streamate.createModel(nickname); + model.setId(Long.toString(p.getLong("PerformerId"))); + model.setPreview("https://m1.nsimg.net/biopic/320x240/" + model.getId()); + boolean online = p.optString("LiveStatus").equals("live"); + model.setOnline(online); + if(online == showOnline) { + models.add(model); + } + } + } else { + throw new IOException("Status: " + json.optString("status")); + } + return models; + } else { + throw new HttpException(response.code(), response.message()); + } + } + } + }; + } + + public void setOnline(boolean online) { + this.showOnline = online; + } +} diff --git a/client/src/main/java/ctbrec/ui/sites/streamate/StreamateFollowedTab.java b/client/src/main/java/ctbrec/ui/sites/streamate/StreamateFollowedTab.java new file mode 100644 index 00000000..f79cc30f --- /dev/null +++ b/client/src/main/java/ctbrec/ui/sites/streamate/StreamateFollowedTab.java @@ -0,0 +1,77 @@ +package ctbrec.ui.sites.streamate; + +import ctbrec.sites.streamate.Streamate; +import ctbrec.ui.FollowedTab; +import ctbrec.ui.ThumbOverviewTab; +import javafx.concurrent.WorkerStateEvent; +import javafx.geometry.Insets; +import javafx.scene.Scene; +import javafx.scene.control.Label; +import javafx.scene.control.RadioButton; +import javafx.scene.control.ToggleGroup; +import javafx.scene.input.KeyCode; +import javafx.scene.input.KeyEvent; +import javafx.scene.layout.HBox; + +public class StreamateFollowedTab extends ThumbOverviewTab implements FollowedTab { + private Label status; + + public StreamateFollowedTab(Streamate streamate) { + super("Favorites", new StreamateFollowedService(streamate), streamate); + + status = new Label("Logging in..."); + grid.getChildren().add(status); + } + + @Override + protected void createGui() { + super.createGui(); + addOnlineOfflineSelector(); + } + + private void addOnlineOfflineSelector() { + ToggleGroup group = new ToggleGroup(); + RadioButton online = new RadioButton("online"); + online.setToggleGroup(group); + RadioButton offline = new RadioButton("offline"); + offline.setToggleGroup(group); + pagination.getChildren().add(online); + pagination.getChildren().add(offline); + HBox.setMargin(online, new Insets(5,5,5,40)); + HBox.setMargin(offline, new Insets(5,5,5,5)); + online.setSelected(true); + group.selectedToggleProperty().addListener((e) -> { + ((StreamateFollowedService)updateService).setOnline(online.isSelected()); + queue.clear(); + updateService.restart(); + }); + } + + @Override + protected void onSuccess() { + grid.getChildren().remove(status); + super.onSuccess(); + } + + @Override + protected void onFail(WorkerStateEvent event) { + status.setText("Login failed"); + super.onFail(event); + } + + @Override + public void selected() { + status.setText("Logging in..."); + super.selected(); + } + + public void setScene(Scene scene) { + scene.addEventFilter(KeyEvent.KEY_PRESSED, event -> { + if(this.isSelected()) { + if(event.getCode() == KeyCode.DELETE) { + follow(selectedThumbCells, false); + } + } + }); + } +} diff --git a/client/src/main/java/ctbrec/ui/sites/streamate/StreamateSiteUi.java b/client/src/main/java/ctbrec/ui/sites/streamate/StreamateSiteUi.java index bd29af2b..c7348a1f 100644 --- a/client/src/main/java/ctbrec/ui/sites/streamate/StreamateSiteUi.java +++ b/client/src/main/java/ctbrec/ui/sites/streamate/StreamateSiteUi.java @@ -11,8 +11,10 @@ public class StreamateSiteUi implements SiteUI { private StreamateTabProvider tabProvider; private StreamateConfigUI configUi; + private Streamate streamate; public StreamateSiteUi(Streamate streamate) { + this.streamate = streamate; tabProvider = new StreamateTabProvider(streamate); configUi = new StreamateConfigUI(streamate); } @@ -29,7 +31,7 @@ public class StreamateSiteUi implements SiteUI { @Override public boolean login() throws IOException { - return false; + return streamate.login(); } } diff --git a/client/src/main/java/ctbrec/ui/sites/streamate/StreamateTabProvider.java b/client/src/main/java/ctbrec/ui/sites/streamate/StreamateTabProvider.java index 137854a5..d43ec25f 100644 --- a/client/src/main/java/ctbrec/ui/sites/streamate/StreamateTabProvider.java +++ b/client/src/main/java/ctbrec/ui/sites/streamate/StreamateTabProvider.java @@ -18,6 +18,7 @@ public class StreamateTabProvider extends TabProvider { private static final transient Logger LOG = LoggerFactory.getLogger(StreamateTabProvider.class); private Streamate streamate; private Recorder recorder; + private ThumbOverviewTab followedTab; public StreamateTabProvider(Streamate streamate) { this.streamate = streamate; @@ -37,6 +38,10 @@ public class StreamateTabProvider extends TabProvider { tabs.add(createTab("Trans female", Streamate.BASE_URL + "/api/search/list?domain=streamate.com&index=availperf&filters=gender:tm2f")); tabs.add(createTab("Trans male", Streamate.BASE_URL + "/api/search/list?domain=streamate.com&index=availperf&filters=gender:tf2m")); tabs.add(createTab("New", Streamate.BASE_URL + "/api/search/list?domain=streamate.com&index=availperf&filters=new:true")); + + followedTab = new StreamateFollowedTab(streamate); + followedTab.setRecorder(recorder); + tabs.add(followedTab); } catch (IOException e) { LOG.error("Couldn't create streamate tab", e); } @@ -45,7 +50,7 @@ public class StreamateTabProvider extends TabProvider { @Override public Tab getFollowedTab() { - return null; + return followedTab; } private Tab createTab(String title, String url) throws IOException { diff --git a/common/src/main/java/ctbrec/sites/streamate/Streamate.java b/common/src/main/java/ctbrec/sites/streamate/Streamate.java index 12cf63c7..390767f0 100644 --- a/common/src/main/java/ctbrec/sites/streamate/Streamate.java +++ b/common/src/main/java/ctbrec/sites/streamate/Streamate.java @@ -15,6 +15,7 @@ import org.slf4j.LoggerFactory; import ctbrec.Config; import ctbrec.Model; +import ctbrec.StringUtil; import ctbrec.io.HttpClient; import ctbrec.io.HttpException; import ctbrec.sites.AbstractSite; @@ -150,6 +151,7 @@ public class Streamate extends AbstractSite { if (response.isSuccessful()) { String body = response.body().string(); JSONObject json = new JSONObject(body); + LOG.debug(json.toString(2)); if (json.optString("status").equals("SM_OK")) { List models = new ArrayList<>(); JSONObject results = json.getJSONObject("results"); @@ -183,7 +185,8 @@ public class Streamate extends AbstractSite { @Override public boolean credentialsAvailable() { - return false; + String username = Config.getInstance().getSettings().username; + return StringUtil.isNotBlank(username); } @Override diff --git a/common/src/main/java/ctbrec/sites/streamate/StreamateHttpClient.java b/common/src/main/java/ctbrec/sites/streamate/StreamateHttpClient.java index aa64b61f..d1242dac 100644 --- a/common/src/main/java/ctbrec/sites/streamate/StreamateHttpClient.java +++ b/common/src/main/java/ctbrec/sites/streamate/StreamateHttpClient.java @@ -3,17 +3,26 @@ package ctbrec.sites.streamate; import java.io.IOException; import java.util.Collections; +import org.json.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import ctbrec.Config; import ctbrec.io.HttpClient; import okhttp3.Cookie; import okhttp3.HttpUrl; +import okhttp3.MediaType; +import okhttp3.Request; +import okhttp3.RequestBody; +import okhttp3.Response; public class StreamateHttpClient extends HttpClient { private static final transient Logger LOG = LoggerFactory.getLogger(StreamateHttpClient.class); + private String userId = ""; + private String saKey = ""; + public StreamateHttpClient() { super("streamate"); @@ -39,7 +48,38 @@ public class StreamateHttpClient extends HttpClient { return true; } - return false; + JSONObject loginRequest = new JSONObject(); + loginRequest.put("email", Config.getInstance().getSettings().streamateUsername); + loginRequest.put("password", Config.getInstance().getSettings().streamatePassword); + loginRequest.put("referrerId", 0); + loginRequest.put("siteId", 1); + RequestBody body = RequestBody.create(MediaType.parse("application/json"), loginRequest.toString()); + Request login = new Request.Builder() + .url(Streamate.BASE_URL + "/api/member/login") + .addHeader("User-Agent", Config.getInstance().getSettings().httpUserAgent) + .addHeader("Accept", "application/json, text/javascript, */*") + .addHeader("Accept-Language", "en") + .addHeader("Referer", Streamate.BASE_URL) + .addHeader("X-Requested-With", "XMLHttpRequest") + .post(body) + .build(); + try (Response response = client.newCall(login).execute()) { + String content = response.body().string(); + //LOG.debug(content); + if(response.isSuccessful()) { + JSONObject json = new JSONObject(content); + LOG.debug(json.toString()); + loggedIn = json.has("sakey"); + saKey = json.optString("sakey"); + JSONObject account = json.getJSONObject("account"); + userId = Long.toString(account.getLong("userid")); + } else { + throw new IOException("Login failed: " + response.code() + " " + response.message()); + } + response.close(); + } + + return loggedIn; } /** @@ -47,6 +87,7 @@ public class StreamateHttpClient extends HttpClient { * @throws IOException */ public boolean checkLoginSuccess() throws IOException { + //https://www.streamate.com/api/search/v1/favorites?host=streamate.com&domain=streamate.com&page_number=1&results_per_page=48&sakey=62857cfd1908cd28 return false; // String modelName = getAnyModelName(); // // we request the roomData of a random model, because it contains @@ -83,4 +124,12 @@ public class StreamateHttpClient extends HttpClient { // } // } } + + public String getSaKey() { + return saKey; + } + + public String getUserId() { + return userId; + } } From 75ab95e1ea8f56c3c7bc98817fa4dcb1fd65c4f3 Mon Sep 17 00:00:00 2001 From: 0xboobface <0xboobface@gmail.com> Date: Fri, 14 Dec 2018 17:43:38 +0100 Subject: [PATCH 11/44] Shut down more gracefully (hopefully) --- client/src/main/java/ctbrec/ui/CamrecApplication.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/client/src/main/java/ctbrec/ui/CamrecApplication.java b/client/src/main/java/ctbrec/ui/CamrecApplication.java index d91a9d5e..b2913d98 100644 --- a/client/src/main/java/ctbrec/ui/CamrecApplication.java +++ b/client/src/main/java/ctbrec/ui/CamrecApplication.java @@ -178,9 +178,13 @@ public class CamrecApplication extends Application { try { Config.getInstance().save(); LOG.info("Shutdown complete. Goodbye!"); - Platform.exit(); - // This is needed, because OkHttp?! seems to block the shutdown with its writer threads. They are not daemon threads :( - System.exit(0); + Platform.runLater(() -> { + primaryStage.close(); + shutdownInfo.close(); + Platform.exit(); + // This is needed, because OkHttp?! seems to block the shutdown with its writer threads. They are not daemon threads :( + System.exit(0); + }); } catch (IOException e1) { Platform.runLater(() -> { Alert alert = new AutosizeAlert(Alert.AlertType.ERROR); From 4d7409f443fc7e4541c6391d49b7758bd7326e77 Mon Sep 17 00:00:00 2001 From: 0xboobface <0xboobface@gmail.com> Date: Fri, 14 Dec 2018 20:25:57 +0100 Subject: [PATCH 12/44] Implement follow/unfollow and login with cookies --- .../streamate/StreamateFollowedService.java | 4 +- .../streamate/StreamateUpdateService.java | 2 +- .../ctbrec/sites/streamate/Streamate.java | 5 +- .../sites/streamate/StreamateHttpClient.java | 82 +++++++++---------- .../sites/streamate/StreamateModel.java | 61 ++++++++++++-- 5 files changed, 98 insertions(+), 56 deletions(-) diff --git a/client/src/main/java/ctbrec/ui/sites/streamate/StreamateFollowedService.java b/client/src/main/java/ctbrec/ui/sites/streamate/StreamateFollowedService.java index 6aab7d5b..2c78e1d7 100644 --- a/client/src/main/java/ctbrec/ui/sites/streamate/StreamateFollowedService.java +++ b/client/src/main/java/ctbrec/ui/sites/streamate/StreamateFollowedService.java @@ -47,7 +47,7 @@ public class StreamateFollowedService extends PaginatedScheduledService { public List call() throws IOException, SAXException, ParserConfigurationException, XPathExpressionException { httpClient.login(); String saKey = httpClient.getSaKey(); - String userId = httpClient.getUserId(); + Long userId = httpClient.getUserId(); String _url = url + "&page_number=" + page + "&results_per_page=" + MODELS_PER_PAGE + "&sakey=" + saKey + "&userid=" + userId; LOG.debug("Fetching page {}", _url); Request request = new Request.Builder() @@ -68,7 +68,7 @@ public class StreamateFollowedService extends PaginatedScheduledService { JSONObject p = performers.getJSONObject(i); String nickname = p.getString("Nickname"); StreamateModel model = (StreamateModel) streamate.createModel(nickname); - model.setId(Long.toString(p.getLong("PerformerId"))); + model.setId(p.getLong("PerformerId")); model.setPreview("https://m1.nsimg.net/biopic/320x240/" + model.getId()); boolean online = p.optString("LiveStatus").equals("live"); model.setOnline(online); diff --git a/client/src/main/java/ctbrec/ui/sites/streamate/StreamateUpdateService.java b/client/src/main/java/ctbrec/ui/sites/streamate/StreamateUpdateService.java index 37ea045c..083a2008 100644 --- a/client/src/main/java/ctbrec/ui/sites/streamate/StreamateUpdateService.java +++ b/client/src/main/java/ctbrec/ui/sites/streamate/StreamateUpdateService.java @@ -61,7 +61,7 @@ public class StreamateUpdateService extends PaginatedScheduledService { JSONObject p = performers.getJSONObject(i); String nickname = p.getString("nickname"); StreamateModel model = (StreamateModel) streamate.createModel(nickname); - model.setId(Long.toString(p.getLong("id"))); + model.setId(p.getLong("id")); model.setPreview(p.getString("thumbnail")); model.setOnline(p.optBoolean("online")); // TODO figure out, what all the states mean diff --git a/common/src/main/java/ctbrec/sites/streamate/Streamate.java b/common/src/main/java/ctbrec/sites/streamate/Streamate.java index 390767f0..a86eb91b 100644 --- a/common/src/main/java/ctbrec/sites/streamate/Streamate.java +++ b/common/src/main/java/ctbrec/sites/streamate/Streamate.java @@ -125,7 +125,7 @@ public class Streamate extends AbstractSite { @Override public boolean supportsFollow() { - return false; + return true; } @Override @@ -151,7 +151,6 @@ public class Streamate extends AbstractSite { if (response.isSuccessful()) { String body = response.body().string(); JSONObject json = new JSONObject(body); - LOG.debug(json.toString(2)); if (json.optString("status").equals("SM_OK")) { List models = new ArrayList<>(); JSONObject results = json.getJSONObject("results"); @@ -159,7 +158,7 @@ public class Streamate extends AbstractSite { for (int i = 0; i < nickname.length(); i++) { JSONObject result = nickname.getJSONObject(i); StreamateModel model = (StreamateModel) createModel(result.getString("nickname")); - model.setId(result.getString("performerId")); + model.setId(Long.parseLong(result.getString("performerId"))); String thumb = result.getString("thumbnail"); if (thumb != null) { model.setPreview(thumb); diff --git a/common/src/main/java/ctbrec/sites/streamate/StreamateHttpClient.java b/common/src/main/java/ctbrec/sites/streamate/StreamateHttpClient.java index d1242dac..b901b385 100644 --- a/common/src/main/java/ctbrec/sites/streamate/StreamateHttpClient.java +++ b/common/src/main/java/ctbrec/sites/streamate/StreamateHttpClient.java @@ -2,6 +2,7 @@ package ctbrec.sites.streamate; import java.io.IOException; import java.util.Collections; +import java.util.NoSuchElementException; import org.json.JSONObject; import org.slf4j.Logger; @@ -20,7 +21,7 @@ public class StreamateHttpClient extends HttpClient { private static final transient Logger LOG = LoggerFactory.getLogger(StreamateHttpClient.class); - private String userId = ""; + private Long userId; private String saKey = ""; public StreamateHttpClient() { @@ -33,6 +34,14 @@ public class StreamateHttpClient extends HttpClient { .value("1") .build(); getCookieJar().saveFromResponse(HttpUrl.parse(Streamate.BASE_URL), Collections.singletonList(searchCookie)); + + // try to load sakey from cookie + try { + Cookie cookie = getCookieJar().getCookie(HttpUrl.parse("https://www.streamate.com"), "sakey"); + saKey = cookie.value(); + } catch (NoSuchElementException e) { + // ignore + } } @Override @@ -65,14 +74,12 @@ public class StreamateHttpClient extends HttpClient { .build(); try (Response response = client.newCall(login).execute()) { String content = response.body().string(); - //LOG.debug(content); if(response.isSuccessful()) { JSONObject json = new JSONObject(content); - LOG.debug(json.toString()); loggedIn = json.has("sakey"); saKey = json.optString("sakey"); JSONObject account = json.getJSONObject("account"); - userId = Long.toString(account.getLong("userid")); + userId = account.getLong("userid"); } else { throw new IOException("Login failed: " + response.code() + " " + response.message()); } @@ -83,53 +90,40 @@ public class StreamateHttpClient extends HttpClient { } /** - * Check, if the login worked - * @throws IOException + * Check, if the login worked by loading the favorites */ - public boolean checkLoginSuccess() throws IOException { - //https://www.streamate.com/api/search/v1/favorites?host=streamate.com&domain=streamate.com&page_number=1&results_per_page=48&sakey=62857cfd1908cd28 - return false; - // String modelName = getAnyModelName(); - // // we request the roomData of a random model, because it contains - // // user data, if the user is logged in, which we can use to verify, that the login worked - // String url = Streamate.BASE_URL + "/tools/amf.php"; - // RequestBody body = new FormBody.Builder() - // .add("method", "getRoomData") - // .add("args[]", modelName) - // .add("args[]", "false") - // //.add("method", "ping") // TODO alternative request, but - // //.add("args[]", ) // where to get the userId - // .build(); - // Request request = new Request.Builder() - // .url(url) - // .addHeader("User-Agent", Config.getInstance().getSettings().httpUserAgent) - // .addHeader("Accept", "application/json, text/javascript, */*") - // .addHeader("Accept-Language", "en") - // .addHeader("Referer", Streamate.BASE_URL) - // .addHeader("X-Requested-With", "XMLHttpRequest") - // .post(body) - // .build(); - // try(Response response = execute(request)) { - // if(response.isSuccessful()) { - // JSONObject json = new JSONObject(response.body().string()); - // if(json.optString("status").equals("success")) { - // JSONObject userData = json.getJSONObject("userData"); - // userId = userData.optInt("userId"); - // return userId > 0; - // } else { - // throw new IOException("Request was not successful: " + json.toString(2)); - // } - // } else { - // throw new HttpException(response.code(), response.message()); - // } - // } + public boolean checkLoginSuccess() { + String url = Streamate.BASE_URL + "/api/search/v1/favorites?host=streamate.com&domain=streamate.com"; + url = url + "&page_number=1&results_per_page=48&sakey=" + saKey + "&userid=" + userId; + Request request = new Request.Builder() + .url(url) + .addHeader("User-Agent", Config.getInstance().getSettings().httpUserAgent) + .addHeader("Accept", "application/json, */*") + .addHeader("Accept-Language", "en") + .addHeader("Referer", Streamate.BASE_URL) + .build(); + try(Response response = execute(request)) { + if (response.isSuccessful()) { + String content = response.body().string(); + JSONObject json = new JSONObject(content); + if(json.optString("status").equals("SM_OK")) { + return true; + } else { + return false; + } + } else { + return false; + } + } catch(Exception e) { + return false; + } } public String getSaKey() { return saKey; } - public String getUserId() { + public Long getUserId() { return userId; } } diff --git a/common/src/main/java/ctbrec/sites/streamate/StreamateModel.java b/common/src/main/java/ctbrec/sites/streamate/StreamateModel.java index 66e58376..4ea1b54b 100644 --- a/common/src/main/java/ctbrec/sites/streamate/StreamateModel.java +++ b/common/src/main/java/ctbrec/sites/streamate/StreamateModel.java @@ -15,12 +15,16 @@ import org.slf4j.LoggerFactory; import com.iheartradio.m3u8.ParseException; import com.iheartradio.m3u8.PlaylistException; +import com.squareup.moshi.JsonReader; +import com.squareup.moshi.JsonWriter; import ctbrec.AbstractModel; import ctbrec.Config; import ctbrec.io.HttpException; import ctbrec.recorder.download.StreamSource; +import okhttp3.MediaType; import okhttp3.Request; +import okhttp3.RequestBody; import okhttp3.Response; public class StreamateModel extends AbstractModel { @@ -30,7 +34,7 @@ public class StreamateModel extends AbstractModel { private boolean online = false; private List streamSources = new ArrayList<>(); private int[] resolution; - private String id; + private Long id; @Override public boolean isOnline(boolean ignoreCache) throws IOException, ExecutionException, InterruptedException { @@ -180,19 +184,64 @@ public class StreamateModel extends AbstractModel { @Override public boolean follow() throws IOException { - return false; + return follow(true); } @Override public boolean unfollow() throws IOException { - return false; + return follow(false); } - public String getId() { + private boolean follow(boolean follow) throws IOException { + StreamateHttpClient client = (StreamateHttpClient) getSite().getHttpClient(); + client.login(); + String saKey = client.getSaKey(); + Long userId = client.getUserId(); + + JSONObject requestParams = new JSONObject(); + requestParams.put("sakey", saKey); + requestParams.put("userid", userId); + requestParams.put("pid", id); + requestParams.put("domain", "streamate.com"); + requestParams.put("fav", follow); + RequestBody body = RequestBody.create(MediaType.parse("application/json"), requestParams.toString()); + + String url = site.getBaseUrl() + "/ajax/fav-notify.php?userid="+userId+"&sakey="+saKey+"&pid="+id+"&fav="+follow+"&domain=streamate.com"; + Request request = new Request.Builder() + .url(url) + .addHeader("User-Agent", Config.getInstance().getSettings().httpUserAgent) + .addHeader("Accept", "application/json, */*") + .addHeader("Accept-Language", "en") + .addHeader("Referer", getSite().getBaseUrl()) + .post(body) + .build(); + try(Response response = getSite().getHttpClient().execute(request)) { + String content = response.body().string(); + if (response.isSuccessful()) { + JSONObject json = new JSONObject(content); + return json.optBoolean("success"); + } else { + throw new HttpException(response.code(), response.message()); + } + } + } + + public Long getId() { return id; } - public void setId(String id) { + public void setId(Long id) { this.id = id; } -} + + @Override + public void readSiteSpecificData(JsonReader reader) throws IOException { + reader.nextName(); + id = reader.nextLong(); + } + + @Override + public void writeSiteSpecificData(JsonWriter writer) throws IOException { + writer.name("id").value(id); + } +} \ No newline at end of file From bd719eac08b0a1b474b30603aaa720df0fd09dd8 Mon Sep 17 00:00:00 2001 From: 0xboobface <0xboobface@gmail.com> Date: Fri, 14 Dec 2018 23:35:44 +0100 Subject: [PATCH 13/44] Remove direct refences to chaturbate in TipDialog This dialog is used for other sites, too. So we have to use the site object to get the name and the affiliate link --- client/src/main/java/ctbrec/ui/TipDialog.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/client/src/main/java/ctbrec/ui/TipDialog.java b/client/src/main/java/ctbrec/ui/TipDialog.java index 2b4dfcf0..8aaefb93 100644 --- a/client/src/main/java/ctbrec/ui/TipDialog.java +++ b/client/src/main/java/ctbrec/ui/TipDialog.java @@ -8,7 +8,6 @@ import org.slf4j.LoggerFactory; import ctbrec.Model; import ctbrec.sites.Site; -import ctbrec.sites.chaturbate.Chaturbate; import javafx.application.Platform; import javafx.concurrent.Task; import javafx.scene.control.Alert; @@ -48,7 +47,7 @@ public class TipDialog extends TextInputDialog { int tokens = get(); Platform.runLater(() -> { if (tokens <= 0) { - String msg = "Do you want to buy tokens now?\n\nIf you agree, Chaturbate will open in a browser. " + String msg = "Do you want to buy tokens now?\n\nIf you agree, "+site.getName()+" will open in a browser. " + "The used address is an affiliate link, which supports me, but doesn't cost you anything more."; Alert buyTokens = new AutosizeAlert(Alert.AlertType.CONFIRMATION, msg, ButtonType.NO, ButtonType.YES); buyTokens.setTitle("No tokens"); @@ -56,7 +55,7 @@ public class TipDialog extends TextInputDialog { buyTokens.showAndWait(); TipDialog.this.close(); if(buyTokens.getResult() == ButtonType.YES) { - DesktopIntegration.open(Chaturbate.AFFILIATE_LINK); + DesktopIntegration.open(site.getAffiliateLink()); } } else { getEditor().setDisable(false); From b83235a32f2c6b3aa386a6edf8fb210ce13be543 Mon Sep 17 00:00:00 2001 From: 0xboobface <0xboobface@gmail.com> Date: Fri, 14 Dec 2018 23:36:00 +0100 Subject: [PATCH 14/44] Log error, if sending tip failed --- client/src/main/java/ctbrec/ui/ThumbOverviewTab.java | 1 + 1 file changed, 1 insertion(+) diff --git a/client/src/main/java/ctbrec/ui/ThumbOverviewTab.java b/client/src/main/java/ctbrec/ui/ThumbOverviewTab.java index aa06a2b7..6ce0cdde 100644 --- a/client/src/main/java/ctbrec/ui/ThumbOverviewTab.java +++ b/client/src/main/java/ctbrec/ui/ThumbOverviewTab.java @@ -479,6 +479,7 @@ public class ThumbOverviewTab extends Tab implements TabSelectionListener { event.put("amount", tokens); EventBusHolder.BUS.post(event); } catch (Exception e1) { + LOG.error("An error occured while sending tip", e1); showError("Couldn't send tip", "An error occured while sending tip:", e1); } } else { From 1ce9a111a98d7a906b0d87476e9de013af822fd0 Mon Sep 17 00:00:00 2001 From: 0xboobface <0xboobface@gmail.com> Date: Fri, 14 Dec 2018 23:37:55 +0100 Subject: [PATCH 15/44] Add tipping for Streamate Tipping does not work, yet. The server returns success: false. I don't know, what the parameters have to look like --- .../streamate/StreamateFollowedService.java | 3 +- .../sites/streamate/StreamateHttpClient.java | 17 ++- .../sites/streamate/StreamateModel.java | 136 ++++++++++++++---- 3 files changed, 124 insertions(+), 32 deletions(-) diff --git a/client/src/main/java/ctbrec/ui/sites/streamate/StreamateFollowedService.java b/client/src/main/java/ctbrec/ui/sites/streamate/StreamateFollowedService.java index 2c78e1d7..d78b04c1 100644 --- a/client/src/main/java/ctbrec/ui/sites/streamate/StreamateFollowedService.java +++ b/client/src/main/java/ctbrec/ui/sites/streamate/StreamateFollowedService.java @@ -47,8 +47,7 @@ public class StreamateFollowedService extends PaginatedScheduledService { public List call() throws IOException, SAXException, ParserConfigurationException, XPathExpressionException { httpClient.login(); String saKey = httpClient.getSaKey(); - Long userId = httpClient.getUserId(); - String _url = url + "&page_number=" + page + "&results_per_page=" + MODELS_PER_PAGE + "&sakey=" + saKey + "&userid=" + userId; + String _url = url + "&page_number=" + page + "&results_per_page=" + MODELS_PER_PAGE + "&sakey=" + saKey; LOG.debug("Fetching page {}", _url); Request request = new Request.Builder() .url(_url) diff --git a/common/src/main/java/ctbrec/sites/streamate/StreamateHttpClient.java b/common/src/main/java/ctbrec/sites/streamate/StreamateHttpClient.java index b901b385..3f056e73 100644 --- a/common/src/main/java/ctbrec/sites/streamate/StreamateHttpClient.java +++ b/common/src/main/java/ctbrec/sites/streamate/StreamateHttpClient.java @@ -23,6 +23,7 @@ public class StreamateHttpClient extends HttpClient { private Long userId; private String saKey = ""; + private String userNickname = ""; public StreamateHttpClient() { super("streamate"); @@ -57,6 +58,11 @@ public class StreamateHttpClient extends HttpClient { return true; } + loggedIn = loginWithoutCookies(); + return loggedIn; + } + + private synchronized boolean loginWithoutCookies() throws IOException { JSONObject loginRequest = new JSONObject(); loginRequest.put("email", Config.getInstance().getSettings().streamateUsername); loginRequest.put("password", Config.getInstance().getSettings().streamatePassword); @@ -76,10 +82,12 @@ public class StreamateHttpClient extends HttpClient { String content = response.body().string(); if(response.isSuccessful()) { JSONObject json = new JSONObject(content); + LOG.debug(json.toString(2)); loggedIn = json.has("sakey"); saKey = json.optString("sakey"); JSONObject account = json.getJSONObject("account"); userId = account.getLong("userid"); + userNickname = account.getString("nickname"); } else { throw new IOException("Login failed: " + response.code() + " " + response.message()); } @@ -123,7 +131,14 @@ public class StreamateHttpClient extends HttpClient { return saKey; } - public Long getUserId() { + public Long getUserId() throws IOException { + if(userId == null) { + loginWithoutCookies(); + } return userId; } + + public String getUserNickname() { + return userNickname; + } } diff --git a/common/src/main/java/ctbrec/sites/streamate/StreamateModel.java b/common/src/main/java/ctbrec/sites/streamate/StreamateModel.java index 4ea1b54b..c4e04670 100644 --- a/common/src/main/java/ctbrec/sites/streamate/StreamateModel.java +++ b/common/src/main/java/ctbrec/sites/streamate/StreamateModel.java @@ -22,10 +22,12 @@ import ctbrec.AbstractModel; import ctbrec.Config; import ctbrec.io.HttpException; import ctbrec.recorder.download.StreamSource; +import okhttp3.FormBody; import okhttp3.MediaType; import okhttp3.Request; import okhttp3.RequestBody; import okhttp3.Response; +import okio.Buffer; public class StreamateModel extends AbstractModel { @@ -128,35 +130,111 @@ public class StreamateModel extends AbstractModel { @Override public void receiveTip(int tokens) throws IOException { - // String url = Streamate.BASE_URL + "/chat-ajax-amf-service?" + System.currentTimeMillis(); - // int userId = ((StreamateHttpClient)site.getHttpClient()).getUserId(); - // RequestBody body = new FormBody.Builder() - // .add("method", "tipModel") - // .add("args[]", getName()) - // .add("args[]", Integer.toString(tokens)) - // .add("args[]", Integer.toString(userId)) - // .add("args[3]", "") - // .build(); - // Request request = new Request.Builder() - // .url(url) - // .addHeader("User-Agent", Config.getInstance().getSettings().httpUserAgent) - // .addHeader("Accept", "application/json, text/javascript, */*") - // .addHeader("Accept-Language", "en") - // .addHeader("Referer", Streamate.BASE_URL + '/' + getName()) - // .addHeader("X-Requested-With", "XMLHttpRequest") - // .post(body) - // .build(); - // try(Response response = site.getHttpClient().execute(request)) { - // if(response.isSuccessful()) { - // JSONObject json = new JSONObject(response.body().string()); - // if(!json.optString("status").equals("success")) { - // LOG.error("Sending tip failed {}", json.toString(2)); - // throw new IOException("Sending tip failed"); - // } - // } else { - // throw new IOException(response.code() + ' ' + response.message()); - // } - // } + /* + Mt._giveGoldAjax = function(e, t) { + var n = _t.getState(), + a = n.nickname, + o = n.id, + i = Ds.getState(), + r = i.userStreamId, + s = i.sakey, + l = i.userId, + c = i.nickname, + u = ""; + switch (Ot.getState().streamType) { + case z.STREAM_TYPE_PRIVATE: + case z.STREAM_TYPE_BLOCK: + u = "premium"; + break; + case z.STREAM_TYPE_EXCLUSIVE: + case z.STREAM_TYPE_BLOCK_EXCLUSIVE: + u = "exclusive" + } + if (!l) return ae.a.reject("no userId!"); + var d = { + amt: e, + isprepopulated: t, + modelname: a, + nickname: c, + performernickname: a, + sakey: s, + session: u, + smid: o, + streamid: r, + userid: l, + username: c + }, + p = de.a.getBaseUrl() + "/api/v1/givegold/"; + return de.a.postPromise(p, d, "json") + }, + */ + + StreamateHttpClient client = (StreamateHttpClient) getSite().getHttpClient(); + client.login(); + String saKey = client.getSaKey(); + Long userId = client.getUserId(); + String nickname = client.getUserNickname(); + + String url = "https://hybridclient.naiadsystems.com/api/v1/givegold/"; // this returns 404 at the moment. not sure if it's the wrong server, or if this is not used anymore + RequestBody body = new FormBody.Builder() + .add("amt", Integer.toString(tokens)) // amount + .add("isprepopulated", "1") // ? + .add("modelname", getName()) // model's name + .add("nickname", nickname) // user's nickname + .add("performernickname", getName()) // model's name + .add("sakey", saKey) // sakey from login + .add("session", "") // is related to gold an private shows, for normal tips keep it empty + .add("smid", Long.toString(getId())) // model id + .add("streamid", getStreamId()) // id of the current stream + .add("userid", Long.toString(userId)) // user's id + .add("username", nickname) // user's nickname + .build(); + Buffer b = new Buffer(); + body.writeTo(b); + LOG.debug("tip params {}", b.readUtf8()); + Request request = new Request.Builder() + .url(url) + .addHeader("User-Agent", Config.getInstance().getSettings().httpUserAgent) + .addHeader("Accept", "application/json, text/javascript, */*") + .addHeader("Accept-Language", "en") + .addHeader("Referer", Streamate.BASE_URL + '/' + getName()) + .addHeader("X-Requested-With", "XMLHttpRequest") + .post(body) + .build(); + try(Response response = site.getHttpClient().execute(request)) { + if(response.isSuccessful()) { + JSONObject json = new JSONObject(response.body().string()); + LOG.debug(json.toString(2)); + if(!json.optString("status").equals("success")) { + LOG.error("Sending tip failed {}", json.toString(2)); + throw new IOException("Sending tip failed"); + } + } else { + throw new HttpException(response.code(), response.message()); + } + } + } + + private String getStreamId() throws IOException { + String url = "https://hybridclient.naiadsystems.com/api/v1/config/?name=" + getName() + + "&sabasic=&sakey=&sk=www.streamate.com&userid=0&version=6.3.17&ajax=1"; + Request request = new Request.Builder() + .url(url) + .addHeader("User-Agent", Config.getInstance().getSettings().httpUserAgent) + .addHeader("Accept", "application/json, text/javascript, */*") + .addHeader("Accept-Language", "en") + .addHeader("Referer", Streamate.BASE_URL + '/' + getName()) + .addHeader("X-Requested-With", "XMLHttpRequest") + .build(); + try(Response response = site.getHttpClient().execute(request)) { + if(response.isSuccessful()) { + JSONObject json = new JSONObject(response.body().string()); + JSONObject stream = json.getJSONObject("stream"); + return stream.getString("streamId"); + } else { + throw new HttpException(response.code(), response.message()); + } + } } @Override From b2d1d41abc6c14f445f831178c421b15dfebf65d Mon Sep 17 00:00:00 2001 From: 0xboobface <0xboobface@gmail.com> Date: Fri, 14 Dec 2018 23:53:10 +0100 Subject: [PATCH 16/44] Remove ordering by sequence This was used for Chaturbate, because the filename format was known. With several camsites the filename format can differ and this is not a good solution anymore. Instead we now just sort filename. To make sure, the files have the right order, HlsDownload now creates a prefix for each segment. --- .../java/ctbrec/recorder/PlaylistGenerator.java | 16 +--------------- .../ctbrec/recorder/download/HlsDownload.java | 15 +++++++++++---- 2 files changed, 12 insertions(+), 19 deletions(-) diff --git a/common/src/main/java/ctbrec/recorder/PlaylistGenerator.java b/common/src/main/java/ctbrec/recorder/PlaylistGenerator.java index a4180765..1c02a614 100644 --- a/common/src/main/java/ctbrec/recorder/PlaylistGenerator.java +++ b/common/src/main/java/ctbrec/recorder/PlaylistGenerator.java @@ -9,8 +9,6 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import java.util.regex.Matcher; -import java.util.regex.Pattern; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -48,10 +46,8 @@ public class PlaylistGenerator { Arrays.sort(files, (f1, f2) -> { String n1 = f1.getName(); - int seq1 = getSequence(n1); String n2 = f2.getName(); - int seq2 = getSequence(n2); - return seq1 - seq2; + return n1.compareTo(n2); }); // create a track containing all files @@ -102,16 +98,6 @@ public class PlaylistGenerator { return output; } - private int getSequence(String filename) { - filename = filename.substring(0, filename.lastIndexOf('.')); // cut off file suffix - Matcher matcher = Pattern.compile(".*?(\\d+)").matcher(filename); - if(matcher.matches()) { - return Integer.parseInt(matcher.group(1)); - } else { - return -1; - } - } - private void updateProgressListeners(double percentage) { int p = (int) (percentage*100); if(p > lastPercentage) { diff --git a/common/src/main/java/ctbrec/recorder/download/HlsDownload.java b/common/src/main/java/ctbrec/recorder/download/HlsDownload.java index 94aa5a4d..b99f3be7 100644 --- a/common/src/main/java/ctbrec/recorder/download/HlsDownload.java +++ b/common/src/main/java/ctbrec/recorder/download/HlsDownload.java @@ -13,6 +13,8 @@ import java.nio.file.FileSystems; import java.nio.file.Files; import java.nio.file.LinkOption; import java.nio.file.Path; +import java.text.DecimalFormat; +import java.text.NumberFormat; import java.text.SimpleDateFormat; import java.time.Instant; import java.util.Date; @@ -39,6 +41,9 @@ public class HlsDownload extends AbstractHlsDownload { protected Path downloadDir; + private int segmentCounter = 1; + private NumberFormat nf = new DecimalFormat("000000"); + public HlsDownload(HttpClient client) { super(client); } @@ -78,7 +83,8 @@ public class HlsDownload extends AbstractHlsDownload { for (int i = nextSegment; i < lsp.seq; i++) { URL segmentUrl = new URL(first.replaceAll(Integer.toString(seq), Integer.toString(i))); LOG.debug("Reloading segment {} for model {}", i, model.getName()); - downloadThreadPool.submit(new SegmentDownload(segmentUrl, downloadDir, client)); + String prefix = nf.format(segmentCounter++); + downloadThreadPool.submit(new SegmentDownload(segmentUrl, downloadDir, client, prefix)); } // TODO switch to a lower bitrate/resolution ?!? } @@ -88,7 +94,8 @@ public class HlsDownload extends AbstractHlsDownload { skip--; } else { URL segmentUrl = new URL(segment); - downloadThreadPool.submit(new SegmentDownload(segmentUrl, downloadDir, client)); + String prefix = nf.format(segmentCounter++); + downloadThreadPool.submit(new SegmentDownload(segmentUrl, downloadDir, client, prefix)); //new SegmentDownload(segment, downloadDir).call(); } } @@ -150,11 +157,11 @@ public class HlsDownload extends AbstractHlsDownload { private Path file; private HttpClient client; - public SegmentDownload(URL url, Path dir, HttpClient client) { + public SegmentDownload(URL url, Path dir, HttpClient client, String prefix) { this.url = url; this.client = client; File path = new File(url.getPath()); - file = FileSystems.getDefault().getPath(dir.toString(), path.getName()); + file = FileSystems.getDefault().getPath(dir.toString(), prefix + '_' + path.getName()); } @Override From e1c16cda9b290febf0de9dc2750e22d3c2c68ab8 Mon Sep 17 00:00:00 2001 From: 0xboobface <0xboobface@gmail.com> Date: Fri, 14 Dec 2018 23:53:19 +0100 Subject: [PATCH 17/44] Add Streamate --- server/src/main/java/ctbrec/recorder/server/HttpServer.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/src/main/java/ctbrec/recorder/server/HttpServer.java b/server/src/main/java/ctbrec/recorder/server/HttpServer.java index ee57211d..4262c6ff 100644 --- a/server/src/main/java/ctbrec/recorder/server/HttpServer.java +++ b/server/src/main/java/ctbrec/recorder/server/HttpServer.java @@ -33,6 +33,7 @@ import ctbrec.sites.cam4.Cam4; import ctbrec.sites.camsoda.Camsoda; import ctbrec.sites.chaturbate.Chaturbate; import ctbrec.sites.mfc.MyFreeCams; +import ctbrec.sites.streamate.Streamate; public class HttpServer { @@ -82,6 +83,7 @@ public class HttpServer { sites.add(new Camsoda()); sites.add(new Cam4()); sites.add(new BongaCams()); + sites.add(new Streamate()); } private void addShutdownHook() { From 465e417b6c2695a5a4db8ce8f8eaa4ac5a982b27 Mon Sep 17 00:00:00 2001 From: 0xboobface <0xboobface@gmail.com> Date: Sat, 15 Dec 2018 13:18:15 +0100 Subject: [PATCH 18/44] Ignore models without username in JSON response Fix for #120 There are objects in the JSON response, which don't look like regular model entries. If an object doesn't have a username, ignore it. --- .../java/ctbrec/ui/sites/bonga/BongaCamsUpdateService.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/client/src/main/java/ctbrec/ui/sites/bonga/BongaCamsUpdateService.java b/client/src/main/java/ctbrec/ui/sites/bonga/BongaCamsUpdateService.java index e7845956..beb2f07c 100644 --- a/client/src/main/java/ctbrec/ui/sites/bonga/BongaCamsUpdateService.java +++ b/client/src/main/java/ctbrec/ui/sites/bonga/BongaCamsUpdateService.java @@ -56,7 +56,10 @@ public class BongaCamsUpdateService extends PaginatedScheduledService { JSONArray _models = json.getJSONArray("models"); for (int i = 0; i < _models.length(); i++) { JSONObject m = _models.getJSONObject(i); - String name = m.getString("username"); + String name = m.optString("username"); + if(name.isEmpty()) { + continue; + } BongaCamsModel model = (BongaCamsModel) bongaCams.createModel(name); boolean away = m.optBoolean("is_away"); boolean online = m.optBoolean("online"); From d09aad1bf630ed8acf7f1e5a18863c0c110fb2af Mon Sep 17 00:00:00 2001 From: 0xboobface <0xboobface@gmail.com> Date: Sat, 15 Dec 2018 15:55:17 +0100 Subject: [PATCH 19/44] Move stream preview to its own control Move stream preview to its own control, so that it can be used in the ThumbCell, too --- .../java/ctbrec/ui/PreviewPopupHandler.java | 168 +---------------- .../ctbrec/ui/controls/StreamPreview.java | 178 ++++++++++++++++++ 2 files changed, 188 insertions(+), 158 deletions(-) create mode 100644 client/src/main/java/ctbrec/ui/controls/StreamPreview.java diff --git a/client/src/main/java/ctbrec/ui/PreviewPopupHandler.java b/client/src/main/java/ctbrec/ui/PreviewPopupHandler.java index e6ffc72a..16de2224 100644 --- a/client/src/main/java/ctbrec/ui/PreviewPopupHandler.java +++ b/client/src/main/java/ctbrec/ui/PreviewPopupHandler.java @@ -1,39 +1,23 @@ package ctbrec.ui; -import java.io.InterruptedIOException; -import java.util.Collections; -import java.util.List; import java.util.Objects; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import ctbrec.Config; -import ctbrec.io.HttpException; -import ctbrec.recorder.download.StreamSource; +import ctbrec.ui.controls.StreamPreview; import javafx.application.Platform; import javafx.event.EventHandler; import javafx.geometry.Insets; import javafx.geometry.Point2D; import javafx.scene.Node; -import javafx.scene.control.ProgressIndicator; import javafx.scene.control.TableColumn; import javafx.scene.control.TableRow; import javafx.scene.control.TableView; -import javafx.scene.image.Image; -import javafx.scene.image.ImageView; import javafx.scene.input.MouseButton; import javafx.scene.input.MouseEvent; -import javafx.scene.layout.Region; import javafx.scene.layout.StackPane; -import javafx.scene.media.Media; -import javafx.scene.media.MediaPlayer; -import javafx.scene.media.MediaView; import javafx.stage.Popup; public class PreviewPopupHandler implements EventHandler { @@ -44,53 +28,24 @@ public class PreviewPopupHandler implements EventHandler { private long timeForPopupClose = 400; private Popup popup = new Popup(); private Node parent; - private ImageView preview = new ImageView(); - private MediaView videoPreview; - private MediaPlayer videoPlayer; - private Media video; + private StreamPreview streamPreview; private JavaFxModel model; private volatile long openCountdown = -1; private volatile long closeCountdown = -1; private volatile long lastModelChange = -1; private volatile boolean changeModel = false; - private ExecutorService executor = Executors.newSingleThreadExecutor(); - private Future future; - private ProgressIndicator progressIndicator; - private StackPane pane; public PreviewPopupHandler(Node parent) { this.parent = parent; - videoPreview = new MediaView(); - videoPreview.setFitWidth(Config.getInstance().getSettings().thumbWidth); - videoPreview.setFitHeight(videoPreview.getFitWidth() * 9 / 16); - videoPreview.setPreserveRatio(true); - StackPane.setMargin(videoPreview, new Insets(5)); - - preview.setFitWidth(Config.getInstance().getSettings().thumbWidth); - preview.setPreserveRatio(true); - preview.setSmooth(true); - preview.setStyle("-fx-background-radius: 10px, 10px, 10px, 10px;"); - preview.visibleProperty().bind(videoPreview.visibleProperty().not()); - StackPane.setMargin(preview, new Insets(5)); - - progressIndicator = new ProgressIndicator(); - progressIndicator.setVisible(false); - progressIndicator.prefWidthProperty().bind(videoPreview.fitWidthProperty()); - - Region veil = new Region(); - veil.setStyle("-fx-background-color: rgba(0, 0, 0, 0.8)"); - veil.visibleProperty().bind(progressIndicator.visibleProperty()); - StackPane.setMargin(veil, new Insets(5)); - - pane = new StackPane(); - pane.getChildren().addAll(preview, videoPreview, veil, progressIndicator); - pane.setStyle("-fx-background-color: -fx-outer-border, -fx-inner-border, -fx-base;"+ + streamPreview = new StreamPreview(); + streamPreview.setStyle("-fx-background-color: -fx-outer-border, -fx-inner-border, -fx-base;"+ "-fx-background-insets: 0 0 -1 0, 0, 1, 2;" + "-fx-background-radius: 10px, 10px, 10px, 10px;" + "-fx-padding: 1;" + "-fx-effect: dropshadow(gaussian, rgba(0, 0, 0, 0.8), 20, 0, 0, 0);"); - popup.getContent().add(pane); + popup.getContent().add(streamPreview); + StackPane.setMargin(streamPreview, new Insets(5)); createTimerThread(); } @@ -121,8 +76,7 @@ public class PreviewPopupHandler implements EventHandler { if(modelChanged) { lastModelChange = System.currentTimeMillis(); changeModel = true; - future.cancel(true); - progressIndicator.setVisible(true); + streamPreview.stop(); } } else { openCountdown = timeForPopupOpen; @@ -173,121 +127,19 @@ public class PreviewPopupHandler implements EventHandler { } private void startStream(JavaFxModel model) { - if(future != null && !future.isDone()) { - future.cancel(true); - } - future = executor.submit(() -> { - try { - Platform.runLater(() -> { - progressIndicator.setVisible(true); - popup.show(parent.getScene().getWindow()); - }); - List sources = model.getStreamSources(); - Collections.sort(sources); - StreamSource best = sources.get(0); - checkInterrupt(); - LOG.debug("Preview url for {} is {}", model.getName(), best.getMediaPlaylistUrl()); - video = new Media(best.getMediaPlaylistUrl()); - if(videoPlayer != null) { - videoPlayer.dispose(); - } - videoPlayer = new MediaPlayer(video); - videoPlayer.setMute(true); - checkInterrupt(); - videoPlayer.setOnReady(() -> { - if(!future.isCancelled()) { - Platform.runLater(() -> { - double aspect = (double)video.getWidth() / video.getHeight(); - double w = Config.getInstance().getSettings().thumbWidth; - double h = w / aspect; - resize(w, h); - progressIndicator.setVisible(false); - videoPreview.setVisible(true); - videoPreview.setMediaPlayer(videoPlayer); - videoPlayer.play(); - }); - } - }); - videoPlayer.setOnError(() -> onError(videoPlayer)); - } catch (IllegalStateException e) { - if(e.getMessage().equals("Stream url unknown")) { - // fine hls url for mfc not known yet - } else { - LOG.warn("Couldn't start preview video: {}", e.getMessage()); - } - showTestImage(); - } catch (HttpException e) { - if(e.getResponseCode() != 404) { - LOG.warn("Couldn't start preview video: {}", e.getMessage()); - } - showTestImage(); - } catch (InterruptedException | InterruptedIOException e) { - // future has been canceled, that's fine - } catch (ExecutionException e) { - if(e.getCause() instanceof InterruptedException || e.getCause() instanceof InterruptedIOException) { - // future has been canceled, that's fine - } else { - LOG.warn("Couldn't start preview video: {}", e.getMessage()); - showTestImage(); - } - } catch (Exception e) { - LOG.warn("Couldn't start preview video: {}", e.getMessage()); - showTestImage(); - } - }); - } - - private void onError(MediaPlayer videoPlayer) { - LOG.error("Error while starting preview stream", videoPlayer.getError()); - if(videoPlayer.getError().getCause() != null) { - LOG.error("Error while starting preview stream root cause:", videoPlayer.getError().getCause()); - } - videoPlayer.dispose(); Platform.runLater(() -> { - showTestImage(); + streamPreview.startStream(model); + popup.show(parent.getScene().getWindow()); }); - } - private void resize(double w, double h) { - preview.setFitWidth(w); - preview.setFitHeight(h); - videoPreview.setFitWidth(w); - videoPreview.setFitHeight(h); - pane.setPrefSize(w, h); - popup.setWidth(w); - popup.setHeight(h); - } - - private void checkInterrupt() throws InterruptedException { - if(Thread.interrupted()) { - throw new InterruptedException(); - } - } - - private void showTestImage() { - Platform.runLater(() -> { - videoPreview.setVisible(false); - Image img = new Image(getClass().getResource("/image_not_found.png").toString(), true); - preview.setImage(img); - double aspect = img.getWidth() / img.getHeight(); - double w = Config.getInstance().getSettings().thumbWidth; - double h = w / aspect; - resize(w, h); - progressIndicator.setVisible(false); - }); } private void hidePopup() { - if(future != null && !future.isDone()) { - future.cancel(true); - } Platform.runLater(() -> { popup.setX(-1000); popup.setY(-1000); popup.hide(); - if(videoPlayer != null) { - videoPlayer.dispose(); - } + streamPreview.stop(); }); } diff --git a/client/src/main/java/ctbrec/ui/controls/StreamPreview.java b/client/src/main/java/ctbrec/ui/controls/StreamPreview.java new file mode 100644 index 00000000..258ec42b --- /dev/null +++ b/client/src/main/java/ctbrec/ui/controls/StreamPreview.java @@ -0,0 +1,178 @@ +package ctbrec.ui.controls; + +import java.io.InterruptedIOException; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import ctbrec.Config; +import ctbrec.Model; +import ctbrec.io.HttpException; +import ctbrec.recorder.download.StreamSource; +import javafx.application.Platform; +import javafx.geometry.Insets; +import javafx.scene.control.ProgressIndicator; +import javafx.scene.image.Image; +import javafx.scene.image.ImageView; +import javafx.scene.layout.Region; +import javafx.scene.layout.StackPane; +import javafx.scene.media.Media; +import javafx.scene.media.MediaPlayer; +import javafx.scene.media.MediaView; + +public class StreamPreview extends StackPane { + private static final transient Logger LOG = LoggerFactory.getLogger(StreamPreview.class); + + private ImageView preview = new ImageView(); + private MediaView videoPreview; + private MediaPlayer videoPlayer; + private Media video; + private ProgressIndicator progressIndicator; + private ExecutorService executor = Executors.newSingleThreadExecutor(); + private Future future; + + public StreamPreview() { + videoPreview = new MediaView(); + videoPreview.setFitWidth(Config.getInstance().getSettings().thumbWidth); + videoPreview.setFitHeight(videoPreview.getFitWidth() * 9 / 16); + videoPreview.setPreserveRatio(true); + StackPane.setMargin(videoPreview, new Insets(5)); + + preview.setFitWidth(Config.getInstance().getSettings().thumbWidth); + preview.setPreserveRatio(true); + preview.setSmooth(true); + preview.setStyle("-fx-background-radius: 10px, 10px, 10px, 10px;"); + preview.visibleProperty().bind(videoPreview.visibleProperty().not()); + StackPane.setMargin(preview, new Insets(5)); + + progressIndicator = new ProgressIndicator(); + progressIndicator.setVisible(false); + progressIndicator.prefWidthProperty().bind(videoPreview.fitWidthProperty()); + + Region veil = new Region(); + veil.setStyle("-fx-background-color: rgba(0, 0, 0, 0.8)"); + veil.visibleProperty().bind(progressIndicator.visibleProperty()); + StackPane.setMargin(veil, new Insets(5)); + + getChildren().addAll(preview, videoPreview, veil, progressIndicator); + } + + public void startStream(Model model) { + if(future != null && !future.isDone()) { + future.cancel(true); + } + future = executor.submit(() -> { + try { + Platform.runLater(() -> { + progressIndicator.setVisible(true); + }); + List sources = model.getStreamSources(); + Collections.sort(sources); + StreamSource best = sources.get(0); + checkInterrupt(); + LOG.debug("Preview url for {} is {}", model.getName(), best.getMediaPlaylistUrl()); + video = new Media(best.getMediaPlaylistUrl()); + if(videoPlayer != null) { + videoPlayer.dispose(); + } + videoPlayer = new MediaPlayer(video); + videoPlayer.setMute(true); + checkInterrupt(); + videoPlayer.setOnReady(() -> { + if(!future.isCancelled()) { + Platform.runLater(() -> { + double aspect = (double)video.getWidth() / video.getHeight(); + double w = Config.getInstance().getSettings().thumbWidth; + double h = w / aspect; + resizeToFitContent(w, h); + progressIndicator.setVisible(false); + videoPreview.setVisible(true); + videoPreview.setMediaPlayer(videoPlayer); + videoPlayer.play(); + }); + } + }); + videoPlayer.setOnError(() -> onError(videoPlayer)); + } catch (IllegalStateException e) { + if(e.getMessage().equals("Stream url unknown")) { + // fine hls url for mfc not known yet + } else { + LOG.warn("Couldn't start preview video: {}", e.getMessage()); + } + showTestImage(); + } catch (HttpException e) { + if(e.getResponseCode() != 404) { + LOG.warn("Couldn't start preview video: {}", e.getMessage()); + } + showTestImage(); + } catch (InterruptedException | InterruptedIOException e) { + // future has been canceled, that's fine + } catch (ExecutionException e) { + if(e.getCause() instanceof InterruptedException || e.getCause() instanceof InterruptedIOException) { + // future has been canceled, that's fine + } else { + LOG.warn("Couldn't start preview video: {}", e.getMessage()); + showTestImage(); + } + } catch (Exception e) { + LOG.warn("Couldn't start preview video: {}", e.getMessage()); + showTestImage(); + } + }); + } + + private void resizeToFitContent(double w, double h) { + setPrefSize(w, h); + preview.setFitWidth(w); + preview.setFitHeight(h); + videoPreview.setFitWidth(w); + videoPreview.setFitHeight(h); + } + + public void stop() { + if(future != null && !future.isDone()) { + future.cancel(true); + } + Platform.runLater(() -> { + if(videoPlayer != null) { + videoPlayer.dispose(); + } + }); + } + + private void onError(MediaPlayer videoPlayer) { + LOG.error("Error while starting preview stream", videoPlayer.getError()); + if(videoPlayer.getError().getCause() != null) { + LOG.error("Error while starting preview stream root cause:", videoPlayer.getError().getCause()); + } + videoPlayer.dispose(); + Platform.runLater(() -> { + showTestImage(); + }); + } + + private void showTestImage() { + Platform.runLater(() -> { + videoPreview.setVisible(false); + Image img = new Image(getClass().getResource("/image_not_found.png").toString(), true); + preview.setImage(img); + double aspect = img.getWidth() / img.getHeight(); + double w = Config.getInstance().getSettings().thumbWidth; + double h = w / aspect; + resizeToFitContent(w, h); + progressIndicator.setVisible(false); + }); + } + + private void checkInterrupt() throws InterruptedException { + if(Thread.interrupted()) { + throw new InterruptedException(); + } + } +} From f6313067682ef2815f61624807100c73949d1cfa Mon Sep 17 00:00:00 2001 From: 0xboobface <0xboobface@gmail.com> Date: Sat, 15 Dec 2018 20:33:57 +0100 Subject: [PATCH 20/44] Tweak video preview in thumb cell --- client/src/main/java/ctbrec/ui/ThumbCell.java | 66 ++++++++++++++++++- .../ctbrec/ui/controls/StreamPreview.java | 36 ++++++---- 2 files changed, 87 insertions(+), 15 deletions(-) diff --git a/client/src/main/java/ctbrec/ui/ThumbCell.java b/client/src/main/java/ctbrec/ui/ThumbCell.java index 898f3516..cb67fa21 100644 --- a/client/src/main/java/ctbrec/ui/ThumbCell.java +++ b/client/src/main/java/ctbrec/ui/ThumbCell.java @@ -22,6 +22,7 @@ import ctbrec.Model; import ctbrec.io.HttpException; import ctbrec.recorder.Recorder; import ctbrec.ui.action.PlayAction; +import ctbrec.ui.controls.StreamPreview; import javafx.animation.FadeTransition; import javafx.animation.FillTransition; import javafx.animation.ParallelTransition; @@ -43,6 +44,7 @@ import javafx.scene.layout.StackPane; import javafx.scene.paint.Color; import javafx.scene.paint.Paint; import javafx.scene.shape.Circle; +import javafx.scene.shape.Polygon; import javafx.scene.shape.Rectangle; import javafx.scene.shape.Shape; import javafx.scene.text.Font; @@ -58,6 +60,7 @@ public class ThumbCell extends StackPane { private static final Duration ANIMATION_DURATION = new Duration(250); private Model model; + private StreamPreview streamPreview; private ImageView iv; private Rectangle resolutionBackground; private final Paint resolutionOnlineColor = new Color(0.22, 0.8, 0.29, 1); @@ -87,8 +90,10 @@ public class ThumbCell extends StackPane { .expireAfterAccess(4, TimeUnit.HOURS) .maximumSize(1000) .build(); + private ThumbOverviewTab parent; public ThumbCell(ThumbOverviewTab parent, Model model, Recorder recorder) { + this.parent = parent; this.thumbCellList = parent.grid.getChildren(); this.model = model; this.recorder = recorder; @@ -96,6 +101,11 @@ public class ThumbCell extends StackPane { model.setSuspended(recorder.isSuspended(model)); this.setStyle("-fx-background-color: -fx-base"); + streamPreview = new StreamPreview(); + streamPreview.prefWidthProperty().bind(widthProperty()); + streamPreview.prefHeightProperty().bind(heightProperty()); + getChildren().add(streamPreview); + iv = new ImageView(); iv.setSmooth(true); iv.setPreserveRatio(true); @@ -164,8 +174,10 @@ public class ThumbCell extends StackPane { StackPane.setAlignment(pausedIndicator, Pos.TOP_LEFT); getChildren().add(pausedIndicator); + getChildren().add(createPreviewTrigger()); + selectionOverlay = new Rectangle(); - selectionOverlay.setOpacity(0); + selectionOverlay.visibleProperty().bind(selectionProperty); selectionOverlay.widthProperty().bind(widthProperty()); selectionOverlay.heightProperty().bind(heightProperty()); StackPane.setAlignment(selectionOverlay, Pos.TOP_LEFT); @@ -197,6 +209,50 @@ public class ThumbCell extends StackPane { update(); } + private Node createPreviewTrigger() { + int s = 32; + StackPane previewTrigger = new StackPane(); + previewTrigger.setStyle("-fx-background-color: white;"); + previewTrigger.setOpacity(.8); + previewTrigger.setMaxSize(s, s); + + Polygon play = new Polygon(new double[] { + 16, 8, + 26, 15, + 16, 22 + }); + StackPane.setMargin(play, new Insets(0, 0, 0, 3)); + play.setStyle("-fx-background-color: black;"); + previewTrigger.getChildren().add(play); + + Circle clip = new Circle(s / 2); + clip.setTranslateX(clip.getRadius()); + clip.setTranslateY(clip.getRadius()); + previewTrigger.setClip(clip); + StackPane.setAlignment(previewTrigger, Pos.BOTTOM_LEFT); + StackPane.setMargin(previewTrigger, new Insets(0, 0, 24, 4)); + previewTrigger.setOnMouseEntered(evt -> setPreviewVisible(previewTrigger, true)); + previewTrigger.setOnMouseExited(evt -> setPreviewVisible(previewTrigger, false)); + return previewTrigger; + } + + private void setPreviewVisible(Node previewTrigger, boolean visible) { + parent.suspendUpdates(visible); + iv.setVisible(!visible); + topic.setVisible(!visible); + topicBackground.setVisible(!visible); + name.setVisible(!visible); + nameBackground.setVisible(!visible); + streamPreview.setVisible(visible); + streamPreview.startStream(model); + recordingIndicator.setVisible(!visible); + pausedIndicator.setVisible(!visible); + if(!visible) { + updateRecordingIndicator(); + } + previewTrigger.setCursor(visible ? Cursor.HAND : Cursor.DEFAULT); + } + public void setSelected(boolean selected) { selectionProperty.set(selected); selectionOverlay.getStyleClass().add("selection-background"); @@ -356,6 +412,10 @@ public class ThumbCell extends StackPane { nameBackground.setFill(c); } + updateRecordingIndicator(); + } + + private void updateRecordingIndicator() { if(recording) { recordingIndicator.setVisible(!model.isSuspended()); pausedIndicator.setVisible(model.isSuspended()); @@ -574,13 +634,15 @@ public class ThumbCell extends StackPane { nameBackground.setWidth(w); nameBackground.setHeight(20); topicBackground.setWidth(w); - topicBackground.setHeight(getHeight()-nameBackground.getHeight()); + topicBackground.setHeight(h - nameBackground.getHeight()); topic.prefHeight(getHeight()-25); topic.maxHeight(getHeight()-25); int margin = 4; topic.maxWidth(w-margin*2); topic.setWrappingWidth(w-margin*2); + streamPreview.resizeTo(w, h); + Rectangle clip = new Rectangle(w, h); clip.setArcWidth(10); clip.arcHeightProperty().bind(clip.arcWidthProperty()); diff --git a/client/src/main/java/ctbrec/ui/controls/StreamPreview.java b/client/src/main/java/ctbrec/ui/controls/StreamPreview.java index 258ec42b..6b7bc022 100644 --- a/client/src/main/java/ctbrec/ui/controls/StreamPreview.java +++ b/client/src/main/java/ctbrec/ui/controls/StreamPreview.java @@ -34,8 +34,8 @@ public class StreamPreview extends StackPane { private MediaPlayer videoPlayer; private Media video; private ProgressIndicator progressIndicator; - private ExecutorService executor = Executors.newSingleThreadExecutor(); - private Future future; + private static ExecutorService executor = Executors.newSingleThreadExecutor(); + private static Future future; public StreamPreview() { videoPreview = new MediaView(); @@ -53,7 +53,6 @@ public class StreamPreview extends StackPane { progressIndicator = new ProgressIndicator(); progressIndicator.setVisible(false); - progressIndicator.prefWidthProperty().bind(videoPreview.fitWidthProperty()); Region veil = new Region(); veil.setStyle("-fx-background-color: rgba(0, 0, 0, 0.8)"); @@ -64,14 +63,25 @@ public class StreamPreview extends StackPane { } public void startStream(Model model) { + Platform.runLater(() -> { + progressIndicator.setVisible(true); + if(model.getPreview() != null) { + try { + videoPreview.setVisible(false); + Image img = new Image(model.getPreview(), true); + preview.setImage(img); + double aspect = img.getWidth() / img.getHeight(); + double w = Config.getInstance().getSettings().thumbWidth; + double h = w / aspect; + resizeTo(w, h); + } catch (Exception e) {} + } + }); if(future != null && !future.isDone()) { future.cancel(true); } future = executor.submit(() -> { try { - Platform.runLater(() -> { - progressIndicator.setVisible(true); - }); List sources = model.getStreamSources(); Collections.sort(sources); StreamSource best = sources.get(0); @@ -90,7 +100,7 @@ public class StreamPreview extends StackPane { double aspect = (double)video.getWidth() / video.getHeight(); double w = Config.getInstance().getSettings().thumbWidth; double h = w / aspect; - resizeToFitContent(w, h); + resizeTo(w, h); progressIndicator.setVisible(false); videoPreview.setVisible(true); videoPreview.setMediaPlayer(videoPlayer); @@ -127,23 +137,23 @@ public class StreamPreview extends StackPane { }); } - private void resizeToFitContent(double w, double h) { - setPrefSize(w, h); + public void resizeTo(double w, double h) { preview.setFitWidth(w); preview.setFitHeight(h); videoPreview.setFitWidth(w); videoPreview.setFitHeight(h); + progressIndicator.setPrefSize(w, h); } public void stop() { if(future != null && !future.isDone()) { future.cancel(true); } - Platform.runLater(() -> { + new Thread(() -> { if(videoPlayer != null) { videoPlayer.dispose(); } - }); + }).start(); } private void onError(MediaPlayer videoPlayer) { @@ -151,7 +161,7 @@ public class StreamPreview extends StackPane { if(videoPlayer.getError().getCause() != null) { LOG.error("Error while starting preview stream root cause:", videoPlayer.getError().getCause()); } - videoPlayer.dispose(); + stop(); Platform.runLater(() -> { showTestImage(); }); @@ -165,7 +175,7 @@ public class StreamPreview extends StackPane { double aspect = img.getWidth() / img.getHeight(); double w = Config.getInstance().getSettings().thumbWidth; double h = w / aspect; - resizeToFitContent(w, h); + resizeTo(w, h); progressIndicator.setVisible(false); }); } From e621e49e0052f8b9e5dc147a495e9b8be8800902 Mon Sep 17 00:00:00 2001 From: 0xboobface <0xboobface@gmail.com> Date: Sun, 16 Dec 2018 02:38:21 +0100 Subject: [PATCH 21/44] Wait for segment download thread pool to finish ... when the download terminates --- .../ctbrec/recorder/download/HlsDownload.java | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/common/src/main/java/ctbrec/recorder/download/HlsDownload.java b/common/src/main/java/ctbrec/recorder/download/HlsDownload.java index b99f3be7..9148f198 100644 --- a/common/src/main/java/ctbrec/recorder/download/HlsDownload.java +++ b/common/src/main/java/ctbrec/recorder/download/HlsDownload.java @@ -19,6 +19,7 @@ import java.text.SimpleDateFormat; import java.time.Instant; import java.util.Date; import java.util.concurrent.Callable; +import java.util.concurrent.TimeUnit; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -74,19 +75,14 @@ public class HlsDownload extends AbstractHlsDownload { } int lastSegment = 0; int nextSegment = 0; + boolean sleep = true; // this enables sleeping between playlist requests + // once we miss a segment, this is set to false, so that no sleeping happens anymore while(running) { SegmentPlaylist lsp = getNextSegments(segments); if(nextSegment > 0 && lsp.seq > nextSegment) { - LOG.warn("Missed segments {} < {} in download for {}", nextSegment, lsp.seq, model); - String first = lsp.segments.get(0); - int seq = lsp.seq; - for (int i = nextSegment; i < lsp.seq; i++) { - URL segmentUrl = new URL(first.replaceAll(Integer.toString(seq), Integer.toString(i))); - LOG.debug("Reloading segment {} for model {}", i, model.getName()); - String prefix = nf.format(segmentCounter++); - downloadThreadPool.submit(new SegmentDownload(segmentUrl, downloadDir, client, prefix)); - } // TODO switch to a lower bitrate/resolution ?!? + LOG.warn("Missed segments {} < {} in download for {}", nextSegment, lsp.seq, model); + sleep = false; } int skip = nextSegment - lsp.seq; for (String segment : lsp.segments) { @@ -101,7 +97,7 @@ public class HlsDownload extends AbstractHlsDownload { } long wait = 0; - if(lastSegment == lsp.seq) { + if(sleep && lastSegment == lsp.seq) { // playlist didn't change -> wait for at least half the target duration wait = (long) lsp.targetDuration * 1000 / 2; LOG.trace("Playlist didn't change... waiting for {}ms", wait); @@ -142,6 +138,11 @@ public class HlsDownload extends AbstractHlsDownload { throw new IOException("Couldn't download segment", e); } finally { alive = false; + downloadThreadPool.shutdown(); + try { + LOG.debug("Waiting for last segments for {}", model); + downloadThreadPool.awaitTermination(60, TimeUnit.SECONDS); + } catch (InterruptedException e) {} LOG.debug("Download for {} terminated", model); } } From ebb5310d262134265643be7dcf9bcf8e0c7a0032 Mon Sep 17 00:00:00 2001 From: 0xboobface <0xboobface@gmail.com> Date: Sun, 16 Dec 2018 16:14:53 +0100 Subject: [PATCH 22/44] Wait for the download to terminate before starting PP Sometimes the PP was started before the last segments were downloaded. This could cause unexpected effects. E.g. the playlist generator would fail, because the number of segments chained during playlist generation. --- .../java/ctbrec/recorder/LocalRecorder.java | 10 +++++-- .../ctbrec/recorder/download/HlsDownload.java | 17 ++++++++--- .../recorder/download/MergedHlsDownload.java | 28 +++++++++++++++++-- 3 files changed, 46 insertions(+), 9 deletions(-) diff --git a/common/src/main/java/ctbrec/recorder/LocalRecorder.java b/common/src/main/java/ctbrec/recorder/LocalRecorder.java index 6e16c809..5dafee65 100644 --- a/common/src/main/java/ctbrec/recorder/LocalRecorder.java +++ b/common/src/main/java/ctbrec/recorder/LocalRecorder.java @@ -216,10 +216,14 @@ public class LocalRecorder implements Recorder { private void stopRecordingProcess(Model model) { Download download = recordingProcesses.get(model); - download.stop(); recordingProcesses.remove(model); fireRecordingStateChanged(download.getTarget(), STOPPED, model, download.getStartTime()); - ppThreadPool.submit(createPostProcessor(download)); + + Runnable stopAndThePostProcess = () -> { + download.stop(); + createPostProcessor(download).run(); + }; + ppThreadPool.submit(stopAndThePostProcess); } private void postprocess(Download download) { @@ -551,6 +555,8 @@ public class LocalRecorder implements Recorder { continue; } + // TODO don't list recordings, which currently get deleted + Date startDate = sdf.parse(rec.getName()); Recording recording = new Recording(); recording.setModelName(subdir.getName()); diff --git a/common/src/main/java/ctbrec/recorder/download/HlsDownload.java b/common/src/main/java/ctbrec/recorder/download/HlsDownload.java index 9148f198..f682acda 100644 --- a/common/src/main/java/ctbrec/recorder/download/HlsDownload.java +++ b/common/src/main/java/ctbrec/recorder/download/HlsDownload.java @@ -44,6 +44,7 @@ public class HlsDownload extends AbstractHlsDownload { private int segmentCounter = 1; private NumberFormat nf = new DecimalFormat("000000"); + private Object downloadFinished = new Object(); public HlsDownload(HttpClient client) { super(client); @@ -75,8 +76,7 @@ public class HlsDownload extends AbstractHlsDownload { } int lastSegment = 0; int nextSegment = 0; - boolean sleep = true; // this enables sleeping between playlist requests - // once we miss a segment, this is set to false, so that no sleeping happens anymore + boolean sleep = true; // this enables sleeping between playlist requests. once we miss a segment, this is set to false, so that no sleeping happens anymore while(running) { SegmentPlaylist lsp = getNextSegments(segments); if(nextSegment > 0 && lsp.seq > nextSegment) { @@ -137,12 +137,15 @@ public class HlsDownload extends AbstractHlsDownload { } catch(Exception e) { throw new IOException("Couldn't download segment", e); } finally { - alive = false; downloadThreadPool.shutdown(); try { LOG.debug("Waiting for last segments for {}", model); downloadThreadPool.awaitTermination(60, TimeUnit.SECONDS); } catch (InterruptedException e) {} + alive = false; + synchronized (downloadFinished) { + downloadFinished.notifyAll(); + } LOG.debug("Download for {} terminated", model); } } @@ -150,7 +153,13 @@ public class HlsDownload extends AbstractHlsDownload { @Override public void stop() { running = false; - alive = false; + try { + synchronized (downloadFinished) { + downloadFinished.wait(); + } + } catch (InterruptedException e) { + LOG.error("Couldn't wait for download to finish", e); + } } private static class SegmentDownload implements Callable { diff --git a/common/src/main/java/ctbrec/recorder/download/MergedHlsDownload.java b/common/src/main/java/ctbrec/recorder/download/MergedHlsDownload.java index e298d48a..958fae17 100644 --- a/common/src/main/java/ctbrec/recorder/download/MergedHlsDownload.java +++ b/common/src/main/java/ctbrec/recorder/download/MergedHlsDownload.java @@ -66,6 +66,7 @@ public class MergedHlsDownload extends AbstractHlsDownload { private BlockingQueue downloadQueue = new LinkedBlockingQueue<>(50); private ExecutorService downloadThreadPool = new ThreadPoolExecutor(5, 5, 2, TimeUnit.MINUTES, downloadQueue); private FileChannel fileChannel = null; + private Object downloadFinished = new Object(); public MergedHlsDownload(HttpClient client) { super(client); @@ -105,13 +106,20 @@ public class MergedHlsDownload extends AbstractHlsDownload { } catch (InvalidKeyException | NoSuchAlgorithmException | IllegalStateException e) { throw new IOException("Couldn't add HMAC to playlist url", e); } finally { - alive = false; try { streamer.stop(); } catch(Exception e) { LOG.error("Couldn't stop streamer", e); } downloadThreadPool.shutdown(); + try { + LOG.debug("Waiting for last segments for {}", model); + downloadThreadPool.awaitTermination(60, TimeUnit.SECONDS); + } catch (InterruptedException e) {} + alive = false; + synchronized (downloadFinished) { + downloadFinished.notifyAll(); + } LOG.debug("Download terminated for {}", segmentPlaylistUri); } } @@ -155,7 +163,6 @@ public class MergedHlsDownload extends AbstractHlsDownload { } catch(Exception e) { throw new IOException("Couldn't download segment", e); } finally { - alive = false; if(streamer != null) { try { streamer.stop(); @@ -163,6 +170,15 @@ public class MergedHlsDownload extends AbstractHlsDownload { LOG.error("Couldn't stop streamer", e); } } + downloadThreadPool.shutdown(); + try { + LOG.debug("Waiting for last segments for {}", model); + downloadThreadPool.awaitTermination(60, TimeUnit.SECONDS); + } catch (InterruptedException e) {} + alive = false; + synchronized (downloadFinished) { + downloadFinished.notifyAll(); + } LOG.debug("Download for {} terminated", model); } } @@ -353,10 +369,16 @@ public class MergedHlsDownload extends AbstractHlsDownload { @Override public void stop() { running = false; - alive = false; if(streamer != null) { streamer.stop(); } + try { + synchronized (downloadFinished) { + downloadFinished.wait(); + } + } catch (InterruptedException e) { + LOG.error("Couldn't wait for download to finish", e); + } LOG.debug("Download stopped"); } From f75687752c0fd24ec59532322a42d762e8be525b Mon Sep 17 00:00:00 2001 From: 0xboobface <0xboobface@gmail.com> Date: Sun, 16 Dec 2018 17:19:57 +0100 Subject: [PATCH 23/44] Add config setting for stream previews in thumbnails This setting allows to switch stream previews of in the thumbnail views. The little play circle will not show up. --- client/src/main/java/ctbrec/ui/ThumbCell.java | 4 ++- .../java/ctbrec/ui/settings/SettingsTab.java | 26 ++++++++++++++----- common/src/main/java/ctbrec/Settings.java | 1 + 3 files changed, 23 insertions(+), 8 deletions(-) diff --git a/client/src/main/java/ctbrec/ui/ThumbCell.java b/client/src/main/java/ctbrec/ui/ThumbCell.java index cb67fa21..fa3d4746 100644 --- a/client/src/main/java/ctbrec/ui/ThumbCell.java +++ b/client/src/main/java/ctbrec/ui/ThumbCell.java @@ -174,7 +174,9 @@ public class ThumbCell extends StackPane { StackPane.setAlignment(pausedIndicator, Pos.TOP_LEFT); getChildren().add(pausedIndicator); - getChildren().add(createPreviewTrigger()); + if(Config.getInstance().getSettings().previewInThumbnails) { + getChildren().add(createPreviewTrigger()); + } selectionOverlay = new Rectangle(); selectionOverlay.visibleProperty().bind(selectionProperty); diff --git a/client/src/main/java/ctbrec/ui/settings/SettingsTab.java b/client/src/main/java/ctbrec/ui/settings/SettingsTab.java index c06733b2..d0794572 100644 --- a/client/src/main/java/ctbrec/ui/settings/SettingsTab.java +++ b/client/src/main/java/ctbrec/ui/settings/SettingsTab.java @@ -64,6 +64,7 @@ public class SettingsTab extends Tab implements TabSelectionListener { private CheckBox chooseStreamQuality = new CheckBox(); private CheckBox multiplePlayers = new CheckBox(); private CheckBox updateThumbnails = new CheckBox(); + private CheckBox previewInThumbnails = new CheckBox(); private CheckBox showPlayerStarting = new CheckBox(); private RadioButton recordLocal; private RadioButton recordRemote; @@ -422,7 +423,7 @@ public class SettingsTab extends Tab implements TabSelectionListener { Config.getInstance().getSettings().showPlayerStarting = showPlayerStarting.isSelected(); saveConfig(); }); - GridPane.setMargin(l, new Insets(CHECKBOX_MARGIN, 0, 0, 0)); + GridPane.setMargin(l, new Insets(3, 0, 0, 0)); GridPane.setMargin(showPlayerStarting, new Insets(CHECKBOX_MARGIN, 0, 0, CHECKBOX_MARGIN)); layout.add(showPlayerStarting, 1, row++); @@ -434,7 +435,7 @@ public class SettingsTab extends Tab implements TabSelectionListener { Config.getInstance().getSettings().determineResolution = loadResolution.isSelected(); saveConfig(); }); - GridPane.setMargin(l, new Insets(CHECKBOX_MARGIN, 0, 0, 0)); + GridPane.setMargin(l, new Insets(3, 0, 0, 0)); GridPane.setMargin(loadResolution, new Insets(CHECKBOX_MARGIN, 0, 0, CHECKBOX_MARGIN)); layout.add(loadResolution, 1, row++); @@ -445,7 +446,7 @@ public class SettingsTab extends Tab implements TabSelectionListener { Config.getInstance().getSettings().chooseStreamQuality = chooseStreamQuality.isSelected(); saveConfig(); }); - GridPane.setMargin(l, new Insets(CHECKBOX_MARGIN, 0, 0, 0)); + GridPane.setMargin(l, new Insets(3, 0, 0, 0)); GridPane.setMargin(chooseStreamQuality, new Insets(CHECKBOX_MARGIN, 0, 0, CHECKBOX_MARGIN)); layout.add(chooseStreamQuality, 1, row++); @@ -456,10 +457,21 @@ public class SettingsTab extends Tab implements TabSelectionListener { Config.getInstance().getSettings().updateThumbnails = updateThumbnails.isSelected(); saveConfig(); }); - GridPane.setMargin(l, new Insets(CHECKBOX_MARGIN, 0, 0, 0)); - GridPane.setMargin(updateThumbnails, new Insets(CHECKBOX_MARGIN, 0, CHECKBOX_MARGIN, CHECKBOX_MARGIN)); + GridPane.setMargin(l, new Insets(3, 0, 0, 0)); + GridPane.setMargin(updateThumbnails, new Insets(CHECKBOX_MARGIN, 0, 0, CHECKBOX_MARGIN)); layout.add(updateThumbnails, 1, row++); + l = new Label("Preview in thumbnails"); + layout.add(l, 0, row); + previewInThumbnails.setSelected(Config.getInstance().getSettings().previewInThumbnails); + previewInThumbnails.setOnAction((e) -> { + Config.getInstance().getSettings().previewInThumbnails = previewInThumbnails.isSelected(); + saveConfig(); + }); + GridPane.setMargin(l, new Insets(3, 0, 0, 0)); + GridPane.setMargin(previewInThumbnails, new Insets(CHECKBOX_MARGIN, 0, 0, CHECKBOX_MARGIN)); + layout.add(previewInThumbnails, 1, row++); + l = new Label("Start Tab"); layout.add(l, 0, row); startTab = new ComboBox<>(); @@ -468,8 +480,8 @@ public class SettingsTab extends Tab implements TabSelectionListener { saveConfig(); }); layout.add(startTab, 1, row++); - GridPane.setMargin(l, new Insets(0, 0, 0, 0)); - GridPane.setMargin(startTab, new Insets(0, 0, 0, CHECKBOX_MARGIN)); + GridPane.setMargin(l, new Insets(3, 0, 0, 0)); + GridPane.setMargin(startTab, new Insets(CHECKBOX_MARGIN, 0, 0, CHECKBOX_MARGIN)); l = new Label("Colors (Base / Accent)"); layout.add(l, 0, row); diff --git a/common/src/main/java/ctbrec/Settings.java b/common/src/main/java/ctbrec/Settings.java index 168f2e25..eb480855 100644 --- a/common/src/main/java/ctbrec/Settings.java +++ b/common/src/main/java/ctbrec/Settings.java @@ -69,6 +69,7 @@ public class Settings { public List models = new ArrayList<>(); public List eventHandlers = new ArrayList<>(); public boolean determineResolution = false; + public boolean previewInThumbnails = true; public boolean requireAuthentication = false; public boolean chooseStreamQuality = false; public int maximumResolution = 0; From 3d7fc64bf54d40020e6ce7e537397f7ab17e232b Mon Sep 17 00:00:00 2001 From: 0xboobface <0xboobface@gmail.com> Date: Sun, 16 Dec 2018 17:20:27 +0100 Subject: [PATCH 24/44] Improve error handling in the StreamPreview --- .../ctbrec/ui/controls/StreamPreview.java | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/client/src/main/java/ctbrec/ui/controls/StreamPreview.java b/client/src/main/java/ctbrec/ui/controls/StreamPreview.java index 6b7bc022..2dd21c52 100644 --- a/client/src/main/java/ctbrec/ui/controls/StreamPreview.java +++ b/client/src/main/java/ctbrec/ui/controls/StreamPreview.java @@ -88,6 +88,7 @@ public class StreamPreview extends StackPane { checkInterrupt(); LOG.debug("Preview url for {} is {}", model.getName(), best.getMediaPlaylistUrl()); video = new Media(best.getMediaPlaylistUrl()); + video.setOnError(() -> onError(videoPlayer)); if(videoPlayer != null) { videoPlayer.dispose(); } @@ -128,8 +129,8 @@ public class StreamPreview extends StackPane { // future has been canceled, that's fine } else { LOG.warn("Couldn't start preview video: {}", e.getMessage()); - showTestImage(); } + showTestImage(); } catch (Exception e) { LOG.warn("Couldn't start preview video: {}", e.getMessage()); showTestImage(); @@ -146,12 +147,14 @@ public class StreamPreview extends StackPane { } public void stop() { - if(future != null && !future.isDone()) { - future.cancel(true); - } + MediaPlayer old = videoPlayer; + Future oldFuture = future; new Thread(() -> { - if(videoPlayer != null) { - videoPlayer.dispose(); + if(oldFuture != null && !oldFuture.isDone()) { + oldFuture.cancel(true); + } + if(old != null) { + old.dispose(); } }).start(); } @@ -161,13 +164,11 @@ public class StreamPreview extends StackPane { if(videoPlayer.getError().getCause() != null) { LOG.error("Error while starting preview stream root cause:", videoPlayer.getError().getCause()); } - stop(); - Platform.runLater(() -> { - showTestImage(); - }); + showTestImage(); } private void showTestImage() { + stop(); Platform.runLater(() -> { videoPreview.setVisible(false); Image img = new Image(getClass().getResource("/image_not_found.png").toString(), true); From 1e4743271402b44cff0947a4aa955c4851239857 Mon Sep 17 00:00:00 2001 From: 0xboobface <0xboobface@gmail.com> Date: Sun, 16 Dec 2018 17:21:11 +0100 Subject: [PATCH 25/44] Add origin stream source only, if mp4-ws sources are available --- .../sites/streamate/StreamateModel.java | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/common/src/main/java/ctbrec/sites/streamate/StreamateModel.java b/common/src/main/java/ctbrec/sites/streamate/StreamateModel.java index c4e04670..55497e8a 100644 --- a/common/src/main/java/ctbrec/sites/streamate/StreamateModel.java +++ b/common/src/main/java/ctbrec/sites/streamate/StreamateModel.java @@ -91,7 +91,6 @@ public class StreamateModel extends AbstractModel { if(response.isSuccessful()) { JSONObject json = new JSONObject(response.body().string()); JSONObject formats = json.getJSONObject("formats"); - JSONObject ws = formats.getJSONObject("mp4-ws"); JSONObject hls = formats.getJSONObject("mp4-hls"); // add encodings @@ -108,14 +107,17 @@ public class StreamateModel extends AbstractModel { } // add raw source stream - JSONObject origin = hls.getJSONObject("origin"); - StreamSource src = new StreamSource(); - src.mediaPlaylistUrl = origin.getString("location"); - origin = ws.getJSONObject("origin"); // switch to web socket origin, because it has width, height and bitrates - src.width = origin.optInt("videoWidth"); - src.height = origin.optInt("videoHeight"); - src.bandwidth = (origin.optInt("videoKbps") + origin.optInt("audioKbps")) * 1024; - streamSources.add(src); + if(formats.has("mp4-ws")) { + JSONObject ws = formats.getJSONObject("mp4-ws"); + JSONObject origin = hls.getJSONObject("origin"); + StreamSource src = new StreamSource(); + src.mediaPlaylistUrl = origin.getString("location"); + origin = ws.getJSONObject("origin"); // switch to web socket origin, because it has width, height and bitrates + src.width = origin.optInt("videoWidth"); + src.height = origin.optInt("videoHeight"); + src.bandwidth = (origin.optInt("videoKbps") + origin.optInt("audioKbps")) * 1024; + streamSources.add(src); + } } else { throw new HttpException(response.code(), response.message()); } From a7ab34c9d76b6c31e27424414a50795753814ab1 Mon Sep 17 00:00:00 2001 From: 0xboobface <0xboobface@gmail.com> Date: Sun, 16 Dec 2018 17:25:05 +0100 Subject: [PATCH 26/44] Set user data directory for WebbrowserTab --- client/src/main/java/ctbrec/ui/WebbrowserTab.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/client/src/main/java/ctbrec/ui/WebbrowserTab.java b/client/src/main/java/ctbrec/ui/WebbrowserTab.java index 3def3feb..2e44c556 100644 --- a/client/src/main/java/ctbrec/ui/WebbrowserTab.java +++ b/client/src/main/java/ctbrec/ui/WebbrowserTab.java @@ -1,5 +1,6 @@ package ctbrec.ui; +import ctbrec.Config; import javafx.scene.control.Tab; import javafx.scene.web.WebEngine; import javafx.scene.web.WebView; @@ -10,6 +11,7 @@ public class WebbrowserTab extends Tab { WebView browser = new WebView(); WebEngine webEngine = browser.getEngine(); webEngine.load(uri); + webEngine.setUserDataDirectory(Config.getInstance().getConfigDir()); setContent(browser); } } From 7b7c7b24b1bd3f902823ceeb98ab8c0992ef8caf Mon Sep 17 00:00:00 2001 From: 0xboobface <0xboobface@gmail.com> Date: Sun, 16 Dec 2018 17:50:56 +0100 Subject: [PATCH 27/44] Replace Exception parameter with Throwable --- client/src/main/java/ctbrec/ui/controls/Dialogs.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/client/src/main/java/ctbrec/ui/controls/Dialogs.java b/client/src/main/java/ctbrec/ui/controls/Dialogs.java index 558f6e0f..653bbc91 100644 --- a/client/src/main/java/ctbrec/ui/controls/Dialogs.java +++ b/client/src/main/java/ctbrec/ui/controls/Dialogs.java @@ -5,14 +5,14 @@ import javafx.application.Platform; import javafx.scene.control.Alert; public class Dialogs { - public static void showError(String header, String text, Exception e) { + public static void showError(String header, String text, Throwable t) { Runnable r = () -> { Alert alert = new AutosizeAlert(Alert.AlertType.ERROR); alert.setTitle("Error"); alert.setHeaderText(header); String content = text; - if(e != null) { - content += " " + e.getLocalizedMessage(); + if(t != null) { + content += " " + t.getLocalizedMessage(); } alert.setContentText(content); alert.showAndWait(); From 10184176b0cddfa65335c0d012b2d83ed3c13c7c Mon Sep 17 00:00:00 2001 From: 0xboobface <0xboobface@gmail.com> Date: Sun, 16 Dec 2018 19:53:54 +0100 Subject: [PATCH 28/44] Enable JavaScript and register an error handler --- .../src/main/java/ctbrec/ui/WebbrowserTab.java | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/client/src/main/java/ctbrec/ui/WebbrowserTab.java b/client/src/main/java/ctbrec/ui/WebbrowserTab.java index 2e44c556..cf904a3e 100644 --- a/client/src/main/java/ctbrec/ui/WebbrowserTab.java +++ b/client/src/main/java/ctbrec/ui/WebbrowserTab.java @@ -1,17 +1,31 @@ package ctbrec.ui; -import ctbrec.Config; +import java.io.File; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import ctbrec.OS; +import ctbrec.ui.controls.Dialogs; import javafx.scene.control.Tab; import javafx.scene.web.WebEngine; import javafx.scene.web.WebView; public class WebbrowserTab extends Tab { + private static final transient Logger LOG = LoggerFactory.getLogger(WebbrowserTab.class); + public WebbrowserTab(String uri) { WebView browser = new WebView(); WebEngine webEngine = browser.getEngine(); + webEngine.setUserDataDirectory(new File(OS.getConfigDir(), "webengine")); + webEngine.setJavaScriptEnabled(true); webEngine.load(uri); - webEngine.setUserDataDirectory(Config.getInstance().getConfigDir()); setContent(browser); + + webEngine.setOnError(evt -> { + LOG.error("Couldn't load {}", uri, evt.getException()); + Dialogs.showError("Error", "Couldn't load " + uri, evt.getException()); + }); } } From d74737113a9be646046624cc102e74e945d7e67a Mon Sep 17 00:00:00 2001 From: 0xboobface <0xboobface@gmail.com> Date: Sun, 16 Dec 2018 20:04:26 +0100 Subject: [PATCH 29/44] Change max resolution input to textfield ... to allow arbitrary values --- .../java/ctbrec/ui/settings/SettingsTab.java | 52 ++++++++----------- 1 file changed, 22 insertions(+), 30 deletions(-) diff --git a/client/src/main/java/ctbrec/ui/settings/SettingsTab.java b/client/src/main/java/ctbrec/ui/settings/SettingsTab.java index d0794572..4916974f 100644 --- a/client/src/main/java/ctbrec/ui/settings/SettingsTab.java +++ b/client/src/main/java/ctbrec/ui/settings/SettingsTab.java @@ -70,7 +70,7 @@ public class SettingsTab extends Tab implements TabSelectionListener { private RadioButton recordRemote; private ToggleGroup recordLocation; private ProxySettingsPane proxySettingsPane; - private ComboBox maxResolution; + private TextField maxResolution; private ComboBox splitAfter; private ComboBox directoryStructure; private ComboBox startTab; @@ -264,26 +264,7 @@ public class SettingsTab extends Tab implements TabSelectionListener { layout.add(directoryStructure, 1, row++); recordingsDirectory.prefWidthProperty().bind(directoryStructure.widthProperty()); - Label l = new Label("Maximum resolution (0 = unlimited)"); - layout.add(l, 0, row); - List resolutionOptions = new ArrayList<>(); - resolutionOptions.add(1080); - resolutionOptions.add(720); - resolutionOptions.add(600); - resolutionOptions.add(480); - resolutionOptions.add(0); - maxResolution = new ComboBox<>(FXCollections.observableList(resolutionOptions)); - setMaxResolutionValue(); - maxResolution.setOnAction((e) -> { - Config.getInstance().getSettings().maximumResolution = maxResolution.getSelectionModel().getSelectedItem(); - saveConfig(); - }); - maxResolution.prefWidthProperty().bind(directoryStructure.widthProperty()); - layout.add(maxResolution, 1, row++); - GridPane.setMargin(l, new Insets(0, 0, 0, 0)); - GridPane.setMargin(maxResolution, new Insets(0, 0, 0, CHECKBOX_MARGIN)); - - l = new Label("Split recordings after (minutes)"); + Label l = new Label("Split recordings after (minutes)"); layout.add(l, 0, row); List splitOptions = new ArrayList<>(); splitOptions.add(new SplitAfterOption("disabled", 0)); @@ -308,6 +289,26 @@ public class SettingsTab extends Tab implements TabSelectionListener { GridPane.setMargin(l, new Insets(0, 0, 0, 0)); GridPane.setMargin(splitAfter, new Insets(0, 0, 0, CHECKBOX_MARGIN)); + l = new Label("Maximum resolution (0 = unlimited)"); + layout.add(l, 0, row); + maxResolution = new TextField(Integer.toString(Config.getInstance().getSettings().maximumResolution)); + maxResolution.textProperty().addListener((observable, oldValue, newValue) -> { + if (!newValue.matches("\\d*")) { + maxResolution.setText(newValue.replaceAll("[^\\d]", "")); + } + if (!maxResolution.getText().isEmpty()) { + int newRes = Integer.parseInt(maxResolution.getText()); + if (newRes != Config.getInstance().getSettings().maximumResolution) { + Config.getInstance().getSettings().maximumResolution = newRes; + saveConfig(); + } + } + }); + maxResolution.prefWidthProperty().bind(directoryStructure.widthProperty()); + layout.add(maxResolution, 1, row++); + GridPane.setMargin(l, new Insets(0, 0, 0, 0)); + GridPane.setMargin(maxResolution, new Insets(0, 0, 0, CHECKBOX_MARGIN)); + layout.add(new Label("Post-Processing"), 0, row); // TODO allow empty strings to remove post-processing scripts postProcessing = new ProgramSelectionBox(Config.getInstance().getSettings().postProcessing); @@ -504,15 +505,6 @@ public class SettingsTab extends Tab implements TabSelectionListener { } } - private void setMaxResolutionValue() { - int value = Config.getInstance().getSettings().maximumResolution; - for (Integer option : maxResolution.getItems()) { - if(option == value) { - maxResolution.getSelectionModel().select(option); - } - } - } - void showRestartRequired() { restartLabel.setVisible(true); } From 910d21463a75aae99ea1a843b62ca82ff6414436 Mon Sep 17 00:00:00 2001 From: 0xboobface <0xboobface@gmail.com> Date: Sun, 16 Dec 2018 20:29:48 +0100 Subject: [PATCH 30/44] Fix: allow empty input / deletion of post-processing script --- .../ui/controls/AbstractFileSelectionBox.java | 57 ++++++++++++------- .../ui/controls/DirectorySelectionBox.java | 2 +- .../ui/settings/ActionSettingsPanel.java | 4 +- .../java/ctbrec/ui/settings/SettingsTab.java | 8 +-- 4 files changed, 44 insertions(+), 27 deletions(-) diff --git a/client/src/main/java/ctbrec/ui/controls/AbstractFileSelectionBox.java b/client/src/main/java/ctbrec/ui/controls/AbstractFileSelectionBox.java index 556c1393..f377998b 100644 --- a/client/src/main/java/ctbrec/ui/controls/AbstractFileSelectionBox.java +++ b/client/src/main/java/ctbrec/ui/controls/AbstractFileSelectionBox.java @@ -6,9 +6,10 @@ import java.io.IOException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import ctbrec.StringUtil; import ctbrec.ui.AutosizeAlert; -import javafx.beans.property.ObjectProperty; -import javafx.beans.property.ObjectPropertyBase; +import javafx.beans.property.SimpleStringProperty; +import javafx.beans.property.StringProperty; import javafx.beans.value.ChangeListener; import javafx.geometry.Point2D; import javafx.scene.Node; @@ -29,18 +30,20 @@ import javafx.stage.FileChooser; public abstract class AbstractFileSelectionBox extends HBox { private static final transient Logger LOG = LoggerFactory.getLogger(AbstractFileSelectionBox.class); - private ObjectProperty fileProperty = new ObjectPropertyBase() { - @Override - public Object getBean() { - return null; - } - - @Override - public String getName() { - return "file"; - } - }; + // private ObjectProperty fileProperty = new ObjectPropertyBase() { + // @Override + // public Object getBean() { + // return null; + // } + // + // @Override + // public String getName() { + // return "file"; + // } + // }; + private StringProperty fileProperty = new SimpleStringProperty(); protected TextField fileInput; + protected boolean allowEmptyValue = false; private Tooltip validationError = new Tooltip(); public AbstractFileSelectionBox() { @@ -67,8 +70,14 @@ public abstract class AbstractFileSelectionBox extends HBox { private ChangeListener textListener() { return (obs, o, n) -> { String input = fileInput.getText(); - File program = new File(input); - setFile(program); + if(StringUtil.isBlank(input) && allowEmptyValue) { + fileProperty.set(""); + hideValidationHints(); + return; + } else { + File program = new File(input); + setFile(program); + } }; } @@ -83,13 +92,17 @@ public abstract class AbstractFileSelectionBox extends HBox { validationError.show(getScene().getWindow(), p.getX(), p.getY() + fileInput.getHeight() + 4); } } else { - fileInput.setBorder(Border.EMPTY); - fileInput.setTooltip(null); - fileProperty.set(file); - validationError.hide(); + fileProperty.set(file.getAbsolutePath()); + hideValidationHints(); } } + private void hideValidationHints() { + fileInput.setBorder(Border.EMPTY); + fileInput.setTooltip(null); + validationError.hide(); + } + protected String validate(File file) { if (file == null || !file.exists()) { return "File does not exist"; @@ -98,6 +111,10 @@ public abstract class AbstractFileSelectionBox extends HBox { } } + public void allowEmptyValue() { + this.allowEmptyValue = true; + } + private Button createBrowseButton() { Button button = new Button("Select"); button.setOnAction((e) -> { @@ -123,7 +140,7 @@ public abstract class AbstractFileSelectionBox extends HBox { } } - public ObjectProperty fileProperty() { + public StringProperty fileProperty() { return fileProperty; } } diff --git a/client/src/main/java/ctbrec/ui/controls/DirectorySelectionBox.java b/client/src/main/java/ctbrec/ui/controls/DirectorySelectionBox.java index f3f1a5e5..ca65a7c4 100644 --- a/client/src/main/java/ctbrec/ui/controls/DirectorySelectionBox.java +++ b/client/src/main/java/ctbrec/ui/controls/DirectorySelectionBox.java @@ -12,7 +12,7 @@ public class DirectorySelectionBox extends AbstractFileSelectionBox { @Override protected void choose() { DirectoryChooser chooser = new DirectoryChooser(); - File currentDir = fileProperty().get(); + File currentDir = new File(fileProperty().get()); if (currentDir.exists() && currentDir.isDirectory()) { chooser.setInitialDirectory(currentDir); } diff --git a/client/src/main/java/ctbrec/ui/settings/ActionSettingsPanel.java b/client/src/main/java/ctbrec/ui/settings/ActionSettingsPanel.java index a163a19e..182ca2d5 100644 --- a/client/src/main/java/ctbrec/ui/settings/ActionSettingsPanel.java +++ b/client/src/main/java/ctbrec/ui/settings/ActionSettingsPanel.java @@ -173,7 +173,7 @@ public class ActionSettingsPanel extends TitledPane { if(playSound.isSelected()) { ActionConfiguration ac = new ActionConfiguration(); ac.setType(PlaySound.class.getName()); - File file = sound.fileProperty().get(); + File file = new File(sound.fileProperty().get()); ac.getConfiguration().put("file", file.getAbsolutePath()); ac.setName("play " + file.getName()); config.getActions().add(ac); @@ -181,7 +181,7 @@ public class ActionSettingsPanel extends TitledPane { if(executeProgram.isSelected()) { ActionConfiguration ac = new ActionConfiguration(); ac.setType(ExecuteProgram.class.getName()); - File file = program.fileProperty().get(); + File file = new File(program.fileProperty().get()); ac.getConfiguration().put("file", file.getAbsolutePath()); ac.setName("execute " + file.getName()); config.getActions().add(ac); diff --git a/client/src/main/java/ctbrec/ui/settings/SettingsTab.java b/client/src/main/java/ctbrec/ui/settings/SettingsTab.java index 4916974f..4585978f 100644 --- a/client/src/main/java/ctbrec/ui/settings/SettingsTab.java +++ b/client/src/main/java/ctbrec/ui/settings/SettingsTab.java @@ -238,7 +238,7 @@ public class SettingsTab extends Tab implements TabSelectionListener { recordingsDirectory = new DirectorySelectionBox(Config.getInstance().getSettings().recordingsDir); recordingsDirectory.prefWidth(400); recordingsDirectory.fileProperty().addListener((obs, o, n) -> { - String path = n.getAbsolutePath(); + String path = n; if(!Objects.equals(path, Config.getInstance().getSettings().recordingsDir)) { Config.getInstance().getSettings().recordingsDir = path; saveConfig(); @@ -310,10 +310,10 @@ public class SettingsTab extends Tab implements TabSelectionListener { GridPane.setMargin(maxResolution, new Insets(0, 0, 0, CHECKBOX_MARGIN)); layout.add(new Label("Post-Processing"), 0, row); - // TODO allow empty strings to remove post-processing scripts postProcessing = new ProgramSelectionBox(Config.getInstance().getSettings().postProcessing); + postProcessing.allowEmptyValue(); postProcessing.fileProperty().addListener((obs, o, n) -> { - String path = n.getAbsolutePath(); + String path = n; if(!Objects.equals(path, Config.getInstance().getSettings().postProcessing)) { Config.getInstance().getSettings().postProcessing = path; saveConfig(); @@ -395,7 +395,7 @@ public class SettingsTab extends Tab implements TabSelectionListener { layout.add(new Label("Player"), 0, row); mediaPlayer = new ProgramSelectionBox(Config.getInstance().getSettings().mediaPlayer); mediaPlayer.fileProperty().addListener((obs, o, n) -> { - String path = n.getAbsolutePath(); + String path = n; if (!Objects.equals(path, Config.getInstance().getSettings().mediaPlayer)) { Config.getInstance().getSettings().mediaPlayer = path; saveConfig(); From d1cf6a681b9933cca9f28dd16b0fda9e09802142 Mon Sep 17 00:00:00 2001 From: 0xboobface <0xboobface@gmail.com> Date: Sun, 16 Dec 2018 20:31:09 +0100 Subject: [PATCH 31/44] Remove outdated comment --- common/src/main/java/ctbrec/OS.java | 1 - 1 file changed, 1 deletion(-) diff --git a/common/src/main/java/ctbrec/OS.java b/common/src/main/java/ctbrec/OS.java index d5181887..e86842e4 100644 --- a/common/src/main/java/ctbrec/OS.java +++ b/common/src/main/java/ctbrec/OS.java @@ -93,7 +93,6 @@ public class OS { } else if(OS.getOsType() == OS.TYPE.WINDOWS) { notifyWindows(title, header, msg); } else if(OS.getOsType() == OS.TYPE.MAC) { - // TODO find out, if it makes a sound or if we have to play a sound notifyMac(title, header, msg); } else { // unknown system, try systemtray notification anyways From bfbd6b17828f090629b34c16619cf975bfdb56a5 Mon Sep 17 00:00:00 2001 From: 0xboobface <0xboobface@gmail.com> Date: Sun, 16 Dec 2018 20:38:58 +0100 Subject: [PATCH 32/44] Open the player on double-click in the Recording tab Implements #121 --- client/src/main/java/ctbrec/ui/RecordedModelsTab.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/client/src/main/java/ctbrec/ui/RecordedModelsTab.java b/client/src/main/java/ctbrec/ui/RecordedModelsTab.java index ae41af76..ab693d3f 100644 --- a/client/src/main/java/ctbrec/ui/RecordedModelsTab.java +++ b/client/src/main/java/ctbrec/ui/RecordedModelsTab.java @@ -58,6 +58,7 @@ import javafx.scene.input.ClipboardContent; import javafx.scene.input.ContextMenuEvent; import javafx.scene.input.KeyCode; import javafx.scene.input.KeyEvent; +import javafx.scene.input.MouseButton; import javafx.scene.input.MouseEvent; import javafx.scene.layout.BorderPane; import javafx.scene.layout.FlowPane; @@ -149,6 +150,14 @@ public class RecordedModelsTab extends Tab implements TabSelectionListener { } event.consume(); }); + table.addEventHandler(MouseEvent.MOUSE_CLICKED, event -> { + if (event.getButton() == MouseButton.PRIMARY && event.getClickCount() == 2) { + JavaFxModel model = table.getSelectionModel().getSelectedItem(); + if(model != null) { + new PlayAction(table, model).execute(); + } + } + }); table.addEventHandler(MouseEvent.MOUSE_PRESSED, event -> { if (popup != null) { popup.hide(); From 5145ed0ce25529e62b728f332e902f8edfaa4e2b Mon Sep 17 00:00:00 2001 From: 0xboobface <0xboobface@gmail.com> Date: Mon, 17 Dec 2018 13:00:46 +0100 Subject: [PATCH 33/44] Update changelog --- CHANGELOG.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9235ca88..41422077 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,14 @@ +1.16.0 +======================== +* Thumbnails can show a live preview. Can be switched off in the settings. +* Added Streamate (metcams, xhamstercams, pornhublive) +* Maximum resolution can be an arbitrary value now +* Added setting for minimal recording length. Recordings, which are shorter + than this value, get deleted automatically. +* Double-click in Recording tab starts the player +* Fix: BongaCams friends tab not working +* Fix: In some cases MFC models got confused + 1.15.0 ======================== * Fix: BongaCams overview didn't work anymore From 47d8101ce89327ec0a2ed6e7abc19e092d627dc3 Mon Sep 17 00:00:00 2001 From: 0xboobface <0xboobface@gmail.com> Date: Mon, 17 Dec 2018 15:21:36 +0100 Subject: [PATCH 34/44] Avoid NPE in onError --- client/src/main/java/ctbrec/ui/controls/StreamPreview.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/client/src/main/java/ctbrec/ui/controls/StreamPreview.java b/client/src/main/java/ctbrec/ui/controls/StreamPreview.java index 2dd21c52..a7b5911f 100644 --- a/client/src/main/java/ctbrec/ui/controls/StreamPreview.java +++ b/client/src/main/java/ctbrec/ui/controls/StreamPreview.java @@ -3,6 +3,7 @@ package ctbrec.ui.controls; import java.io.InterruptedIOException; import java.util.Collections; import java.util.List; +import java.util.Optional; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -161,8 +162,9 @@ public class StreamPreview extends StackPane { private void onError(MediaPlayer videoPlayer) { LOG.error("Error while starting preview stream", videoPlayer.getError()); - if(videoPlayer.getError().getCause() != null) { - LOG.error("Error while starting preview stream root cause:", videoPlayer.getError().getCause()); + Optional cause = Optional.ofNullable(videoPlayer).map(v -> v.getError()).map(e -> e.getCause()); + if(cause.isPresent()) { + LOG.error("Error while starting preview stream root cause:", cause.get()); } showTestImage(); } From e362980028b3bbd6c72a024141fe9cd6189cbaa1 Mon Sep 17 00:00:00 2001 From: 0xboobface <0xboobface@gmail.com> Date: Mon, 17 Dec 2018 17:11:45 +0100 Subject: [PATCH 35/44] Make live previews experimental Add a setting to toggle live previews. When switched off, the thumbnails won't show the trigger and the recording table won't show the column. I did this, because the MediaPlayer sometimes bugs out. It then opens a lot of connections and further tries to open a stream end in an exception in the playlist parser. --- .../main/java/ctbrec/ui/RecordedModelsTab.java | 3 +++ client/src/main/java/ctbrec/ui/ThumbCell.java | 2 +- .../main/java/ctbrec/ui/settings/SettingsTab.java | 15 ++++++++------- common/src/main/java/ctbrec/Settings.java | 2 +- 4 files changed, 13 insertions(+), 9 deletions(-) diff --git a/client/src/main/java/ctbrec/ui/RecordedModelsTab.java b/client/src/main/java/ctbrec/ui/RecordedModelsTab.java index ab693d3f..020512c5 100644 --- a/client/src/main/java/ctbrec/ui/RecordedModelsTab.java +++ b/client/src/main/java/ctbrec/ui/RecordedModelsTab.java @@ -118,6 +118,9 @@ public class RecordedModelsTab extends Tab implements TabSelectionListener { preview.setCellValueFactory(cdf -> new SimpleStringProperty(" ▶ ")); preview.setEditable(false); preview.setId("preview"); + if(!Config.getInstance().getSettings().livePreviews) { + preview.setVisible(false); + } TableColumn name = new TableColumn<>("Model"); name.setPrefWidth(200); name.setCellValueFactory(new PropertyValueFactory("displayName")); diff --git a/client/src/main/java/ctbrec/ui/ThumbCell.java b/client/src/main/java/ctbrec/ui/ThumbCell.java index fa3d4746..4f1cecc8 100644 --- a/client/src/main/java/ctbrec/ui/ThumbCell.java +++ b/client/src/main/java/ctbrec/ui/ThumbCell.java @@ -174,7 +174,7 @@ public class ThumbCell extends StackPane { StackPane.setAlignment(pausedIndicator, Pos.TOP_LEFT); getChildren().add(pausedIndicator); - if(Config.getInstance().getSettings().previewInThumbnails) { + if(Config.getInstance().getSettings().livePreviews) { getChildren().add(createPreviewTrigger()); } diff --git a/client/src/main/java/ctbrec/ui/settings/SettingsTab.java b/client/src/main/java/ctbrec/ui/settings/SettingsTab.java index 4585978f..6ced7598 100644 --- a/client/src/main/java/ctbrec/ui/settings/SettingsTab.java +++ b/client/src/main/java/ctbrec/ui/settings/SettingsTab.java @@ -64,7 +64,7 @@ public class SettingsTab extends Tab implements TabSelectionListener { private CheckBox chooseStreamQuality = new CheckBox(); private CheckBox multiplePlayers = new CheckBox(); private CheckBox updateThumbnails = new CheckBox(); - private CheckBox previewInThumbnails = new CheckBox(); + private CheckBox livePreviews = new CheckBox(); private CheckBox showPlayerStarting = new CheckBox(); private RadioButton recordLocal; private RadioButton recordRemote; @@ -462,16 +462,17 @@ public class SettingsTab extends Tab implements TabSelectionListener { GridPane.setMargin(updateThumbnails, new Insets(CHECKBOX_MARGIN, 0, 0, CHECKBOX_MARGIN)); layout.add(updateThumbnails, 1, row++); - l = new Label("Preview in thumbnails"); + l = new Label("Enable live previews (experimental)"); layout.add(l, 0, row); - previewInThumbnails.setSelected(Config.getInstance().getSettings().previewInThumbnails); - previewInThumbnails.setOnAction((e) -> { - Config.getInstance().getSettings().previewInThumbnails = previewInThumbnails.isSelected(); + livePreviews.setSelected(Config.getInstance().getSettings().livePreviews); + livePreviews.setOnAction((e) -> { + Config.getInstance().getSettings().livePreviews = livePreviews.isSelected(); saveConfig(); + showRestartRequired(); }); GridPane.setMargin(l, new Insets(3, 0, 0, 0)); - GridPane.setMargin(previewInThumbnails, new Insets(CHECKBOX_MARGIN, 0, 0, CHECKBOX_MARGIN)); - layout.add(previewInThumbnails, 1, row++); + GridPane.setMargin(livePreviews, new Insets(CHECKBOX_MARGIN, 0, 0, CHECKBOX_MARGIN)); + layout.add(livePreviews, 1, row++); l = new Label("Start Tab"); layout.add(l, 0, row); diff --git a/common/src/main/java/ctbrec/Settings.java b/common/src/main/java/ctbrec/Settings.java index eb480855..c96cf985 100644 --- a/common/src/main/java/ctbrec/Settings.java +++ b/common/src/main/java/ctbrec/Settings.java @@ -69,7 +69,7 @@ public class Settings { public List models = new ArrayList<>(); public List eventHandlers = new ArrayList<>(); public boolean determineResolution = false; - public boolean previewInThumbnails = true; + public boolean livePreviews = false; public boolean requireAuthentication = false; public boolean chooseStreamQuality = false; public int maximumResolution = 0; From 9958e04ef86d119f55165e8ec978058f8f241435 Mon Sep 17 00:00:00 2001 From: 0xboobface <0xboobface@gmail.com> Date: Mon, 17 Dec 2018 17:13:15 +0100 Subject: [PATCH 36/44] Make sure, that the model ID is set When a Streamate model is added by URL, make sure, that the ID is loaded, so that saving and loading works properly, since the ID is saved as site specific data. --- .../main/java/ctbrec/sites/streamate/Streamate.java | 8 +++++++- .../java/ctbrec/sites/streamate/StreamateModel.java | 10 +++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/common/src/main/java/ctbrec/sites/streamate/Streamate.java b/common/src/main/java/ctbrec/sites/streamate/Streamate.java index a86eb91b..d8870509 100644 --- a/common/src/main/java/ctbrec/sites/streamate/Streamate.java +++ b/common/src/main/java/ctbrec/sites/streamate/Streamate.java @@ -193,7 +193,13 @@ public class Streamate extends AbstractSite { Matcher m = Pattern.compile("https?://.*?streamate.com/cam/([^/]*?)/?").matcher(url); if (m.matches()) { String modelName = m.group(1); - return createModel(modelName); + StreamateModel model = (StreamateModel) createModel(modelName); + try { + model.loadModelInfo(); + } catch (IOException e) { + LOG.error("Couldn't load model info. This can cause problems with saving / loading the model"); + } + return model; } else { return super.createModelFromUrl(url); } diff --git a/common/src/main/java/ctbrec/sites/streamate/StreamateModel.java b/common/src/main/java/ctbrec/sites/streamate/StreamateModel.java index 55497e8a..cda334bb 100644 --- a/common/src/main/java/ctbrec/sites/streamate/StreamateModel.java +++ b/common/src/main/java/ctbrec/sites/streamate/StreamateModel.java @@ -37,6 +37,7 @@ public class StreamateModel extends AbstractModel { private List streamSources = new ArrayList<>(); private int[] resolution; private Long id; + private String streamId; @Override public boolean isOnline(boolean ignoreCache) throws IOException, ExecutionException, InterruptedException { @@ -218,6 +219,11 @@ public class StreamateModel extends AbstractModel { } private String getStreamId() throws IOException { + loadModelInfo(); + return streamId; + } + + void loadModelInfo() throws IOException { String url = "https://hybridclient.naiadsystems.com/api/v1/config/?name=" + getName() + "&sabasic=&sakey=&sk=www.streamate.com&userid=0&version=6.3.17&ajax=1"; Request request = new Request.Builder() @@ -232,7 +238,9 @@ public class StreamateModel extends AbstractModel { if(response.isSuccessful()) { JSONObject json = new JSONObject(response.body().string()); JSONObject stream = json.getJSONObject("stream"); - return stream.getString("streamId"); + streamId = stream.getString("streamId"); + JSONObject performer = json.getJSONObject("performer"); + id = performer.getLong("id"); } else { throw new HttpException(response.code(), response.message()); } From 8a8327a2a5ec2069e2f1050a24410bc4d2e07b33 Mon Sep 17 00:00:00 2001 From: 0xboobface <0xboobface@gmail.com> Date: Mon, 17 Dec 2018 17:13:54 +0100 Subject: [PATCH 37/44] If loading site specific fails, print out the model name --- common/src/main/java/ctbrec/io/ModelJsonAdapter.java | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/common/src/main/java/ctbrec/io/ModelJsonAdapter.java b/common/src/main/java/ctbrec/io/ModelJsonAdapter.java index 5296f3e6..54794087 100644 --- a/common/src/main/java/ctbrec/io/ModelJsonAdapter.java +++ b/common/src/main/java/ctbrec/io/ModelJsonAdapter.java @@ -5,6 +5,9 @@ import java.lang.reflect.InvocationTargetException; import java.util.List; import java.util.Optional; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import com.squareup.moshi.JsonAdapter; import com.squareup.moshi.JsonReader; import com.squareup.moshi.JsonReader.Token; @@ -16,6 +19,8 @@ import ctbrec.sites.chaturbate.ChaturbateModel; public class ModelJsonAdapter extends JsonAdapter { + private static final transient Logger LOG = LoggerFactory.getLogger(ModelJsonAdapter.class); + private List sites; public ModelJsonAdapter() { @@ -62,7 +67,12 @@ public class ModelJsonAdapter extends JsonAdapter { model.setSuspended(suspended); } else if(key.equals("siteSpecific")) { reader.beginObject(); - model.readSiteSpecificData(reader); + try { + model.readSiteSpecificData(reader); + } catch(Exception e) { + LOG.error("Couldn't read site specific data for model {}", model.getName()); + throw e; + } reader.endObject(); } } else { From eedb2379234ff9faba63d8ff3c035a40b6120052 Mon Sep 17 00:00:00 2001 From: 0xboobface <0xboobface@gmail.com> Date: Mon, 17 Dec 2018 17:22:21 +0100 Subject: [PATCH 38/44] Make sure, that the model ID is set --- .../src/main/java/ctbrec/sites/streamate/Streamate.java | 8 +------- .../main/java/ctbrec/sites/streamate/StreamateModel.java | 7 +++++++ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/common/src/main/java/ctbrec/sites/streamate/Streamate.java b/common/src/main/java/ctbrec/sites/streamate/Streamate.java index d8870509..a86eb91b 100644 --- a/common/src/main/java/ctbrec/sites/streamate/Streamate.java +++ b/common/src/main/java/ctbrec/sites/streamate/Streamate.java @@ -193,13 +193,7 @@ public class Streamate extends AbstractSite { Matcher m = Pattern.compile("https?://.*?streamate.com/cam/([^/]*?)/?").matcher(url); if (m.matches()) { String modelName = m.group(1); - StreamateModel model = (StreamateModel) createModel(modelName); - try { - model.loadModelInfo(); - } catch (IOException e) { - LOG.error("Couldn't load model info. This can cause problems with saving / loading the model"); - } - return model; + return createModel(modelName); } else { return super.createModelFromUrl(url); } diff --git a/common/src/main/java/ctbrec/sites/streamate/StreamateModel.java b/common/src/main/java/ctbrec/sites/streamate/StreamateModel.java index cda334bb..519bce2c 100644 --- a/common/src/main/java/ctbrec/sites/streamate/StreamateModel.java +++ b/common/src/main/java/ctbrec/sites/streamate/StreamateModel.java @@ -330,6 +330,13 @@ public class StreamateModel extends AbstractModel { @Override public void writeSiteSpecificData(JsonWriter writer) throws IOException { + if(id == null) { + try { + loadModelInfo(); + } catch (IOException e) { + LOG.error("Couldn't load model ID for {}. This can cause problems with saving / loading the model", getName()); + } + } writer.name("id").value(id); } } \ No newline at end of file From b473782b9225d125a134fc51d895c951dc23e0b9 Mon Sep 17 00:00:00 2001 From: 0xboobface <0xboobface@gmail.com> Date: Mon, 17 Dec 2018 17:38:55 +0100 Subject: [PATCH 39/44] Don't list recordings, which currently get deleted --- common/src/main/java/ctbrec/recorder/LocalRecorder.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/common/src/main/java/ctbrec/recorder/LocalRecorder.java b/common/src/main/java/ctbrec/recorder/LocalRecorder.java index 5dafee65..6c9ab6df 100644 --- a/common/src/main/java/ctbrec/recorder/LocalRecorder.java +++ b/common/src/main/java/ctbrec/recorder/LocalRecorder.java @@ -554,8 +554,10 @@ public class LocalRecorder implements Recorder { if (rec.listFiles().length == 0) { continue; } - - // TODO don't list recordings, which currently get deleted + // don't list recordings, which currently get deleted + if (deleteInProgress.contains(rec)) { + continue; + } Date startDate = sdf.parse(rec.getName()); Recording recording = new Recording(); From cef3ca351e408b444515227da46dcc2a2bcc18ea Mon Sep 17 00:00:00 2001 From: 0xboobface <0xboobface@gmail.com> Date: Mon, 17 Dec 2018 17:52:13 +0100 Subject: [PATCH 40/44] Update changelog --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 41422077..2d401867 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,8 @@ 1.16.0 ======================== -* Thumbnails can show a live preview. Can be switched off in the settings. +* Thumbnails can show a live preview. Can be switched on in the settings. +* Live preview is experimental for now, because I noticed some funky behavior + of the the internal media player. You can use it on your own risk. * Added Streamate (metcams, xhamstercams, pornhublive) * Maximum resolution can be an arbitrary value now * Added setting for minimal recording length. Recordings, which are shorter From 2a1dc77a89d92b36b28877ab601ae88c119ca3ab Mon Sep 17 00:00:00 2001 From: 0xboobface <0xboobface@gmail.com> Date: Mon, 17 Dec 2018 17:53:53 +0100 Subject: [PATCH 41/44] Increase version number to 1.16.0 --- client/pom.xml | 2 +- common/pom.xml | 2 +- master/pom.xml | 2 +- server/pom.xml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/client/pom.xml b/client/pom.xml index 9ea0e01e..435967b8 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -8,7 +8,7 @@ ctbrec master - 1.15.0 + 1.16.0 ../master diff --git a/common/pom.xml b/common/pom.xml index 4f20e4fa..4308d633 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -8,7 +8,7 @@ ctbrec master - 1.15.0 + 1.16.0 ../master diff --git a/master/pom.xml b/master/pom.xml index 78ea5606..bc994b4a 100644 --- a/master/pom.xml +++ b/master/pom.xml @@ -6,7 +6,7 @@ ctbrec master pom - 1.15.0 + 1.16.0 ../common diff --git a/server/pom.xml b/server/pom.xml index 24075158..4b71406f 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -8,7 +8,7 @@ ctbrec master - 1.15.0 + 1.16.0 ../master From 6dd7de9762ab25d2bfe57d4c2832f86745a31f36 Mon Sep 17 00:00:00 2001 From: 0xboobface <0xboobface@gmail.com> Date: Mon, 17 Dec 2018 18:10:55 +0100 Subject: [PATCH 42/44] Fix: BongaCams search fails with JSON exceptions --- .../java/ctbrec/sites/bonga/BongaCams.java | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/common/src/main/java/ctbrec/sites/bonga/BongaCams.java b/common/src/main/java/ctbrec/sites/bonga/BongaCams.java index 573c4d67..52b2d084 100644 --- a/common/src/main/java/ctbrec/sites/bonga/BongaCams.java +++ b/common/src/main/java/ctbrec/sites/bonga/BongaCams.java @@ -157,15 +157,17 @@ public class BongaCams extends AbstractSite { JSONArray results = json.getJSONArray("models"); for (int i = 0; i < results.length(); i++) { JSONObject result = results.getJSONObject(i); - Model model = createModel(result.getString("username")); - String thumb = result.getString("thumb_image"); - if(thumb != null) { - model.setPreview("https:" + thumb); + if(result.has("username")) { + Model model = createModel(result.getString("username")); + String thumb = result.getString("thumb_image"); + if(thumb != null) { + model.setPreview("https:" + thumb); + } + if(result.has("display_name")) { + model.setDisplayName(result.getString("display_name")); + } + models.add(model); } - if(result.has("display_name")) { - model.setDisplayName(result.getString("display_name")); - } - models.add(model); } return models; } else { From 75fedfcddd16fbfcb182cdbe002d680e27be8446 Mon Sep 17 00:00:00 2001 From: 0xboobface <0xboobface@gmail.com> Date: Mon, 17 Dec 2018 18:28:29 +0100 Subject: [PATCH 43/44] Disable minimum length in remote mode --- client/src/main/java/ctbrec/ui/settings/SettingsTab.java | 1 + 1 file changed, 1 insertion(+) diff --git a/client/src/main/java/ctbrec/ui/settings/SettingsTab.java b/client/src/main/java/ctbrec/ui/settings/SettingsTab.java index 6ced7598..68c797b3 100644 --- a/client/src/main/java/ctbrec/ui/settings/SettingsTab.java +++ b/client/src/main/java/ctbrec/ui/settings/SettingsTab.java @@ -529,6 +529,7 @@ public class SettingsTab extends Tab implements TabSelectionListener { onlineCheckIntervalInSecs.setDisable(!local); leaveSpaceOnDevice.setDisable(!local); postProcessing.setDisable(!local); + minimumLengthInSecs.setDisable(!local); } @Override From ad71f0cf11a9389a5b4a9089e1af13e985c47d77 Mon Sep 17 00:00:00 2001 From: 0xboobface <0xboobface@gmail.com> Date: Mon, 17 Dec 2018 19:04:57 +0100 Subject: [PATCH 44/44] Make sure to not download segments multiple times For some reason streamate playlists can go back in time. This change makes sure, that we don't download segments multiple times as a consequence of the time travelling ;) --- CHANGELOG.md | 1 + .../recorder/download/AbstractHlsDownload.java | 3 +++ .../ctbrec/recorder/download/HlsDownload.java | 16 ++++++++++------ 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2d401867..773b637e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ than this value, get deleted automatically. * Double-click in Recording tab starts the player * Fix: BongaCams friends tab not working +* Fix: BongaCams search fails with JSON exception * Fix: In some cases MFC models got confused 1.15.0 diff --git a/common/src/main/java/ctbrec/recorder/download/AbstractHlsDownload.java b/common/src/main/java/ctbrec/recorder/download/AbstractHlsDownload.java index 1fb6333d..b4ab0507 100644 --- a/common/src/main/java/ctbrec/recorder/download/AbstractHlsDownload.java +++ b/common/src/main/java/ctbrec/recorder/download/AbstractHlsDownload.java @@ -52,6 +52,9 @@ public abstract class AbstractHlsDownload implements Download { Request request = new Request.Builder().url(segmentsUrl).addHeader("connection", "keep-alive").build(); try(Response response = client.execute(request)) { if(response.isSuccessful()) { + // String body = response.body().string(); + // InputStream inputStream = new ByteArrayInputStream(body.getBytes("utf-8")); + // LOG.debug("Segments {}", body); InputStream inputStream = response.body().byteStream(); PlaylistParser parser = new PlaylistParser(inputStream, Format.EXT_M3U, Encoding.UTF_8, ParsingMode.LENIENT); Playlist playlist = parser.parse(); diff --git a/common/src/main/java/ctbrec/recorder/download/HlsDownload.java b/common/src/main/java/ctbrec/recorder/download/HlsDownload.java index f682acda..9eef5d7c 100644 --- a/common/src/main/java/ctbrec/recorder/download/HlsDownload.java +++ b/common/src/main/java/ctbrec/recorder/download/HlsDownload.java @@ -76,13 +76,13 @@ public class HlsDownload extends AbstractHlsDownload { } int lastSegment = 0; int nextSegment = 0; - boolean sleep = true; // this enables sleeping between playlist requests. once we miss a segment, this is set to false, so that no sleeping happens anymore + int waitFactor = 1; while(running) { SegmentPlaylist lsp = getNextSegments(segments); if(nextSegment > 0 && lsp.seq > nextSegment) { // TODO switch to a lower bitrate/resolution ?!? - LOG.warn("Missed segments {} < {} in download for {}", nextSegment, lsp.seq, model); - sleep = false; + waitFactor *= 2; + LOG.warn("Missed segments {} < {} in download for {} - setting wait factor to 1/{}", nextSegment, lsp.seq, model, waitFactor); } int skip = nextSegment - lsp.seq; for (String segment : lsp.segments) { @@ -97,9 +97,9 @@ public class HlsDownload extends AbstractHlsDownload { } long wait = 0; - if(sleep && lastSegment == lsp.seq) { + if(lastSegment == lsp.seq) { // playlist didn't change -> wait for at least half the target duration - wait = (long) lsp.targetDuration * 1000 / 2; + wait = (long) lsp.targetDuration * 1000 / waitFactor; LOG.trace("Playlist didn't change... waiting for {}ms", wait); } else { // playlist did change -> wait for at least last segment duration @@ -115,8 +115,12 @@ public class HlsDownload extends AbstractHlsDownload { } } + // this if check makes sure, that we don't decrease nextSegment. for some reason + // streamate playlists sometimes jump back. e.g. max sequence = 79 -> 80 -> 79 lastSegment = lsp.seq; - nextSegment = lastSegment + lsp.segments.size(); + if(lastSegment + lsp.segments.size() > nextSegment) { + nextSegment = lastSegment + lsp.segments.size(); + } } } else { throw new IOException("Couldn't determine segments uri");