From 9bb18426a6742e12e3950cbe8023fa04ea119ba8 Mon Sep 17 00:00:00 2001 From: 0xboobface <0xboobface@gmail.com> Date: Sat, 11 Jul 2020 14:15:34 +0200 Subject: [PATCH 01/56] Interrupt keep alive thread on reconnect On reconnect interrupt the current keep alive thread, so that we don't pile up a bunch keep alive threads, if the there is a problem with the websocket connection. --- .../src/main/java/ctbrec/sites/mfc/MyFreeCamsClient.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/common/src/main/java/ctbrec/sites/mfc/MyFreeCamsClient.java b/common/src/main/java/ctbrec/sites/mfc/MyFreeCamsClient.java index ade01494..1e4f5ca5 100644 --- a/common/src/main/java/ctbrec/sites/mfc/MyFreeCamsClient.java +++ b/common/src/main/java/ctbrec/sites/mfc/MyFreeCamsClient.java @@ -56,6 +56,7 @@ public class MyFreeCamsClient { private static MyFreeCamsClient instance; private MyFreeCams mfc; private WebSocket ws; + private Thread keepAlive; private Moshi moshi; private volatile boolean running = false; @@ -562,8 +563,11 @@ public class MyFreeCamsClient { } private void startKeepAlive(WebSocket ws) { - Thread keepAlive = new Thread(() -> { - while (running) { + if (keepAlive != null) { + keepAlive.interrupt(); + } + keepAlive = new Thread(() -> { + while (running && !Thread.currentThread().isInterrupted()) { try { if (!connecting) { LOG.trace("--> NULL to keep the connection alive"); From 3b9fb87d047c310bc39e8ab0bc518c46e76b8267 Mon Sep 17 00:00:00 2001 From: 0xboobface <0xboobface@gmail.com> Date: Sun, 12 Jul 2020 11:33:53 +0200 Subject: [PATCH 02/56] Fire recording finished event for downloads --- client/src/main/java/ctbrec/ui/tabs/RecordingsTab.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/client/src/main/java/ctbrec/ui/tabs/RecordingsTab.java b/client/src/main/java/ctbrec/ui/tabs/RecordingsTab.java index 2f4be5c5..2df62b99 100644 --- a/client/src/main/java/ctbrec/ui/tabs/RecordingsTab.java +++ b/client/src/main/java/ctbrec/ui/tabs/RecordingsTab.java @@ -31,6 +31,8 @@ import ctbrec.Config; import ctbrec.Recording; import ctbrec.Recording.State; import ctbrec.StringUtil; +import ctbrec.event.EventBusHolder; +import ctbrec.event.RecordingStateChangedEvent; import ctbrec.recorder.ProgressListener; import ctbrec.recorder.Recorder; import ctbrec.recorder.RecordingPinnedException; @@ -633,6 +635,8 @@ public class RecordingsTab extends Tab implements TabSelectionListener { Platform.runLater(() -> { recording.setStatus(FINISHED); recording.setProgress(-1); + RecordingStateChangedEvent evt = new RecordingStateChangedEvent(target, recording.getStatus(), recording.getModel(), recording.getStartDate()); + EventBusHolder.BUS.post(evt); }); } }); From 59697c600f4446d5a151eea13bf2894fac5b3b11 Mon Sep 17 00:00:00 2001 From: 0xboobface <0xboobface@gmail.com> Date: Sun, 12 Jul 2020 12:37:03 +0200 Subject: [PATCH 03/56] Reduce global connection pool size from 50 to 20 Also remove the second connectionPool call, which accidentally was left in and rendered the global connection pool useless --- common/src/main/java/ctbrec/io/HttpClient.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/common/src/main/java/ctbrec/io/HttpClient.java b/common/src/main/java/ctbrec/io/HttpClient.java index 16cc2b4f..0b34897e 100644 --- a/common/src/main/java/ctbrec/io/HttpClient.java +++ b/common/src/main/java/ctbrec/io/HttpClient.java @@ -46,8 +46,8 @@ import okhttp3.WebSocketListener; public abstract class HttpClient { private static final Logger LOG = LoggerFactory.getLogger(HttpClient.class); - private static final ConnectionPool GLOBAL_HTTP_CONN_POOL = new ConnectionPool(10, 2, TimeUnit.MINUTES); - + private static final ConnectionPool GLOBAL_HTTP_CONN_POOL = new ConnectionPool(20, 2, TimeUnit.MINUTES); + protected OkHttpClient client; protected CookieJarImpl cookieJar = new CookieJarImpl(); protected boolean loggedIn = false; @@ -125,8 +125,7 @@ public abstract class HttpClient { .cookieJar(cookieJar) .connectionPool(GLOBAL_HTTP_CONN_POOL) .connectTimeout(Config.getInstance().getSettings().httpTimeout, TimeUnit.MILLISECONDS) - .readTimeout(Config.getInstance().getSettings().httpTimeout, TimeUnit.MILLISECONDS) - .connectionPool(new ConnectionPool(50, 10, TimeUnit.MINUTES)); + .readTimeout(Config.getInstance().getSettings().httpTimeout, TimeUnit.MILLISECONDS); //.addInterceptor(new LoggingInterceptor()); ProxyType proxyType = Config.getInstance().getSettings().proxyType; From 764119e20aae4211a701b25408cc3036604e7f17 Mon Sep 17 00:00:00 2001 From: 0xboobface <0xboobface@gmail.com> Date: Sun, 12 Jul 2020 12:37:47 +0200 Subject: [PATCH 04/56] Reduce event bus thread pool size from 10 to 2 --- common/src/main/java/ctbrec/event/EventBusHolder.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/src/main/java/ctbrec/event/EventBusHolder.java b/common/src/main/java/ctbrec/event/EventBusHolder.java index 4e0a7a42..12e265ac 100644 --- a/common/src/main/java/ctbrec/event/EventBusHolder.java +++ b/common/src/main/java/ctbrec/event/EventBusHolder.java @@ -17,7 +17,7 @@ public class EventBusHolder { private EventBusHolder() {} - public static final EventBus BUS = new AsyncEventBus(Executors.newFixedThreadPool(10, r -> { + public static final EventBus BUS = new AsyncEventBus(Executors.newFixedThreadPool(2, r -> { Thread t = new Thread(r); t.setName("EventBus-" + UUID.randomUUID().toString().substring(0, 8)); t.setPriority(Thread.NORM_PRIORITY - 1); From fa3512621c2d3e353fded5c5bf9b14c966b07a8a Mon Sep 17 00:00:00 2001 From: 0xboobface <0xboobface@gmail.com> Date: Sun, 12 Jul 2020 12:39:15 +0200 Subject: [PATCH 05/56] Release resources, if the tab is deselected This allows the garbage collector to work properly and reduces the minimum heap size --- client/src/main/java/ctbrec/ui/tabs/ThumbCell.java | 4 ++++ .../src/main/java/ctbrec/ui/tabs/ThumbOverviewTab.java | 9 +++++++++ 2 files changed, 13 insertions(+) diff --git a/client/src/main/java/ctbrec/ui/tabs/ThumbCell.java b/client/src/main/java/ctbrec/ui/tabs/ThumbCell.java index 1f26d841..bb01c4d5 100644 --- a/client/src/main/java/ctbrec/ui/tabs/ThumbCell.java +++ b/client/src/main/java/ctbrec/ui/tabs/ThumbCell.java @@ -644,4 +644,8 @@ public class ThumbCell extends StackPane { return new int[2]; } } + + public void releaseResources() { + iv.setImage(null); + } } diff --git a/client/src/main/java/ctbrec/ui/tabs/ThumbOverviewTab.java b/client/src/main/java/ctbrec/ui/tabs/ThumbOverviewTab.java index 6d052701..7623cf2b 100644 --- a/client/src/main/java/ctbrec/ui/tabs/ThumbOverviewTab.java +++ b/client/src/main/java/ctbrec/ui/tabs/ThumbOverviewTab.java @@ -872,6 +872,15 @@ public class ThumbOverviewTab extends Tab implements TabSelectionListener { updateService.cancel(); } queue.clear(); + + for (Iterator iterator = grid.getChildren().iterator(); iterator.hasNext();) { + Node node = iterator.next(); + if(node instanceof ThumbCell) { + ThumbCell thumbCell = (ThumbCell) node; + thumbCell.releaseResources(); + iterator.remove(); + } + } } void suspendUpdates(boolean suspend) { From f79b5eddc5118dcb0fdefee481870d96762b57b3 Mon Sep 17 00:00:00 2001 From: 0xboobface <0xboobface@gmail.com> Date: Sun, 12 Jul 2020 12:39:49 +0200 Subject: [PATCH 06/56] Reduce HTTP connection keep-alive to 1 minute --- common/src/main/java/ctbrec/io/HttpClient.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/src/main/java/ctbrec/io/HttpClient.java b/common/src/main/java/ctbrec/io/HttpClient.java index 0b34897e..b2577453 100644 --- a/common/src/main/java/ctbrec/io/HttpClient.java +++ b/common/src/main/java/ctbrec/io/HttpClient.java @@ -46,7 +46,7 @@ import okhttp3.WebSocketListener; public abstract class HttpClient { private static final Logger LOG = LoggerFactory.getLogger(HttpClient.class); - private static final ConnectionPool GLOBAL_HTTP_CONN_POOL = new ConnectionPool(20, 2, TimeUnit.MINUTES); + private static final ConnectionPool GLOBAL_HTTP_CONN_POOL = new ConnectionPool(20, 1, TimeUnit.MINUTES); protected OkHttpClient client; protected CookieJarImpl cookieJar = new CookieJarImpl(); From 49feade0c062e9ab75ba636599574850a42c68e6 Mon Sep 17 00:00:00 2001 From: 0xboobface <0xboobface@gmail.com> Date: Sun, 12 Jul 2020 13:08:55 +0200 Subject: [PATCH 07/56] Empty table, if tab is deselected --- .../main/java/ctbrec/ui/sites/myfreecams/MyFreeCamsTableTab.java | 1 + 1 file changed, 1 insertion(+) diff --git a/client/src/main/java/ctbrec/ui/sites/myfreecams/MyFreeCamsTableTab.java b/client/src/main/java/ctbrec/ui/sites/myfreecams/MyFreeCamsTableTab.java index db3d7e94..8581ca7f 100644 --- a/client/src/main/java/ctbrec/ui/sites/myfreecams/MyFreeCamsTableTab.java +++ b/client/src/main/java/ctbrec/ui/sites/myfreecams/MyFreeCamsTableTab.java @@ -526,6 +526,7 @@ public class MyFreeCamsTableTab extends Tab implements TabSelectionListener { updateService.cancel(); } saveData(); + observableModels.clear(); } private void saveData() { From ed26228d7b89f08ca176e432973f78dcfe446c67 Mon Sep 17 00:00:00 2001 From: 0xboobface <0xboobface@gmail.com> Date: Sun, 12 Jul 2020 13:11:19 +0200 Subject: [PATCH 08/56] Increase version number to 3.8.2 --- CHANGELOG.md | 16 ++++++++++++++-- client/pom.xml | 2 +- common/pom.xml | 2 +- master/pom.xml | 2 +- server/pom.xml | 2 +- 5 files changed, 18 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ffea0876..9d04f90d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,15 @@ +3.8.2 +======================== +* Fixed misconfiguration in global connection pool, which caused a lot of + threads to spawn while browsing in the thumbnail overviews +* Improved memory handling for the thumbnail overviews. Thumbnail images were + not released, when a tab was switched. This caused a huge memory consumption, + if you opened a lot of different tabs. +* Fixed a bug in MFC websocket client, which caused to spawn a bunch of + "keep-alive" threads, if there was a problem with the connection +* Reworked the settings tab +* Fire recording finished event, if a download from the server is finished + 3.8.1 ======================== * Fixed recent MFC error @@ -10,7 +22,7 @@ * Models can be added by name in the web-interface * Added a bandwidth monitor * Added possibility to add notes to recordings -* Added range slider to restrict the recording resolution in a range; e.g. 480p - 1080p +* Added range slider to restrict the recording resolution in a range; e.g. 480p - 1080p * Improved MFC SD downloads (much less blocking, I think) 3.7.3 @@ -26,7 +38,7 @@ 3.7.1 ======================== * Server now logs in on startup, if credentials are set -* Show confirmation dialog on shutdown, if the are active downloads from the +* Show confirmation dialog on shutdown, if the are active downloads from the server * Added setting to remove recordings after post-processing * Added max resolution setting for the player (click on the gear!) diff --git a/client/pom.xml b/client/pom.xml index 76f36cf6..a26a2ed9 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -8,7 +8,7 @@ ctbrec master - 3.8.1 + 3.8.2 ../master diff --git a/common/pom.xml b/common/pom.xml index ac6c1fdd..779b1639 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -8,7 +8,7 @@ ctbrec master - 3.8.1 + 3.8.2 ../master diff --git a/master/pom.xml b/master/pom.xml index ebbb31be..f405a4a4 100644 --- a/master/pom.xml +++ b/master/pom.xml @@ -6,7 +6,7 @@ ctbrec master pom - 3.8.1 + 3.8.2 ../common diff --git a/server/pom.xml b/server/pom.xml index 26cf509e..48d56be2 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -8,7 +8,7 @@ ctbrec master - 3.8.1 + 3.8.2 ../master From 296585463ad62cea73a2ffa66862e8d1953143b6 Mon Sep 17 00:00:00 2001 From: 0xboobface <0xboobface@gmail.com> Date: Sun, 12 Jul 2020 13:32:06 +0200 Subject: [PATCH 09/56] Don't apply min/max resolution settings, if resolution is unknown --- CHANGELOG.md | 1 + .../main/java/ctbrec/recorder/download/StreamSource.java | 5 +++-- .../ctbrec/recorder/download/hls/AbstractHlsDownload.java | 8 +++++--- .../java/ctbrec/sites/mfc/HlsStreamSourceProvider.java | 4 ++-- 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9d04f90d..2928127a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ "keep-alive" threads, if there was a problem with the connection * Reworked the settings tab * Fire recording finished event, if a download from the server is finished +* Ignore min/max resolution, if the resolution is unknown 3.8.1 ======================== diff --git a/common/src/main/java/ctbrec/recorder/download/StreamSource.java b/common/src/main/java/ctbrec/recorder/download/StreamSource.java index 5bd01a0c..c9ed2211 100644 --- a/common/src/main/java/ctbrec/recorder/download/StreamSource.java +++ b/common/src/main/java/ctbrec/recorder/download/StreamSource.java @@ -4,6 +4,7 @@ import java.text.DecimalFormat; public class StreamSource implements Comparable { public static final int ORIGIN = Integer.MAX_VALUE - 1; + public static final int UNKNOWN = Integer.MAX_VALUE; public int bandwidth; public int width; public int height; @@ -45,7 +46,7 @@ public class StreamSource implements Comparable { public String toString() { DecimalFormat df = new DecimalFormat("0.00"); float mbit = bandwidth / 1024.0f / 1024.0f; - if (height == Integer.MAX_VALUE) { + if (height == UNKNOWN) { return "unknown resolution (" + df.format(mbit) + " Mbit/s)"; } else if (height == ORIGIN) { return "Origin"; @@ -61,7 +62,7 @@ public class StreamSource implements Comparable { @Override public int compareTo(StreamSource o) { int heightDiff = height - o.height; - if(heightDiff != 0 && height != Integer.MAX_VALUE && o.height != Integer.MAX_VALUE) { + if(heightDiff != 0 && height != UNKNOWN && o.height != UNKNOWN) { return heightDiff; } else { return bandwidth - o.bandwidth; diff --git a/common/src/main/java/ctbrec/recorder/download/hls/AbstractHlsDownload.java b/common/src/main/java/ctbrec/recorder/download/hls/AbstractHlsDownload.java index fee4c801..d5f260cf 100644 --- a/common/src/main/java/ctbrec/recorder/download/hls/AbstractHlsDownload.java +++ b/common/src/main/java/ctbrec/recorder/download/hls/AbstractHlsDownload.java @@ -1,6 +1,8 @@ package ctbrec.recorder.download.hls; import static ctbrec.io.HttpConstants.*; +import static ctbrec.io.HttpConstants.ORIGIN; +import static ctbrec.recorder.download.StreamSource.*; import java.io.ByteArrayInputStream; import java.io.IOException; @@ -160,12 +162,12 @@ public abstract class AbstractHlsDownload extends AbstractDownload { LOG.debug("{} selected {}", model.getName(), streamSources.get(model.getStreamUrlIndex())); url = streamSources.get(model.getStreamUrlIndex()).getMediaPlaylistUrl(); } else { - // filter out stream resolutions, which are too high + // filter out stream resolutions, which are out of range of the configured min and max int minRes = Config.getInstance().getSettings().minimumResolution; int maxRes = Config.getInstance().getSettings().maximumResolution; List filteredStreamSources = streamSources.stream() - .filter(src -> src.height == 0 || minRes <= src.height) - .filter(src -> src.height == 0 || maxRes >= src.height) + .filter(src -> src.height == 0 || src.height == UNKNOWN || minRes <= src.height) + .filter(src -> src.height == 0 || src.height == UNKNOWN || maxRes >= src.height) .collect(Collectors.toList()); if (filteredStreamSources.isEmpty()) { diff --git a/common/src/main/java/ctbrec/sites/mfc/HlsStreamSourceProvider.java b/common/src/main/java/ctbrec/sites/mfc/HlsStreamSourceProvider.java index d8a3efc5..3b1ff192 100644 --- a/common/src/main/java/ctbrec/sites/mfc/HlsStreamSourceProvider.java +++ b/common/src/main/java/ctbrec/sites/mfc/HlsStreamSourceProvider.java @@ -50,8 +50,8 @@ public class HlsStreamSourceProvider implements StreamSourceProvider { src.width = playlist.getStreamInfo().getResolution().width; src.height = playlist.getStreamInfo().getResolution().height; } else { - src.width = Integer.MAX_VALUE; - src.height = Integer.MAX_VALUE; + src.width = StreamSource.UNKNOWN; + src.height = StreamSource.UNKNOWN; } String masterUrl = streamUrl; String baseUrl = masterUrl.substring(0, masterUrl.lastIndexOf('/') + 1); From 3f4973137c8f635f261192a29eee7e9ce1510edb Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Thu, 16 Jul 2020 18:52:34 +0200 Subject: [PATCH 10/56] Reduce core pool size to 0 for the download thread pool Most of the time only one thread is used, so we can save resources by reducing the core pool size --- .../java/ctbrec/recorder/download/hls/AbstractHlsDownload.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/src/main/java/ctbrec/recorder/download/hls/AbstractHlsDownload.java b/common/src/main/java/ctbrec/recorder/download/hls/AbstractHlsDownload.java index d5f260cf..33b68abd 100644 --- a/common/src/main/java/ctbrec/recorder/download/hls/AbstractHlsDownload.java +++ b/common/src/main/java/ctbrec/recorder/download/hls/AbstractHlsDownload.java @@ -66,7 +66,7 @@ public abstract class AbstractHlsDownload extends AbstractDownload { protected volatile boolean running = false; protected Model model = new UnknownModel(); protected transient LinkedBlockingQueue downloadQueue = new LinkedBlockingQueue<>(50); - protected transient ExecutorService downloadThreadPool = new ThreadPoolExecutor(5, 5, 2, TimeUnit.MINUTES, downloadQueue, createThreadFactory()); + protected transient ExecutorService downloadThreadPool = new ThreadPoolExecutor(0, 5, 20, TimeUnit.SECONDS, downloadQueue, createThreadFactory()); protected State state = State.UNKNOWN; private int playlistEmptyCount = 0; From 03b6de626cf774d418fa0bd7e599c79e89683098 Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Thu, 16 Jul 2020 18:53:22 +0200 Subject: [PATCH 11/56] Fix Streamate The way to obtains the xsrf token has changed --- common/src/main/java/ctbrec/io/HttpConstants.java | 1 + .../java/ctbrec/sites/streamate/StreamateHttpClient.java | 8 +++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/common/src/main/java/ctbrec/io/HttpConstants.java b/common/src/main/java/ctbrec/io/HttpConstants.java index 92826433..15c95891 100644 --- a/common/src/main/java/ctbrec/io/HttpConstants.java +++ b/common/src/main/java/ctbrec/io/HttpConstants.java @@ -6,6 +6,7 @@ public class HttpConstants { public static final String ACCEPT_LANGUAGE = "Accept-Language"; public static final String CONNECTION = "Connection"; public static final String CONTENT_TYPE = "Content-Type"; + public static final String COOKIE = "Cookie"; public static final String KEEP_ALIVE = "keep-alive"; public static final String MIMETYPE_APPLICATION_JSON = "application/json"; public static final String ORIGIN = "Origin"; diff --git a/common/src/main/java/ctbrec/sites/streamate/StreamateHttpClient.java b/common/src/main/java/ctbrec/sites/streamate/StreamateHttpClient.java index 2184abc2..497ee5d5 100644 --- a/common/src/main/java/ctbrec/sites/streamate/StreamateHttpClient.java +++ b/common/src/main/java/ctbrec/sites/streamate/StreamateHttpClient.java @@ -6,6 +6,7 @@ import java.io.IOException; import java.util.Collections; import java.util.Locale; import java.util.NoSuchElementException; +import java.util.UUID; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -15,7 +16,6 @@ import org.slf4j.LoggerFactory; import ctbrec.Config; import ctbrec.io.HttpClient; -import ctbrec.io.HttpConstants; import ctbrec.io.HttpException; import okhttp3.Cookie; import okhttp3.HttpUrl; @@ -60,8 +60,10 @@ public class StreamateHttpClient extends HttpClient { private void loadXsrfToken() { // do a first request to get cookies and stuff Request req = new Request.Builder() // - .url(Streamate.BASE_URL) // - .header(HttpConstants.USER_AGENT, Config.getInstance().getSettings().httpUserAgent) // + .url(Streamate.BASE_URL + "/initialData.js") // + .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) // + .header(COOKIE, "smtid="+UUID.randomUUID().toString()+"; Xld_rct=1;") // + .header(REFERER, Streamate.BASE_URL) .build(); try (Response resp = execute(req)) { if (resp.code() == 200) { From 4b2e17d0b10100934728c19ca447d8bd875183fc Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Thu, 16 Jul 2020 19:51:14 +0200 Subject: [PATCH 12/56] Fix Cam4 favorites tab --- CHANGELOG.md | 7 +++ .../sites/cam4/Cam4FollowedUpdateService.java | 55 +++++-------------- .../src/main/java/ctbrec/sites/cam4/Cam4.java | 2 +- .../java/ctbrec/sites/cam4/Cam4Model.java | 2 +- 4 files changed, 24 insertions(+), 42 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2928127a..5fae3e6f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +3.8.3 +======================== +* Fixed Streamate +* Fixed favorites tab for Cam4; kind of, because only the online tab works. I currently don't see + a way to retrieve the offline favorites + + 3.8.2 ======================== * Fixed misconfiguration in global connection pool, which caused a lot of diff --git a/client/src/main/java/ctbrec/ui/sites/cam4/Cam4FollowedUpdateService.java b/client/src/main/java/ctbrec/ui/sites/cam4/Cam4FollowedUpdateService.java index 05144d23..e96b8b64 100644 --- a/client/src/main/java/ctbrec/ui/sites/cam4/Cam4FollowedUpdateService.java +++ b/client/src/main/java/ctbrec/ui/sites/cam4/Cam4FollowedUpdateService.java @@ -3,19 +3,13 @@ package ctbrec.ui.sites.cam4; import java.io.IOException; import java.util.ArrayList; import java.util.List; -import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.stream.Collectors; -import org.jsoup.nodes.Element; -import org.jsoup.select.Elements; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.json.JSONArray; +import org.json.JSONObject; -import ctbrec.Config; import ctbrec.Model; -import ctbrec.io.HtmlParser; import ctbrec.io.HttpException; import ctbrec.sites.cam4.Cam4; import ctbrec.sites.cam4.Cam4Model; @@ -27,7 +21,6 @@ import okhttp3.Response; public class Cam4FollowedUpdateService extends PaginatedScheduledService { - private static final Logger LOG = LoggerFactory.getLogger(Cam4FollowedUpdateService.class); private Cam4 site; private boolean showOnline = true; @@ -50,46 +43,28 @@ public class Cam4FollowedUpdateService extends PaginatedScheduledService { // login first SiteUiFactory.getUi(site).login(); List models = new ArrayList<>(); - String username = Config.getInstance().getSettings().cam4Username; - String url = site.getBaseUrl() + '/' + username + "/edit/friends_favorites"; + String url = site.getBaseUrl() + "/directoryCams?directoryJson=true&online=" + showOnline + "&url=true&friends=true&favorites=true&resultsPerPage=90"; Request req = new Request.Builder().url(url).build(); - try(Response response = site.getHttpClient().execute(req)) { - if(response.isSuccessful()) { + try (Response response = site.getHttpClient().execute(req)) { + if (response.isSuccessful()) { String content = response.body().string(); - Elements cells = HtmlParser.getTags(content, "div#favorites div.ff_thumb"); - for (Element cell : cells) { - String cellHtml = cell.html(); - Element link = HtmlParser.getTag(cellHtml, "div.ff_img a"); - String path = link.attr("href"); - String modelName = path.substring(1); - Cam4Model model = (Cam4Model) site.createModel(modelName); - model.setPreview("https://snapshots.xcdnpro.com/thumbnails/"+model.getName()+"?s=" + System.currentTimeMillis()); - model.setOnlineStateByShowType(parseOnlineState(cellHtml)); + JSONObject json = new JSONObject(content); + JSONArray users = json.getJSONArray("users"); + for (int i = 0; i < users.length(); i++) { + JSONObject modelJson = users.getJSONObject(i); + String username = modelJson.optString("username"); + Cam4Model model = site.createModel(username); + model.setPreview(modelJson.optString("snapshotImageLink")); + model.setOnlineStateByShowType(modelJson.optString("showType")); + model.setDescription(modelJson.optString("statusMessage")); models.add(model); } - return models.stream() - .filter(m -> { - try { - return m.isOnline() == showOnline; - } catch (IOException | ExecutionException e) { - LOG.error("Couldn't determine online state", e); - return false; - } catch(InterruptedException e) { - Thread.currentThread().interrupt(); - LOG.error("Couldn't determine online state", e); - return false; - } - }).collect(Collectors.toList()); + return models; } else { throw new HttpException(response.code(), response.message()); } } } - - private String parseOnlineState(String cellHtml) { - Element state = HtmlParser.getTag(cellHtml, "div.ff_name div"); - return state.attr("class").equals("online") ? "NORMAL" : "OFFLINE"; - } }; } diff --git a/common/src/main/java/ctbrec/sites/cam4/Cam4.java b/common/src/main/java/ctbrec/sites/cam4/Cam4.java index f099bfcd..6a8b7562 100644 --- a/common/src/main/java/ctbrec/sites/cam4/Cam4.java +++ b/common/src/main/java/ctbrec/sites/cam4/Cam4.java @@ -44,7 +44,7 @@ public class Cam4 extends AbstractSite { } @Override - public Model createModel(String name) { + public Cam4Model createModel(String name) { Cam4Model m = new Cam4Model(); m.setSite(this); m.setName(name); diff --git a/common/src/main/java/ctbrec/sites/cam4/Cam4Model.java b/common/src/main/java/ctbrec/sites/cam4/Cam4Model.java index 5272b78b..63cf1afc 100644 --- a/common/src/main/java/ctbrec/sites/cam4/Cam4Model.java +++ b/common/src/main/java/ctbrec/sites/cam4/Cam4Model.java @@ -103,7 +103,7 @@ public class Cam4Model extends AbstractModel { onlineState = OFFLINE; break; default: - LOG.debug("Unknown show type {}", showType); + LOG.debug("Unknown show type [{}]", showType); onlineState = UNKNOWN; } From e202c946ac5987fe691d18ddee351340ec51adce Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Thu, 16 Jul 2020 21:28:47 +0200 Subject: [PATCH 13/56] Fix CamSoda followed tab --- .../ui/sites/camsoda/CamsodaFollowedTab.java | 24 +++++- .../camsoda/CamsodaFollowedUpdateService.java | 79 ------------------- .../sites/camsoda/CamsodaUpdateService.java | 34 +++++--- 3 files changed, 42 insertions(+), 95 deletions(-) delete mode 100644 client/src/main/java/ctbrec/ui/sites/camsoda/CamsodaFollowedUpdateService.java diff --git a/client/src/main/java/ctbrec/ui/sites/camsoda/CamsodaFollowedTab.java b/client/src/main/java/ctbrec/ui/sites/camsoda/CamsodaFollowedTab.java index d9904195..f3872970 100644 --- a/client/src/main/java/ctbrec/ui/sites/camsoda/CamsodaFollowedTab.java +++ b/client/src/main/java/ctbrec/ui/sites/camsoda/CamsodaFollowedTab.java @@ -1,6 +1,9 @@ package ctbrec.ui.sites.camsoda; +import java.util.function.Predicate; + import ctbrec.sites.camsoda.Camsoda; +import ctbrec.sites.camsoda.CamsodaModel; import ctbrec.ui.tabs.FollowedTab; import ctbrec.ui.tabs.ThumbOverviewTab; import javafx.concurrent.WorkerStateEvent; @@ -18,9 +21,10 @@ public class CamsodaFollowedTab extends ThumbOverviewTab implements FollowedTab boolean showOnline = true; public CamsodaFollowedTab(String title, Camsoda camsoda) { - super(title, new CamsodaFollowedUpdateService(camsoda), camsoda); + super(title, new CamsodaUpdateService(camsoda.getBaseUrl() + "/api/v1/browse/following", true, camsoda, m -> true), camsoda); status = new Label("Logging in..."); grid.getChildren().add(status); + ((CamsodaUpdateService)updateService).setFilter(createFilter(this)); } @Override @@ -40,9 +44,9 @@ public class CamsodaFollowedTab extends ThumbOverviewTab implements FollowedTab HBox.setMargin(online, new Insets(5, 5, 5, 40)); HBox.setMargin(offline, new Insets(5, 5, 5, 5)); online.setSelected(true); - group.selectedToggleProperty().addListener((e) -> { + group.selectedToggleProperty().addListener(e -> { + showOnline = online.isSelected(); queue.clear(); - ((CamsodaFollowedUpdateService)updateService).showOnline(online.isSelected()); updateService.restart(); }); } @@ -78,4 +82,18 @@ public class CamsodaFollowedTab extends ThumbOverviewTab implements FollowedTab } }); } + + private static Predicate createFilter(CamsodaFollowedTab tab) { + return m -> { + try { + return m.isOnline() == tab.showOnline; + } catch(InterruptedException e) { + Thread.currentThread().interrupt(); + return false; + } catch (Exception e) { + return false; + } + }; + + } } diff --git a/client/src/main/java/ctbrec/ui/sites/camsoda/CamsodaFollowedUpdateService.java b/client/src/main/java/ctbrec/ui/sites/camsoda/CamsodaFollowedUpdateService.java deleted file mode 100644 index ca3ff432..00000000 --- a/client/src/main/java/ctbrec/ui/sites/camsoda/CamsodaFollowedUpdateService.java +++ /dev/null @@ -1,79 +0,0 @@ -package ctbrec.ui.sites.camsoda; - -import static ctbrec.Model.State.*; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.concurrent.ExecutionException; -import java.util.stream.Collectors; - -import org.json.JSONArray; -import org.json.JSONObject; - -import ctbrec.Model; -import ctbrec.io.HttpException; -import ctbrec.sites.camsoda.Camsoda; -import ctbrec.sites.camsoda.CamsodaModel; -import ctbrec.ui.SiteUiFactory; -import ctbrec.ui.tabs.PaginatedScheduledService; -import javafx.concurrent.Task; -import okhttp3.Request; -import okhttp3.Response; - -public class CamsodaFollowedUpdateService extends PaginatedScheduledService { - private Camsoda camsoda; - private boolean showOnline = true; - - public CamsodaFollowedUpdateService(Camsoda camsoda) { - this.camsoda = camsoda; - } - - @Override - protected Task> createTask() { - return new Task>() { - @Override - public List call() throws IOException { - List models = new ArrayList<>(); - String url = camsoda.getBaseUrl() + "/api/v1/user/current"; - SiteUiFactory.getUi(camsoda).login(); - Request request = new Request.Builder().url(url).build(); - try(Response response = camsoda.getHttpClient().execute(request)) { - if (response.isSuccessful()) { - JSONObject json = new JSONObject(response.body().string()); - if(json.has("status") && json.getBoolean("status")) { - JSONObject user = json.getJSONObject("user"); - JSONArray following = user.getJSONArray("following"); - for (int i = 0; i < following.length(); i++) { - JSONObject m = following.getJSONObject(i); - CamsodaModel model = (CamsodaModel) camsoda.createModel(m.getString("followname")); - boolean online = m.getInt("online") == 1; - model.setOnlineState(online ? ONLINE : OFFLINE); - model.setPreview("https://md.camsoda.com/thumbs/" + model.getName() + ".jpg"); - models.add(model); - } - return models.stream() - .filter((m) -> { - try { - return m.isOnline() == showOnline; - } catch (IOException | ExecutionException | InterruptedException e) { - return false; - } - }).collect(Collectors.toList()); - } else { - response.close(); - return Collections.emptyList(); - } - } else { - throw new HttpException(response.code(), response.message()); - } - } - } - }; - } - - void showOnline(boolean online) { - this.showOnline = online; - } -} diff --git a/client/src/main/java/ctbrec/ui/sites/camsoda/CamsodaUpdateService.java b/client/src/main/java/ctbrec/ui/sites/camsoda/CamsodaUpdateService.java index 53822bcc..ab362568 100644 --- a/client/src/main/java/ctbrec/ui/sites/camsoda/CamsodaUpdateService.java +++ b/client/src/main/java/ctbrec/ui/sites/camsoda/CamsodaUpdateService.java @@ -1,5 +1,7 @@ package ctbrec.ui.sites.camsoda; +import static ctbrec.Model.State.*; + import java.io.IOException; import java.util.ArrayList; import java.util.Collections; @@ -60,53 +62,55 @@ public class CamsodaUpdateService extends PaginatedScheduledService { protected List loadOnlineModels() throws IOException { List models = new ArrayList<>(); - if(loginRequired && StringUtil.isBlank(ctbrec.Config.getInstance().getSettings().username)) { + if(loginRequired && StringUtil.isBlank(ctbrec.Config.getInstance().getSettings().camsodaUsername)) { return Collections.emptyList(); } else { - String url = CamsodaUpdateService.this.url; LOG.debug("Fetching page {}", url); if(loginRequired) { SiteUiFactory.getUi(camsoda).login(); } Request request = new Request.Builder().url(url).build(); - try(Response response = camsoda.getHttpClient().execute(request)) { + try (Response response = camsoda.getHttpClient().execute(request)) { if (response.isSuccessful()) { - JSONObject json = new JSONObject(response.body().string()); - if(json.has("status") && json.getBoolean("status")) { + String body = response.body().string(); + JSONObject json = new JSONObject(body); + if (json.optBoolean("status")) { JSONArray template = json.getJSONArray("template"); JSONArray results = json.getJSONArray("results"); for (int i = 0; i < results.length(); i++) { JSONObject result = results.getJSONObject(i); try { - if(result.has("tpl")) { + CamsodaModel model; + if (result.has("tpl")) { JSONArray tpl = result.getJSONArray("tpl"); String name = tpl.getString(getTemplateIndex(template, "username")); - CamsodaModel model = (CamsodaModel) camsoda.createModel(name); + model = (CamsodaModel) camsoda.createModel(name); model.setDescription(tpl.getString(getTemplateIndex(template, "subject_html"))); model.setSortOrder(tpl.getFloat(getTemplateIndex(template, "sort_value"))); String preview = "https:" + tpl.getString(getTemplateIndex(template, "thumb")); model.setPreview(preview); String displayName = tpl.getString(getTemplateIndex(template, "display_name")); model.setDisplayName(displayName.replaceAll("[^a-zA-Z0-9]", "")); - if(model.getDisplayName().isBlank()) { + if (model.getDisplayName().isBlank()) { model.setDisplayName(name); } model.setNew(result.optBoolean("new")); + model.setOnlineState(tpl.getString(getTemplateIndex(template, "stream_name")).isEmpty() ? OFFLINE : ONLINE); models.add(model); } else { String name = result.getString("username"); - CamsodaModel model = (CamsodaModel) camsoda.createModel(name); + model = (CamsodaModel) camsoda.createModel(name); model.setSortOrder(result.getFloat("sort_value")); - if(result.has("status")) { + if (result.has("status")) { model.setOnlineStateByStatus(result.getString("status")); } - if(result.has("display_name")) { + if (result.has("display_name")) { model.setDisplayName(result.getString("display_name").replaceAll("[^a-zA-Z0-9]", "")); - if(model.getDisplayName().isBlank()) { + if (model.getDisplayName().isBlank()) { model.setDisplayName(name); } } - if(result.has("thumb")) { + if (result.has("thumb")) { String previewUrl = "https:" + result.getString("thumb"); model.setPreview(previewUrl); } @@ -138,4 +142,8 @@ public class CamsodaUpdateService extends PaginatedScheduledService { } throw new NoSuchElementException(string + " not found in template: " + template.toString()); } + + public void setFilter(Predicate filter) { + this.filter = filter; + } } From c50519be82040e4d515e7323790e957c3234874f Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Sat, 18 Jul 2020 12:48:00 +0200 Subject: [PATCH 14/56] Fix CamSoda recordings --- common/src/main/java/ctbrec/sites/camsoda/CamsodaModel.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/common/src/main/java/ctbrec/sites/camsoda/CamsodaModel.java b/common/src/main/java/ctbrec/sites/camsoda/CamsodaModel.java index 9b182202..b5988b04 100644 --- a/common/src/main/java/ctbrec/sites/camsoda/CamsodaModel.java +++ b/common/src/main/java/ctbrec/sites/camsoda/CamsodaModel.java @@ -158,7 +158,11 @@ public class CamsodaModel extends AbstractModel { if (oldStreamUrl && chat.has(EDGE_SERVERS)) { String edgeServer = chat.getJSONArray(EDGE_SERVERS).getString(0); String streamName = chat.getString(STREAM_NAME); - streamUrl = "https://" + edgeServer + "/cam/mp4:" + streamName + "_h264_aac_480p/playlist.m3u8"; + if(streamName.contains("/")) { + streamUrl = "https://" + edgeServer + "/" + streamName + "/index.m3u8"; + } else { + streamUrl = "https://" + edgeServer + "/cam/mp4:" + streamName + "_h264_aac_480p/playlist.m3u8"; + } } setOnlineStateByStatus(status); } else { From b1d5d959d48165a3e3201ed94191e0c525b99cb3 Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Sat, 18 Jul 2020 12:48:12 +0200 Subject: [PATCH 15/56] Add URL to HttpException --- .../src/main/java/ctbrec/io/HttpException.java | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/common/src/main/java/ctbrec/io/HttpException.java b/common/src/main/java/ctbrec/io/HttpException.java index e9664bf5..f58e8cd2 100644 --- a/common/src/main/java/ctbrec/io/HttpException.java +++ b/common/src/main/java/ctbrec/io/HttpException.java @@ -4,15 +4,24 @@ import java.io.IOException; public class HttpException extends IOException { - private int code; - private String msg; + private final String url; + private final int code; + private final String msg; public HttpException(int code, String msg) { super(code + " - " + msg); + this.url = ""; this.code = code; this.msg = msg; } + public HttpException(String url, int code, String msg) { + super(code + " - " + msg + " - " + url); + this.code = code; + this.msg = msg; + this.url = url; + } + public int getResponseCode() { return code; } @@ -20,4 +29,8 @@ public class HttpException extends IOException { public String getResponseMessage() { return msg; } + + public String getUrl() { + return url; + } } From 6ca0e61f1f88c08894c41c27f8a250d6be85f58e Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Sat, 18 Jul 2020 19:05:09 +0200 Subject: [PATCH 16/56] Improve exception handling --- .../ctbrec/recorder/download/hls/MergedFfmpegHlsDownload.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/common/src/main/java/ctbrec/recorder/download/hls/MergedFfmpegHlsDownload.java b/common/src/main/java/ctbrec/recorder/download/hls/MergedFfmpegHlsDownload.java index 27e1c242..da63dd00 100644 --- a/common/src/main/java/ctbrec/recorder/download/hls/MergedFfmpegHlsDownload.java +++ b/common/src/main/java/ctbrec/recorder/download/hls/MergedFfmpegHlsDownload.java @@ -6,6 +6,7 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.net.MalformedURLException; import java.net.URL; import java.nio.file.Files; import java.time.Duration; @@ -229,6 +230,9 @@ public class MergedFfmpegHlsDownload extends AbstractHlsDownload { LOG.info("Unexpected error while downloading {}", model, e); } running = false; + } catch (MalformedURLException e) { + LOG.info("Malformed URL {} - {}", model, segmentPlaylistUri, e); + running = false; } catch (Exception e) { LOG.info("Unexpected error while downloading {}", model, e); running = false; From 7ff731ec88a51d803614d63464270d82d5c120a5 Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Sat, 18 Jul 2020 19:05:41 +0200 Subject: [PATCH 17/56] Fix CamSoda downloads --- .../ctbrec/sites/camsoda/CamsodaModel.java | 168 ++++++++++-------- 1 file changed, 90 insertions(+), 78 deletions(-) diff --git a/common/src/main/java/ctbrec/sites/camsoda/CamsodaModel.java b/common/src/main/java/ctbrec/sites/camsoda/CamsodaModel.java index b5988b04..69c3f5f4 100644 --- a/common/src/main/java/ctbrec/sites/camsoda/CamsodaModel.java +++ b/common/src/main/java/ctbrec/sites/camsoda/CamsodaModel.java @@ -10,6 +10,7 @@ import java.util.Collections; import java.util.List; import java.util.Locale; import java.util.Objects; +import java.util.Optional; import java.util.Random; import java.util.concurrent.ExecutionException; @@ -44,99 +45,124 @@ public class CamsodaModel extends AbstractModel { private static final String EDGE_SERVERS = "edge_servers"; private static final String STATUS = "status"; private static final Logger LOG = LoggerFactory.getLogger(CamsodaModel.class); - private String streamUrl; private transient List streamSources = null; private transient boolean isNew; private float sortOrder = 0; private Random random = new Random(); int[] resolution = new int[2]; - boolean oldStreamUrl = true; + public String getStreamUrl() throws IOException { - if (streamUrl == null) { - if(oldStreamUrl) { - loadModel(); - } else { - getNewStreamUrl(); - } + Request req = createJsonRequest(getTokenInfoUrl()); + JSONObject response = executeJsonRequest(req); + if (response.optInt(STATUS) == 1 && response.optJSONArray(EDGE_SERVERS) != null && response.optJSONArray(EDGE_SERVERS).length() > 0) { + String edgeServer = response.getJSONArray(EDGE_SERVERS).getString(0); + String streamName = response.getString(STREAM_NAME); + String token = response.getString("token"); + return constructStreamUrl(edgeServer, streamName, token); + } else { + throw new JSONException("JSON response has not the expected structure"); } - return streamUrl; } - public String getNewStreamUrl() throws IOException { + private String getTokenInfoUrl() { String guestUsername = "guest_" + 10_000 + random.nextInt(50_000); - String url = site.getBaseUrl() + "/api/v1/video/vtoken/" + getName() + "?username=" + guestUsername; - Request req = new Request.Builder() - .url(url) + String tokenInfoUrl = site.getBaseUrl() + "/api/v1/video/vtoken/" + getName() + "?username=" + guestUsername; + return tokenInfoUrl; + } + + private String constructStreamUrl(String edgeServer, String streamName, String token) { + StringBuilder url = new StringBuilder("https://"); + url.append(edgeServer).append('/'); + if (streamName.contains("-flu")) { + url.append(streamName); + url.append("_h264_aac"); + url.append(streamName.contains("-flu-hd") ? "_720p" : "_480p"); + url.append("/index.m3u8"); + if (!isPublic(streamName)) { + url.append("?token=").append(token); + } + } else { + // https://vide7-ord.camsoda.com/cam/mp4:maxandtokio-enc10-ord_h264_aac_480p/playlist.m3u8 + url.append("cam/mp4:"); + url.append(streamName); + url.append("_h264_aac_480p/playlist.m3u8"); + } + LOG.debug("Stream URL: {}", url); + return url.toString(); + } + + private Request createJsonRequest(String tokenInfoUrl) { + return new Request.Builder() + .url(tokenInfoUrl) .header(ACCEPT, MIMETYPE_APPLICATION_JSON) .header(ACCEPT_LANGUAGE, Locale.ENGLISH.getLanguage()) .header(X_REQUESTED_WITH, XML_HTTP_REQUEST) .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) .build(); - try (Response response = site.getHttpClient().execute(req)) { + } + + private JSONObject executeJsonRequest(Request request) throws IOException { + try (Response response = site.getHttpClient().execute(request)) { if (response.isSuccessful()) { JSONObject jsonResponse = new JSONObject(response.body().string()); - if (jsonResponse.optInt(STATUS) == 1) { - String edgeServer = jsonResponse.getJSONArray(EDGE_SERVERS).getString(0); - String streamName = jsonResponse.getString(STREAM_NAME); - String token = jsonResponse.getString("token"); - streamUrl = "https://" + edgeServer + '/' + streamName + "_h264_aac_480p/index.m3u8?token=" + token; - } else { - throw new JSONException("Response does not contain a token"); - } + return jsonResponse; } else { throw new HttpException(response.code(), response.message()); } } - return streamUrl; + } + + private boolean isPublic(String streamName) { + return Optional.ofNullable(streamName).orElse("").contains("_public"); } @Override public List getStreamSources() throws IOException, ExecutionException, ParseException, PlaylistException { - String playlistUrl = getStreamUrl(); - if (playlistUrl == null) { - return Collections.emptyList(); - } - LOG.trace("Loading playlist {}", playlistUrl); - Request req = new Request.Builder() - .url(playlistUrl) - .header(ACCEPT_LANGUAGE, Locale.ENGLISH.getLanguage()) - .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) - .build(); - try (Response response = site.getHttpClient().execute(req)) { - if (response.isSuccessful()) { - InputStream inputStream = response.body().byteStream(); - PlaylistParser parser = new PlaylistParser(inputStream, Format.EXT_M3U, Encoding.UTF_8, ParsingMode.LENIENT); - Playlist playlist = parser.parse(); - MasterPlaylist master = playlist.getMasterPlaylist(); - PlaylistData playlistData = master.getPlaylists().get(0); - StreamSource streamsource = new StreamSource(); - if (oldStreamUrl) { - streamsource.mediaPlaylistUrl = streamUrl.replace("playlist.m3u8", playlistData.getUri()); - } else { - int cutOffAt = playlistUrl.indexOf("index.m3u8"); + try { + String playlistUrl = getStreamUrl(); + if (playlistUrl == null) { + return Collections.emptyList(); + } + LOG.info("Loading playlist {}", playlistUrl); + Request req = new Request.Builder() + .url(playlistUrl) + .header(ACCEPT_LANGUAGE, Locale.ENGLISH.getLanguage()) + .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) + .build(); + try (Response response = site.getHttpClient().execute(req)) { + if (response.isSuccessful()) { + InputStream inputStream = response.body().byteStream(); + PlaylistParser parser = new PlaylistParser(inputStream, Format.EXT_M3U, Encoding.UTF_8, ParsingMode.LENIENT); + Playlist playlist = parser.parse(); + MasterPlaylist master = playlist.getMasterPlaylist(); + PlaylistData playlistData = master.getPlaylists().get(0); + StreamSource streamsource = new StreamSource(); + int cutOffAt = Math.max(playlistUrl.indexOf("index.m3u8"), playlistUrl.indexOf("playlist.m3u8")); String segmentPlaylistUrl = playlistUrl.substring(0, cutOffAt) + playlistData.getUri(); streamsource.mediaPlaylistUrl = segmentPlaylistUrl; - } - if (playlistData.hasStreamInfo()) { - StreamInfo info = playlistData.getStreamInfo(); - streamsource.bandwidth = info.getBandwidth(); - streamsource.width = info.hasResolution() ? info.getResolution().width : 0; - streamsource.height = info.hasResolution() ? info.getResolution().height : 0; + if (playlistData.hasStreamInfo()) { + StreamInfo info = playlistData.getStreamInfo(); + streamsource.bandwidth = info.getBandwidth(); + streamsource.width = info.hasResolution() ? info.getResolution().width : 0; + streamsource.height = info.hasResolution() ? info.getResolution().height : 0; + } else { + streamsource.bandwidth = 0; + streamsource.width = 0; + streamsource.height = 0; + } + streamSources = new ArrayList<>(); + streamSources.add(streamsource); } else { - streamsource.bandwidth = 0; - streamsource.width = 0; - streamsource.height = 0; + LOG.trace("Response: {}", response.body().string()); + throw new HttpException(playlistUrl, response.code(), response.message()); } - streamSources = new ArrayList<>(); - streamSources.add(streamsource); - } else { - LOG.trace("Response: {}", response.body().string()); - throw new HttpException(response.code(), response.message()); } + return streamSources; + } catch (JSONException e) { + return Collections.emptyList(); } - return streamSources; } private void loadModel() throws IOException { @@ -151,19 +177,9 @@ public class CamsodaModel extends AbstractModel { try (Response response = site.getHttpClient().execute(req)) { if (response.isSuccessful()) { JSONObject result = new JSONObject(response.body().string()); - if (result.getBoolean(STATUS)) { + if (result.optBoolean(STATUS)) { JSONObject chat = result.getJSONObject("user").getJSONObject("chat"); String status = chat.getString(STATUS); - oldStreamUrl = !chat.getString(STREAM_NAME).contains("/"); - if (oldStreamUrl && chat.has(EDGE_SERVERS)) { - String edgeServer = chat.getJSONArray(EDGE_SERVERS).getString(0); - String streamName = chat.getString(STREAM_NAME); - if(streamName.contains("/")) { - streamUrl = "https://" + edgeServer + "/" + streamName + "/index.m3u8"; - } else { - streamUrl = "https://" + edgeServer + "/cam/mp4:" + streamName + "_h264_aac_480p/playlist.m3u8"; - } - } setOnlineStateByStatus(status); } else { throw new IOException("Result was not ok"); @@ -226,11 +242,11 @@ public class CamsodaModel extends AbstractModel { return resolution; } else { try { - List streamSources = getStreamSources(); - if (streamSources.isEmpty()) { + List sources = getStreamSources(); + if (sources.isEmpty()) { return new int[] { 0, 0 }; } else { - StreamSource src = streamSources.get(0); + StreamSource src = sources.get(0); resolution = new int[] { src.width, src.height }; return resolution; } @@ -313,10 +329,6 @@ public class CamsodaModel extends AbstractModel { } } - public void setStreamUrl(String streamUrl) { - this.streamUrl = streamUrl; - } - public float getSortOrder() { return sortOrder; } From 010c4a04cd9ef4941b0a224ecfce124328ffeda9 Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Sat, 18 Jul 2020 19:15:09 +0200 Subject: [PATCH 18/56] Adjust log levels --- common/src/main/java/ctbrec/sites/camsoda/CamsodaModel.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common/src/main/java/ctbrec/sites/camsoda/CamsodaModel.java b/common/src/main/java/ctbrec/sites/camsoda/CamsodaModel.java index 69c3f5f4..e980eb78 100644 --- a/common/src/main/java/ctbrec/sites/camsoda/CamsodaModel.java +++ b/common/src/main/java/ctbrec/sites/camsoda/CamsodaModel.java @@ -89,7 +89,7 @@ public class CamsodaModel extends AbstractModel { url.append(streamName); url.append("_h264_aac_480p/playlist.m3u8"); } - LOG.debug("Stream URL: {}", url); + LOG.trace("Stream URL: {}", url); return url.toString(); } @@ -125,7 +125,7 @@ public class CamsodaModel extends AbstractModel { if (playlistUrl == null) { return Collections.emptyList(); } - LOG.info("Loading playlist {}", playlistUrl); + LOG.trace("Loading playlist {}", playlistUrl); Request req = new Request.Builder() .url(playlistUrl) .header(ACCEPT_LANGUAGE, Locale.ENGLISH.getLanguage()) From 882742ce3b135c51659b6a601ea8023d2823baf5 Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Sat, 18 Jul 2020 19:30:24 +0200 Subject: [PATCH 19/56] Make MFC client and getOnlineModels more robust --- .../src/main/java/ctbrec/recorder/NextGenLocalRecorder.java | 4 ++-- common/src/main/java/ctbrec/sites/mfc/ServerConfig.java | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/common/src/main/java/ctbrec/recorder/NextGenLocalRecorder.java b/common/src/main/java/ctbrec/recorder/NextGenLocalRecorder.java index 7d2122c9..c2d94fa0 100644 --- a/common/src/main/java/ctbrec/recorder/NextGenLocalRecorder.java +++ b/common/src/main/java/ctbrec/recorder/NextGenLocalRecorder.java @@ -555,11 +555,11 @@ public class NextGenLocalRecorder implements Recorder { return getModels().stream().filter(m -> { try { return m.isOnline(); - } catch (IOException | ExecutionException e) { - return false; } catch (InterruptedException e) { Thread.currentThread().interrupt(); return false; + } catch (Exception e) { + return false; } }).collect(Collectors.toList()); } diff --git a/common/src/main/java/ctbrec/sites/mfc/ServerConfig.java b/common/src/main/java/ctbrec/sites/mfc/ServerConfig.java index 96c7fd65..52019d44 100644 --- a/common/src/main/java/ctbrec/sites/mfc/ServerConfig.java +++ b/common/src/main/java/ctbrec/sites/mfc/ServerConfig.java @@ -8,6 +8,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import org.json.JSONArray; import org.json.JSONObject; @@ -86,7 +87,7 @@ public class ServerConfig { } public boolean isOnWzObsVideoServer(SessionState state) { - int camserv = Objects.requireNonNull(Objects.requireNonNull(state.getU()).getCamserv()); + int camserv = Optional.ofNullable(state).map(SessionState::getU).map(User::getCamserv).orElse(-1); return wzobsServers.containsKey(Integer.toString(camserv)); } From 7462d68d7b13c3d7f56f233253ce357a8d83e638 Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Sun, 19 Jul 2020 15:17:31 +0200 Subject: [PATCH 20/56] Add external login dialog for stripchat This also enables us to support xhamsterlive --- CHANGELOG.md | 7 +- .../StripchatElectronLoginDialog.java | 124 ++++++++++++++++++ .../StripchatFollowedUpdateService.java | 1 + .../ui/sites/stripchat/StripchatSiteUi.java | 51 ++++++- .../sites/stripchat/StripchatHttpClient.java | 49 ++++++- 5 files changed, 221 insertions(+), 11 deletions(-) create mode 100644 client/src/main/java/ctbrec/ui/sites/stripchat/StripchatElectronLoginDialog.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 5fae3e6f..32fd7d05 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,11 @@ 3.8.3 ======================== * Fixed Streamate -* Fixed favorites tab for Cam4; kind of, because only the online tab works. I currently don't see - a way to retrieve the offline favorites +* Fixed favorites tab for Cam4; kind of, because only the online tab works. + I currently don't see a way to retrieve the offline favorites +* Fixed favorites tab for CamSoda +* Fixed CamSoda recordings +* Added external login dialog to Stripchat to support the captcha 3.8.2 diff --git a/client/src/main/java/ctbrec/ui/sites/stripchat/StripchatElectronLoginDialog.java b/client/src/main/java/ctbrec/ui/sites/stripchat/StripchatElectronLoginDialog.java new file mode 100644 index 00000000..7eb49cc5 --- /dev/null +++ b/client/src/main/java/ctbrec/ui/sites/stripchat/StripchatElectronLoginDialog.java @@ -0,0 +1,124 @@ +package ctbrec.ui.sites.stripchat; + +import java.io.IOException; +import java.util.Collections; +import java.util.function.Consumer; + +import org.json.JSONArray; +import org.json.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import ctbrec.Config; +import ctbrec.sites.stripchat.Stripchat; +import ctbrec.ui.ExternalBrowser; +import okhttp3.Cookie; +import okhttp3.Cookie.Builder; +import okhttp3.CookieJar; +import okhttp3.HttpUrl; + +public class StripchatElectronLoginDialog { + + private static final transient Logger LOG = LoggerFactory.getLogger(StripchatElectronLoginDialog.class); + public static final String DOMAIN = "stripchat.com"; + public static final String URL = Stripchat.BASE_URI; + private CookieJar cookieJar; + private ExternalBrowser browser; + + public StripchatElectronLoginDialog(CookieJar cookieJar) throws IOException { + this.cookieJar = cookieJar; + browser = ExternalBrowser.getInstance(); + try { + JSONObject config = new JSONObject(); + config.put("url", URL); + config.put("w", 640); + config.put("h", 640); + JSONObject msg = new JSONObject(); + msg.put("config", config); + browser.run(msg, msgHandler); + } catch (InterruptedException e) { + throw new IOException("Couldn't wait for login dialog", e); + } finally { + browser.close(); + } + } + + private Consumer msgHandler = (line) -> { + if(!line.startsWith("{")) { + System.err.println(line); + } else { + JSONObject json = new JSONObject(line); + if(json.has("url")) { + String url = json.getString("url"); + + if(url.endsWith(DOMAIN) || url.endsWith(DOMAIN + '/')) { + try { + browser.executeJavaScript("document.querySelector('button[class~=\"btn-visitors-agreement-accept\"]').click();"); + browser.executeJavaScript("document.querySelector('div[class~=\"header-dropdown\"] a[class~=\"dropdown-link\"]').click();"); + browser.executeJavaScript("document.querySelector('a[class~=\"btn\"][href*=\"login\"]').click();"); + String username = Config.getInstance().getSettings().stripchatUsername; + if (username != null && !username.trim().isEmpty()) { + browser.executeJavaScript("document.querySelector('#login_login_or_email').value = '" + username + "';"); + } + String password = Config.getInstance().getSettings().stripchatPassword; + if (password != null && !password.trim().isEmpty()) { + browser.executeJavaScript("document.querySelector('#login_password').value = '" + password + "';"); + } + browser.executeJavaScript("document.querySelector('#recaptcha-checkbox-border').click();"); + browser.executeJavaScript("document.querySelector('*[class~=btn-login]').addEventListener('click', function() {window.setTimeout(function() {location.reload()}, 2000)});"); + } catch(Exception e) { + LOG.warn("Couldn't auto fill username and password for Stripchat", e); + } + } + + if (json.has("cookies")) { + JSONArray _cookies = json.getJSONArray("cookies"); + boolean sessionCookieFound = false; + for (int i = 0; i < _cookies.length(); i++) { + JSONObject cookie = _cookies.getJSONObject(i); + if (cookie.getString("domain").contains(DOMAIN)) { + String domain = cookie.getString("domain"); + if (domain.startsWith(".")) { + domain = domain.substring(1); + } + Cookie c = createCookie(domain, cookie); + cookieJar.saveFromResponse(HttpUrl.parse(url), Collections.singletonList(c)); + c = createCookie(DOMAIN, cookie); + cookieJar.saveFromResponse(HttpUrl.parse(Stripchat.BASE_URI), Collections.singletonList(c)); + if (c.name().equals("stripchat_com_sessionId")) { + sessionCookieFound = true; + } + } + } + + if(sessionCookieFound) { + try { + browser.close(); + } catch (IOException e) { + LOG.error("Couldn't send close request to browser", e); + } + } + } + } + } + }; + + private Cookie createCookie(String domain, JSONObject cookie) { + Builder b = new Cookie.Builder() + .path(cookie.getString("path")) + .domain(domain) + .name(cookie.getString("name")) + .value(cookie.getString("value")) + .expiresAt(Double.valueOf(cookie.optDouble("expirationDate")).longValue()); + if(cookie.optBoolean("hostOnly")) { + b.hostOnlyDomain(domain); + } + if(cookie.optBoolean("httpOnly")) { + b.httpOnly(); + } + if(cookie.optBoolean("secure")) { + b.secure(); + } + return b.build(); + } +} diff --git a/client/src/main/java/ctbrec/ui/sites/stripchat/StripchatFollowedUpdateService.java b/client/src/main/java/ctbrec/ui/sites/stripchat/StripchatFollowedUpdateService.java index 2dcc28c2..938be470 100644 --- a/client/src/main/java/ctbrec/ui/sites/stripchat/StripchatFollowedUpdateService.java +++ b/client/src/main/java/ctbrec/ui/sites/stripchat/StripchatFollowedUpdateService.java @@ -79,6 +79,7 @@ public class StripchatFollowedUpdateService extends PaginatedScheduledService { private JSONArray loadFavoriteModelIds() throws IOException { SiteUiFactory.getUi(stripchat).login(); + stripchat.getHttpClient().login(); long userId = ((StripchatHttpClient) stripchat.getHttpClient()).getUserId(); String url = stripchat.getBaseUrl() + "/api/front/users/" + userId + "/favorites"; Request request = new Request.Builder() diff --git a/client/src/main/java/ctbrec/ui/sites/stripchat/StripchatSiteUi.java b/client/src/main/java/ctbrec/ui/sites/stripchat/StripchatSiteUi.java index a7462903..af898505 100644 --- a/client/src/main/java/ctbrec/ui/sites/stripchat/StripchatSiteUi.java +++ b/client/src/main/java/ctbrec/ui/sites/stripchat/StripchatSiteUi.java @@ -1,20 +1,30 @@ package ctbrec.ui.sites.stripchat; import java.io.IOException; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import ctbrec.sites.stripchat.Stripchat; +import ctbrec.sites.stripchat.StripchatHttpClient; +import ctbrec.ui.controls.Dialogs; import ctbrec.ui.sites.AbstractSiteUi; import ctbrec.ui.sites.ConfigUI; import ctbrec.ui.tabs.TabProvider; +import javafx.application.Platform; public class StripchatSiteUi extends AbstractSiteUi { + private static final Logger LOG = LoggerFactory.getLogger(StripchatSiteUi.class); + private StripchatTabProvider tabProvider; private StripchatConfigUI configUi; - private Stripchat stripchat; + private Stripchat site; public StripchatSiteUi(Stripchat stripchat) { - this.stripchat = stripchat; + this.site = stripchat; tabProvider = new StripchatTabProvider(stripchat); configUi = new StripchatConfigUI(stripchat); } @@ -31,7 +41,40 @@ public class StripchatSiteUi extends AbstractSiteUi { @Override public synchronized boolean login() throws IOException { - boolean automaticLogin = stripchat.login(); - return automaticLogin; + boolean automaticLogin = site.login(); + if (automaticLogin) { + return true; + } else { + + BlockingQueue queue = new LinkedBlockingQueue<>(); + + Runnable showDialog = () -> { + // login with external browser + try { + new StripchatElectronLoginDialog(site.getHttpClient().getCookieJar()); + } catch (Exception e1) { + LOG.error("Error logging in with external browser", e1); + Dialogs.showError("Login error", "Couldn't login to " + site.getName(), e1); + } + + try { + queue.put(true); + } catch (InterruptedException e) { + LOG.error("Error while signaling termination", e); + } + }; + + Platform.runLater(showDialog); + try { + queue.take(); + } catch (InterruptedException e) { + LOG.error("Error while waiting for login dialog to close", e); + throw new IOException(e); + } + + StripchatHttpClient httpClient = (StripchatHttpClient) site.getHttpClient(); + boolean loggedIn = httpClient.checkLoginSuccess(); + return loggedIn; + } } } diff --git a/common/src/main/java/ctbrec/sites/stripchat/StripchatHttpClient.java b/common/src/main/java/ctbrec/sites/stripchat/StripchatHttpClient.java index 71561a15..94beb70f 100644 --- a/common/src/main/java/ctbrec/sites/stripchat/StripchatHttpClient.java +++ b/common/src/main/java/ctbrec/sites/stripchat/StripchatHttpClient.java @@ -4,6 +4,7 @@ import static ctbrec.io.HttpConstants.*; import java.io.IOException; +import org.json.JSONException; import org.json.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -36,7 +37,7 @@ public class StripchatHttpClient extends HttpClient { } // persisted cookies might let us log in - if(checkLoginSuccess()) { + if (checkLoginSuccess()) { loggedIn = true; LOG.debug("Logged in with cookies"); return true; @@ -75,7 +76,8 @@ public class StripchatHttpClient extends HttpClient { return false; } } else { - throw new HttpException(response.code(), response.message()); + LOG.info("Auto-Login failed: {} {} {}", response.code(), response.message(), url); + return false; } } } @@ -107,11 +109,48 @@ public class StripchatHttpClient extends HttpClient { * check, if the login worked * @throws IOException */ - public boolean checkLoginSuccess() { - return userId > 0; + public boolean checkLoginSuccess() throws IOException { + long userId = getUserId(); + String url = Stripchat.BASE_URI + "/api/front/users/" + userId + "/favorites"; + Request request = new Request.Builder() + .url(url) + .header(ACCEPT, MIMETYPE_APPLICATION_JSON) + .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) + .header(ORIGIN, Stripchat.BASE_URI) + .header(REFERER, Stripchat.BASE_URI + "/favorites") + .header(CONTENT_TYPE, MIMETYPE_APPLICATION_JSON) + .build(); + try (Response response = execute(request)) { + if (response.isSuccessful()) { + return true; + } + } catch (Exception e) { + LOG.info("Login check returned unsuccessful: {}", e.getLocalizedMessage()); + } + return false; } - public long getUserId() { + public long getUserId() throws JSONException, IOException { + if (userId == 0) { + String url = Stripchat.BASE_URI + "/api/front/users/username/" + Config.getInstance().getSettings().stripchatUsername; + Request request = new Request.Builder() + .url(url) + .header(ACCEPT, MIMETYPE_APPLICATION_JSON) + .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) + .header(ORIGIN, Stripchat.BASE_URI) + .header(REFERER, Stripchat.BASE_URI) + .header(CONTENT_TYPE, MIMETYPE_APPLICATION_JSON) + .build(); + try (Response response = execute(request)) { + if (response.isSuccessful()) { + JSONObject resp = new JSONObject(response.body().string()); + JSONObject user = resp.getJSONObject("user"); + userId = user.optLong("id"); + } else { + throw new HttpException(url, response.code(), response.message()); + } + } + } return userId; } From d96b9a1380ad8ae4b7c377bee3e7e0e010b5a22a Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Sun, 19 Jul 2020 16:35:38 +0200 Subject: [PATCH 21/56] Escape passwords before injecting them Escape quotes in passwords before injecting them into the external browser, so that the injected javascript is valid and doesn't break --- .../java/ctbrec/ui/sites/bonga/BongaCamsElectronLoginDialog.java | 1 + .../main/java/ctbrec/ui/sites/cam4/Cam4ElectronLoginDialog.java | 1 + .../ctbrec/ui/sites/jasmin/LiveJasminElectronLoginDialog.java | 1 + .../java/ctbrec/ui/sites/showup/ShowupElectronLoginDialog.java | 1 + .../ctbrec/ui/sites/stripchat/StripchatElectronLoginDialog.java | 1 + 5 files changed, 5 insertions(+) diff --git a/client/src/main/java/ctbrec/ui/sites/bonga/BongaCamsElectronLoginDialog.java b/client/src/main/java/ctbrec/ui/sites/bonga/BongaCamsElectronLoginDialog.java index 0627e4aa..21e60397 100644 --- a/client/src/main/java/ctbrec/ui/sites/bonga/BongaCamsElectronLoginDialog.java +++ b/client/src/main/java/ctbrec/ui/sites/bonga/BongaCamsElectronLoginDialog.java @@ -63,6 +63,7 @@ public class BongaCamsElectronLoginDialog { } String password = Config.getInstance().getSettings().bongaPassword; if (password != null && !password.trim().isEmpty()) { + password = password.replace("'", "\\'"); browser.executeJavaScript("$('input[name=\"log_in[password]\"]').attr('value','" + password + "')"); } String[] simplify = new String[] { diff --git a/client/src/main/java/ctbrec/ui/sites/cam4/Cam4ElectronLoginDialog.java b/client/src/main/java/ctbrec/ui/sites/cam4/Cam4ElectronLoginDialog.java index 6adc8f0d..7f79ea0a 100644 --- a/client/src/main/java/ctbrec/ui/sites/cam4/Cam4ElectronLoginDialog.java +++ b/client/src/main/java/ctbrec/ui/sites/cam4/Cam4ElectronLoginDialog.java @@ -62,6 +62,7 @@ public class Cam4ElectronLoginDialog { } String password = Config.getInstance().getSettings().cam4Password; if (password != null && !password.trim().isEmpty()) { + password = password.replace("'", "\\'"); browser.executeJavaScript("document.querySelector('#loginPageForm input[name=\"password\"]').value = '" + password + "';"); } browser.executeJavaScript("document.getElementById('footer').setAttribute('style', 'display:none');"); diff --git a/client/src/main/java/ctbrec/ui/sites/jasmin/LiveJasminElectronLoginDialog.java b/client/src/main/java/ctbrec/ui/sites/jasmin/LiveJasminElectronLoginDialog.java index ebd97ca4..ab665a8e 100644 --- a/client/src/main/java/ctbrec/ui/sites/jasmin/LiveJasminElectronLoginDialog.java +++ b/client/src/main/java/ctbrec/ui/sites/jasmin/LiveJasminElectronLoginDialog.java @@ -60,6 +60,7 @@ public class LiveJasminElectronLoginDialog { } String password = Config.getInstance().getSettings().livejasminPassword; if (password != null && !password.trim().isEmpty()) { + password = password.replace("'", "\\'"); browser.executeJavaScript("document.querySelector('#login_form input[name=\"password\"]').value = '" + password + "';"); } browser.executeJavaScript("document.getElementById('header_container').setAttribute('style', 'display:none');"); diff --git a/client/src/main/java/ctbrec/ui/sites/showup/ShowupElectronLoginDialog.java b/client/src/main/java/ctbrec/ui/sites/showup/ShowupElectronLoginDialog.java index 64a56c18..28375850 100644 --- a/client/src/main/java/ctbrec/ui/sites/showup/ShowupElectronLoginDialog.java +++ b/client/src/main/java/ctbrec/ui/sites/showup/ShowupElectronLoginDialog.java @@ -84,6 +84,7 @@ public class ShowupElectronLoginDialog { } String password = Config.getInstance().getSettings().showupPassword; if (password != null && !password.trim().isEmpty()) { + password = password.replace("'", "\\'"); browser.executeJavaScript("$('input[name=\"password\"]').attr('value','" + password + "')"); } browser.executeJavaScript("$('input[name=\"remember\"]').attr('value','true')"); diff --git a/client/src/main/java/ctbrec/ui/sites/stripchat/StripchatElectronLoginDialog.java b/client/src/main/java/ctbrec/ui/sites/stripchat/StripchatElectronLoginDialog.java index 7eb49cc5..12c410d2 100644 --- a/client/src/main/java/ctbrec/ui/sites/stripchat/StripchatElectronLoginDialog.java +++ b/client/src/main/java/ctbrec/ui/sites/stripchat/StripchatElectronLoginDialog.java @@ -62,6 +62,7 @@ public class StripchatElectronLoginDialog { } String password = Config.getInstance().getSettings().stripchatPassword; if (password != null && !password.trim().isEmpty()) { + password = password.replace("'", "\\'"); browser.executeJavaScript("document.querySelector('#login_password').value = '" + password + "';"); } browser.executeJavaScript("document.querySelector('#recaptcha-checkbox-border').click();"); From e50b9bcc71faf786b06022dcb2597eb9f9599da5 Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Sun, 19 Jul 2020 18:23:16 +0200 Subject: [PATCH 22/56] Close response in onClosed in F4F websocket --- .../src/main/java/ctbrec/sites/flirt4free/Flirt4FreeModel.java | 1 + 1 file changed, 1 insertion(+) diff --git a/common/src/main/java/ctbrec/sites/flirt4free/Flirt4FreeModel.java b/common/src/main/java/ctbrec/sites/flirt4free/Flirt4FreeModel.java index 13357872..dc332d17 100644 --- a/common/src/main/java/ctbrec/sites/flirt4free/Flirt4FreeModel.java +++ b/common/src/main/java/ctbrec/sites/flirt4free/Flirt4FreeModel.java @@ -262,6 +262,7 @@ public class Flirt4FreeModel extends AbstractModel { synchronized (monitor) { monitor.notify(); } + response.close(); } @Override From bc872b1ed57ddbfcafcbfa8c72406ddf7855c820 Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Sun, 19 Jul 2020 18:23:32 +0200 Subject: [PATCH 23/56] Set version to 3.8.3 --- CHANGELOG.md | 3 +-- client/pom.xml | 2 +- common/pom.xml | 2 +- master/pom.xml | 2 +- server/pom.xml | 2 +- 5 files changed, 5 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 32fd7d05..42a9b471 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,8 +5,7 @@ I currently don't see a way to retrieve the offline favorites * Fixed favorites tab for CamSoda * Fixed CamSoda recordings -* Added external login dialog to Stripchat to support the captcha - +* Added external login dialog for Stripchat to support the captcha 3.8.2 ======================== diff --git a/client/pom.xml b/client/pom.xml index a26a2ed9..f6d8fa00 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -8,7 +8,7 @@ ctbrec master - 3.8.2 + 3.8.3 ../master diff --git a/common/pom.xml b/common/pom.xml index 779b1639..9ec7ece6 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -8,7 +8,7 @@ ctbrec master - 3.8.2 + 3.8.3 ../master diff --git a/master/pom.xml b/master/pom.xml index f405a4a4..3329685c 100644 --- a/master/pom.xml +++ b/master/pom.xml @@ -6,7 +6,7 @@ ctbrec master pom - 3.8.2 + 3.8.3 ../common diff --git a/server/pom.xml b/server/pom.xml index 48d56be2..4b9289fa 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -8,7 +8,7 @@ ctbrec master - 3.8.2 + 3.8.3 ../master From c80230cee70e8954076dbfd34ae41f01fe91f6f5 Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Mon, 20 Jul 2020 18:44:25 +0200 Subject: [PATCH 24/56] Add support for xHamsterLive --- CHANGELOG.md | 6 ++++ .../ui/sites/stripchat/StripchatConfigUI.java | 23 +++++++++++++ .../StripchatElectronLoginDialog.java | 8 ++--- .../StripchatFollowedUpdateService.java | 6 ++-- common/src/main/java/ctbrec/Settings.java | 1 + .../ctbrec/sites/stripchat/Stripchat.java | 33 +++++++++++-------- .../sites/stripchat/StripchatHttpClient.java | 32 ++++++++++-------- .../sites/stripchat/StripchatModel.java | 14 ++++---- 8 files changed, 83 insertions(+), 40 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 42a9b471..c497b3fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +3.8.4 +======================== +* Added support for xHamsterLive (go to Settings -> Sites -> Stripchat, + switch to xHamsterLive, enter your credentials and restart) +* Fixed follow / unfollow for Stripchat + 3.8.3 ======================== * Fixed Streamate diff --git a/client/src/main/java/ctbrec/ui/sites/stripchat/StripchatConfigUI.java b/client/src/main/java/ctbrec/ui/sites/stripchat/StripchatConfigUI.java index 1a0f745e..e860f2f8 100644 --- a/client/src/main/java/ctbrec/ui/sites/stripchat/StripchatConfigUI.java +++ b/client/src/main/java/ctbrec/ui/sites/stripchat/StripchatConfigUI.java @@ -12,8 +12,11 @@ import javafx.scene.control.Button; import javafx.scene.control.CheckBox; import javafx.scene.control.Label; import javafx.scene.control.PasswordField; +import javafx.scene.control.RadioButton; import javafx.scene.control.TextField; +import javafx.scene.control.ToggleGroup; import javafx.scene.layout.GridPane; +import javafx.scene.layout.HBox; import javafx.scene.layout.Priority; public class StripchatConfigUI extends AbstractConfigUI { @@ -44,6 +47,26 @@ public class StripchatConfigUI extends AbstractConfigUI { GridPane.setMargin(enabled, new Insets(0, 0, 0, SettingsTab.CHECKBOX_MARGIN)); layout.add(enabled, 1, row++); + l = new Label("Site"); + layout.add(l, 0, row); + ToggleGroup toggleGroup = new ToggleGroup(); + RadioButton optionA = new RadioButton("Stripchat"); + optionA.setSelected(!Config.getInstance().getSettings().stripchatUseXhamster); + optionA.setToggleGroup(toggleGroup); + RadioButton optionB = new RadioButton("xHamsterLive"); + optionB.setSelected(!optionA.isSelected()); + optionB.setToggleGroup(toggleGroup); + optionA.selectedProperty().addListener((obs, oldV, newV) -> { + Config.getInstance().getSettings().stripchatUseXhamster = !newV; + save(); + }); + HBox hbox = new HBox(); + hbox.getChildren().addAll(optionA, optionB); + HBox.setMargin(optionA, new Insets(5)); + HBox.setMargin(optionB, new Insets(5)); + GridPane.setMargin(hbox, new Insets(0, 0, 0, SettingsTab.CHECKBOX_MARGIN)); + layout.add(hbox, 1, row++); + layout.add(new Label("Stripchat User"), 0, row); TextField username = new TextField(Config.getInstance().getSettings().stripchatUsername); username.textProperty().addListener((ob, o, n) -> { diff --git a/client/src/main/java/ctbrec/ui/sites/stripchat/StripchatElectronLoginDialog.java b/client/src/main/java/ctbrec/ui/sites/stripchat/StripchatElectronLoginDialog.java index 12c410d2..c326c85c 100644 --- a/client/src/main/java/ctbrec/ui/sites/stripchat/StripchatElectronLoginDialog.java +++ b/client/src/main/java/ctbrec/ui/sites/stripchat/StripchatElectronLoginDialog.java @@ -20,8 +20,8 @@ import okhttp3.HttpUrl; public class StripchatElectronLoginDialog { private static final transient Logger LOG = LoggerFactory.getLogger(StripchatElectronLoginDialog.class); - public static final String DOMAIN = "stripchat.com"; - public static final String URL = Stripchat.BASE_URI; + public String DOMAIN = Stripchat.domain; + public String URL = Stripchat.baseUri; private CookieJar cookieJar; private ExternalBrowser browser; @@ -85,8 +85,8 @@ public class StripchatElectronLoginDialog { Cookie c = createCookie(domain, cookie); cookieJar.saveFromResponse(HttpUrl.parse(url), Collections.singletonList(c)); c = createCookie(DOMAIN, cookie); - cookieJar.saveFromResponse(HttpUrl.parse(Stripchat.BASE_URI), Collections.singletonList(c)); - if (c.name().equals("stripchat_com_sessionId")) { + cookieJar.saveFromResponse(HttpUrl.parse(Stripchat.baseUri), Collections.singletonList(c)); + if (c.name().contains("_com_sessionId")) { sessionCookieFound = true; } } diff --git a/client/src/main/java/ctbrec/ui/sites/stripchat/StripchatFollowedUpdateService.java b/client/src/main/java/ctbrec/ui/sites/stripchat/StripchatFollowedUpdateService.java index 938be470..901c4053 100644 --- a/client/src/main/java/ctbrec/ui/sites/stripchat/StripchatFollowedUpdateService.java +++ b/client/src/main/java/ctbrec/ui/sites/stripchat/StripchatFollowedUpdateService.java @@ -51,7 +51,7 @@ public class StripchatFollowedUpdateService extends PaginatedScheduledService { .url(urlBuilder.build()) .header(ACCEPT, "*/*") .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) - .header(REFERER, Stripchat.BASE_URI + "/favorites") + .header(REFERER, Stripchat.baseUri + "/favorites") .header(CONTENT_TYPE, MIMETYPE_APPLICATION_JSON) .build(); try (Response response = stripchat.getHttpClient().execute(request)) { @@ -86,8 +86,8 @@ public class StripchatFollowedUpdateService extends PaginatedScheduledService { .url(url) .header(ACCEPT, MIMETYPE_APPLICATION_JSON) .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) - .header(ORIGIN, Stripchat.BASE_URI) - .header(REFERER, Stripchat.BASE_URI + "/favorites") + .header(ORIGIN, Stripchat.baseUri) + .header(REFERER, Stripchat.baseUri + "/favorites") .header(CONTENT_TYPE, MIMETYPE_APPLICATION_JSON) .build(); try (Response response = stripchat.getHttpClient().execute(request)) { diff --git a/common/src/main/java/ctbrec/Settings.java b/common/src/main/java/ctbrec/Settings.java index e62e54d6..9094117b 100644 --- a/common/src/main/java/ctbrec/Settings.java +++ b/common/src/main/java/ctbrec/Settings.java @@ -122,6 +122,7 @@ public class Settings { public String streamateUsername = ""; public String stripchatUsername = ""; public String stripchatPassword = ""; + public boolean stripchatUseXhamster = false; public boolean transportLayerSecurity = true; public int thumbWidth = 180; public boolean updateThumbnails = true; diff --git a/common/src/main/java/ctbrec/sites/stripchat/Stripchat.java b/common/src/main/java/ctbrec/sites/stripchat/Stripchat.java index 7919ef23..31942434 100644 --- a/common/src/main/java/ctbrec/sites/stripchat/Stripchat.java +++ b/common/src/main/java/ctbrec/sites/stripchat/Stripchat.java @@ -23,9 +23,19 @@ import okhttp3.Response; public class Stripchat extends AbstractSite { - public static final String BASE_URI = "https://stripchat.com"; + public static String domain = "stripchat.com"; + public static String baseUri = "https://stripchat.com"; private HttpClient httpClient; + @Override + public void init() throws IOException { + boolean hamster = Config.getInstance().getSettings().stripchatUseXhamster; + if (hamster) { + domain = "xhamsterlive.com"; + baseUri = "https://" + domain; + } + } + @Override public String getName() { return "Stripchat"; @@ -33,7 +43,7 @@ public class Stripchat extends AbstractSite { @Override public String getBaseUrl() { - return BASE_URI; + return baseUri; } @Override @@ -62,14 +72,14 @@ public class Stripchat extends AbstractSite { } String username = Config.getInstance().getSettings().camsodaUsername; - String url = BASE_URI + "/api/v1/user/" + username; + String url = baseUri + "/api/v1/user/" + username; Request request = new Request.Builder().url(url).build(); - try(Response response = getHttpClient().execute(request)) { - if(response.isSuccessful()) { + try (Response response = getHttpClient().execute(request)) { + if (response.isSuccessful()) { JSONObject json = new JSONObject(response.body().string()); - if(json.has("user")) { + if (json.has("user")) { JSONObject user = json.getJSONObject("user"); - if(user.has("tokens")) { + if (user.has("tokens")) { return (double) user.getInt("tokens"); } } @@ -93,11 +103,6 @@ public class Stripchat extends AbstractSite { return httpClient; } - @Override - public void init() throws IOException { - // noop - } - @Override public void shutdown() { if (httpClient != null) { @@ -122,7 +127,7 @@ public class Stripchat extends AbstractSite { @Override public List search(String q) throws IOException, InterruptedException { - String url = BASE_URI + "/api/front/v2/models/search?limit=20&query=" + URLEncoder.encode(q, "utf-8"); + String url = baseUri + "/api/front/v2/models/search?limit=20&query=" + URLEncoder.encode(q, "utf-8"); Request req = new Request.Builder() .url(url) .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) @@ -162,7 +167,7 @@ public class Stripchat extends AbstractSite { @Override public Model createModelFromUrl(String url) { - Matcher m = Pattern.compile("https?://(?:.*?\\.)?stripchat.com/([^/]*?)/?").matcher(url); + Matcher m = Pattern.compile("https?://(?:.*?\\.)?(?:stripchat.com|xhamsterlive.com)/([^/]*?)/?").matcher(url); if (m.matches()) { String modelName = m.group(1); return createModel(modelName); diff --git a/common/src/main/java/ctbrec/sites/stripchat/StripchatHttpClient.java b/common/src/main/java/ctbrec/sites/stripchat/StripchatHttpClient.java index 94beb70f..5702459b 100644 --- a/common/src/main/java/ctbrec/sites/stripchat/StripchatHttpClient.java +++ b/common/src/main/java/ctbrec/sites/stripchat/StripchatHttpClient.java @@ -32,7 +32,10 @@ public class StripchatHttpClient extends HttpClient { @Override public boolean login() throws IOException { - if(loggedIn) { + if (loggedIn) { + if (csrfToken == null) { + loadCsrfToken(); + } return true; } @@ -40,6 +43,9 @@ public class StripchatHttpClient extends HttpClient { if (checkLoginSuccess()) { loggedIn = true; LOG.debug("Logged in with cookies"); + if (csrfToken == null) { + loadCsrfToken(); + } return true; } @@ -47,7 +53,7 @@ public class StripchatHttpClient extends HttpClient { loadCsrfToken(); } - String url = Stripchat.BASE_URI + "/api/front/auth/login"; + String url = Stripchat.baseUri + "/api/front/auth/login"; JSONObject requestParams = new JSONObject(); requestParams.put("loginOrEmail", Config.getInstance().getSettings().stripchatUsername); requestParams.put("password", Config.getInstance().getSettings().stripchatPassword); @@ -60,8 +66,8 @@ public class StripchatHttpClient extends HttpClient { .url(url) .header(ACCEPT, MIMETYPE_APPLICATION_JSON) .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) - .header(ORIGIN, Stripchat.BASE_URI) - .header(REFERER, Stripchat.BASE_URI) + .header(ORIGIN, Stripchat.baseUri) + .header(REFERER, Stripchat.baseUri) .header(CONTENT_TYPE, MIMETYPE_APPLICATION_JSON) .post(body) .build(); @@ -83,13 +89,13 @@ public class StripchatHttpClient extends HttpClient { } private void loadCsrfToken() throws IOException { - String url = Stripchat.BASE_URI + "/api/front/v2/config/data?requestPath=%2F&timezoneOffset=0"; + String url = Stripchat.baseUri + "/api/front/v2/config/data?requestPath=%2F&timezoneOffset=0"; Request request = new Request.Builder() .url(url) .header(ACCEPT, MIMETYPE_APPLICATION_JSON) .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) - .header(ORIGIN, Stripchat.BASE_URI) - .header(REFERER, Stripchat.BASE_URI) + .header(ORIGIN, Stripchat.baseUri) + .header(REFERER, Stripchat.baseUri) .header(CONTENT_TYPE, MIMETYPE_APPLICATION_JSON) .build(); try (Response response = execute(request)) { @@ -111,13 +117,13 @@ public class StripchatHttpClient extends HttpClient { */ public boolean checkLoginSuccess() throws IOException { long userId = getUserId(); - String url = Stripchat.BASE_URI + "/api/front/users/" + userId + "/favorites"; + String url = Stripchat.baseUri + "/api/front/users/" + userId + "/favorites"; Request request = new Request.Builder() .url(url) .header(ACCEPT, MIMETYPE_APPLICATION_JSON) .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) - .header(ORIGIN, Stripchat.BASE_URI) - .header(REFERER, Stripchat.BASE_URI + "/favorites") + .header(ORIGIN, Stripchat.baseUri) + .header(REFERER, Stripchat.baseUri + "/favorites") .header(CONTENT_TYPE, MIMETYPE_APPLICATION_JSON) .build(); try (Response response = execute(request)) { @@ -132,13 +138,13 @@ public class StripchatHttpClient extends HttpClient { public long getUserId() throws JSONException, IOException { if (userId == 0) { - String url = Stripchat.BASE_URI + "/api/front/users/username/" + Config.getInstance().getSettings().stripchatUsername; + String url = Stripchat.baseUri + "/api/front/users/username/" + Config.getInstance().getSettings().stripchatUsername; Request request = new Request.Builder() .url(url) .header(ACCEPT, MIMETYPE_APPLICATION_JSON) .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) - .header(ORIGIN, Stripchat.BASE_URI) - .header(REFERER, Stripchat.BASE_URI) + .header(ORIGIN, Stripchat.baseUri) + .header(REFERER, Stripchat.baseUri) .header(CONTENT_TYPE, MIMETYPE_APPLICATION_JSON) .build(); try (Response response = execute(request)) { diff --git a/common/src/main/java/ctbrec/sites/stripchat/StripchatModel.java b/common/src/main/java/ctbrec/sites/stripchat/StripchatModel.java index 174fa7ed..72b9d30c 100644 --- a/common/src/main/java/ctbrec/sites/stripchat/StripchatModel.java +++ b/common/src/main/java/ctbrec/sites/stripchat/StripchatModel.java @@ -145,12 +145,13 @@ public class StripchatModel extends AbstractModel { @Override public boolean follow() throws IOException { + getSite().getHttpClient().login(); JSONObject modelInfo = loadModelInfo(); JSONObject user = modelInfo.getJSONObject("user"); long modelId = user.optLong("id"); StripchatHttpClient client = (StripchatHttpClient) getSite().getHttpClient(); - String url = Stripchat.BASE_URI + "/api/front/users/" + client.getUserId() + "/favorites/" + modelId; + String url = Stripchat.baseUri + "/api/front/users/" + client.getUserId() + "/favorites/" + modelId; JSONObject requestParams = new JSONObject(); requestParams.put("csrfToken", client.getCsrfToken()); requestParams.put("csrfTimestamp", client.getCsrfTimestamp()); @@ -160,8 +161,8 @@ public class StripchatModel extends AbstractModel { .url(url) .header(ACCEPT, "*/*") .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) - .header(ORIGIN, Stripchat.BASE_URI) - .header(REFERER, Stripchat.BASE_URI + '/' + getName()) + .header(ORIGIN, Stripchat.baseUri) + .header(REFERER, Stripchat.baseUri + '/' + getName()) .header(CONTENT_TYPE, MIMETYPE_APPLICATION_JSON) .put(body) .build(); @@ -176,6 +177,7 @@ public class StripchatModel extends AbstractModel { @Override public boolean unfollow() throws IOException { + getSite().getHttpClient().login(); JSONObject modelInfo = loadModelInfo(); JSONObject user = modelInfo.getJSONObject("user"); long modelId = user.optLong("id"); @@ -183,7 +185,7 @@ public class StripchatModel extends AbstractModel { favoriteIds.put(modelId); StripchatHttpClient client = (StripchatHttpClient) getSite().getHttpClient(); - String url = Stripchat.BASE_URI + "/api/front/users/" + client.getUserId() + "/favorites"; + String url = Stripchat.baseUri + "/api/front/users/" + client.getUserId() + "/favorites"; JSONObject requestParams = new JSONObject(); requestParams.put("favoriteIds", favoriteIds); requestParams.put("csrfToken", client.getCsrfToken()); @@ -194,8 +196,8 @@ public class StripchatModel extends AbstractModel { .url(url) .header(ACCEPT, MIMETYPE_APPLICATION_JSON) .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) - .header(ORIGIN, Stripchat.BASE_URI) - .header(REFERER, Stripchat.BASE_URI) + .header(ORIGIN, Stripchat.baseUri) + .header(REFERER, Stripchat.baseUri) .header(CONTENT_TYPE, MIMETYPE_APPLICATION_JSON) .delete(body) .build(); From eaa26fa1bfc35261fa1cc90c1e6685f5af93b451 Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Wed, 22 Jul 2020 19:45:12 +0200 Subject: [PATCH 25/56] Reduce flexmark dependencies --- client/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/pom.xml b/client/pom.xml index f6d8fa00..6d123db1 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -91,7 +91,7 @@ com.vladsch.flexmark - flexmark-all + flexmark 0.40.34 From 64c6b9aa4fcc17caad0290bc9bc0ea2bfa401f4f Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Sun, 26 Jul 2020 11:07:55 +0200 Subject: [PATCH 26/56] Enable rerun PP for multiple recordings --- .../java/ctbrec/ui/tabs/RecordingsTab.java | 22 +++++++++---------- common/src/main/java/ctbrec/Recording.java | 6 +++++ 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/client/src/main/java/ctbrec/ui/tabs/RecordingsTab.java b/client/src/main/java/ctbrec/ui/tabs/RecordingsTab.java index 2df62b99..5fff704d 100644 --- a/client/src/main/java/ctbrec/ui/tabs/RecordingsTab.java +++ b/client/src/main/java/ctbrec/ui/tabs/RecordingsTab.java @@ -439,15 +439,13 @@ public class RecordingsTab extends Tab implements TabSelectionListener { } MenuItem rerunPostProcessing = new MenuItem("Rerun Post-Processing"); - rerunPostProcessing.setOnAction(e -> triggerPostProcessing(first)); - if (first.getStatus() == FAILED || first.getStatus() == WAITING || first.getStatus() == FINISHED) { - contextMenu.getItems().add(rerunPostProcessing); - } + rerunPostProcessing.setOnAction(e -> triggerPostProcessing(recordings)); + contextMenu.getItems().add(rerunPostProcessing); + rerunPostProcessing.setDisable(!recordings.stream().allMatch(Recording::canBePostProcessed)); if(recordings.size() > 1) { openInPlayer.setDisable(true); openDir.setDisable(true); - rerunPostProcessing.setDisable(true); } return contextMenu; @@ -567,13 +565,15 @@ public class RecordingsTab extends Tab implements TabSelectionListener { new Thread(() -> DesktopIntegration.open(tsFile.getParent())).start(); } - private void triggerPostProcessing(JavaFxRecording first) { + private void triggerPostProcessing(List recs) { new Thread(() -> { - try { - recorder.rerunPostProcessing(first.getDelegate()); - } catch (IOException | InvalidKeyException | NoSuchAlgorithmException | IllegalStateException e1) { - showErrorDialog("Error while starting post-processing", "The post-processing could not be started", e1); - LOG.error("Error while starting post-processing", e1); + for (JavaFxRecording rec : recs) { + try { + recorder.rerunPostProcessing(rec.getDelegate()); + } catch (IOException | InvalidKeyException | NoSuchAlgorithmException | IllegalStateException e1) { + showErrorDialog("Error while starting post-processing", "The post-processing could not be started", e1); + LOG.error("Error while starting post-processing", e1); + } } }).start(); } diff --git a/common/src/main/java/ctbrec/Recording.java b/common/src/main/java/ctbrec/Recording.java index 9195d801..e6f60bf8 100644 --- a/common/src/main/java/ctbrec/Recording.java +++ b/common/src/main/java/ctbrec/Recording.java @@ -1,5 +1,7 @@ package ctbrec; +import static ctbrec.Recording.State.*; + import java.io.File; import java.io.IOException; import java.io.Serializable; @@ -272,4 +274,8 @@ public class Recording implements Serializable { public void refresh() { sizeInByte = getSize(); } + + public boolean canBePostProcessed() { + return getStatus() == FAILED || getStatus() == WAITING || getStatus() == FINISHED; + } } From 5629f5103ebce88695a2019ae8180f7dde75bc56 Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Sun, 26 Jul 2020 11:08:44 +0200 Subject: [PATCH 27/56] Update exe meta information --- client/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/pom.xml b/client/pom.xml index 6d123db1..e83932a2 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -140,8 +140,8 @@ ${project.version}.0 ${project.version} - Recorder for Charturbate streams - 2018 0xboobface + Software to record live streams + 2020 0xboobface ${project.version}.0 ${project.version} CTB Recorder From 787d4301befdc24be0d158ad9fb0ed057c0df4a4 Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Sun, 26 Jul 2020 11:10:57 +0200 Subject: [PATCH 28/56] Update JavaFX to 14.0.2.1 --- master/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/master/pom.xml b/master/pom.xml index 3329685c..fa70ad81 100644 --- a/master/pom.xml +++ b/master/pom.xml @@ -16,7 +16,7 @@ UTF-8 - 14-ea+4 + 14.0.2.1 From d63c98cf368e13fd11cb9b8db491c8e68d28ebef Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Sun, 26 Jul 2020 15:14:22 +0200 Subject: [PATCH 29/56] Fix NPEs in MFCs ServerConfig --- .../src/main/java/ctbrec/sites/mfc/ServerConfig.java | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/common/src/main/java/ctbrec/sites/mfc/ServerConfig.java b/common/src/main/java/ctbrec/sites/mfc/ServerConfig.java index 52019d44..469e0488 100644 --- a/common/src/main/java/ctbrec/sites/mfc/ServerConfig.java +++ b/common/src/main/java/ctbrec/sites/mfc/ServerConfig.java @@ -7,7 +7,6 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Optional; import org.json.JSONArray; @@ -75,7 +74,7 @@ public class ServerConfig { } public boolean isOnHtml5VideoServer(SessionState state) { - int camserv = Objects.requireNonNull(Objects.requireNonNull(state.getU()).getCamserv()); + int camserv = getCamServ(state); return isOnObsServer(state) || h5Servers.containsKey(Integer.toString(camserv)) || (camserv >= 904 && camserv <= 915 @@ -87,12 +86,17 @@ public class ServerConfig { } public boolean isOnWzObsVideoServer(SessionState state) { - int camserv = Optional.ofNullable(state).map(SessionState::getU).map(User::getCamserv).orElse(-1); + int camserv = getCamServ(state); return wzobsServers.containsKey(Integer.toString(camserv)); } public boolean isOnNgServer(SessionState state) { - int camserv = Objects.requireNonNull(Objects.requireNonNull(state.getU()).getCamserv()); + int camserv = getCamServ(state); return ngVideoServers.containsKey(Integer.toString(camserv)); } + + private static int getCamServ(SessionState state) { + int camserv = Optional.ofNullable(state).map(SessionState::getU).map(User::getCamserv).orElse(-1); + return camserv; + } } From 963f0f0f5ff2dfff5fda850d1f986fbf38e5cafc Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Sun, 26 Jul 2020 15:14:44 +0200 Subject: [PATCH 30/56] Change order how things are shutdown --- .../main/java/ctbrec/recorder/NextGenLocalRecorder.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/common/src/main/java/ctbrec/recorder/NextGenLocalRecorder.java b/common/src/main/java/ctbrec/recorder/NextGenLocalRecorder.java index c2d94fa0..46b4bc84 100644 --- a/common/src/main/java/ctbrec/recorder/NextGenLocalRecorder.java +++ b/common/src/main/java/ctbrec/recorder/NextGenLocalRecorder.java @@ -456,7 +456,7 @@ public class NextGenLocalRecorder implements Recorder { recorderLock.unlock(); } - // wait for post-processing to finish + // wait for downloads to finish LOG.info("Waiting for downloads to finish"); for (int i = 0; i < 60; i++) { if (!recordingProcesses.isEmpty()) { @@ -471,11 +471,12 @@ public class NextGenLocalRecorder implements Recorder { // shutdown threadpools try { - LOG.info("Shutting down pools"); + LOG.info("Shutting down download pool"); downloadPool.shutdown(); - ppPool.shutdown(); client.shutdown(); downloadPool.awaitTermination(1, TimeUnit.MINUTES); + LOG.info("Shutting down post-processing pool"); + ppPool.shutdown(); int minutesToWait = 10; LOG.info("Waiting {} minutes (max) for post-processing to finish", minutesToWait); ppPool.awaitTermination(minutesToWait, TimeUnit.MINUTES); From 2154aacdbeca8915582976d93731d9db6594bddd Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Sun, 26 Jul 2020 15:40:03 +0200 Subject: [PATCH 31/56] Fix problem, that downloads wouldn't finish properly Some downloads couldn't be stopped properly, because they would wait for segment data to arrive to write to disk indefinitely. We now only wait for a max of 30 seconds and also cancel all futures, which are waiting for segment data. --- .../download/hls/MergedFfmpegHlsDownload.java | 44 ++++++++++++++----- 1 file changed, 34 insertions(+), 10 deletions(-) diff --git a/common/src/main/java/ctbrec/recorder/download/hls/MergedFfmpegHlsDownload.java b/common/src/main/java/ctbrec/recorder/download/hls/MergedFfmpegHlsDownload.java index da63dd00..08d8a641 100644 --- a/common/src/main/java/ctbrec/recorder/download/hls/MergedFfmpegHlsDownload.java +++ b/common/src/main/java/ctbrec/recorder/download/hls/MergedFfmpegHlsDownload.java @@ -1,5 +1,7 @@ package ctbrec.recorder.download.hls; +import static java.util.Optional.*; + import java.io.EOFException; import java.io.File; import java.io.FileOutputStream; @@ -15,12 +17,13 @@ import java.time.ZonedDateTime; import java.util.Arrays; import java.util.HashMap; import java.util.LinkedList; -import java.util.Optional; import java.util.Queue; import java.util.concurrent.Callable; +import java.util.concurrent.CancellationException; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.regex.Pattern; import org.slf4j.Logger; @@ -57,6 +60,7 @@ public class MergedFfmpegHlsDownload extends AbstractHlsDownload { private transient OutputStream ffmpegStdIn; protected transient Thread ffmpegThread; private transient Object ffmpegStartMonitor = new Object(); + private Queue> downloads = new LinkedList<>(); public MergedFfmpegHlsDownload(HttpClient client) { super(client); @@ -105,6 +109,7 @@ public class MergedFfmpegHlsDownload extends AbstractHlsDownload { LOG.debug("Starting to download segments"); downloadSegments(segments, true); ffmpegThread.join(); + LOG.debug("FFmpeg thread terminated"); } } catch (ParseException e) { throw new IOException("Couldn't parse stream information", e); @@ -179,7 +184,8 @@ public class MergedFfmpegHlsDownload extends AbstractHlsDownload { } } }); - ffmpegThread.setName("FFmpeg"); + String name = "FFmpeg " + ofNullable(model).map(Model::getName).orElse("").trim(); + ffmpegThread.setName(name); ffmpegThread.start(); } @@ -254,7 +260,7 @@ public class MergedFfmpegHlsDownload extends AbstractHlsDownload { int skip = nextSegment - lsp.seq; // add segments to download threadpool - Queue> downloads = new LinkedList<>(); + downloads.clear(); if (downloadQueue.remainingCapacity() == 0) { LOG.warn("Download to slow for this stream. Download queue is full. Skipping segment"); } else { @@ -278,11 +284,15 @@ public class MergedFfmpegHlsDownload extends AbstractHlsDownload { private void writeFinishedSegments(Queue> downloads) throws ExecutionException, IOException { for (Future downloadFuture : downloads) { try { - byte[] segmentData = downloadFuture.get(); + byte[] segmentData = downloadFuture.get(30, TimeUnit.SECONDS); writeSegment(segmentData); } catch (InterruptedException e) { Thread.currentThread().interrupt(); LOG.error("Error while downloading segment", e); + } catch (TimeoutException e) { + LOG.info("Segment download took too long for {}. Not waiting for it any longer", getModel()); + } catch (CancellationException e) { + LOG.info("Segment download cancelled"); } catch (ExecutionException e) { Throwable cause = e.getCause(); if (cause instanceof MissingSegmentException) { @@ -290,7 +300,7 @@ public class MergedFfmpegHlsDownload extends AbstractHlsDownload { LOG.debug("Error while downloading segment, because model {} is offline. Stopping now", model.getName()); running = false; } else { - LOG.debug("Segment not available, but model {} still online. Going on", Optional.ofNullable(model).map(Model::getName).orElse("n/a")); + LOG.debug("Segment not available, but model {} still online. Going on", ofNullable(model).map(Model::getName).orElse("n/a")); } } else if (cause instanceof HttpException) { HttpException he = (HttpException) cause; @@ -299,10 +309,10 @@ public class MergedFfmpegHlsDownload extends AbstractHlsDownload { running = false; } else { if (he.getResponseCode() == 404) { - LOG.info("Playlist for {} not found [HTTP 404]. Stopping now", Optional.ofNullable(model).map(Model::getName).orElse("n/a")); + LOG.info("Playlist for {} not found [HTTP 404]. Stopping now", ofNullable(model).map(Model::getName).orElse("n/a")); running = false; } else if (he.getResponseCode() == 403) { - LOG.info("Playlist for {} not accessible [HTTP 403]. Stopping now", Optional.ofNullable(model).map(Model::getName).orElse("n/a")); + LOG.info("Playlist for {} not accessible [HTTP 403]. Stopping now", ofNullable(model).map(Model::getName).orElse("n/a")); running = false; } else { throw he; @@ -378,9 +388,23 @@ public class MergedFfmpegHlsDownload extends AbstractHlsDownload { @Override synchronized void internalStop() { running = false; + + try { + downloadQueue.clear(); + downloadThreadPool.shutdownNow(); + LOG.debug("Waiting for segment download thread pool to terminate for model {}", getModel()); + downloadThreadPool.awaitTermination(60, TimeUnit.SECONDS); + LOG.debug("Segment download thread pool terminated for model {}", getModel()); + for (Future future : downloads) { + future.cancel(true); + } + } catch (InterruptedException e) { + LOG.error("Interrupted while waiting for segment pool to shutdown"); + Thread.currentThread().interrupt(); + } + if (ffmpegStdIn != null) { try { - downloadQueue.clear(); ffmpegStdIn.close(); } catch (IOException e) { LOG.error("Couldn't terminate FFmpeg by closing stdin", e); @@ -389,7 +413,7 @@ public class MergedFfmpegHlsDownload extends AbstractHlsDownload { if (ffmpeg != null) { try { - boolean waitFor = ffmpeg.waitFor(5, TimeUnit.MINUTES); + boolean waitFor = ffmpeg.waitFor(45, TimeUnit.SECONDS); if (!waitFor && ffmpeg.isAlive()) { LOG.info("FFmpeg didn't terminate. Destroying the process with force!"); ffmpeg.destroyForcibly(); @@ -419,7 +443,7 @@ public class MergedFfmpegHlsDownload extends AbstractHlsDownload { int maxTries = 3; for (int i = 1; i <= maxTries && running; i++) { Builder builder = new Request.Builder().url(url); - addHeaders(builder, Optional.ofNullable(model).map(Model::getHttpHeaderFactory).map(HttpHeaderFactory::createSegmentHeaders).orElse(new HashMap<>())); + addHeaders(builder, ofNullable(model).map(Model::getHttpHeaderFactory).map(HttpHeaderFactory::createSegmentHeaders).orElse(new HashMap<>())); Request request = builder.build(); try (Response response = client.execute(request)) { if (response.isSuccessful()) { From 81643545d2394ee61142d1a887098a0817b93a11 Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Sun, 26 Jul 2020 15:42:36 +0200 Subject: [PATCH 32/56] Set version to 3.8.4 --- CHANGELOG.md | 3 +++ client/pom.xml | 2 +- common/pom.xml | 2 +- master/pom.xml | 2 +- server/pom.xml | 2 +- 5 files changed, 7 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c497b3fc..4327a5d0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,9 @@ * Added support for xHamsterLive (go to Settings -> Sites -> Stripchat, switch to xHamsterLive, enter your credentials and restart) * Fixed follow / unfollow for Stripchat +* Enable rerun PP for multiple recordings +* Fixed bug, which prevented recordings to finish properly on app + shutdown. Recordings now shouldn't end up in state waiting anymore 3.8.3 ======================== diff --git a/client/pom.xml b/client/pom.xml index e83932a2..3bd74eef 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -8,7 +8,7 @@ ctbrec master - 3.8.3 + 3.8.4 ../master diff --git a/common/pom.xml b/common/pom.xml index 9ec7ece6..9b9f6a73 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -8,7 +8,7 @@ ctbrec master - 3.8.3 + 3.8.4 ../master diff --git a/master/pom.xml b/master/pom.xml index fa70ad81..4031a2f0 100644 --- a/master/pom.xml +++ b/master/pom.xml @@ -6,7 +6,7 @@ ctbrec master pom - 3.8.3 + 3.8.4 ../common diff --git a/server/pom.xml b/server/pom.xml index 4b9289fa..0b3ab7b0 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -8,7 +8,7 @@ ctbrec master - 3.8.3 + 3.8.4 ../master From e2fdda32dbf72bf294f7a549cc261a31a82a037f Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Sun, 26 Jul 2020 16:00:12 +0200 Subject: [PATCH 33/56] Remove JAVA_HOME variable --- client/build.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/client/build.sh b/client/build.sh index c87fc3cc..ec3723aa 100755 --- a/client/build.sh +++ b/client/build.sh @@ -1,5 +1,4 @@ #!/bin/bash -export JAVA_HOME=/opt/jdk-11.0.1 mvn clean mvn -Djavafx.platform=win package verify mvn -Djavafx.platform=linux package verify From 48964cc85fe09dbb4d26d93f551c5a0770797296 Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Wed, 29 Jul 2020 20:20:45 +0200 Subject: [PATCH 34/56] Fix Stripchat followed tab --- .../sites/stripchat/StripchatFollowedTab.java | 23 ------------ .../StripchatFollowedUpdateService.java | 36 ++++++++++--------- 2 files changed, 20 insertions(+), 39 deletions(-) diff --git a/client/src/main/java/ctbrec/ui/sites/stripchat/StripchatFollowedTab.java b/client/src/main/java/ctbrec/ui/sites/stripchat/StripchatFollowedTab.java index f82525ab..5b0cc16b 100644 --- a/client/src/main/java/ctbrec/ui/sites/stripchat/StripchatFollowedTab.java +++ b/client/src/main/java/ctbrec/ui/sites/stripchat/StripchatFollowedTab.java @@ -4,14 +4,10 @@ import ctbrec.sites.stripchat.Stripchat; import ctbrec.ui.tabs.FollowedTab; import ctbrec.ui.tabs.ThumbOverviewTab; import javafx.concurrent.WorkerStateEvent; -import javafx.geometry.Insets; import javafx.scene.Scene; import javafx.scene.control.Label; -import javafx.scene.control.RadioButton; -import javafx.scene.control.ToggleGroup; import javafx.scene.input.KeyCode; import javafx.scene.input.KeyEvent; -import javafx.scene.layout.HBox; public class StripchatFollowedTab extends ThumbOverviewTab implements FollowedTab { private Label status; @@ -26,25 +22,6 @@ public class StripchatFollowedTab extends ThumbOverviewTab implements FollowedTa @Override protected void createGui() { super.createGui(); - addOnlineOfflineSelector(); - } - - private void addOnlineOfflineSelector() { - ToggleGroup group = new ToggleGroup(); - RadioButton online = new RadioButton("online"); - online.setToggleGroup(group); - RadioButton offline = new RadioButton("offline"); - offline.setToggleGroup(group); - pagination.getChildren().add(online); - pagination.getChildren().add(offline); - HBox.setMargin(online, new Insets(5, 5, 5, 40)); - HBox.setMargin(offline, new Insets(5, 5, 5, 5)); - online.setSelected(true); - group.selectedToggleProperty().addListener(e -> { - queue.clear(); - ((StripchatFollowedUpdateService)updateService).showOnline(online.isSelected()); - updateService.restart(); - }); } @Override diff --git a/client/src/main/java/ctbrec/ui/sites/stripchat/StripchatFollowedUpdateService.java b/client/src/main/java/ctbrec/ui/sites/stripchat/StripchatFollowedUpdateService.java index 901c4053..80b788ba 100644 --- a/client/src/main/java/ctbrec/ui/sites/stripchat/StripchatFollowedUpdateService.java +++ b/client/src/main/java/ctbrec/ui/sites/stripchat/StripchatFollowedUpdateService.java @@ -4,8 +4,8 @@ import static ctbrec.io.HttpConstants.*; import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.List; -import java.util.Objects; import org.json.JSONArray; import org.json.JSONObject; @@ -24,8 +24,8 @@ import okhttp3.Request; import okhttp3.Response; public class StripchatFollowedUpdateService extends PaginatedScheduledService { + private static final int PAGE_SIZE = 30; private Stripchat stripchat; - private boolean showOnline = true; public StripchatFollowedUpdateService(Stripchat stripchat) { this.stripchat = stripchat; @@ -36,16 +36,27 @@ public class StripchatFollowedUpdateService extends PaginatedScheduledService { return new Task>() { @Override public List call() throws IOException { + int startIndex = (getPage() - 1) * PAGE_SIZE; JSONArray favoriteModelIds = loadFavoriteModelIds(); - List models = loadModels(favoriteModelIds); + List modelIdsToLoad = new ArrayList<>(PAGE_SIZE); + List models; + if (startIndex < favoriteModelIds.length()) { + int modelsOnPage = Math.min(PAGE_SIZE, favoriteModelIds.length() - startIndex - 1); + for (int i = 0; i < modelsOnPage; i++) { + modelIdsToLoad.add(favoriteModelIds.getInt(startIndex + i)); + } + models = loadModels(modelIdsToLoad); + } else { + models = Collections.emptyList(); + } return models; } - private List loadModels(JSONArray favoriteModelIds) throws IOException { + private List loadModels(List modelIdsToLoad) throws IOException { List models = new ArrayList<>(); HttpUrl.Builder urlBuilder = HttpUrl.parse(stripchat.getBaseUrl() + "/api/front/models/list").newBuilder(); - for (int i = 0; i < favoriteModelIds.length(); i++) { - urlBuilder.addQueryParameter("modelIds["+i+"]", Integer.toString(favoriteModelIds.getInt(i))); + for (int i = 0; i < modelIdsToLoad.size(); i++) { + urlBuilder.addQueryParameter("modelIds["+i+"]", modelIdsToLoad.get(i).toString()); } Request request = new Request.Builder() .url(urlBuilder.build()) @@ -64,10 +75,7 @@ public class StripchatFollowedUpdateService extends PaginatedScheduledService { StripchatModel model = stripchat.createModel(user.optString("username")); model.setDescription(user.optString("description")); model.setPreview(user.optString("previewUrlThumbBig")); - boolean online = Objects.equals(user.optString("status"), "public"); - if (showOnline == online) { - models.add(model); - } + models.add(model); } } } else { @@ -93,8 +101,8 @@ public class StripchatFollowedUpdateService extends PaginatedScheduledService { try (Response response = stripchat.getHttpClient().execute(request)) { if (response.isSuccessful()) { JSONObject json = new JSONObject(response.body().string()); - if(json.has("userIds")) { - JSONArray userIds = json.getJSONArray("userIds"); + if (json.has("modelIds")) { + JSONArray userIds = json.getJSONArray("modelIds"); return userIds; } else { return new JSONArray(); @@ -106,8 +114,4 @@ public class StripchatFollowedUpdateService extends PaginatedScheduledService { } }; } - - void showOnline(boolean online) { - this.showOnline = online; - } } From cbd529d0017fa9dcfd60aa88e0668587121a0998 Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Sat, 8 Aug 2020 13:37:21 +0200 Subject: [PATCH 35/56] Replace wrong username used to retrieve the token balance --- common/src/main/java/ctbrec/sites/stripchat/Stripchat.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/src/main/java/ctbrec/sites/stripchat/Stripchat.java b/common/src/main/java/ctbrec/sites/stripchat/Stripchat.java index 31942434..7aba56c6 100644 --- a/common/src/main/java/ctbrec/sites/stripchat/Stripchat.java +++ b/common/src/main/java/ctbrec/sites/stripchat/Stripchat.java @@ -71,7 +71,7 @@ public class Stripchat extends AbstractSite { throw new IOException("Account settings not available"); } - String username = Config.getInstance().getSettings().camsodaUsername; + String username = Config.getInstance().getSettings().stripchatPassword; String url = baseUri + "/api/v1/user/" + username; Request request = new Request.Builder().url(url).build(); try (Response response = getHttpClient().execute(request)) { From 729319dfd23cfe981fbce10de6423d36c2f5bb95 Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Sat, 8 Aug 2020 15:28:29 +0200 Subject: [PATCH 36/56] Add mechanism to record a model only up to a certain timestamp --- .../src/main/java/ctbrec/ui/JavaFxModel.java | 21 +++ .../src/main/java/ctbrec/AbstractModel.java | 22 +++ common/src/main/java/ctbrec/Model.java | 6 + .../main/java/ctbrec/SubsequentAction.java | 6 + .../main/java/ctbrec/io/ModelJsonAdapter.java | 7 + .../ctbrec/recorder/NextGenLocalRecorder.java | 162 ++++++++++++------ .../recorder/PreconditionNotMetException.java | 15 ++ .../recorder/RecordUntilExpiredException.java | 21 +++ 8 files changed, 203 insertions(+), 57 deletions(-) create mode 100644 common/src/main/java/ctbrec/SubsequentAction.java create mode 100644 common/src/main/java/ctbrec/recorder/PreconditionNotMetException.java create mode 100644 common/src/main/java/ctbrec/recorder/RecordUntilExpiredException.java diff --git a/client/src/main/java/ctbrec/ui/JavaFxModel.java b/client/src/main/java/ctbrec/ui/JavaFxModel.java index bc404953..21885b73 100644 --- a/client/src/main/java/ctbrec/ui/JavaFxModel.java +++ b/client/src/main/java/ctbrec/ui/JavaFxModel.java @@ -13,6 +13,7 @@ import com.squareup.moshi.JsonReader; import com.squareup.moshi.JsonWriter; import ctbrec.Model; +import ctbrec.SubsequentAction; import ctbrec.recorder.download.Download; import ctbrec.recorder.download.HttpHeaderFactory; import ctbrec.recorder.download.StreamSource; @@ -286,4 +287,24 @@ public class JavaFxModel implements Model { public HttpHeaderFactory getHttpHeaderFactory() { return delegate.getHttpHeaderFactory(); } + + @Override + public Instant getRecordUntil() { + return delegate.getRecordUntil(); + } + + @Override + public void setRecordUntil(Instant instant) { + delegate.setRecordUntil(instant); + } + + @Override + public SubsequentAction getRecordUntilSubsequentAction() { + return delegate.getRecordUntilSubsequentAction(); + } + + @Override + public void setRecordUntilSubsequentAction(SubsequentAction action) { + delegate.setRecordUntilSubsequentAction(action); + } } diff --git a/common/src/main/java/ctbrec/AbstractModel.java b/common/src/main/java/ctbrec/AbstractModel.java index a9d48b3a..22dc81cc 100644 --- a/common/src/main/java/ctbrec/AbstractModel.java +++ b/common/src/main/java/ctbrec/AbstractModel.java @@ -33,6 +33,8 @@ public abstract class AbstractModel implements Model { protected State onlineState = State.UNKNOWN; private Instant lastSeen; private Instant lastRecorded; + private Instant recordUntil; + private SubsequentAction recordUntilSubsequentAction; @Override public boolean isOnline() throws IOException, ExecutionException, InterruptedException { @@ -231,6 +233,26 @@ public abstract class AbstractModel implements Model { this.lastRecorded = lastRecorded; } + @Override + public Instant getRecordUntil() { + return Optional.ofNullable(recordUntil).orElse(Instant.ofEpochMilli(Long.MAX_VALUE)); + } + + @Override + public void setRecordUntil(Instant recordUntil) { + this.recordUntil = recordUntil; + } + + @Override + public SubsequentAction getRecordUntilSubsequentAction() { + return Optional.ofNullable(recordUntilSubsequentAction).orElse(SubsequentAction.PAUSE); + } + + @Override + public void setRecordUntilSubsequentAction(SubsequentAction recordUntilSubsequentAction) { + this.recordUntilSubsequentAction = recordUntilSubsequentAction; + } + @Override public Download createDownload() { if (Config.isServerMode() && !Config.getInstance().getSettings().recordSingleFile) { diff --git a/common/src/main/java/ctbrec/Model.java b/common/src/main/java/ctbrec/Model.java index 6fb12ff5..fec54ca7 100644 --- a/common/src/main/java/ctbrec/Model.java +++ b/common/src/main/java/ctbrec/Model.java @@ -128,4 +128,10 @@ public interface Model extends Comparable, Serializable { public HttpHeaderFactory getHttpHeaderFactory(); + public Instant getRecordUntil(); + public void setRecordUntil(Instant instant); + + public SubsequentAction getRecordUntilSubsequentAction(); + public void setRecordUntilSubsequentAction(SubsequentAction action); + } \ No newline at end of file diff --git a/common/src/main/java/ctbrec/SubsequentAction.java b/common/src/main/java/ctbrec/SubsequentAction.java new file mode 100644 index 00000000..ddfe6228 --- /dev/null +++ b/common/src/main/java/ctbrec/SubsequentAction.java @@ -0,0 +1,6 @@ +package ctbrec; + +public enum SubsequentAction { + PAUSE, + REMOVE +} diff --git a/common/src/main/java/ctbrec/io/ModelJsonAdapter.java b/common/src/main/java/ctbrec/io/ModelJsonAdapter.java index 6551b94e..a15172ff 100644 --- a/common/src/main/java/ctbrec/io/ModelJsonAdapter.java +++ b/common/src/main/java/ctbrec/io/ModelJsonAdapter.java @@ -15,6 +15,7 @@ import com.squareup.moshi.JsonReader.Token; import com.squareup.moshi.JsonWriter; import ctbrec.Model; +import ctbrec.SubsequentAction; import ctbrec.sites.Site; import ctbrec.sites.chaturbate.ChaturbateModel; @@ -74,6 +75,10 @@ public class ModelJsonAdapter extends JsonAdapter { model.setLastSeen(Instant.ofEpochMilli(reader.nextLong())); } else if(key.equals("lastRecorded")) { model.setLastRecorded(Instant.ofEpochMilli(reader.nextLong())); + } else if(key.equals("recordUntil")) { + model.setRecordUntil(Instant.ofEpochMilli(reader.nextLong())); + } else if(key.equals("recordUntilSubsequentAction")) { + model.setRecordUntilSubsequentAction(SubsequentAction.valueOf(reader.nextString())); } else if(key.equals("siteSpecific")) { reader.beginObject(); try { @@ -115,6 +120,8 @@ public class ModelJsonAdapter extends JsonAdapter { writer.name("suspended").value(model.isSuspended()); writer.name("lastSeen").value(model.getLastSeen().toEpochMilli()); writer.name("lastRecorded").value(model.getLastRecorded().toEpochMilli()); + writer.name("recordUntil").value(model.getRecordUntil().toEpochMilli()); + writer.name("recordUntilSubsequentAction").value(model.getRecordUntilSubsequentAction().name()); writer.name("siteSpecific"); writer.beginObject(); model.writeSiteSpecificData(writer); diff --git a/common/src/main/java/ctbrec/recorder/NextGenLocalRecorder.java b/common/src/main/java/ctbrec/recorder/NextGenLocalRecorder.java index 46b4bc84..79161d58 100644 --- a/common/src/main/java/ctbrec/recorder/NextGenLocalRecorder.java +++ b/common/src/main/java/ctbrec/recorder/NextGenLocalRecorder.java @@ -42,6 +42,7 @@ import ctbrec.Config; import ctbrec.Model; import ctbrec.Recording; import ctbrec.Recording.State; +import ctbrec.SubsequentAction; import ctbrec.event.Event; import ctbrec.event.EventBusHolder; import ctbrec.event.ModelIsOnlineEvent; @@ -59,7 +60,7 @@ public class NextGenLocalRecorder implements Recorder { private volatile boolean recording = true; private ReentrantLock recorderLock = new ReentrantLock(); private RecorderHttpClient client = new RecorderHttpClient(); - private long lastSpaceMessage = 0; + private long lastPreconditionMessage = 0; private Map recordingProcesses = Collections.synchronizedMap(new HashMap<>()); private RecordingManager recordingManager; @@ -213,54 +214,7 @@ public class NextGenLocalRecorder implements Recorder { private void startRecordingProcess(Model model) throws IOException { recorderLock.lock(); try { - if (!recording) { - // recorder is not in recording mode - return; - } - - if (model.isSuspended()) { - LOG.info("Recording for model {} is suspended.", model); - return; - } - - if (recordingProcesses.containsKey(model)) { - LOG.error("A recording for model {} is already running", model); - return; - } - - if (!models.contains(model)) { - LOG.info("Model {} has been removed. Restarting of recording cancelled.", model); - return; - } - - if (!enoughSpaceForRecording()) { - long now = System.currentTimeMillis(); - if ((now - lastSpaceMessage) > TimeUnit.MINUTES.toMillis(1)) { - LOG.info("Not enough space for recording, not starting recording for {}", model); - lastSpaceMessage = now; - } - return; - } - - if (!downloadSlotAvailable()) { - long now = System.currentTimeMillis(); - if ((now - lastSpaceMessage) > TimeUnit.MINUTES.toMillis(1)) { - LOG.info("The number of downloads is maxed out"); - } - // check, if we can stop a recording for a model with lower priority - Optional lowerPrioRecordingProcess = recordingProcessWithLowerPrio(model.getPriority()); - if (lowerPrioRecordingProcess.isPresent()) { - Download download = lowerPrioRecordingProcess.get().getDownload(); - Model lowerPrioModel = download.getModel(); - LOG.info("Stopping recording for {}. Prio {} < {}", lowerPrioModel.getName(), lowerPrioModel.getPriority(), model.getPriority()); - stopRecordingProcess(lowerPrioModel); - } else { - if ((now - lastSpaceMessage) > TimeUnit.MINUTES.toMillis(1)) { - LOG.info("Other models have higher prio, not starting recording for {}", model.getName()); - } - return; - } - } + checkRecordingPreconditions(model); LOG.info("Starting recording for model {}", model.getName()); Download download = model.createDownload(); @@ -269,14 +223,7 @@ public class NextGenLocalRecorder implements Recorder { "At this point the download should have set a startTime. Make sure to set a startTime in " + download.getClass() + ".init()"); LOG.debug("Downloading with {}", download.getClass().getSimpleName()); - Recording rec = new Recording(); - rec.setDownload(download); - rec.setPath(download.getPath(model).replaceAll("\\\\", "/")); - rec.setModel(model); - rec.setStartDate(download.getStartTime()); - rec.setSingleFile(download.isSingleFile()); - recordingProcesses.put(model, rec); - recordingManager.add(rec); + Recording rec = createRecording(model, download); completionService.submit(() -> { try { setRecordingStatus(rec, State.RECORDING); @@ -294,11 +241,112 @@ public class NextGenLocalRecorder implements Recorder { } return rec; }); + } catch (RecordUntilExpiredException e) { + LOG.info("Precondition not met to record {}: {}", model, e.getLocalizedMessage()); + executeRecordUntilSubsequentAction(model); + } catch (PreconditionNotMetException e) { + // long now = System.currentTimeMillis(); + // if ((now - lastSpaceMessage) > TimeUnit.MINUTES.toMillis(1)) { + // LOG.info("Not enough space for recording, not starting recording for {}", model); + // lastSpaceMessage = now; + // } + LOG.info("Precondition not met to record {}: {}", model, e.getLocalizedMessage()); + return; } finally { recorderLock.unlock(); } } + private void executeRecordUntilSubsequentAction(Model model) throws IOException { + if (model.getRecordUntilSubsequentAction() == SubsequentAction.PAUSE) { + model.setSuspended(true); + } else if (model.getRecordUntilSubsequentAction() == SubsequentAction.REMOVE) { + try { + LOG.info("Recording timeframe expired for model {} - {}", model, model.getRecordUntil()); + stopRecording(model); + } catch (InvalidKeyException | NoSuchAlgorithmException e1) { + LOG.error("Error while stopping recording", e1); + } + } + } + + private Recording createRecording(Model model, Download download) throws IOException { + Recording rec = new Recording(); + rec.setDownload(download); + rec.setPath(download.getPath(model).replaceAll("\\\\", "/")); + rec.setModel(model); + rec.setStartDate(download.getStartTime()); + rec.setSingleFile(download.isSingleFile()); + recordingProcesses.put(model, rec); + recordingManager.add(rec); + return rec; + } + + private void checkRecordingPreconditions(Model model) throws IOException { + ensureRecorderIsActive(); + ensureModelIsNotSuspended(model); + ensureRecordUntilIsInFuture(model); + ensureNoRecordingRunningForModel(model); + ensureModelShouldBeRecorded(model); + ensureEnoughSpaceForRecording(); + ensureDownloadSlotAvailable(model); + } + + private void ensureRecordUntilIsInFuture(Model model) { + if (Instant.now().isAfter(model.getRecordUntil())) { + throw new RecordUntilExpiredException(model.getRecordUntil()); + } + } + + private void ensureEnoughSpaceForRecording() throws IOException { + if (!enoughSpaceForRecording()) { + throw new PreconditionNotMetException("Not enough disk space for recording"); + } + } + + private void ensureDownloadSlotAvailable(Model model) { + if (!downloadSlotAvailable()) { + long now = System.currentTimeMillis(); + if ((now - lastPreconditionMessage) > TimeUnit.MINUTES.toMillis(1)) { + LOG.info("The number of downloads is maxed out"); + } + // check, if we can stop a recording for a model with lower priority + Optional lowerPrioRecordingProcess = recordingProcessWithLowerPrio(model.getPriority()); + if (lowerPrioRecordingProcess.isPresent()) { + Download download = lowerPrioRecordingProcess.get().getDownload(); + Model lowerPrioModel = download.getModel(); + LOG.info("Stopping recording for {}. Prio {} < {}", lowerPrioModel.getName(), lowerPrioModel.getPriority(), model.getPriority()); + stopRecordingProcess(lowerPrioModel); + } else { + throw new PreconditionNotMetException("Other models have higher prio, not starting recording for " + model.getName()); + } + } + } + + private void ensureModelShouldBeRecorded(Model model) { + if (!models.contains(model)) { + throw new PreconditionNotMetException("Model " + model + " has been removed. Restarting of recording cancelled."); + } + } + + private void ensureNoRecordingRunningForModel(Model model) { + if (recordingProcesses.containsKey(model)) { + throw new PreconditionNotMetException("A recording for model " + model + " is already running"); + } + } + + private void ensureModelIsNotSuspended(Model model) { + if (model.isSuspended()) { + throw new PreconditionNotMetException("Recording for model " + model + " is suspended"); + } + } + + private void ensureRecorderIsActive() { + if (!recording) { + throw new PreconditionNotMetException("Recorder is not in recording mode"); + } + } + private Optional recordingProcessWithLowerPrio(int priority) { Model lowest = null; for (Model m : recordingProcesses.keySet()) { diff --git a/common/src/main/java/ctbrec/recorder/PreconditionNotMetException.java b/common/src/main/java/ctbrec/recorder/PreconditionNotMetException.java new file mode 100644 index 00000000..89ae5f45 --- /dev/null +++ b/common/src/main/java/ctbrec/recorder/PreconditionNotMetException.java @@ -0,0 +1,15 @@ +package ctbrec.recorder; + +public class PreconditionNotMetException extends RuntimeException { + public PreconditionNotMetException() { + super("Precondition not met"); + } + + public PreconditionNotMetException(String message) { + super(message); + } + + public PreconditionNotMetException(String message, Throwable t) { + super(message, t); + } +} diff --git a/common/src/main/java/ctbrec/recorder/RecordUntilExpiredException.java b/common/src/main/java/ctbrec/recorder/RecordUntilExpiredException.java new file mode 100644 index 00000000..69434860 --- /dev/null +++ b/common/src/main/java/ctbrec/recorder/RecordUntilExpiredException.java @@ -0,0 +1,21 @@ +package ctbrec.recorder; + +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; + +public class RecordUntilExpiredException extends PreconditionNotMetException { + private Instant until; + + public RecordUntilExpiredException(Instant until) { + this.until = until; + } + + @Override + public String getMessage() { + LocalDateTime dateTime = LocalDateTime.ofInstant(until, ZoneId.systemDefault()); + String date = DateTimeFormatter.ISO_LOCAL_DATE.format(dateTime); + return "Recording expired at " + date; + } +} From e55daa0772f78e42e95d6fe2a65cc680721d1244 Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Sat, 8 Aug 2020 17:51:03 +0200 Subject: [PATCH 37/56] Add GUI and remote support for temporary recordings --- .../main/java/ctbrec/ui/controls/Dialogs.java | 18 +++ .../ctbrec/ui/tabs/RecordedModelsTab.java | 52 +++++++++ .../ctbrec/recorder/NextGenLocalRecorder.java | 109 ++++++++++++------ .../main/java/ctbrec/recorder/Recorder.java | 1 + .../java/ctbrec/recorder/RemoteRecorder.java | 5 + .../recorder/server/RecorderServlet.java | 11 ++ 6 files changed, 159 insertions(+), 37 deletions(-) diff --git a/client/src/main/java/ctbrec/ui/controls/Dialogs.java b/client/src/main/java/ctbrec/ui/controls/Dialogs.java index c96ae2c1..fec5c51b 100644 --- a/client/src/main/java/ctbrec/ui/controls/Dialogs.java +++ b/client/src/main/java/ctbrec/ui/controls/Dialogs.java @@ -16,6 +16,7 @@ import javafx.scene.control.Dialog; import javafx.scene.control.TextArea; import javafx.scene.image.Image; import javafx.scene.layout.GridPane; +import javafx.scene.layout.Region; import javafx.stage.Modality; import javafx.stage.Stage; @@ -87,6 +88,23 @@ public class Dialogs { return dialog.showAndWait(); } + public static Boolean showCustomInput(Scene parent, String title, Region region) { + Dialog dialog = new Dialog<>(); + dialog.setTitle(title); + dialog.getDialogPane().getButtonTypes().addAll(ButtonType.OK, ButtonType.CANCEL); + dialog.initModality(Modality.APPLICATION_MODAL); + dialog.setResizable(true); + InputStream icon = Dialogs.class.getResourceAsStream("/icon.png"); + Stage stage = (Stage) dialog.getDialogPane().getScene().getWindow(); + stage.getIcons().add(new Image(icon)); + if (parent != null) { + stage.getScene().getStylesheets().addAll(parent.getStylesheets()); + } + dialog.getDialogPane().setContent(region); + dialog.showAndWait(); + return dialog.getResult() == ButtonType.OK; + } + public static boolean showConfirmDialog(String title, String message, String header, Scene parent) { AutosizeAlert confirm = new AutosizeAlert(AlertType.CONFIRMATION, message, parent, YES, NO); confirm.setTitle(title); diff --git a/client/src/main/java/ctbrec/ui/tabs/RecordedModelsTab.java b/client/src/main/java/ctbrec/ui/tabs/RecordedModelsTab.java index 48faf307..9922ca5a 100644 --- a/client/src/main/java/ctbrec/ui/tabs/RecordedModelsTab.java +++ b/client/src/main/java/ctbrec/ui/tabs/RecordedModelsTab.java @@ -1,9 +1,13 @@ package ctbrec.ui.tabs; +import static ctbrec.SubsequentAction.*; + import java.io.IOException; import java.security.InvalidKeyException; import java.security.NoSuchAlgorithmException; import java.time.Instant; +import java.time.LocalDate; +import java.time.ZoneId; import java.util.ArrayList; import java.util.Collections; import java.util.Iterator; @@ -25,6 +29,7 @@ import ctbrec.Config; import ctbrec.Model; import ctbrec.Recording; import ctbrec.StringUtil; +import ctbrec.SubsequentAction; import ctbrec.recorder.Recorder; import ctbrec.sites.Site; import ctbrec.ui.AutosizeAlert; @@ -59,8 +64,10 @@ import javafx.geometry.Pos; import javafx.scene.control.Alert; import javafx.scene.control.Button; import javafx.scene.control.ContextMenu; +import javafx.scene.control.DatePicker; import javafx.scene.control.Label; import javafx.scene.control.MenuItem; +import javafx.scene.control.RadioButton; import javafx.scene.control.ScrollPane; import javafx.scene.control.SelectionMode; import javafx.scene.control.Tab; @@ -71,6 +78,7 @@ import javafx.scene.control.TableColumn.SortType; import javafx.scene.control.TableRow; import javafx.scene.control.TableView; import javafx.scene.control.TextField; +import javafx.scene.control.ToggleGroup; import javafx.scene.control.Tooltip; import javafx.scene.control.cell.CheckBoxTableCell; import javafx.scene.control.cell.PropertyValueFactory; @@ -84,6 +92,7 @@ import javafx.scene.input.MouseButton; import javafx.scene.input.MouseEvent; import javafx.scene.layout.BorderPane; import javafx.scene.layout.FlowPane; +import javafx.scene.layout.GridPane; import javafx.scene.layout.HBox; import javafx.scene.layout.Priority; import javafx.util.Callback; @@ -612,6 +621,8 @@ public class RecordedModelsTab extends Tab implements TabSelectionListener { pauseRecording.setOnAction(e -> pauseRecording(selectedModels)); MenuItem resumeRecording = new MenuItem("Resume Recording"); resumeRecording.setOnAction(e -> resumeRecording(selectedModels)); + MenuItem stopRecordingAt = new MenuItem("Stop Recording at Date"); + stopRecordingAt.setOnAction(e -> setStopDate(selectedModels.get(0))); MenuItem openInBrowser = new MenuItem("Open in Browser"); openInBrowser.setOnAction(e -> DesktopIntegration.open(selectedModels.get(0).getUrl())); MenuItem openInPlayer = new MenuItem("Open in Player"); @@ -630,6 +641,7 @@ public class RecordedModelsTab extends Tab implements TabSelectionListener { ContextMenu menu = new ContextMenu(stop); if (selectedModels.size() == 1) { menu.getItems().add(selectedModels.get(0).isSuspended() ? resumeRecording : pauseRecording); + menu.getItems().add(stopRecordingAt); } else { menu.getItems().addAll(resumeRecording, pauseRecording); } @@ -646,6 +658,46 @@ public class RecordedModelsTab extends Tab implements TabSelectionListener { return menu; } + private void setStopDate(JavaFxModel model) { + DatePicker datePicker = new DatePicker(); + GridPane grid = new GridPane(); + grid.setHgap(10); + grid.setVgap(10); + grid.setPadding(new Insets(20, 150, 10, 10)); + grid.add(new Label("Stop at"), 0, 0); + grid.add(datePicker, 1, 0); + grid.add(new Label("And then"), 0, 1); + ToggleGroup toggleGroup = new ToggleGroup(); + RadioButton pauseButton = new RadioButton("pause recording"); + pauseButton.setSelected(model.getRecordUntilSubsequentAction() == PAUSE); + pauseButton.setToggleGroup(toggleGroup); + RadioButton removeButton = new RadioButton("remove model"); + removeButton.setSelected(model.getRecordUntilSubsequentAction() == REMOVE); + removeButton.setToggleGroup(toggleGroup); + HBox row = new HBox(); + row.getChildren().addAll(pauseButton, removeButton); + HBox.setMargin(pauseButton, new Insets(5)); + HBox.setMargin(removeButton, new Insets(5)); + grid.add(row, 1, 1); + if (model.getRecordUntil().toEpochMilli() != Long.MAX_VALUE) { + LocalDate localDate = LocalDate.ofInstant(model.getRecordUntil(), ZoneId.systemDefault()); + datePicker.setValue(localDate); + } + boolean userClickedOk = Dialogs.showCustomInput(getTabPane().getScene(), "Stop Recording at", grid); + if (userClickedOk) { + SubsequentAction action = pauseButton.isSelected() ? PAUSE : REMOVE; + LOG.info("Stop at {} and {}", datePicker.getValue(), action); + Instant stopAt = Instant.from(datePicker.getValue().atStartOfDay().atZone(ZoneId.systemDefault())); + model.setRecordUntil(stopAt); + model.setRecordUntilSubsequentAction(action); + try { + recorder.stopRecordingAt(model.getDelegate()); + } catch (InvalidKeyException | NoSuchAlgorithmException | IOException e) { + Dialogs.showError(getTabPane().getScene(), "Error", "Couln't set stop date", e); + } + } + } + private void ignore(ObservableList selectedModels) { for (JavaFxModel fxModel : selectedModels) { Model modelToIgnore = fxModel.getDelegate(); diff --git a/common/src/main/java/ctbrec/recorder/NextGenLocalRecorder.java b/common/src/main/java/ctbrec/recorder/NextGenLocalRecorder.java index 79161d58..c7826077 100644 --- a/common/src/main/java/ctbrec/recorder/NextGenLocalRecorder.java +++ b/common/src/main/java/ctbrec/recorder/NextGenLocalRecorder.java @@ -1,5 +1,6 @@ package ctbrec.recorder; +import static ctbrec.SubsequentAction.*; import static ctbrec.event.Event.Type.*; import java.io.File; @@ -10,6 +11,7 @@ import java.security.InvalidKeyException; import java.security.NoSuchAlgorithmException; import java.time.Duration; import java.time.Instant; +import java.time.ZoneId; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -20,6 +22,7 @@ import java.util.Objects; import java.util.Optional; import java.util.UUID; import java.util.concurrent.BlockingQueue; +import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorCompletionService; import java.util.concurrent.Executors; @@ -42,7 +45,6 @@ import ctbrec.Config; import ctbrec.Model; import ctbrec.Recording; import ctbrec.Recording.State; -import ctbrec.SubsequentAction; import ctbrec.event.Event; import ctbrec.event.EventBusHolder; import ctbrec.event.ModelIsOnlineEvent; @@ -215,41 +217,14 @@ public class NextGenLocalRecorder implements Recorder { recorderLock.lock(); try { checkRecordingPreconditions(model); - LOG.info("Starting recording for model {}", model.getName()); - Download download = model.createDownload(); - download.init(config, model, Instant.now()); - Objects.requireNonNull(download.getStartTime(), - "At this point the download should have set a startTime. Make sure to set a startTime in " + download.getClass() + ".init()"); - LOG.debug("Downloading with {}", download.getClass().getSimpleName()); - - Recording rec = createRecording(model, download); - completionService.submit(() -> { - try { - setRecordingStatus(rec, State.RECORDING); - model.setLastRecorded(rec.getStartDate()); - recordingManager.saveRecording(rec); - download.start(); - } catch (Exception e) { - LOG.error("Download for {} failed. Download state: {}", model.getName(), rec.getStatus(), e); - } - boolean deleted = deleteIfEmpty(rec); - setRecordingStatus(rec, deleted ? State.DELETED : State.WAITING); - if (!deleted) { - // only save the status, if the recording has not been deleted, otherwise we recreate the metadata file - recordingManager.saveRecording(rec); - } - return rec; - }); + Download download = createDownload(model); + Recording rec = createRecording(download); + completionService.submit(createDownloadJob(rec)); } catch (RecordUntilExpiredException e) { LOG.info("Precondition not met to record {}: {}", model, e.getLocalizedMessage()); executeRecordUntilSubsequentAction(model); } catch (PreconditionNotMetException e) { - // long now = System.currentTimeMillis(); - // if ((now - lastSpaceMessage) > TimeUnit.MINUTES.toMillis(1)) { - // LOG.info("Not enough space for recording, not starting recording for {}", model); - // lastSpaceMessage = now; - // } LOG.info("Precondition not met to record {}: {}", model, e.getLocalizedMessage()); return; } finally { @@ -257,20 +232,53 @@ public class NextGenLocalRecorder implements Recorder { } } - private void executeRecordUntilSubsequentAction(Model model) throws IOException { - if (model.getRecordUntilSubsequentAction() == SubsequentAction.PAUSE) { - model.setSuspended(true); - } else if (model.getRecordUntilSubsequentAction() == SubsequentAction.REMOVE) { + private Download createDownload(Model model) { + Download download = model.createDownload(); + download.init(config, model, Instant.now()); + Objects.requireNonNull(download.getStartTime(), + "At this point the download should have set a startTime. Make sure to set a startTime in " + download.getClass() + ".init()"); + LOG.debug("Downloading with {}", download.getClass().getSimpleName()); + return download; + } + + private Callable createDownloadJob(Recording rec) { + return () -> { try { - LOG.info("Recording timeframe expired for model {} - {}", model, model.getRecordUntil()); + setRecordingStatus(rec, State.RECORDING); + rec.getModel().setLastRecorded(rec.getStartDate()); + recordingManager.saveRecording(rec); + rec.getDownload().start(); + } catch (Exception e) { + LOG.error("Download for {} failed. Download state: {}", rec.getModel().getName(), rec.getStatus(), e); + } + boolean deleted = deleteIfEmpty(rec); + setRecordingStatus(rec, deleted ? State.DELETED : State.WAITING); + if (!deleted) { + // only save the status, if the recording has not been deleted, otherwise we recreate the metadata file + recordingManager.saveRecording(rec); + } + return rec; + }; + } + + private void executeRecordUntilSubsequentAction(Model model) throws IOException { + if (model.getRecordUntilSubsequentAction() == PAUSE) { + model.setSuspended(true); + } else if (model.getRecordUntilSubsequentAction() == REMOVE) { + try { + LOG.info("Removing {} because the recording timeframe ended at {}", model, model.getRecordUntil().atZone(ZoneId.systemDefault())); stopRecording(model); } catch (InvalidKeyException | NoSuchAlgorithmException e1) { LOG.error("Error while stopping recording", e1); } } + // reset values, so that model can be recorded again + model.setRecordUntil(null); + model.setRecordUntilSubsequentAction(PAUSE); } - private Recording createRecording(Model model, Download download) throws IOException { + private Recording createRecording(Download download) throws IOException { + Model model = download.getModel(); Recording rec = new Recording(); rec.setDownload(download); rec.setPath(download.getPath(model).replaceAll("\\\\", "/")); @@ -749,4 +757,31 @@ public class NextGenLocalRecorder implements Recorder { rec.setNote(note); recordingManager.saveRecording(rec); } + + @Override + public void stopRecordingAt(Model model) throws IOException, InvalidKeyException, NoSuchAlgorithmException { + recorderLock.lock(); + try { + int index = models.indexOf(model); + if (index >= 0) { + Model m = models.get(index); + m.setRecordUntil(model.getRecordUntil()); + m.setRecordUntilSubsequentAction(model.getRecordUntilSubsequentAction()); + config.save(); + } else { + throw new NoSuchElementException("Model " + model.getName() + " [" + model.getUrl() + "] not found in list of recorded models"); + } + + if (recordingProcesses.containsKey(model)) { + Recording rec = recordingProcesses.get(model); + rec.getDownload().stop(); + } + } finally { + recorderLock.unlock(); + } + + if (Instant.now().isAfter(model.getRecordUntil())) { + executeRecordUntilSubsequentAction(model); + } + } } diff --git a/common/src/main/java/ctbrec/recorder/Recorder.java b/common/src/main/java/ctbrec/recorder/Recorder.java index 7dee8c4d..dd8238a0 100644 --- a/common/src/main/java/ctbrec/recorder/Recorder.java +++ b/common/src/main/java/ctbrec/recorder/Recorder.java @@ -14,6 +14,7 @@ public interface Recorder { public void startRecording(Model model) throws IOException, InvalidKeyException, NoSuchAlgorithmException; public void stopRecording(Model model) throws IOException, InvalidKeyException, NoSuchAlgorithmException; + public void stopRecordingAt(Model model) throws IOException, InvalidKeyException, NoSuchAlgorithmException; public void switchStreamSource(Model model) throws IOException, InvalidKeyException, NoSuchAlgorithmException; diff --git a/common/src/main/java/ctbrec/recorder/RemoteRecorder.java b/common/src/main/java/ctbrec/recorder/RemoteRecorder.java index 84a4b036..10fae6b5 100644 --- a/common/src/main/java/ctbrec/recorder/RemoteRecorder.java +++ b/common/src/main/java/ctbrec/recorder/RemoteRecorder.java @@ -86,6 +86,11 @@ public class RemoteRecorder implements Recorder { sendRequest("stop", model); } + @Override + public void stopRecordingAt(Model model) throws IOException, InvalidKeyException, NoSuchAlgorithmException { + sendRequest("stopAt", model); + } + private void sendRequest(String action, Model model) throws IOException, InvalidKeyException, NoSuchAlgorithmException { String payload = modelRequestAdapter.toJson(new ModelRequest(action, model)); LOG.debug("Sending request to recording server: {}", payload); diff --git a/server/src/main/java/ctbrec/recorder/server/RecorderServlet.java b/server/src/main/java/ctbrec/recorder/server/RecorderServlet.java index 5c71abec..fb42c4a0 100644 --- a/server/src/main/java/ctbrec/recorder/server/RecorderServlet.java +++ b/server/src/main/java/ctbrec/recorder/server/RecorderServlet.java @@ -96,6 +96,17 @@ public class RecorderServlet extends AbstractCtbrecServlet { response = "{\"status\": \"success\", \"msg\": \"Stopping recording\"}"; resp.getWriter().write(response); break; + case "stopAt": + new Thread(() -> { + try { + recorder.stopRecordingAt(request.model); + } catch (InvalidKeyException | NoSuchAlgorithmException | IllegalStateException | IOException e) { + LOG.error("Couldn't stop recording for model {}", request.model, e); + } + }).start(); + response = "{\"status\": \"success\", \"msg\": \"Stopping recording\"}"; + resp.getWriter().write(response); + break; case "list": resp.getWriter().write("{\"status\": \"success\", \"msg\": \"List of models\", \"models\": ["); JsonAdapter modelAdapter = new ModelJsonAdapter(); From caf329eb2370fab7411fa206c7feeefe3000b82a Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Sat, 8 Aug 2020 20:11:48 +0200 Subject: [PATCH 38/56] Change look of the model table in the web interface --- .../java/ctbrec/ui/tabs/RecordedModelsTab.java | 2 +- common/src/main/java/ctbrec/AbstractModel.java | 2 +- common/src/main/java/ctbrec/Model.java | 2 ++ server/src/main/resources/html/static/custom.css | 8 ++++++-- server/src/main/resources/html/static/index.html | 15 +++++++++------ server/src/main/resources/html/static/models.js | 2 ++ .../src/main/resources/html/static/recordings.js | 1 - 7 files changed, 21 insertions(+), 11 deletions(-) diff --git a/client/src/main/java/ctbrec/ui/tabs/RecordedModelsTab.java b/client/src/main/java/ctbrec/ui/tabs/RecordedModelsTab.java index 9922ca5a..96721f8c 100644 --- a/client/src/main/java/ctbrec/ui/tabs/RecordedModelsTab.java +++ b/client/src/main/java/ctbrec/ui/tabs/RecordedModelsTab.java @@ -679,7 +679,7 @@ public class RecordedModelsTab extends Tab implements TabSelectionListener { HBox.setMargin(pauseButton, new Insets(5)); HBox.setMargin(removeButton, new Insets(5)); grid.add(row, 1, 1); - if (model.getRecordUntil().toEpochMilli() != Long.MAX_VALUE) { + if (model.getRecordUntil().toEpochMilli() != Model.RECORD_INDEFINITELY) { LocalDate localDate = LocalDate.ofInstant(model.getRecordUntil(), ZoneId.systemDefault()); datePicker.setValue(localDate); } diff --git a/common/src/main/java/ctbrec/AbstractModel.java b/common/src/main/java/ctbrec/AbstractModel.java index 22dc81cc..f87bb843 100644 --- a/common/src/main/java/ctbrec/AbstractModel.java +++ b/common/src/main/java/ctbrec/AbstractModel.java @@ -235,7 +235,7 @@ public abstract class AbstractModel implements Model { @Override public Instant getRecordUntil() { - return Optional.ofNullable(recordUntil).orElse(Instant.ofEpochMilli(Long.MAX_VALUE)); + return Optional.ofNullable(recordUntil).orElse(Instant.ofEpochMilli(RECORD_INDEFINITELY)); } @Override diff --git a/common/src/main/java/ctbrec/Model.java b/common/src/main/java/ctbrec/Model.java index fec54ca7..42ccd10e 100644 --- a/common/src/main/java/ctbrec/Model.java +++ b/common/src/main/java/ctbrec/Model.java @@ -20,6 +20,8 @@ import ctbrec.sites.Site; public interface Model extends Comparable, Serializable { + public static final long RECORD_INDEFINITELY = 9000000000000000000l; + public enum State { ONLINE("online"), OFFLINE("offline"), diff --git a/server/src/main/resources/html/static/custom.css b/server/src/main/resources/html/static/custom.css index 820f8651..a1df2311 100644 --- a/server/src/main/resources/html/static/custom.css +++ b/server/src/main/resources/html/static/custom.css @@ -58,6 +58,10 @@ th a:hover { text-decoration: none; } +.checkmark-green { + color: #28a745; +} - - +.red { + color: #dc4444; +} \ No newline at end of file diff --git a/server/src/main/resources/html/static/index.html b/server/src/main/resources/html/static/index.html index dd0d9aeb..e9cc978b 100644 --- a/server/src/main/resources/html/static/index.html +++ b/server/src/main/resources/html/static/index.html @@ -96,19 +96,22 @@ Model - Paused Online Recording - Actions + + - - - - + + + + + + + diff --git a/server/src/main/resources/html/static/models.js b/server/src/main/resources/html/static/models.js index 55ac777a..4534b531 100644 --- a/server/src/main/resources/html/static/models.js +++ b/server/src/main/resources/html/static/models.js @@ -69,6 +69,7 @@ function syncModels(models) { } } model.ko_recording = ko.observable(model.online && !model.suspended); + //model.ko_recording_class = ko.observable( (model.online && !model.suspended) ? 'fa fa-circle red' : '' ); model.ko_suspended = ko.observable(model.suspended); model.swallowEvents = false; model.ko_suspended.subscribe(function(checked) { @@ -102,6 +103,7 @@ function syncModels(models) { } } m.ko_online(onlineState); + //m.ko_recording_class( (model.online && !model.suspended) ? 'fa fa-circle red' : ''); m.swallowEvents = true; m.ko_suspended(model.suspended); m.swallowEvents = false; diff --git a/server/src/main/resources/html/static/recordings.js b/server/src/main/resources/html/static/recordings.js index 9380eb82..6c446355 100644 --- a/server/src/main/resources/html/static/recordings.js +++ b/server/src/main/resources/html/static/recordings.js @@ -195,7 +195,6 @@ function updateDiskSpace() { throughput.bytes(data.throughput); throughput.timeframe(data.throughputTimeframe); let bytesPerSecond = data.throughput / data.throughputTimeframe; - console.log(data.throughput, data.throughputTimeframe, bytesPerSecond, calculateSize(bytesPerSecond) + '/s'); throughput.text(calculateSize(bytesPerSecond) + '/s'); } else { if (console) From b6e4bad837d059dc23db344b54ba31c51e6f5070 Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Sat, 8 Aug 2020 21:23:38 +0200 Subject: [PATCH 39/56] Fixed bug in JSON parsing Some models wouldn't get recorded, because of a missing element in the JSON doc --- .../src/main/java/ctbrec/sites/stripchat/StripchatModel.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/common/src/main/java/ctbrec/sites/stripchat/StripchatModel.java b/common/src/main/java/ctbrec/sites/stripchat/StripchatModel.java index 72b9d30c..c07b5b1c 100644 --- a/common/src/main/java/ctbrec/sites/stripchat/StripchatModel.java +++ b/common/src/main/java/ctbrec/sites/stripchat/StripchatModel.java @@ -95,9 +95,8 @@ public class StripchatModel extends AbstractModel { best.width = broadcastSettings.optInt("width"); best.mediaPlaylistUrl = "https://b-" + serverName + ".stripst.com/hls/" + streamName + "/" + streamName + ".m3u8"; sources.add(best); - Object resolutionObject = broadcastSettings.get("resolutions"); - if (resolutionObject instanceof JSONObject) { - JSONObject resolutions = (JSONObject) resolutionObject; + JSONObject resolutions = broadcastSettings.optJSONObject("resolutions"); + if (resolutions instanceof JSONObject) { JSONArray heights = resolutions.names(); for (int i = 0; i < heights.length(); i++) { String h = heights.getString(i); From ee302e49a40e7d770ea8110daa81921416ca711e Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Sun, 9 Aug 2020 11:04:24 +0200 Subject: [PATCH 40/56] Distinguish between performer_id and display_name for LiveJasmin models --- CHANGELOG.md | 11 +++++++++++ client/src/main/java/ctbrec/ui/CamrecApplication.java | 4 ++-- .../src/main/java/ctbrec/sites/jasmin/LiveJasmin.java | 9 +++++++-- .../java/ctbrec/sites/jasmin/LiveJasminModel.java | 7 ++++++- 4 files changed, 26 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4327a5d0..a3b4dfc2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,14 @@ +3.8.5 +======================== +* Fixed Stripchat followed tab. It didn't work, if you have many favorited + models +* Fixed: Some Stripchat models didn't get recorded +* Fixed: Some LiveJasmin models didn't get recorded +* Added support for temporary recordings. On the recording tab you can now set + a date, when to stop recording a model and what to do afterwards + (pause or remove remove the model) +* Changed the look of the of the model table in the web interface a bit + 3.8.4 ======================== * Added support for xHamsterLive (go to Settings -> Sites -> Stripchat, diff --git a/client/src/main/java/ctbrec/ui/CamrecApplication.java b/client/src/main/java/ctbrec/ui/CamrecApplication.java index ca6003f0..7c23ec30 100644 --- a/client/src/main/java/ctbrec/ui/CamrecApplication.java +++ b/client/src/main/java/ctbrec/ui/CamrecApplication.java @@ -141,8 +141,6 @@ public class CamrecApplication extends Application { } private void startOnlineMonitor() { - onlineMonitor = new OnlineMonitor(recorder); - onlineMonitor.start(); for (Site site : sites) { if(site.isEnabled()) { try { @@ -153,6 +151,8 @@ public class CamrecApplication extends Application { } } } + onlineMonitor = new OnlineMonitor(recorder); + onlineMonitor.start(); } private void logEnvironment() { diff --git a/common/src/main/java/ctbrec/sites/jasmin/LiveJasmin.java b/common/src/main/java/ctbrec/sites/jasmin/LiveJasmin.java index 3f534cb7..e6cfd9dd 100644 --- a/common/src/main/java/ctbrec/sites/jasmin/LiveJasmin.java +++ b/common/src/main/java/ctbrec/sites/jasmin/LiveJasmin.java @@ -5,6 +5,7 @@ import static ctbrec.io.HttpConstants.*; import java.io.IOException; import java.net.URLEncoder; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Locale; import java.util.regex.Matcher; @@ -13,6 +14,8 @@ import java.util.regex.Pattern; import org.json.JSONObject; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import ctbrec.Config; import ctbrec.Model; @@ -27,6 +30,7 @@ import okhttp3.Response; public class LiveJasmin extends AbstractSite { + private static final Logger LOG = LoggerFactory.getLogger(LiveJasmin.class); public static String baseUrl = ""; public static String baseDomain = ""; private HttpClient httpClient; @@ -169,7 +173,8 @@ public class LiveJasmin extends AbstractSite { } return models; } else { - throw new IOException("Response was not successful: " + url + "\n" + body); + LOG.debug("Response was not successful: {}\n{}", url, body); + return Collections.emptyList(); } } else { throw new HttpException(response.code(), response.message()); @@ -198,7 +203,7 @@ public class LiveJasmin extends AbstractSite { String name = m.group(1); return createModel(name); } - m = Pattern.compile("http.*?livejasmin\\.com.*?/chat-html5/(.*)").matcher(url); + m = Pattern.compile("http.*?livejasmin\\.com.*?/chat(?:-html5)?/(.*)").matcher(url); if(m.find()) { String name = m.group(1); return createModel(name); diff --git a/common/src/main/java/ctbrec/sites/jasmin/LiveJasminModel.java b/common/src/main/java/ctbrec/sites/jasmin/LiveJasminModel.java index c7aa3d18..5a4f566d 100644 --- a/common/src/main/java/ctbrec/sites/jasmin/LiveJasminModel.java +++ b/common/src/main/java/ctbrec/sites/jasmin/LiveJasminModel.java @@ -72,6 +72,8 @@ public class LiveJasminModel extends AbstractModel { JSONObject config = data.getJSONObject("config"); JSONObject chatRoom = config.getJSONObject("chatRoom"); setId(chatRoom.getString("p_id")); + setName(chatRoom.getString("performer_id")); + setDisplayName(chatRoom.getString("display_name")); if (chatRoom.has("profile_picture_url")) { setPreview(chatRoom.getString("profile_picture_url")); } @@ -80,11 +82,14 @@ public class LiveJasminModel extends AbstractModel { if (chatRoom.optInt("is_on_private", 0) == 1) { onlineState = State.PRIVATE; } + if (chatRoom.optInt("is_video_call_enabled", 0) == 1) { + onlineState = State.PRIVATE; + } resolution = new int[2]; resolution[0] = config.optInt("streamWidth"); resolution[1] = config.optInt("streamHeight"); online = onlineState == State.ONLINE; - LOG.trace("{} - status:{} {} {} {}", getName(), online, onlineState, Arrays.toString(resolution), getUrl()); + LOG.trace("{} - status:{} {} {} {} {}", getName(), online, onlineState, Arrays.toString(resolution), getUrl(), id); } else { throw new IOException("Response was not successful: " + body); } From c02d9562bfce72f66e39afccef54a3a032e2f44b Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Sun, 9 Aug 2020 11:05:56 +0200 Subject: [PATCH 41/56] Set version to 3.8.5 --- client/pom.xml | 2 +- common/pom.xml | 2 +- master/pom.xml | 2 +- server/pom.xml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/client/pom.xml b/client/pom.xml index 3bd74eef..44e39450 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -8,7 +8,7 @@ ctbrec master - 3.8.4 + 3.8.5 ../master diff --git a/common/pom.xml b/common/pom.xml index 9b9f6a73..3ae29f86 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -8,7 +8,7 @@ ctbrec master - 3.8.4 + 3.8.5 ../master diff --git a/master/pom.xml b/master/pom.xml index 4031a2f0..b695f290 100644 --- a/master/pom.xml +++ b/master/pom.xml @@ -6,7 +6,7 @@ ctbrec master pom - 3.8.4 + 3.8.5 ../common diff --git a/server/pom.xml b/server/pom.xml index 0b3ab7b0..37a85098 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -8,7 +8,7 @@ ctbrec master - 3.8.4 + 3.8.5 ../master From 68cf2635df66ac2305c3376cafb7c87c6aa33cea Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Sun, 9 Aug 2020 11:41:22 +0200 Subject: [PATCH 42/56] Fix typo --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a3b4dfc2..8af54f2d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,8 +6,8 @@ * Fixed: Some LiveJasmin models didn't get recorded * Added support for temporary recordings. On the recording tab you can now set a date, when to stop recording a model and what to do afterwards - (pause or remove remove the model) -* Changed the look of the of the model table in the web interface a bit + (pause or remove the model) +* Changed the look of the model table in the web interface a bit 3.8.4 ======================== From 0fe16f8ff8102ff3dacbf50473fe3481f5067232 Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Sun, 9 Aug 2020 12:27:04 +0200 Subject: [PATCH 43/56] Add setting to disable online check for paused models --- .../src/main/java/ctbrec/ui/CamrecApplication.java | 2 +- .../main/java/ctbrec/ui/settings/SettingsTab.java | 7 +++++-- .../main/resources/html/docs/ConfigurationFile.md | 6 +++++- common/src/main/java/ctbrec/Settings.java | 1 + .../main/java/ctbrec/recorder/OnlineMonitor.java | 14 ++++++++++---- .../java/ctbrec/recorder/server/HttpServer.java | 2 +- 6 files changed, 23 insertions(+), 9 deletions(-) diff --git a/client/src/main/java/ctbrec/ui/CamrecApplication.java b/client/src/main/java/ctbrec/ui/CamrecApplication.java index 7c23ec30..29469fb1 100644 --- a/client/src/main/java/ctbrec/ui/CamrecApplication.java +++ b/client/src/main/java/ctbrec/ui/CamrecApplication.java @@ -151,7 +151,7 @@ public class CamrecApplication extends Application { } } } - onlineMonitor = new OnlineMonitor(recorder); + onlineMonitor = new OnlineMonitor(recorder, config); onlineMonitor.start(); } diff --git a/client/src/main/java/ctbrec/ui/settings/SettingsTab.java b/client/src/main/java/ctbrec/ui/settings/SettingsTab.java index 7d3a3c52..3ce04942 100644 --- a/client/src/main/java/ctbrec/ui/settings/SettingsTab.java +++ b/client/src/main/java/ctbrec/ui/settings/SettingsTab.java @@ -83,6 +83,7 @@ public class SettingsTab extends Tab implements TabSelectionListener { private DiscreteRange rangeValues = new DiscreteRange<>(values, labels); private SimpleIntegerProperty concurrentRecordings; private SimpleIntegerProperty onlineCheckIntervalInSecs; + private SimpleBooleanProperty onlineCheckSkipsPausedModels; private SimpleLongProperty leaveSpaceOnDevice; private SimpleIntegerProperty minimumLengthInSecs; private SimpleStringProperty ffmpegParameters; @@ -146,6 +147,7 @@ public class SettingsTab extends Tab implements TabSelectionListener { postProcessing = new SimpleFileProperty(null, "postProcessing", settings.postProcessing); postProcessingThreads = new SimpleIntegerProperty(null, "postProcessingThreads", settings.postProcessingThreads); removeRecordingAfterPp = new SimpleBooleanProperty(null, "removeRecordingAfterPostProcessing", settings.removeRecordingAfterPostProcessing); + onlineCheckSkipsPausedModels = new SimpleBooleanProperty(null, "onlineCheckSkipsPausedModels", settings.onlineCheckSkipsPausedModels); } private void createGui() { @@ -183,10 +185,11 @@ public class SettingsTab extends Tab implements TabSelectionListener { Setting.of("Split recordings after (minutes)", splitAfter).converter(SplitAfterOption.converter()), Setting.of("Restrict Resolution", resolutionRange, "Only record streams with resolution within the given range"), Setting.of("Concurrent Recordings (0 = unlimited)", concurrentRecordings), - Setting.of("Check online state every (seconds)", onlineCheckIntervalInSecs, "Check every x seconds, if a model came online"), Setting.of("Leave space on device (GiB)", leaveSpaceOnDevice, "Stop recording, if the free space on the device gets below this threshold").converter(new GigabytesConverter()), Setting.of("FFmpeg parameters", ffmpegParameters, "FFmpeg parameters to use when merging stream segments"), - Setting.of("File Extension", fileExtension, "File extension to use for recordings") + Setting.of("File Extension", fileExtension, "File extension to use for recordings"), + Setting.of("Check online state every (seconds)", onlineCheckIntervalInSecs, "Check every x seconds, if a model came online"), + Setting.of("Skip online check for paused models", onlineCheckSkipsPausedModels, "Skip online check for paused models") ), Group.of("Location", Setting.of("Record Location", recordLocal), diff --git a/client/src/main/resources/html/docs/ConfigurationFile.md b/client/src/main/resources/html/docs/ConfigurationFile.md index d1312750..e3ea4bf1 100644 --- a/client/src/main/resources/html/docs/ConfigurationFile.md +++ b/client/src/main/resources/html/docs/ConfigurationFile.md @@ -41,13 +41,17 @@ the port ctbrec tries to connect to, if it is run in remote mode. - **livePreviews** (app only) - Enables the live preview feature in the app. +- **minimumResolution** - [1 - 2147483647]. Sets the minimum video height for a recording. ctbrec tries to find a stream quality, which is higher than or equal to this value. If the only provided stream quality is below this threshold, ctbrec won't record the stream. + - **maximumResolution** - [1 - 2147483647]. Sets the maximum video height for a recording. ctbrec tries to find a stream quality, which is lower than or equal to this value. If the only provided stream quality is above this threshold, ctbrec won't record the stream. - **minimumLengthInSeconds** - [0 - 2147483647] Automatically delete recordings, which are shorter than this amount of seconds. 0 disables this feature. - **minimumSpaceLeftInBytes** - [0 - 9223372036854775807] The space in bytes ctbrec should conserve on the hard drive. 1 GiB = 1024 MiB = 1048576 KiB = 1073741824 bytes -- **onlineCheckIntervalInSecs** - [1 - 2147483647] How often ctbrec checks, if a model is online. This is not a guaranteed interval: If you record many models, the online check for all models can take longer than this interval. A minute is a reasonable value, but you can go lower, if you don't want to miss a anything. But don't go too low, or you risk to do too many requests in a short amount of time and get banned by some sites. +- **onlineCheckIntervalInSecs** - [1 - 2147483647] How often ctbrec checks, if a model is online. This is not a guaranteed interval: If you record many models, the online check for all models can take longer than this interval. A minute is a reasonable value, but you can go lower, if you don't want to miss a anything. But don't go too low, or you risk to do too many requests in a short amount of time and get banned by some sites. + +- **onlineCheckSkipsPausedModels** - [`true`,`false`] Skip the online check for paused models. If you have many models in the recording list, this can reduce the delay when a recording starts after a model came online. - **postProcessing** - Absolute path to a script, which is executed once a recording is finished. See [Post-Processing](/docs/PostProcessing.md). diff --git a/common/src/main/java/ctbrec/Settings.java b/common/src/main/java/ctbrec/Settings.java index 9094117b..e6c7e5fc 100644 --- a/common/src/main/java/ctbrec/Settings.java +++ b/common/src/main/java/ctbrec/Settings.java @@ -89,6 +89,7 @@ public class Settings { public List models = new ArrayList<>(); public List modelsIgnored = new ArrayList<>(); public int onlineCheckIntervalInSecs = 60; + public boolean onlineCheckSkipsPausedModels = false; public int overviewUpdateIntervalInSecs = 10; public String password = ""; // chaturbate password TODO maybe rename this onetime public String postProcessing = ""; diff --git a/common/src/main/java/ctbrec/recorder/OnlineMonitor.java b/common/src/main/java/ctbrec/recorder/OnlineMonitor.java index 6d633d61..8cfcc801 100644 --- a/common/src/main/java/ctbrec/recorder/OnlineMonitor.java +++ b/common/src/main/java/ctbrec/recorder/OnlineMonitor.java @@ -38,9 +38,11 @@ public class OnlineMonitor extends Thread { private Map states = new HashMap<>(); private Map executors = new HashMap<>(); + private Config config; - public OnlineMonitor(Recorder recorder) { + public OnlineMonitor(Recorder recorder, Config config) { this.recorder = recorder; + this.config = config; setName("OnlineMonitor"); setDaemon(true); } @@ -80,7 +82,11 @@ public class OnlineMonitor extends Thread { // submit online check jobs to the executor for the model's site List> futures = new LinkedList<>(); for (Model model : models) { - futures.add(updateModel(model)); + if (config.getSettings().onlineCheckSkipsPausedModels && model.isSuspended()) { + continue; + } else { + futures.add(updateModel(model)); + } } // wait for all jobs to finish for (Future future : futures) { @@ -111,7 +117,7 @@ public class OnlineMonitor extends Thread { model.setLastSeen(Instant.now()); } Model.State state = model.getOnlineState(false); - LOG.trace("Model online state: {} {}", model.getName(), state); + LOG.debug("Model online state: {} {}", model.getName(), state); Model.State oldState = states.getOrDefault(model, UNKNOWN); states.put(model, state); if (state != oldState) { @@ -134,7 +140,7 @@ public class OnlineMonitor extends Thread { private void suspendUntilNextIteration(List models, Duration timeCheckTook) { LOG.trace("Online check for {} models took {} seconds", models.size(), timeCheckTook.getSeconds()); - long sleepTime = Config.getInstance().getSettings().onlineCheckIntervalInSecs; + long sleepTime = config.getSettings().onlineCheckIntervalInSecs; if(timeCheckTook.getSeconds() < sleepTime) { try { if (running) { diff --git a/server/src/main/java/ctbrec/recorder/server/HttpServer.java b/server/src/main/java/ctbrec/recorder/server/HttpServer.java index f97d49b2..fb179fbe 100644 --- a/server/src/main/java/ctbrec/recorder/server/HttpServer.java +++ b/server/src/main/java/ctbrec/recorder/server/HttpServer.java @@ -106,7 +106,7 @@ public class HttpServer { safeLogin(site); } } - onlineMonitor = new OnlineMonitor(recorder); + onlineMonitor = new OnlineMonitor(recorder, config); onlineMonitor.start(); startHttpServer(); } From daefe1a7d442b316ae9b0de404241114ae7a448b Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Sun, 9 Aug 2020 12:27:04 +0200 Subject: [PATCH 44/56] Add setting to disable online check for paused models --- client/src/main/java/ctbrec/ui/settings/SettingsTab.java | 1 + 1 file changed, 1 insertion(+) diff --git a/client/src/main/java/ctbrec/ui/settings/SettingsTab.java b/client/src/main/java/ctbrec/ui/settings/SettingsTab.java index 3ce04942..ace58c39 100644 --- a/client/src/main/java/ctbrec/ui/settings/SettingsTab.java +++ b/client/src/main/java/ctbrec/ui/settings/SettingsTab.java @@ -235,6 +235,7 @@ public class SettingsTab extends Tab implements TabSelectionListener { prefs.getSetting("minimumResolution").ifPresent(s -> bindEnabledProperty(s, recordLocal.not())); prefs.getSetting("recordingsDirStructure").ifPresent(s -> bindEnabledProperty(s, recordLocal.not())); prefs.getSetting("onlineCheckIntervalInSecs").ifPresent(s -> bindEnabledProperty(s, recordLocal.not())); + prefs.getSetting("onlineCheckSkipsPausedModels").ifPresent(s -> bindEnabledProperty(s, recordLocal.not())); prefs.getSetting("minimumSpaceLeftInBytes").ifPresent(s -> bindEnabledProperty(s, recordLocal.not())); prefs.getSetting("postProcessing").ifPresent(s -> bindEnabledProperty(s, recordLocal.not())); prefs.getSetting("postProcessingThreads").ifPresent(s -> bindEnabledProperty(s, recordLocal.not())); From a0779c118d521b391059c0bab1a893f9f28c3e58 Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Sun, 9 Aug 2020 14:26:12 +0200 Subject: [PATCH 45/56] Reduce log level again --- common/src/main/java/ctbrec/recorder/OnlineMonitor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/src/main/java/ctbrec/recorder/OnlineMonitor.java b/common/src/main/java/ctbrec/recorder/OnlineMonitor.java index 8cfcc801..d3f10744 100644 --- a/common/src/main/java/ctbrec/recorder/OnlineMonitor.java +++ b/common/src/main/java/ctbrec/recorder/OnlineMonitor.java @@ -117,7 +117,7 @@ public class OnlineMonitor extends Thread { model.setLastSeen(Instant.now()); } Model.State state = model.getOnlineState(false); - LOG.debug("Model online state: {} {}", model.getName(), state); + LOG.trace("Model online state: {} {}", model.getName(), state); Model.State oldState = states.getOrDefault(model, UNKNOWN); states.put(model, state); if (state != oldState) { From 2126b61e99972d6572ed2613f4647cffc06d7201 Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Sun, 9 Aug 2020 18:29:04 +0200 Subject: [PATCH 46/56] Add logging tab --- CHANGELOG.md | 5 + client/pom.xml | 2 +- .../java/ctbrec/ui/CamrecApplication.java | 2 + .../ui/tabs/logging/CtbrecAppender.java | 13 ++ .../ctbrec/ui/tabs/logging/LoggingTab.java | 120 ++++++++++++++++++ client/src/main/resources/logback.xml | 7 + 6 files changed, 148 insertions(+), 1 deletion(-) create mode 100644 client/src/main/java/ctbrec/ui/tabs/logging/CtbrecAppender.java create mode 100644 client/src/main/java/ctbrec/ui/tabs/logging/LoggingTab.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 8af54f2d..70fd84bd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +3.8.6 +======================== +* Added setting to disable the online check for paused models +* Added tab which shows the log output + 3.8.5 ======================== * Fixed Stripchat followed tab. It didn't work, if you have many favorited diff --git a/client/pom.xml b/client/pom.xml index 44e39450..7becf27d 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -71,7 +71,7 @@ ch.qos.logback logback-classic - runtime + compile org.openjfx diff --git a/client/src/main/java/ctbrec/ui/CamrecApplication.java b/client/src/main/java/ctbrec/ui/CamrecApplication.java index 29469fb1..5ccb8fb6 100644 --- a/client/src/main/java/ctbrec/ui/CamrecApplication.java +++ b/client/src/main/java/ctbrec/ui/CamrecApplication.java @@ -61,6 +61,7 @@ import ctbrec.ui.tabs.RecordingsTab; import ctbrec.ui.tabs.SiteTab; import ctbrec.ui.tabs.TabSelectionListener; import ctbrec.ui.tabs.UpdateTab; +import ctbrec.ui.tabs.logging.LoggingTab; import javafx.application.Application; import javafx.application.HostServices; import javafx.application.Platform; @@ -191,6 +192,7 @@ public class CamrecApplication extends Application { tabPane.getTabs().add(new NewsTab()); tabPane.getTabs().add(new DonateTabFx()); tabPane.getTabs().add(new HelpTab()); + tabPane.getTabs().add(new LoggingTab()); switchToStartTab(); writeColorSchemeStyleSheet(); diff --git a/client/src/main/java/ctbrec/ui/tabs/logging/CtbrecAppender.java b/client/src/main/java/ctbrec/ui/tabs/logging/CtbrecAppender.java new file mode 100644 index 00000000..14ff5b9e --- /dev/null +++ b/client/src/main/java/ctbrec/ui/tabs/logging/CtbrecAppender.java @@ -0,0 +1,13 @@ +package ctbrec.ui.tabs.logging; + +import ch.qos.logback.classic.spi.LoggingEvent; +import ch.qos.logback.core.ConsoleAppender; +import ctbrec.event.EventBusHolder; + +public class CtbrecAppender extends ConsoleAppender { + + @Override + protected void append(LoggingEvent event) { + EventBusHolder.BUS.post(event); + } +} diff --git a/client/src/main/java/ctbrec/ui/tabs/logging/LoggingTab.java b/client/src/main/java/ctbrec/ui/tabs/logging/LoggingTab.java new file mode 100644 index 00000000..dc3a4c01 --- /dev/null +++ b/client/src/main/java/ctbrec/ui/tabs/logging/LoggingTab.java @@ -0,0 +1,120 @@ +package ctbrec.ui.tabs.logging; + +import java.time.Instant; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.util.Collections; +import java.util.LinkedList; +import java.util.stream.Collectors; + +import com.google.common.eventbus.Subscribe; + +import ch.qos.logback.classic.spi.IThrowableProxy; +import ch.qos.logback.classic.spi.LoggingEvent; +import ch.qos.logback.classic.spi.StackTraceElementProxy; +import ctbrec.event.EventBusHolder; +import ctbrec.ui.controls.SearchBox; +import javafx.application.Platform; +import javafx.beans.property.SimpleStringProperty; +import javafx.collections.FXCollections; +import javafx.collections.ObservableList; +import javafx.geometry.Insets; +import javafx.scene.control.Tab; +import javafx.scene.control.TableColumn; +import javafx.scene.control.TableView; +import javafx.scene.layout.BorderPane; + +public class LoggingTab extends Tab { + + private static final int HISTORY_LENGTH = 10_000; + private SearchBox filter = new SearchBox(); + private TableView table = new TableView<>(); + private ObservableList history = FXCollections.observableList(Collections.synchronizedList(new LinkedList<>())); + private ObservableList filteredEvents = FXCollections.observableArrayList(); + private DateTimeFormatter timeFormatter = DateTimeFormatter.ISO_LOCAL_DATE_TIME; + private volatile boolean tabClosed = false; + private Object eventBustSubscriber = new Object() { + @Subscribe + public void publishLoggingEevent(LoggingEvent event) { + if (!tabClosed) { + Platform.runLater(() -> { + history.add(event); + if (history.size() > HISTORY_LENGTH - 1) { + history.remove(0); + } + filter(); + }); + } + } + }; + + private void filter() { + filteredEvents.clear(); + filteredEvents.addAll(history.stream().filter(evt -> { + String q = filter.getText().toLowerCase(); + return evt.getLevel().toString().toLowerCase().contains(q) + || createLogMessage(evt).toLowerCase().contains(q); + }).collect(Collectors.toList())); + } + + public LoggingTab() { + setText("Logging"); + subscribeToEventBus(); + + table = new TableView<>(filteredEvents); + + int idx = 0; + TableColumn level = createTableColumn("Level", 65, idx++); + level.setCellValueFactory(cdf -> new SimpleStringProperty(cdf.getValue().getLevel().toString())); + table.getColumns().add(level); + + TableColumn time = createTableColumn("Timestamp", 200, idx++); + time.setCellValueFactory(cdf -> { + Instant instant = Instant.ofEpochMilli(cdf.getValue().getTimeStamp()); + return new SimpleStringProperty(instant.atZone(ZoneId.systemDefault()).format(timeFormatter)); + }); + table.getColumns().add(time); + + TableColumn msg = createTableColumn("Message", 2000, idx++); + msg.setCellValueFactory(cdf -> new SimpleStringProperty(createLogMessage(cdf.getValue()))); + table.getColumns().add(msg); + + BorderPane layout = new BorderPane(); + BorderPane.setMargin(table, new Insets(10)); + BorderPane.setMargin(filter, new Insets(10, 10, 0, 10)); + layout.setCenter(table); + layout.setTop(filter); + + setContent(layout); + setOnClosed(evt -> { + EventBusHolder.BUS.unregister(eventBustSubscriber); + tabClosed = true; + }); + + filter.setPromptText("Search"); + filter.textProperty().addListener( (observableValue, oldValue, newValue) -> filter()); + } + + private String createLogMessage(LoggingEvent evt) { + StringBuilder sb = new StringBuilder(evt.getFormattedMessage()); + if(evt.getThrowableProxy() != null) { + IThrowableProxy throwableProxy = evt.getThrowableProxy(); + sb.append('\n').append(throwableProxy.getClassName()).append(':').append(' ').append(throwableProxy.getMessage()); + for (StackTraceElementProxy step : throwableProxy.getStackTraceElementProxyArray()) { + sb.append('\n').append('\t').append(step.getSTEAsString()); + } + } + return sb.toString(); + } + + private TableColumn createTableColumn(String text, int width, int idx) { + TableColumn tc = new TableColumn<>(text); + tc.setPrefWidth(width); + tc.setUserData(idx); + return tc; + } + + private void subscribeToEventBus() { + EventBusHolder.BUS.register(eventBustSubscriber); + } +} diff --git a/client/src/main/resources/logback.xml b/client/src/main/resources/logback.xml index 6aa57c26..e9ef55dd 100644 --- a/client/src/main/resources/logback.xml +++ b/client/src/main/resources/logback.xml @@ -8,6 +8,12 @@ + + + + %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n + + @@ -32,6 +38,7 @@ + From 0e61421537065bf12bebb6a8618e8cc98f9d6906 Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Fri, 14 Aug 2020 19:16:39 +0200 Subject: [PATCH 47/56] Fix MFC websocket message parser --- .../ctbrec/sites/mfc/MyFreeCamsClient.java | 131 +++++++++++------- .../src/test/java/ctbrec/ReflectionUtil.java | 21 +++ .../sites/mfc/MyFreeCamsClientTest.java | 38 +++++ 3 files changed, 142 insertions(+), 48 deletions(-) create mode 100644 common/src/test/java/ctbrec/ReflectionUtil.java create mode 100644 common/src/test/java/ctbrec/sites/mfc/MyFreeCamsClientTest.java diff --git a/common/src/main/java/ctbrec/sites/mfc/MyFreeCamsClient.java b/common/src/main/java/ctbrec/sites/mfc/MyFreeCamsClient.java index 1e4f5ca5..45178981 100644 --- a/common/src/main/java/ctbrec/sites/mfc/MyFreeCamsClient.java +++ b/common/src/main/java/ctbrec/sites/mfc/MyFreeCamsClient.java @@ -40,6 +40,7 @@ import com.squareup.moshi.Moshi; import ctbrec.Config; import ctbrec.StringUtil; +import ctbrec.io.HttpException; import okhttp3.Cookie; import okhttp3.Request; import okhttp3.Response; @@ -93,6 +94,7 @@ public class MyFreeCamsClient { } public void start() throws IOException { + requestLandingPage(); // to get some cookies running = true; serverConfig = new ServerConfig(mfc); List websocketServers = new ArrayList<>(serverConfig.wsServers.size()); @@ -134,6 +136,21 @@ public class MyFreeCamsClient { watchDog.start(); } + private void requestLandingPage() throws IOException { + Request req = new Request.Builder() + .url(MyFreeCams.baseUrl) + .header(ACCEPT, "*/*") + .header(ACCEPT_LANGUAGE, Locale.ENGLISH.getLanguage()) + .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) + .header(CONNECTION, KEEP_ALIVE) + .build(); + try(Response resp = mfc.getHttpClient().execute(req)) { + if(!resp.isSuccessful()) { + throw new HttpException(resp.code(), resp.message()); + } + } + } + public void stop() { running = false; ws.close(1000, "Good Bye"); // terminate normally (1000) @@ -455,54 +472,6 @@ public class MyFreeCamsClient { model.update(state, getStreamUrl(state)); } - private Message parseMessage(StringBuilder msgBuffer) throws UnsupportedEncodingException { - int packetLengthBytes = 6; - if (msgBuffer.length() < packetLengthBytes) { - // packet size not transmitted completely - return null; - } else { - try { - int packetLength = Integer.parseInt(msgBuffer.substring(0, packetLengthBytes)); - if (packetLength > msgBuffer.length() - packetLengthBytes) { - // packet not complete - return null; - } else { - LOG.trace("<-- {}", msgBuffer); - msgBuffer.delete(0, packetLengthBytes); - StringBuilder rawMessage = new StringBuilder(msgBuffer.substring(0, packetLength)); - int type = parseNextInt(rawMessage); - int sender = parseNextInt(rawMessage); - int receiver = parseNextInt(rawMessage); - int arg1 = parseNextInt(rawMessage); - int arg2 = parseNextInt(rawMessage); - Message message = new Message(type, sender, receiver, arg1, arg2, URLDecoder.decode(rawMessage.toString(), "utf-8")); - msgBuffer.delete(0, packetLength); - return message; - } - } catch (Exception e) { - LOG.error("StringBuilder contains invalid data {}", msgBuffer.toString(), e); - String logfile = "mfc_messages.log"; - try (FileOutputStream fout = new FileOutputStream(logfile)) { - for (String string : receivedTextHistory) { - fout.write(string.getBytes()); - fout.write(10); - } - } catch (Exception e1) { - LOG.error("Couldn't write mfc message history to {}", logfile, e1); - } - msgBuffer.setLength(0); - return null; - } - } - } - - private int parseNextInt(StringBuilder s) { - int nextSpace = s.indexOf(" "); - int i = Integer.parseInt(s.substring(0, nextSpace)); - s.delete(0, nextSpace + 1); - return i; - } - @Override public void onMessage(WebSocket webSocket, ByteString bytes) { super.onMessage(webSocket, bytes); @@ -512,6 +481,58 @@ public class MyFreeCamsClient { return websocket; } + private Message parseMessage(StringBuilder msgBuffer) throws UnsupportedEncodingException { + int packetLengthBytes = parsePacketLengthBytes(msgBuffer); + if (packetLengthBytes < 0) { + // packet size not transmitted completely + return null; + } else { + try { + int packetLength = Integer.parseInt(msgBuffer.substring(0, packetLengthBytes)); + if (packetLength > msgBuffer.length() - packetLengthBytes) { + // packet not complete + return null; + } else { + LOG.trace("<-- {}", msgBuffer); + msgBuffer.delete(0, packetLengthBytes); + StringBuilder rawMessage = new StringBuilder(msgBuffer.substring(0, packetLength)); + int type = parseNextInt(rawMessage); + int sender = parseNextInt(rawMessage); + int receiver = parseNextInt(rawMessage); + int arg1 = parseNextInt(rawMessage); + int arg2 = parseNextInt(rawMessage); + Message message = new Message(type, sender, receiver, arg1, arg2, URLDecoder.decode(rawMessage.toString(), "utf-8")); + msgBuffer.delete(0, packetLength); + return message; + } + } catch (Exception e) { + LOG.error("StringBuilder contains invalid data {}", msgBuffer.toString(), e); + String logfile = "mfc_messages.log"; + try (FileOutputStream fout = new FileOutputStream(logfile)) { + for (String string : receivedTextHistory) { + fout.write(string.getBytes()); + fout.write(10); + } + } catch (Exception e1) { + LOG.error("Couldn't write mfc message history to {}", logfile, e1); + } + msgBuffer.setLength(0); + return null; + } + } + } + + private int parsePacketLengthBytes(StringBuilder msgBuffer) { + return msgBuffer.indexOf(" ") - 2; + } + + private int parseNextInt(StringBuilder s) { + int nextSpace = s.indexOf(" "); + int i = Integer.parseInt(s.substring(0, nextSpace)); + s.delete(0, nextSpace + 1); + return i; + } + protected boolean follow(int uid) { if (ws != null) { return ws.send(ADDFRIENDREQ + " " + sessionId + " 0 " + uid + " 1\n"); @@ -703,4 +724,18 @@ public class MyFreeCamsClient { public Collection getSessionStates() { return Collections.unmodifiableCollection(sessionStates.asMap().values()); } + + public void joinChannel(MyFreeCamsModel model) { + SessionState state = getSessionState(model); + int userChannel = 100000000 + state.getUid(); + LOG.debug("Joining chat channel for model {}", model.getDisplayName()); + try { + search(model.getName()); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + ws.send(MessageTypes.ROOMDATA + " " + sessionId + " 0 1 0\n"); + ws.send(MessageTypes.UEOPT + " " + sessionId + " 0 66 1 111111\n"); + ws.send(MessageTypes.JOINCHAN + " " + sessionId + " 0 " + userChannel + " 9\n"); + } } diff --git a/common/src/test/java/ctbrec/ReflectionUtil.java b/common/src/test/java/ctbrec/ReflectionUtil.java new file mode 100644 index 00000000..37bbba44 --- /dev/null +++ b/common/src/test/java/ctbrec/ReflectionUtil.java @@ -0,0 +1,21 @@ +package ctbrec; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +public class ReflectionUtil { + + private ReflectionUtil () {} + + @SuppressWarnings("unchecked") + public static T call(Object target, String methodName, Object...args) throws NoSuchMethodException, SecurityException, IllegalAccessException, IllegalArgumentException, InvocationTargetException { + List> argTypes = Arrays.stream(args).map(arg -> arg.getClass()).collect(Collectors.toList()); + Class[] argTypeArray = argTypes.toArray(new Class[0]); + Method method = target.getClass().getDeclaredMethod(methodName, argTypeArray); + method.setAccessible(true); + return (T) method.invoke(target, args); + } +} diff --git a/common/src/test/java/ctbrec/sites/mfc/MyFreeCamsClientTest.java b/common/src/test/java/ctbrec/sites/mfc/MyFreeCamsClientTest.java new file mode 100644 index 00000000..d8bbdac3 --- /dev/null +++ b/common/src/test/java/ctbrec/sites/mfc/MyFreeCamsClientTest.java @@ -0,0 +1,38 @@ +package ctbrec.sites.mfc; + +import static org.junit.Assert.*; + +import java.lang.reflect.InvocationTargetException; + +import org.json.JSONObject; +import org.junit.Test; + +import ctbrec.ReflectionUtil; + +public class MyFreeCamsClientTest { + + @Test + public void testMessageParsing() throws NoSuchMethodException, SecurityException, IllegalAccessException, IllegalArgumentException, InvocationTargetException { + MyFreeCamsClient client = MyFreeCamsClient.getInstance(); + StringBuilder input = new StringBuilder("082720 418550763 419337943 0 37761695 %7B%22lv%22%3A4%2C%22nm%22%3A%22AliceArras%22%2C%22pid%22%3A1%2C%22sid%22%3A418550763%2C%22uid%22%3A37761695%2C%22vs%22%3A0%2C%22u%22%3A%7B%22age%22%3A19%2C%22avatar%22%3A1%2C%22blurb%22%3A%22Open%20minded%2C%20willing%20to%20try%20new%20horny%20things%20all%20the%20time!%22%2C%22camserv%22%3A1205%2C%22chat_font%22%3A0%2C%22chat_opt%22%3A1%2C%22city%22%3A%22Tallinn%22%2C%22country%22%3A%22Estonia%22%2C%22creation%22%3A1596715285%2C%22ethnic%22%3A%22Caucasian%22%2C%22photos%22%3A1%2C%22profile%22%3A1%2C%22status%22%3A%22%22%7D%2C%22m%22%3A%7B%22camscore%22%3A629.800%2C%22continent%22%3A%22EU%22%2C%22flags%22%3A3104%2C%22kbit%22%3A0%2C%22lastnews%22%3A0%2C%22mg%22%3A0%2C%22missmfc%22%3A0%2C%22new_model%22%3A1%2C%22rank%22%3A0%2C%22rc%22%3A4%2C%22sfw%22%3A0%2C%22topic%22%3A%22%22%7D%7D"); + Message msg = ReflectionUtil.call(client, "parseMessage", input); + assertEquals(20, msg.getType()); + assertEquals(418550763, msg.getSender()); + assertEquals(419337943, msg.getReceiver()); + assertEquals(0, msg.getArg1()); + assertEquals(37761695, msg.getArg2()); + assertEquals(JSONObject.class, new JSONObject(msg.getMessage()).getClass()); + } + + @Test + public void testMessageLengthParsing() throws NoSuchMethodException, SecurityException, IllegalAccessException, IllegalArgumentException, InvocationTargetException { + MyFreeCamsClient client = MyFreeCamsClient.getInstance(); + StringBuilder input = new StringBuilder("082720 418550763 419337943 0 37761695"); + int packetBytesLenth = ReflectionUtil.call(client, "parsePacketLengthBytes", input); + assertEquals(4, packetBytesLenth); + + input = new StringBuilder("82720 418550763 419337943 0 37761695"); + packetBytesLenth = ReflectionUtil.call(client, "parsePacketLengthBytes", input); + assertEquals(3, packetBytesLenth); + } +} From 192e7093d3912d3388a5a0e08c495052002153d3 Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Sat, 15 Aug 2020 13:14:59 +0200 Subject: [PATCH 48/56] Fix Flirt4Free models losing their name --- .../sites/flirt4free/Flirt4FreeModel.java | 55 ++++++++++++++----- 1 file changed, 41 insertions(+), 14 deletions(-) diff --git a/common/src/main/java/ctbrec/sites/flirt4free/Flirt4FreeModel.java b/common/src/main/java/ctbrec/sites/flirt4free/Flirt4FreeModel.java index dc332d17..a7759a94 100644 --- a/common/src/main/java/ctbrec/sites/flirt4free/Flirt4FreeModel.java +++ b/common/src/main/java/ctbrec/sites/flirt4free/Flirt4FreeModel.java @@ -86,12 +86,11 @@ public class Flirt4FreeModel extends AbstractModel { return false; } JSONObject json = new JSONObject(body); - //LOG.debug("check model status: {}", json.toString(2)); - online = Objects.equals(json.optString("status"), "online"); - id = String.valueOf(json.get("model_id")); + online = Objects.equals(json.optString("status"), "online"); // online is true, even if the model is in private or away + updateModelId(json); if (online) { try { - loadStreamUrl(); + loadModelInfo(); } catch (Exception e) { online = false; onlineState = Model.State.OFFLINE; @@ -109,6 +108,18 @@ public class Flirt4FreeModel extends AbstractModel { return online; } + private void updateModelId(JSONObject json) { + if (json.has("model_id")) { + Object modelId = json.get("model_id"); + if (modelId instanceof Number) { + Number n = (Number) modelId; + if (n.intValue() > 0) { + id = String.valueOf(json.get("model_id")); + } + } + } + } + private void loadModelInfo() throws IOException, InterruptedException { String url = getSite().getBaseUrl() + "/webservices/chat-room-interface.php?a=login_room&model_id=" + id; LOG.trace("Loading url {}", url); @@ -127,13 +138,15 @@ public class Flirt4FreeModel extends AbstractModel { // LOG.debug("chat-room-interface {}", json.toString(2)); JSONObject config = json.getJSONObject("config"); JSONObject performer = config.getJSONObject("performer"); - setName(performer.optString("name_seo", "n/a")); - setDisplayName(performer.optString("name", "n/a")); setUrl(getSite().getBaseUrl() + "/rooms/" + getName() + '/'); + setDisplayName(performer.optString("name", getName())); JSONObject room = config.getJSONObject("room"); chatHost = room.getString("host"); chatPort = room.getString("port_to_be"); chatToken = json.getString("token_enc"); + String status = room.optString("status"); + setOnlineState(mapStatus(status)); + online = onlineState == State.ONLINE; JSONObject user = config.getJSONObject("user"); userIp = user.getString("ip"); } else { @@ -147,6 +160,19 @@ public class Flirt4FreeModel extends AbstractModel { } } + private State mapStatus(String status) { + switch (status) { + case "P": + case "F": + return State.PRIVATE; + case "A": + return State.AWAY; + case "O": + default: + return State.ONLINE; + } + } + @Override public List getStreamSources() throws IOException, ExecutionException, ParseException, PlaylistException { return getStreamSources(true); @@ -238,10 +264,9 @@ public class Flirt4FreeModel extends AbstractModel { streamHost = data.getString("stream_host"); // TODO look, if the stream_host is equal to the one encoded in base64 in some of the ajax requests (parameters) online = true; isInteractiveShow = data.optString("devices").equals("1"); - if(data.optString("room_state").equals("P")) { - onlineState = Model.State.PRIVATE; - online = false; - } + String roomState = data.optString("room_state"); + onlineState = mapStatus(roomState); + online = onlineState == State.ONLINE; if(data.optString("room_state").equals("0") && data.optString("login_group_id").equals("14")) { onlineState = Model.State.GROUP; online = false; @@ -456,8 +481,10 @@ public class Flirt4FreeModel extends AbstractModel { @Override public void readSiteSpecificData(JsonReader reader) throws IOException { - reader.nextName(); - id = reader.nextString(); + if (reader.hasNext()) { + reader.nextName(); + id = reader.nextString(); + } } @Override @@ -482,7 +509,7 @@ public class Flirt4FreeModel extends AbstractModel { } private void acquireSlot() throws InterruptedException { - //LOG.debug("Acquire: {}", requestThrottle.availablePermits()); + LOG.debug("Acquire: {} - Queue: {}", requestThrottle.availablePermits(), requestThrottle.getQueueLength()); requestThrottle.acquire(); long now = System.currentTimeMillis(); long millisSinceLastRequest = now - lastRequest; @@ -495,6 +522,6 @@ public class Flirt4FreeModel extends AbstractModel { private void releaseSlot() { lastRequest = System.currentTimeMillis(); requestThrottle.release(); - //LOG.debug("Release: {}", requestThrottle.availablePermits()); + // LOG.debug("Release: {}", requestThrottle.availablePermits()); } } From 6cfdb59c9604330eb2389f0d3e6b89436979e4fa Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Sat, 15 Aug 2020 16:06:47 +0200 Subject: [PATCH 49/56] Remove log output --- .../src/main/java/ctbrec/sites/flirt4free/Flirt4FreeModel.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/src/main/java/ctbrec/sites/flirt4free/Flirt4FreeModel.java b/common/src/main/java/ctbrec/sites/flirt4free/Flirt4FreeModel.java index a7759a94..22a6e831 100644 --- a/common/src/main/java/ctbrec/sites/flirt4free/Flirt4FreeModel.java +++ b/common/src/main/java/ctbrec/sites/flirt4free/Flirt4FreeModel.java @@ -509,7 +509,7 @@ public class Flirt4FreeModel extends AbstractModel { } private void acquireSlot() throws InterruptedException { - LOG.debug("Acquire: {} - Queue: {}", requestThrottle.availablePermits(), requestThrottle.getQueueLength()); + //LOG.debug("Acquire: {} - Queue: {}", requestThrottle.availablePermits(), requestThrottle.getQueueLength()); requestThrottle.acquire(); long now = System.currentTimeMillis(); long millisSinceLastRequest = now - lastRequest; From 5c0d841474ce58416a9ba6dce6d8b42f11546c69 Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Sat, 15 Aug 2020 16:07:16 +0200 Subject: [PATCH 50/56] Shut down all recordings simultaneously --- .../ctbrec/recorder/NextGenLocalRecorder.java | 17 +++++++++++++++-- .../download/hls/MergedFfmpegHlsDownload.java | 6 +++--- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/common/src/main/java/ctbrec/recorder/NextGenLocalRecorder.java b/common/src/main/java/ctbrec/recorder/NextGenLocalRecorder.java index c7826077..c0c93f57 100644 --- a/common/src/main/java/ctbrec/recorder/NextGenLocalRecorder.java +++ b/common/src/main/java/ctbrec/recorder/NextGenLocalRecorder.java @@ -25,6 +25,7 @@ import java.util.concurrent.BlockingQueue; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorCompletionService; +import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.LinkedBlockingQueue; @@ -505,8 +506,20 @@ public class NextGenLocalRecorder implements Recorder { try { // make a copy to avoid ConcurrentModificationException List toStop = new ArrayList<>(recordingProcesses.values()); - for (Recording rec : toStop) { - Optional.ofNullable(rec.getDownload()).ifPresent(Download::stop); + if (!toStop.isEmpty()) { + ExecutorService shutdownPool = Executors.newFixedThreadPool(toStop.size()); + List> shutdownFutures = new ArrayList<>(toStop.size()); + for (Recording rec : toStop) { + Optional.ofNullable(rec.getDownload()).ifPresent(d -> { + shutdownFutures.add(shutdownPool.submit(() -> d.stop())); + }); + } + shutdownPool.shutdown(); + try { + shutdownPool.awaitTermination(10, TimeUnit.MINUTES); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } } } finally { recorderLock.unlock(); diff --git a/common/src/main/java/ctbrec/recorder/download/hls/MergedFfmpegHlsDownload.java b/common/src/main/java/ctbrec/recorder/download/hls/MergedFfmpegHlsDownload.java index 08d8a641..b90028fe 100644 --- a/common/src/main/java/ctbrec/recorder/download/hls/MergedFfmpegHlsDownload.java +++ b/common/src/main/java/ctbrec/recorder/download/hls/MergedFfmpegHlsDownload.java @@ -391,13 +391,13 @@ public class MergedFfmpegHlsDownload extends AbstractHlsDownload { try { downloadQueue.clear(); + for (Future future : downloads) { + future.cancel(true); + } downloadThreadPool.shutdownNow(); LOG.debug("Waiting for segment download thread pool to terminate for model {}", getModel()); downloadThreadPool.awaitTermination(60, TimeUnit.SECONDS); LOG.debug("Segment download thread pool terminated for model {}", getModel()); - for (Future future : downloads) { - future.cancel(true); - } } catch (InterruptedException e) { LOG.error("Interrupted while waiting for segment pool to shutdown"); Thread.currentThread().interrupt(); From 792a6c10c8089581f871fcfb15a9f31c521a5948 Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Sat, 15 Aug 2020 17:17:31 +0200 Subject: [PATCH 51/56] Revert MFC websocket message parsing change --- .../ctbrec/sites/mfc/MyFreeCamsClient.java | 8 ++--- .../sites/mfc/MyFreeCamsClientTest.java | 34 +++++++++---------- 2 files changed, 18 insertions(+), 24 deletions(-) diff --git a/common/src/main/java/ctbrec/sites/mfc/MyFreeCamsClient.java b/common/src/main/java/ctbrec/sites/mfc/MyFreeCamsClient.java index 45178981..70b7ad8a 100644 --- a/common/src/main/java/ctbrec/sites/mfc/MyFreeCamsClient.java +++ b/common/src/main/java/ctbrec/sites/mfc/MyFreeCamsClient.java @@ -482,8 +482,8 @@ public class MyFreeCamsClient { } private Message parseMessage(StringBuilder msgBuffer) throws UnsupportedEncodingException { - int packetLengthBytes = parsePacketLengthBytes(msgBuffer); - if (packetLengthBytes < 0) { + int packetLengthBytes = 6; + if (msgBuffer.length() < packetLengthBytes) { // packet size not transmitted completely return null; } else { @@ -522,10 +522,6 @@ public class MyFreeCamsClient { } } - private int parsePacketLengthBytes(StringBuilder msgBuffer) { - return msgBuffer.indexOf(" ") - 2; - } - private int parseNextInt(StringBuilder s) { int nextSpace = s.indexOf(" "); int i = Integer.parseInt(s.substring(0, nextSpace)); diff --git a/common/src/test/java/ctbrec/sites/mfc/MyFreeCamsClientTest.java b/common/src/test/java/ctbrec/sites/mfc/MyFreeCamsClientTest.java index d8bbdac3..7e171ffa 100644 --- a/common/src/test/java/ctbrec/sites/mfc/MyFreeCamsClientTest.java +++ b/common/src/test/java/ctbrec/sites/mfc/MyFreeCamsClientTest.java @@ -11,28 +11,26 @@ import ctbrec.ReflectionUtil; public class MyFreeCamsClientTest { + @Test public void testMessageParsing() throws NoSuchMethodException, SecurityException, IllegalAccessException, IllegalArgumentException, InvocationTargetException { MyFreeCamsClient client = MyFreeCamsClient.getInstance(); - StringBuilder input = new StringBuilder("082720 418550763 419337943 0 37761695 %7B%22lv%22%3A4%2C%22nm%22%3A%22AliceArras%22%2C%22pid%22%3A1%2C%22sid%22%3A418550763%2C%22uid%22%3A37761695%2C%22vs%22%3A0%2C%22u%22%3A%7B%22age%22%3A19%2C%22avatar%22%3A1%2C%22blurb%22%3A%22Open%20minded%2C%20willing%20to%20try%20new%20horny%20things%20all%20the%20time!%22%2C%22camserv%22%3A1205%2C%22chat_font%22%3A0%2C%22chat_opt%22%3A1%2C%22city%22%3A%22Tallinn%22%2C%22country%22%3A%22Estonia%22%2C%22creation%22%3A1596715285%2C%22ethnic%22%3A%22Caucasian%22%2C%22photos%22%3A1%2C%22profile%22%3A1%2C%22status%22%3A%22%22%7D%2C%22m%22%3A%7B%22camscore%22%3A629.800%2C%22continent%22%3A%22EU%22%2C%22flags%22%3A3104%2C%22kbit%22%3A0%2C%22lastnews%22%3A0%2C%22mg%22%3A0%2C%22missmfc%22%3A0%2C%22new_model%22%3A1%2C%22rank%22%3A0%2C%22rc%22%3A4%2C%22sfw%22%3A0%2C%22topic%22%3A%22%22%7D%7D"); + StringBuilder input = new StringBuilder("00001933 0 439895060 1 0 00001933 0 439895060 1 0 0000208 0 439895060 0 112 00038..."); Message msg = ReflectionUtil.call(client, "parseMessage", input); + assertEquals(33, msg.getType()); + assertEquals(0, msg.getSender()); + assertEquals(439895060, msg.getReceiver()); + assertEquals(1, msg.getArg1()); + assertEquals(0, msg.getArg2()); + assertTrue(msg.getMessage().isBlank()); + + input = new StringBuilder("00207820 439461784 439895060 12 507930 %7B%22lv%22%3A4%2C%22nm%22%3A%22Nivea%22%2C%22pid%22%3A1%2C%22sid%22%3A439461784%2C%22uid%22%3A507930%2C%22vs%22%3A12%2C%22u%22%3A%7B%22age%22%3A33%2C%22avatar%22%3A1%2C%22blurb%22%3A%22I%20love%20when%20my%20nose%20touch%20your%20belly%20when%20I%20do%20you%20a%20blowjob!%20When%20I%20look%20into%20your%20horny%20eyes%20when%22%2C%22camserv%22%3A1367%2C%22chat_color%22%3A%22FF0000%22%2C%22chat_font%22%3A0%2C%22chat_opt%22%3A1%2C%22ethnic%22%3A%22Caucasian%22%2C%22photos%22%3A74%2C%22profile%22%3A1%2C%22status%22%3A%22%22%7D%2C%22m%22%3A%7B%22camscore%22%3A7505.700%2C%22continent%22%3A%22EU%22%2C%22flags%22%3A605224%2C%22hidecs%22%3Atrue%2C%22kbit%22%3A0%2C%22lastnews%22%3A0%2C%22mg%22%3A0%2C%22missmfc%22%3A2%2C%22new_model%22%3A0%2C%22rank%22%3A0%2C%22rc%22%3A20%2C%22sfw%22%3A0%2C%22topic%22%3A%22hi%253A)%255Bnone%255D-Topless%252C500-snap4life%252C50-spanks%252C180-flash%252C700-10%2520mins%2520of%2520Nora%2520fun%252C666-shot%252C27%252C270%2520%253C3%22%7D%2C%22x%22%3A%7B%22fcext%22%3A%7B%22sm%22%3A%22%22%2C%22sfw%22%3A0%7D%2C%22share%22%3A%7B%22follows%22%3A11%2C%22albums%22%3A0%2C%22clubs%22%3A0%2C%22tm_album%22%3A0%2C%22collections%22%3A0%2C%22stores%22%3A0%2C%22goals%22%3A0%2C%22polls%22%3A0%2C%22things%22%3A0%2C%22recent_album_tm%22%3A0%2C%22recent_club_tm%22%3A0%2C%22recent_collection_tm%22%3A0%2C%22recent_goal_tm%22%3A0%2C%22recent_item_tm%22%3A0%2C%22recent_poll_tm%22%3A0%2C%22recent_story_tm%22%3A0%2C%22recent_album_thumb%22%3A%22%22%2C%22recent_club_thumb%22%3A%22%22%2C%22recent_collection_thumb%22%3A%22%22%2C%22recent_goal_thumb%22%3A%22%22%2C%22recent_item_thumb%22%3A%22%22%2C%22recent_poll_thumb%22%3A%22%22%2C%22recent_story_thumb%22%3A%22%22%2C%22recent_album_title%22%3A%22%22%2C%22recent_club_title%22%3A%22%22%2C%22recent_collection_title%22%3A%22%22%2C%22recent_goal_title%22%3A%22%22%2C%22recent_item_title%22%3A%22%22%2C%22recent_poll_title%22%3A%22%22%2C%22recent_story_title%22%3A%22%22%2C%22recent_album_slug%22%3A%22%22%2C%22recent_collection_slug%22%3A%22%22%2C%22tipmenus%22%3A0%7D%7D%7D"); + msg = ReflectionUtil.call(client, "parseMessage", input); assertEquals(20, msg.getType()); - assertEquals(418550763, msg.getSender()); - assertEquals(419337943, msg.getReceiver()); - assertEquals(0, msg.getArg1()); - assertEquals(37761695, msg.getArg2()); - assertEquals(JSONObject.class, new JSONObject(msg.getMessage()).getClass()); - } - - @Test - public void testMessageLengthParsing() throws NoSuchMethodException, SecurityException, IllegalAccessException, IllegalArgumentException, InvocationTargetException { - MyFreeCamsClient client = MyFreeCamsClient.getInstance(); - StringBuilder input = new StringBuilder("082720 418550763 419337943 0 37761695"); - int packetBytesLenth = ReflectionUtil.call(client, "parsePacketLengthBytes", input); - assertEquals(4, packetBytesLenth); - - input = new StringBuilder("82720 418550763 419337943 0 37761695"); - packetBytesLenth = ReflectionUtil.call(client, "parsePacketLengthBytes", input); - assertEquals(3, packetBytesLenth); + assertEquals(439461784, msg.getSender()); + assertEquals(439895060, msg.getReceiver()); + assertEquals(12, msg.getArg1()); + assertEquals(507930, msg.getArg2()); + assertEquals(JSONObject.class, new JSONObject(msg.getMessage()).getClass()); // make sure we have a parsable JSON message } } From fe9ac0680d379c7f05ef20b096ebd925bb5d6030 Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Sun, 16 Aug 2020 13:09:37 +0200 Subject: [PATCH 52/56] Add context menu to logging table --- .../ctbrec/ui/tabs/logging/LoggingTab.java | 74 +++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/client/src/main/java/ctbrec/ui/tabs/logging/LoggingTab.java b/client/src/main/java/ctbrec/ui/tabs/logging/LoggingTab.java index dc3a4c01..b6482b1a 100644 --- a/client/src/main/java/ctbrec/ui/tabs/logging/LoggingTab.java +++ b/client/src/main/java/ctbrec/ui/tabs/logging/LoggingTab.java @@ -7,8 +7,13 @@ import java.util.Collections; import java.util.LinkedList; import java.util.stream.Collectors; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import com.google.common.eventbus.Subscribe; +import ch.qos.logback.classic.LoggerContext; +import ch.qos.logback.classic.encoder.PatternLayoutEncoder; import ch.qos.logback.classic.spi.IThrowableProxy; import ch.qos.logback.classic.spi.LoggingEvent; import ch.qos.logback.classic.spi.StackTraceElementProxy; @@ -19,9 +24,16 @@ import javafx.beans.property.SimpleStringProperty; import javafx.collections.FXCollections; import javafx.collections.ObservableList; import javafx.geometry.Insets; +import javafx.scene.control.ContextMenu; +import javafx.scene.control.MenuItem; +import javafx.scene.control.SelectionMode; import javafx.scene.control.Tab; import javafx.scene.control.TableColumn; import javafx.scene.control.TableView; +import javafx.scene.input.Clipboard; +import javafx.scene.input.ClipboardContent; +import javafx.scene.input.ContextMenuEvent; +import javafx.scene.input.MouseEvent; import javafx.scene.layout.BorderPane; public class LoggingTab extends Tab { @@ -33,6 +45,7 @@ public class LoggingTab extends Tab { private ObservableList filteredEvents = FXCollections.observableArrayList(); private DateTimeFormatter timeFormatter = DateTimeFormatter.ISO_LOCAL_DATE_TIME; private volatile boolean tabClosed = false; + private ContextMenu popup; private Object eventBustSubscriber = new Object() { @Subscribe public void publishLoggingEevent(LoggingEvent event) { @@ -75,6 +88,14 @@ public class LoggingTab extends Tab { }); table.getColumns().add(time); + TableColumn location = createTableColumn("Location", 250, idx++); + location.setCellValueFactory(cdf -> { + StackTraceElement loc = cdf.getValue().getCallerData()[0]; + String l = loc.getFileName() + ":" + loc.getLineNumber(); + return new SimpleStringProperty(l); + }); + table.getColumns().add(location); + TableColumn msg = createTableColumn("Message", 2000, idx++); msg.setCellValueFactory(cdf -> new SimpleStringProperty(createLogMessage(cdf.getValue()))); table.getColumns().add(msg); @@ -93,6 +114,59 @@ public class LoggingTab extends Tab { filter.setPromptText("Search"); filter.textProperty().addListener( (observableValue, oldValue, newValue) -> filter()); + + table.getSelectionModel().setSelectionMode(SelectionMode.MULTIPLE); + table.addEventHandler(ContextMenuEvent.CONTEXT_MENU_REQUESTED, event -> { + popup = createContextMenu(); + if (popup != null) { + popup.show(table, event.getScreenX(), event.getScreenY()); + } + event.consume(); + }); + table.addEventHandler(MouseEvent.MOUSE_PRESSED, event -> { + if (popup != null) { + popup.hide(); + } + }); + } + + private ContextMenu createContextMenu() { + final ObservableList selectedEvents = table.getSelectionModel().getSelectedItems(); + if (selectedEvents.isEmpty()) { + return null; + } + MenuItem copy = new MenuItem("Copy"); + copy.setOnAction(e -> { + Platform.runLater(() -> { + String formattedMessages = getFormattedMessages(selectedEvents); + final Clipboard clipboard = Clipboard.getSystemClipboard(); + final ClipboardContent content = new ClipboardContent(); + content.putString(formattedMessages); + clipboard.setContent(content); + }); + }); + + ContextMenu menu = new ContextMenu(copy); + return menu; + } + + private String getFormattedMessages(ObservableList selectedEvents) { + StringBuilder sb = new StringBuilder(); + + ch.qos.logback.classic.Logger rootLogger = (ch.qos.logback.classic.Logger)LoggerFactory.getLogger(Logger.ROOT_LOGGER_NAME); + LoggerContext loggerContext = rootLogger.getLoggerContext(); + loggerContext.reset(); + + PatternLayoutEncoder encoder = new PatternLayoutEncoder(); + encoder.setContext(loggerContext); + encoder.setPattern("%date %level [%thread] %logger{10} [%file:%line] %msg%n"); + encoder.start(); + + for (LoggingEvent evt : selectedEvents) { + byte[] encode = encoder.encode(evt); + sb.append(new String(encode)); + } + return sb.toString(); } private String createLogMessage(LoggingEvent evt) { From 4139e42ce20f54c7ea332fbc07b990b3c2e645c8 Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Sun, 16 Aug 2020 14:16:38 +0200 Subject: [PATCH 53/56] Set version to 3.8.6 --- CHANGELOG.md | 3 +++ client/pom.xml | 2 +- common/pom.xml | 2 +- master/pom.xml | 2 +- server/pom.xml | 2 +- 5 files changed, 7 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 70fd84bd..404d9826 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,9 @@ 3.8.6 ======================== * Added setting to disable the online check for paused models +* Speed up shutdown process by stopping all recordings simultaneously +* Fixed: Flirt4Model loose their name after some time +* Made loading of config file more robust for Flirt4Free models * Added tab which shows the log output 3.8.5 diff --git a/client/pom.xml b/client/pom.xml index 7becf27d..518af8db 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -8,7 +8,7 @@ ctbrec master - 3.8.5 + 3.8.6 ../master diff --git a/common/pom.xml b/common/pom.xml index 3ae29f86..4547a408 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -8,7 +8,7 @@ ctbrec master - 3.8.5 + 3.8.6 ../master diff --git a/master/pom.xml b/master/pom.xml index b695f290..d332594f 100644 --- a/master/pom.xml +++ b/master/pom.xml @@ -6,7 +6,7 @@ ctbrec master pom - 3.8.5 + 3.8.6 ../common diff --git a/server/pom.xml b/server/pom.xml index 37a85098..4d56941e 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -8,7 +8,7 @@ ctbrec master - 3.8.5 + 3.8.6 ../master From dbd9e00600b23ac6a662d4fa6512e410a9f95d21 Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Sun, 16 Aug 2020 15:00:49 +0200 Subject: [PATCH 54/56] Fix Streamate followed tab once again --- .../ctbrec/ui/sites/streamate/StreamateFollowedService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/src/main/java/ctbrec/ui/sites/streamate/StreamateFollowedService.java b/client/src/main/java/ctbrec/ui/sites/streamate/StreamateFollowedService.java index 6b4dd129..6548f0e8 100644 --- a/client/src/main/java/ctbrec/ui/sites/streamate/StreamateFollowedService.java +++ b/client/src/main/java/ctbrec/ui/sites/streamate/StreamateFollowedService.java @@ -41,7 +41,7 @@ public class StreamateFollowedService extends PaginatedScheduledService { public StreamateFollowedService(Streamate streamate) { this.streamate = streamate; this.httpClient = (StreamateHttpClient) streamate.getHttpClient(); - this.url = "https://member.naiadsystems.com/search/favorites?domain=streamate.com&skipXmentSelection=true"; + this.url = "https://member.naiadsystems.com/search/v3/favorites?skipXmentSelection=true&skinConfig=%7B%7D&filters="; } @Override From 79a035529702e1fb83e5a6ee068abd7f47b6e1cf Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Sun, 16 Aug 2020 15:02:09 +0200 Subject: [PATCH 55/56] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 404d9826..d5ec653d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ======================== * Added setting to disable the online check for paused models * Speed up shutdown process by stopping all recordings simultaneously +* Fixed Streamate followed tab once again * Fixed: Flirt4Model loose their name after some time * Made loading of config file more robust for Flirt4Free models * Added tab which shows the log output From acec91ee69d1cb5ba04e8c0a9cf5d19d1cb917e0 Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Sun, 16 Aug 2020 15:21:25 +0200 Subject: [PATCH 56/56] Update changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d5ec653d..4e49c833 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ * Added setting to disable the online check for paused models * Speed up shutdown process by stopping all recordings simultaneously * Fixed Streamate followed tab once again -* Fixed: Flirt4Model loose their name after some time +* Fixed: Flirt4Free models loose their name after some time * Made loading of config file more robust for Flirt4Free models * Added tab which shows the log output