diff --git a/CHANGELOG.md b/CHANGELOG.md index ffea0876..4e49c833 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,54 @@ +3.8.6 +======================== +* Added setting to disable the online check for paused models +* Speed up shutdown process by stopping all recordings simultaneously +* Fixed Streamate followed tab once again +* Fixed: Flirt4Free models loose their name after some time +* Made loading of config file more robust for Flirt4Free models +* Added tab which shows the log output + +3.8.5 +======================== +* Fixed Stripchat followed tab. It didn't work, if you have many favorited + models +* Fixed: Some Stripchat models didn't get recorded +* Fixed: Some LiveJasmin models didn't get recorded +* Added support for temporary recordings. On the recording tab you can now set + a date, when to stop recording a model and what to do afterwards + (pause or remove the model) +* Changed the look of the model table in the web interface a bit + +3.8.4 +======================== +* Added support for xHamsterLive (go to Settings -> Sites -> Stripchat, + switch to xHamsterLive, enter your credentials and restart) +* Fixed follow / unfollow for Stripchat +* Enable rerun PP for multiple recordings +* Fixed bug, which prevented recordings to finish properly on app + shutdown. Recordings now shouldn't end up in state waiting anymore + +3.8.3 +======================== +* Fixed Streamate +* Fixed favorites tab for Cam4; kind of, because only the online tab works. + I currently don't see a way to retrieve the offline favorites +* Fixed favorites tab for CamSoda +* Fixed CamSoda recordings +* Added external login dialog for Stripchat to support the captcha + +3.8.2 +======================== +* Fixed misconfiguration in global connection pool, which caused a lot of + threads to spawn while browsing in the thumbnail overviews +* Improved memory handling for the thumbnail overviews. Thumbnail images were + not released, when a tab was switched. This caused a huge memory consumption, + if you opened a lot of different tabs. +* Fixed a bug in MFC websocket client, which caused to spawn a bunch of + "keep-alive" threads, if there was a problem with the connection +* Reworked the settings tab +* Fire recording finished event, if a download from the server is finished +* Ignore min/max resolution, if the resolution is unknown + 3.8.1 ======================== * Fixed recent MFC error @@ -10,7 +61,7 @@ * Models can be added by name in the web-interface * Added a bandwidth monitor * Added possibility to add notes to recordings -* Added range slider to restrict the recording resolution in a range; e.g. 480p - 1080p +* Added range slider to restrict the recording resolution in a range; e.g. 480p - 1080p * Improved MFC SD downloads (much less blocking, I think) 3.7.3 @@ -26,7 +77,7 @@ 3.7.1 ======================== * Server now logs in on startup, if credentials are set -* Show confirmation dialog on shutdown, if the are active downloads from the +* Show confirmation dialog on shutdown, if the are active downloads from the server * Added setting to remove recordings after post-processing * Added max resolution setting for the player (click on the gear!) diff --git a/client/build.sh b/client/build.sh index c87fc3cc..ec3723aa 100755 --- a/client/build.sh +++ b/client/build.sh @@ -1,5 +1,4 @@ #!/bin/bash -export JAVA_HOME=/opt/jdk-11.0.1 mvn clean mvn -Djavafx.platform=win package verify mvn -Djavafx.platform=linux package verify diff --git a/client/pom.xml b/client/pom.xml index 76f36cf6..518af8db 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -8,7 +8,7 @@ ctbrec master - 3.8.1 + 3.8.6 ../master @@ -71,7 +71,7 @@ ch.qos.logback logback-classic - runtime + compile org.openjfx @@ -91,7 +91,7 @@ com.vladsch.flexmark - flexmark-all + flexmark 0.40.34 @@ -140,8 +140,8 @@ ${project.version}.0 ${project.version} - Recorder for Charturbate streams - 2018 0xboobface + Software to record live streams + 2020 0xboobface ${project.version}.0 ${project.version} CTB Recorder diff --git a/client/src/main/java/ctbrec/ui/CamrecApplication.java b/client/src/main/java/ctbrec/ui/CamrecApplication.java index 9eb41adf..8557ae93 100644 --- a/client/src/main/java/ctbrec/ui/CamrecApplication.java +++ b/client/src/main/java/ctbrec/ui/CamrecApplication.java @@ -62,6 +62,7 @@ import ctbrec.ui.tabs.RecordingsTab; import ctbrec.ui.tabs.SiteTab; import ctbrec.ui.tabs.TabSelectionListener; import ctbrec.ui.tabs.UpdateTab; +import ctbrec.ui.tabs.logging.LoggingTab; import javafx.application.Application; import javafx.application.HostServices; import javafx.application.Platform; @@ -143,8 +144,6 @@ public class CamrecApplication extends Application { } private void startOnlineMonitor() { - onlineMonitor = new OnlineMonitor(recorder); - onlineMonitor.start(); for (Site site : sites) { if(site.isEnabled()) { try { @@ -155,6 +154,8 @@ public class CamrecApplication extends Application { } } } + onlineMonitor = new OnlineMonitor(recorder, config); + onlineMonitor.start(); } private void logEnvironment() { @@ -193,6 +194,7 @@ public class CamrecApplication extends Application { tabPane.getTabs().add(new NewsTab()); tabPane.getTabs().add(new DonateTabFx()); tabPane.getTabs().add(new HelpTab()); + tabPane.getTabs().add(new LoggingTab()); switchToStartTab(); writeColorSchemeStyleSheet(); diff --git a/client/src/main/java/ctbrec/ui/JavaFxModel.java b/client/src/main/java/ctbrec/ui/JavaFxModel.java index bc404953..21885b73 100644 --- a/client/src/main/java/ctbrec/ui/JavaFxModel.java +++ b/client/src/main/java/ctbrec/ui/JavaFxModel.java @@ -13,6 +13,7 @@ import com.squareup.moshi.JsonReader; import com.squareup.moshi.JsonWriter; import ctbrec.Model; +import ctbrec.SubsequentAction; import ctbrec.recorder.download.Download; import ctbrec.recorder.download.HttpHeaderFactory; import ctbrec.recorder.download.StreamSource; @@ -286,4 +287,24 @@ public class JavaFxModel implements Model { public HttpHeaderFactory getHttpHeaderFactory() { return delegate.getHttpHeaderFactory(); } + + @Override + public Instant getRecordUntil() { + return delegate.getRecordUntil(); + } + + @Override + public void setRecordUntil(Instant instant) { + delegate.setRecordUntil(instant); + } + + @Override + public SubsequentAction getRecordUntilSubsequentAction() { + return delegate.getRecordUntilSubsequentAction(); + } + + @Override + public void setRecordUntilSubsequentAction(SubsequentAction action) { + delegate.setRecordUntilSubsequentAction(action); + } } diff --git a/client/src/main/java/ctbrec/ui/controls/Dialogs.java b/client/src/main/java/ctbrec/ui/controls/Dialogs.java index c96ae2c1..fec5c51b 100644 --- a/client/src/main/java/ctbrec/ui/controls/Dialogs.java +++ b/client/src/main/java/ctbrec/ui/controls/Dialogs.java @@ -16,6 +16,7 @@ import javafx.scene.control.Dialog; import javafx.scene.control.TextArea; import javafx.scene.image.Image; import javafx.scene.layout.GridPane; +import javafx.scene.layout.Region; import javafx.stage.Modality; import javafx.stage.Stage; @@ -87,6 +88,23 @@ public class Dialogs { return dialog.showAndWait(); } + public static Boolean showCustomInput(Scene parent, String title, Region region) { + Dialog dialog = new Dialog<>(); + dialog.setTitle(title); + dialog.getDialogPane().getButtonTypes().addAll(ButtonType.OK, ButtonType.CANCEL); + dialog.initModality(Modality.APPLICATION_MODAL); + dialog.setResizable(true); + InputStream icon = Dialogs.class.getResourceAsStream("/icon.png"); + Stage stage = (Stage) dialog.getDialogPane().getScene().getWindow(); + stage.getIcons().add(new Image(icon)); + if (parent != null) { + stage.getScene().getStylesheets().addAll(parent.getStylesheets()); + } + dialog.getDialogPane().setContent(region); + dialog.showAndWait(); + return dialog.getResult() == ButtonType.OK; + } + public static boolean showConfirmDialog(String title, String message, String header, Scene parent) { AutosizeAlert confirm = new AutosizeAlert(AlertType.CONFIRMATION, message, parent, YES, NO); confirm.setTitle(title); diff --git a/client/src/main/java/ctbrec/ui/settings/SettingsTab.java b/client/src/main/java/ctbrec/ui/settings/SettingsTab.java index 0f8e7cbc..f83a38f7 100644 --- a/client/src/main/java/ctbrec/ui/settings/SettingsTab.java +++ b/client/src/main/java/ctbrec/ui/settings/SettingsTab.java @@ -86,6 +86,7 @@ public class SettingsTab extends Tab implements TabSelectionListener { private DiscreteRange rangeValues = new DiscreteRange<>(values, labels); private SimpleIntegerProperty concurrentRecordings; private SimpleIntegerProperty onlineCheckIntervalInSecs; + private SimpleBooleanProperty onlineCheckSkipsPausedModels; private SimpleLongProperty leaveSpaceOnDevice; private SimpleIntegerProperty minimumLengthInSecs; private SimpleStringProperty ffmpegParameters; @@ -149,6 +150,7 @@ public class SettingsTab extends Tab implements TabSelectionListener { postProcessing = new SimpleFileProperty(null, "postProcessing", settings.postProcessing); postProcessingThreads = new SimpleIntegerProperty(null, "postProcessingThreads", settings.postProcessingThreads); removeRecordingAfterPp = new SimpleBooleanProperty(null, "removeRecordingAfterPostProcessing", settings.removeRecordingAfterPostProcessing); + onlineCheckSkipsPausedModels = new SimpleBooleanProperty(null, "onlineCheckSkipsPausedModels", settings.onlineCheckSkipsPausedModels); } private void createGui() { @@ -187,10 +189,11 @@ public class SettingsTab extends Tab implements TabSelectionListener { Setting.of("Split recordings after (minutes)", splitAfter).converter(SplitAfterOption.converter()), Setting.of("Restrict Resolution", resolutionRange, "Only record streams with resolution within the given range"), Setting.of("Concurrent Recordings (0 = unlimited)", concurrentRecordings), - Setting.of("Check online state every (seconds)", onlineCheckIntervalInSecs, "Check every x seconds, if a model came online"), Setting.of("Leave space on device (GiB)", leaveSpaceOnDevice, "Stop recording, if the free space on the device gets below this threshold").converter(new GigabytesConverter()), Setting.of("FFmpeg parameters", ffmpegParameters, "FFmpeg parameters to use when merging stream segments"), - Setting.of("File Extension", fileExtension, "File extension to use for recordings") + Setting.of("File Extension", fileExtension, "File extension to use for recordings"), + Setting.of("Check online state every (seconds)", onlineCheckIntervalInSecs, "Check every x seconds, if a model came online"), + Setting.of("Skip online check for paused models", onlineCheckSkipsPausedModels, "Skip online check for paused models") ), Group.of("Location", Setting.of("Record Location", recordLocal), @@ -236,6 +239,7 @@ public class SettingsTab extends Tab implements TabSelectionListener { prefs.getSetting("minimumResolution").ifPresent(s -> bindEnabledProperty(s, recordLocal.not())); prefs.getSetting("recordingsDirStructure").ifPresent(s -> bindEnabledProperty(s, recordLocal.not())); prefs.getSetting("onlineCheckIntervalInSecs").ifPresent(s -> bindEnabledProperty(s, recordLocal.not())); + prefs.getSetting("onlineCheckSkipsPausedModels").ifPresent(s -> bindEnabledProperty(s, recordLocal.not())); prefs.getSetting("minimumSpaceLeftInBytes").ifPresent(s -> bindEnabledProperty(s, recordLocal.not())); prefs.getSetting("postProcessing").ifPresent(s -> bindEnabledProperty(s, recordLocal.not())); prefs.getSetting("postProcessingThreads").ifPresent(s -> bindEnabledProperty(s, recordLocal.not())); diff --git a/client/src/main/java/ctbrec/ui/sites/bonga/BongaCamsElectronLoginDialog.java b/client/src/main/java/ctbrec/ui/sites/bonga/BongaCamsElectronLoginDialog.java index 0627e4aa..21e60397 100644 --- a/client/src/main/java/ctbrec/ui/sites/bonga/BongaCamsElectronLoginDialog.java +++ b/client/src/main/java/ctbrec/ui/sites/bonga/BongaCamsElectronLoginDialog.java @@ -63,6 +63,7 @@ public class BongaCamsElectronLoginDialog { } String password = Config.getInstance().getSettings().bongaPassword; if (password != null && !password.trim().isEmpty()) { + password = password.replace("'", "\\'"); browser.executeJavaScript("$('input[name=\"log_in[password]\"]').attr('value','" + password + "')"); } String[] simplify = new String[] { diff --git a/client/src/main/java/ctbrec/ui/sites/cam4/Cam4ElectronLoginDialog.java b/client/src/main/java/ctbrec/ui/sites/cam4/Cam4ElectronLoginDialog.java index 6adc8f0d..7f79ea0a 100644 --- a/client/src/main/java/ctbrec/ui/sites/cam4/Cam4ElectronLoginDialog.java +++ b/client/src/main/java/ctbrec/ui/sites/cam4/Cam4ElectronLoginDialog.java @@ -62,6 +62,7 @@ public class Cam4ElectronLoginDialog { } String password = Config.getInstance().getSettings().cam4Password; if (password != null && !password.trim().isEmpty()) { + password = password.replace("'", "\\'"); browser.executeJavaScript("document.querySelector('#loginPageForm input[name=\"password\"]').value = '" + password + "';"); } browser.executeJavaScript("document.getElementById('footer').setAttribute('style', 'display:none');"); diff --git a/client/src/main/java/ctbrec/ui/sites/cam4/Cam4FollowedUpdateService.java b/client/src/main/java/ctbrec/ui/sites/cam4/Cam4FollowedUpdateService.java index 05144d23..e96b8b64 100644 --- a/client/src/main/java/ctbrec/ui/sites/cam4/Cam4FollowedUpdateService.java +++ b/client/src/main/java/ctbrec/ui/sites/cam4/Cam4FollowedUpdateService.java @@ -3,19 +3,13 @@ package ctbrec.ui.sites.cam4; import java.io.IOException; import java.util.ArrayList; import java.util.List; -import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.stream.Collectors; -import org.jsoup.nodes.Element; -import org.jsoup.select.Elements; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.json.JSONArray; +import org.json.JSONObject; -import ctbrec.Config; import ctbrec.Model; -import ctbrec.io.HtmlParser; import ctbrec.io.HttpException; import ctbrec.sites.cam4.Cam4; import ctbrec.sites.cam4.Cam4Model; @@ -27,7 +21,6 @@ import okhttp3.Response; public class Cam4FollowedUpdateService extends PaginatedScheduledService { - private static final Logger LOG = LoggerFactory.getLogger(Cam4FollowedUpdateService.class); private Cam4 site; private boolean showOnline = true; @@ -50,46 +43,28 @@ public class Cam4FollowedUpdateService extends PaginatedScheduledService { // login first SiteUiFactory.getUi(site).login(); List models = new ArrayList<>(); - String username = Config.getInstance().getSettings().cam4Username; - String url = site.getBaseUrl() + '/' + username + "/edit/friends_favorites"; + String url = site.getBaseUrl() + "/directoryCams?directoryJson=true&online=" + showOnline + "&url=true&friends=true&favorites=true&resultsPerPage=90"; Request req = new Request.Builder().url(url).build(); - try(Response response = site.getHttpClient().execute(req)) { - if(response.isSuccessful()) { + try (Response response = site.getHttpClient().execute(req)) { + if (response.isSuccessful()) { String content = response.body().string(); - Elements cells = HtmlParser.getTags(content, "div#favorites div.ff_thumb"); - for (Element cell : cells) { - String cellHtml = cell.html(); - Element link = HtmlParser.getTag(cellHtml, "div.ff_img a"); - String path = link.attr("href"); - String modelName = path.substring(1); - Cam4Model model = (Cam4Model) site.createModel(modelName); - model.setPreview("https://snapshots.xcdnpro.com/thumbnails/"+model.getName()+"?s=" + System.currentTimeMillis()); - model.setOnlineStateByShowType(parseOnlineState(cellHtml)); + JSONObject json = new JSONObject(content); + JSONArray users = json.getJSONArray("users"); + for (int i = 0; i < users.length(); i++) { + JSONObject modelJson = users.getJSONObject(i); + String username = modelJson.optString("username"); + Cam4Model model = site.createModel(username); + model.setPreview(modelJson.optString("snapshotImageLink")); + model.setOnlineStateByShowType(modelJson.optString("showType")); + model.setDescription(modelJson.optString("statusMessage")); models.add(model); } - return models.stream() - .filter(m -> { - try { - return m.isOnline() == showOnline; - } catch (IOException | ExecutionException e) { - LOG.error("Couldn't determine online state", e); - return false; - } catch(InterruptedException e) { - Thread.currentThread().interrupt(); - LOG.error("Couldn't determine online state", e); - return false; - } - }).collect(Collectors.toList()); + return models; } else { throw new HttpException(response.code(), response.message()); } } } - - private String parseOnlineState(String cellHtml) { - Element state = HtmlParser.getTag(cellHtml, "div.ff_name div"); - return state.attr("class").equals("online") ? "NORMAL" : "OFFLINE"; - } }; } diff --git a/client/src/main/java/ctbrec/ui/sites/camsoda/CamsodaFollowedTab.java b/client/src/main/java/ctbrec/ui/sites/camsoda/CamsodaFollowedTab.java index d9904195..f3872970 100644 --- a/client/src/main/java/ctbrec/ui/sites/camsoda/CamsodaFollowedTab.java +++ b/client/src/main/java/ctbrec/ui/sites/camsoda/CamsodaFollowedTab.java @@ -1,6 +1,9 @@ package ctbrec.ui.sites.camsoda; +import java.util.function.Predicate; + import ctbrec.sites.camsoda.Camsoda; +import ctbrec.sites.camsoda.CamsodaModel; import ctbrec.ui.tabs.FollowedTab; import ctbrec.ui.tabs.ThumbOverviewTab; import javafx.concurrent.WorkerStateEvent; @@ -18,9 +21,10 @@ public class CamsodaFollowedTab extends ThumbOverviewTab implements FollowedTab boolean showOnline = true; public CamsodaFollowedTab(String title, Camsoda camsoda) { - super(title, new CamsodaFollowedUpdateService(camsoda), camsoda); + super(title, new CamsodaUpdateService(camsoda.getBaseUrl() + "/api/v1/browse/following", true, camsoda, m -> true), camsoda); status = new Label("Logging in..."); grid.getChildren().add(status); + ((CamsodaUpdateService)updateService).setFilter(createFilter(this)); } @Override @@ -40,9 +44,9 @@ public class CamsodaFollowedTab extends ThumbOverviewTab implements FollowedTab HBox.setMargin(online, new Insets(5, 5, 5, 40)); HBox.setMargin(offline, new Insets(5, 5, 5, 5)); online.setSelected(true); - group.selectedToggleProperty().addListener((e) -> { + group.selectedToggleProperty().addListener(e -> { + showOnline = online.isSelected(); queue.clear(); - ((CamsodaFollowedUpdateService)updateService).showOnline(online.isSelected()); updateService.restart(); }); } @@ -78,4 +82,18 @@ public class CamsodaFollowedTab extends ThumbOverviewTab implements FollowedTab } }); } + + private static Predicate createFilter(CamsodaFollowedTab tab) { + return m -> { + try { + return m.isOnline() == tab.showOnline; + } catch(InterruptedException e) { + Thread.currentThread().interrupt(); + return false; + } catch (Exception e) { + return false; + } + }; + + } } diff --git a/client/src/main/java/ctbrec/ui/sites/camsoda/CamsodaFollowedUpdateService.java b/client/src/main/java/ctbrec/ui/sites/camsoda/CamsodaFollowedUpdateService.java deleted file mode 100644 index ca3ff432..00000000 --- a/client/src/main/java/ctbrec/ui/sites/camsoda/CamsodaFollowedUpdateService.java +++ /dev/null @@ -1,79 +0,0 @@ -package ctbrec.ui.sites.camsoda; - -import static ctbrec.Model.State.*; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.concurrent.ExecutionException; -import java.util.stream.Collectors; - -import org.json.JSONArray; -import org.json.JSONObject; - -import ctbrec.Model; -import ctbrec.io.HttpException; -import ctbrec.sites.camsoda.Camsoda; -import ctbrec.sites.camsoda.CamsodaModel; -import ctbrec.ui.SiteUiFactory; -import ctbrec.ui.tabs.PaginatedScheduledService; -import javafx.concurrent.Task; -import okhttp3.Request; -import okhttp3.Response; - -public class CamsodaFollowedUpdateService extends PaginatedScheduledService { - private Camsoda camsoda; - private boolean showOnline = true; - - public CamsodaFollowedUpdateService(Camsoda camsoda) { - this.camsoda = camsoda; - } - - @Override - protected Task> createTask() { - return new Task>() { - @Override - public List call() throws IOException { - List models = new ArrayList<>(); - String url = camsoda.getBaseUrl() + "/api/v1/user/current"; - SiteUiFactory.getUi(camsoda).login(); - Request request = new Request.Builder().url(url).build(); - try(Response response = camsoda.getHttpClient().execute(request)) { - if (response.isSuccessful()) { - JSONObject json = new JSONObject(response.body().string()); - if(json.has("status") && json.getBoolean("status")) { - JSONObject user = json.getJSONObject("user"); - JSONArray following = user.getJSONArray("following"); - for (int i = 0; i < following.length(); i++) { - JSONObject m = following.getJSONObject(i); - CamsodaModel model = (CamsodaModel) camsoda.createModel(m.getString("followname")); - boolean online = m.getInt("online") == 1; - model.setOnlineState(online ? ONLINE : OFFLINE); - model.setPreview("https://md.camsoda.com/thumbs/" + model.getName() + ".jpg"); - models.add(model); - } - return models.stream() - .filter((m) -> { - try { - return m.isOnline() == showOnline; - } catch (IOException | ExecutionException | InterruptedException e) { - return false; - } - }).collect(Collectors.toList()); - } else { - response.close(); - return Collections.emptyList(); - } - } else { - throw new HttpException(response.code(), response.message()); - } - } - } - }; - } - - void showOnline(boolean online) { - this.showOnline = online; - } -} diff --git a/client/src/main/java/ctbrec/ui/sites/camsoda/CamsodaUpdateService.java b/client/src/main/java/ctbrec/ui/sites/camsoda/CamsodaUpdateService.java index 53822bcc..ab362568 100644 --- a/client/src/main/java/ctbrec/ui/sites/camsoda/CamsodaUpdateService.java +++ b/client/src/main/java/ctbrec/ui/sites/camsoda/CamsodaUpdateService.java @@ -1,5 +1,7 @@ package ctbrec.ui.sites.camsoda; +import static ctbrec.Model.State.*; + import java.io.IOException; import java.util.ArrayList; import java.util.Collections; @@ -60,53 +62,55 @@ public class CamsodaUpdateService extends PaginatedScheduledService { protected List loadOnlineModels() throws IOException { List models = new ArrayList<>(); - if(loginRequired && StringUtil.isBlank(ctbrec.Config.getInstance().getSettings().username)) { + if(loginRequired && StringUtil.isBlank(ctbrec.Config.getInstance().getSettings().camsodaUsername)) { return Collections.emptyList(); } else { - String url = CamsodaUpdateService.this.url; LOG.debug("Fetching page {}", url); if(loginRequired) { SiteUiFactory.getUi(camsoda).login(); } Request request = new Request.Builder().url(url).build(); - try(Response response = camsoda.getHttpClient().execute(request)) { + try (Response response = camsoda.getHttpClient().execute(request)) { if (response.isSuccessful()) { - JSONObject json = new JSONObject(response.body().string()); - if(json.has("status") && json.getBoolean("status")) { + String body = response.body().string(); + JSONObject json = new JSONObject(body); + if (json.optBoolean("status")) { JSONArray template = json.getJSONArray("template"); JSONArray results = json.getJSONArray("results"); for (int i = 0; i < results.length(); i++) { JSONObject result = results.getJSONObject(i); try { - if(result.has("tpl")) { + CamsodaModel model; + if (result.has("tpl")) { JSONArray tpl = result.getJSONArray("tpl"); String name = tpl.getString(getTemplateIndex(template, "username")); - CamsodaModel model = (CamsodaModel) camsoda.createModel(name); + model = (CamsodaModel) camsoda.createModel(name); model.setDescription(tpl.getString(getTemplateIndex(template, "subject_html"))); model.setSortOrder(tpl.getFloat(getTemplateIndex(template, "sort_value"))); String preview = "https:" + tpl.getString(getTemplateIndex(template, "thumb")); model.setPreview(preview); String displayName = tpl.getString(getTemplateIndex(template, "display_name")); model.setDisplayName(displayName.replaceAll("[^a-zA-Z0-9]", "")); - if(model.getDisplayName().isBlank()) { + if (model.getDisplayName().isBlank()) { model.setDisplayName(name); } model.setNew(result.optBoolean("new")); + model.setOnlineState(tpl.getString(getTemplateIndex(template, "stream_name")).isEmpty() ? OFFLINE : ONLINE); models.add(model); } else { String name = result.getString("username"); - CamsodaModel model = (CamsodaModel) camsoda.createModel(name); + model = (CamsodaModel) camsoda.createModel(name); model.setSortOrder(result.getFloat("sort_value")); - if(result.has("status")) { + if (result.has("status")) { model.setOnlineStateByStatus(result.getString("status")); } - if(result.has("display_name")) { + if (result.has("display_name")) { model.setDisplayName(result.getString("display_name").replaceAll("[^a-zA-Z0-9]", "")); - if(model.getDisplayName().isBlank()) { + if (model.getDisplayName().isBlank()) { model.setDisplayName(name); } } - if(result.has("thumb")) { + if (result.has("thumb")) { String previewUrl = "https:" + result.getString("thumb"); model.setPreview(previewUrl); } @@ -138,4 +142,8 @@ public class CamsodaUpdateService extends PaginatedScheduledService { } throw new NoSuchElementException(string + " not found in template: " + template.toString()); } + + public void setFilter(Predicate filter) { + this.filter = filter; + } } diff --git a/client/src/main/java/ctbrec/ui/sites/jasmin/LiveJasminElectronLoginDialog.java b/client/src/main/java/ctbrec/ui/sites/jasmin/LiveJasminElectronLoginDialog.java index ebd97ca4..ab665a8e 100644 --- a/client/src/main/java/ctbrec/ui/sites/jasmin/LiveJasminElectronLoginDialog.java +++ b/client/src/main/java/ctbrec/ui/sites/jasmin/LiveJasminElectronLoginDialog.java @@ -60,6 +60,7 @@ public class LiveJasminElectronLoginDialog { } String password = Config.getInstance().getSettings().livejasminPassword; if (password != null && !password.trim().isEmpty()) { + password = password.replace("'", "\\'"); browser.executeJavaScript("document.querySelector('#login_form input[name=\"password\"]').value = '" + password + "';"); } browser.executeJavaScript("document.getElementById('header_container').setAttribute('style', 'display:none');"); diff --git a/client/src/main/java/ctbrec/ui/sites/myfreecams/MyFreeCamsTableTab.java b/client/src/main/java/ctbrec/ui/sites/myfreecams/MyFreeCamsTableTab.java index db3d7e94..8581ca7f 100644 --- a/client/src/main/java/ctbrec/ui/sites/myfreecams/MyFreeCamsTableTab.java +++ b/client/src/main/java/ctbrec/ui/sites/myfreecams/MyFreeCamsTableTab.java @@ -526,6 +526,7 @@ public class MyFreeCamsTableTab extends Tab implements TabSelectionListener { updateService.cancel(); } saveData(); + observableModels.clear(); } private void saveData() { diff --git a/client/src/main/java/ctbrec/ui/sites/showup/ShowupElectronLoginDialog.java b/client/src/main/java/ctbrec/ui/sites/showup/ShowupElectronLoginDialog.java index 64a56c18..28375850 100644 --- a/client/src/main/java/ctbrec/ui/sites/showup/ShowupElectronLoginDialog.java +++ b/client/src/main/java/ctbrec/ui/sites/showup/ShowupElectronLoginDialog.java @@ -84,6 +84,7 @@ public class ShowupElectronLoginDialog { } String password = Config.getInstance().getSettings().showupPassword; if (password != null && !password.trim().isEmpty()) { + password = password.replace("'", "\\'"); browser.executeJavaScript("$('input[name=\"password\"]').attr('value','" + password + "')"); } browser.executeJavaScript("$('input[name=\"remember\"]').attr('value','true')"); diff --git a/client/src/main/java/ctbrec/ui/sites/streamate/StreamateFollowedService.java b/client/src/main/java/ctbrec/ui/sites/streamate/StreamateFollowedService.java index 6b4dd129..6548f0e8 100644 --- a/client/src/main/java/ctbrec/ui/sites/streamate/StreamateFollowedService.java +++ b/client/src/main/java/ctbrec/ui/sites/streamate/StreamateFollowedService.java @@ -41,7 +41,7 @@ public class StreamateFollowedService extends PaginatedScheduledService { public StreamateFollowedService(Streamate streamate) { this.streamate = streamate; this.httpClient = (StreamateHttpClient) streamate.getHttpClient(); - this.url = "https://member.naiadsystems.com/search/favorites?domain=streamate.com&skipXmentSelection=true"; + this.url = "https://member.naiadsystems.com/search/v3/favorites?skipXmentSelection=true&skinConfig=%7B%7D&filters="; } @Override diff --git a/client/src/main/java/ctbrec/ui/sites/stripchat/StripchatConfigUI.java b/client/src/main/java/ctbrec/ui/sites/stripchat/StripchatConfigUI.java index 1a0f745e..e860f2f8 100644 --- a/client/src/main/java/ctbrec/ui/sites/stripchat/StripchatConfigUI.java +++ b/client/src/main/java/ctbrec/ui/sites/stripchat/StripchatConfigUI.java @@ -12,8 +12,11 @@ import javafx.scene.control.Button; import javafx.scene.control.CheckBox; import javafx.scene.control.Label; import javafx.scene.control.PasswordField; +import javafx.scene.control.RadioButton; import javafx.scene.control.TextField; +import javafx.scene.control.ToggleGroup; import javafx.scene.layout.GridPane; +import javafx.scene.layout.HBox; import javafx.scene.layout.Priority; public class StripchatConfigUI extends AbstractConfigUI { @@ -44,6 +47,26 @@ public class StripchatConfigUI extends AbstractConfigUI { GridPane.setMargin(enabled, new Insets(0, 0, 0, SettingsTab.CHECKBOX_MARGIN)); layout.add(enabled, 1, row++); + l = new Label("Site"); + layout.add(l, 0, row); + ToggleGroup toggleGroup = new ToggleGroup(); + RadioButton optionA = new RadioButton("Stripchat"); + optionA.setSelected(!Config.getInstance().getSettings().stripchatUseXhamster); + optionA.setToggleGroup(toggleGroup); + RadioButton optionB = new RadioButton("xHamsterLive"); + optionB.setSelected(!optionA.isSelected()); + optionB.setToggleGroup(toggleGroup); + optionA.selectedProperty().addListener((obs, oldV, newV) -> { + Config.getInstance().getSettings().stripchatUseXhamster = !newV; + save(); + }); + HBox hbox = new HBox(); + hbox.getChildren().addAll(optionA, optionB); + HBox.setMargin(optionA, new Insets(5)); + HBox.setMargin(optionB, new Insets(5)); + GridPane.setMargin(hbox, new Insets(0, 0, 0, SettingsTab.CHECKBOX_MARGIN)); + layout.add(hbox, 1, row++); + layout.add(new Label("Stripchat User"), 0, row); TextField username = new TextField(Config.getInstance().getSettings().stripchatUsername); username.textProperty().addListener((ob, o, n) -> { diff --git a/client/src/main/java/ctbrec/ui/sites/stripchat/StripchatElectronLoginDialog.java b/client/src/main/java/ctbrec/ui/sites/stripchat/StripchatElectronLoginDialog.java new file mode 100644 index 00000000..c326c85c --- /dev/null +++ b/client/src/main/java/ctbrec/ui/sites/stripchat/StripchatElectronLoginDialog.java @@ -0,0 +1,125 @@ +package ctbrec.ui.sites.stripchat; + +import java.io.IOException; +import java.util.Collections; +import java.util.function.Consumer; + +import org.json.JSONArray; +import org.json.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import ctbrec.Config; +import ctbrec.sites.stripchat.Stripchat; +import ctbrec.ui.ExternalBrowser; +import okhttp3.Cookie; +import okhttp3.Cookie.Builder; +import okhttp3.CookieJar; +import okhttp3.HttpUrl; + +public class StripchatElectronLoginDialog { + + private static final transient Logger LOG = LoggerFactory.getLogger(StripchatElectronLoginDialog.class); + public String DOMAIN = Stripchat.domain; + public String URL = Stripchat.baseUri; + private CookieJar cookieJar; + private ExternalBrowser browser; + + public StripchatElectronLoginDialog(CookieJar cookieJar) throws IOException { + this.cookieJar = cookieJar; + browser = ExternalBrowser.getInstance(); + try { + JSONObject config = new JSONObject(); + config.put("url", URL); + config.put("w", 640); + config.put("h", 640); + JSONObject msg = new JSONObject(); + msg.put("config", config); + browser.run(msg, msgHandler); + } catch (InterruptedException e) { + throw new IOException("Couldn't wait for login dialog", e); + } finally { + browser.close(); + } + } + + private Consumer msgHandler = (line) -> { + if(!line.startsWith("{")) { + System.err.println(line); + } else { + JSONObject json = new JSONObject(line); + if(json.has("url")) { + String url = json.getString("url"); + + if(url.endsWith(DOMAIN) || url.endsWith(DOMAIN + '/')) { + try { + browser.executeJavaScript("document.querySelector('button[class~=\"btn-visitors-agreement-accept\"]').click();"); + browser.executeJavaScript("document.querySelector('div[class~=\"header-dropdown\"] a[class~=\"dropdown-link\"]').click();"); + browser.executeJavaScript("document.querySelector('a[class~=\"btn\"][href*=\"login\"]').click();"); + String username = Config.getInstance().getSettings().stripchatUsername; + if (username != null && !username.trim().isEmpty()) { + browser.executeJavaScript("document.querySelector('#login_login_or_email').value = '" + username + "';"); + } + String password = Config.getInstance().getSettings().stripchatPassword; + if (password != null && !password.trim().isEmpty()) { + password = password.replace("'", "\\'"); + browser.executeJavaScript("document.querySelector('#login_password').value = '" + password + "';"); + } + browser.executeJavaScript("document.querySelector('#recaptcha-checkbox-border').click();"); + browser.executeJavaScript("document.querySelector('*[class~=btn-login]').addEventListener('click', function() {window.setTimeout(function() {location.reload()}, 2000)});"); + } catch(Exception e) { + LOG.warn("Couldn't auto fill username and password for Stripchat", e); + } + } + + if (json.has("cookies")) { + JSONArray _cookies = json.getJSONArray("cookies"); + boolean sessionCookieFound = false; + for (int i = 0; i < _cookies.length(); i++) { + JSONObject cookie = _cookies.getJSONObject(i); + if (cookie.getString("domain").contains(DOMAIN)) { + String domain = cookie.getString("domain"); + if (domain.startsWith(".")) { + domain = domain.substring(1); + } + Cookie c = createCookie(domain, cookie); + cookieJar.saveFromResponse(HttpUrl.parse(url), Collections.singletonList(c)); + c = createCookie(DOMAIN, cookie); + cookieJar.saveFromResponse(HttpUrl.parse(Stripchat.baseUri), Collections.singletonList(c)); + if (c.name().contains("_com_sessionId")) { + sessionCookieFound = true; + } + } + } + + if(sessionCookieFound) { + try { + browser.close(); + } catch (IOException e) { + LOG.error("Couldn't send close request to browser", e); + } + } + } + } + } + }; + + private Cookie createCookie(String domain, JSONObject cookie) { + Builder b = new Cookie.Builder() + .path(cookie.getString("path")) + .domain(domain) + .name(cookie.getString("name")) + .value(cookie.getString("value")) + .expiresAt(Double.valueOf(cookie.optDouble("expirationDate")).longValue()); + if(cookie.optBoolean("hostOnly")) { + b.hostOnlyDomain(domain); + } + if(cookie.optBoolean("httpOnly")) { + b.httpOnly(); + } + if(cookie.optBoolean("secure")) { + b.secure(); + } + return b.build(); + } +} diff --git a/client/src/main/java/ctbrec/ui/sites/stripchat/StripchatFollowedTab.java b/client/src/main/java/ctbrec/ui/sites/stripchat/StripchatFollowedTab.java index f82525ab..5b0cc16b 100644 --- a/client/src/main/java/ctbrec/ui/sites/stripchat/StripchatFollowedTab.java +++ b/client/src/main/java/ctbrec/ui/sites/stripchat/StripchatFollowedTab.java @@ -4,14 +4,10 @@ import ctbrec.sites.stripchat.Stripchat; import ctbrec.ui.tabs.FollowedTab; import ctbrec.ui.tabs.ThumbOverviewTab; import javafx.concurrent.WorkerStateEvent; -import javafx.geometry.Insets; import javafx.scene.Scene; import javafx.scene.control.Label; -import javafx.scene.control.RadioButton; -import javafx.scene.control.ToggleGroup; import javafx.scene.input.KeyCode; import javafx.scene.input.KeyEvent; -import javafx.scene.layout.HBox; public class StripchatFollowedTab extends ThumbOverviewTab implements FollowedTab { private Label status; @@ -26,25 +22,6 @@ public class StripchatFollowedTab extends ThumbOverviewTab implements FollowedTa @Override protected void createGui() { super.createGui(); - addOnlineOfflineSelector(); - } - - private void addOnlineOfflineSelector() { - ToggleGroup group = new ToggleGroup(); - RadioButton online = new RadioButton("online"); - online.setToggleGroup(group); - RadioButton offline = new RadioButton("offline"); - offline.setToggleGroup(group); - pagination.getChildren().add(online); - pagination.getChildren().add(offline); - HBox.setMargin(online, new Insets(5, 5, 5, 40)); - HBox.setMargin(offline, new Insets(5, 5, 5, 5)); - online.setSelected(true); - group.selectedToggleProperty().addListener(e -> { - queue.clear(); - ((StripchatFollowedUpdateService)updateService).showOnline(online.isSelected()); - updateService.restart(); - }); } @Override diff --git a/client/src/main/java/ctbrec/ui/sites/stripchat/StripchatFollowedUpdateService.java b/client/src/main/java/ctbrec/ui/sites/stripchat/StripchatFollowedUpdateService.java index 2dcc28c2..80b788ba 100644 --- a/client/src/main/java/ctbrec/ui/sites/stripchat/StripchatFollowedUpdateService.java +++ b/client/src/main/java/ctbrec/ui/sites/stripchat/StripchatFollowedUpdateService.java @@ -4,8 +4,8 @@ import static ctbrec.io.HttpConstants.*; import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.List; -import java.util.Objects; import org.json.JSONArray; import org.json.JSONObject; @@ -24,8 +24,8 @@ import okhttp3.Request; import okhttp3.Response; public class StripchatFollowedUpdateService extends PaginatedScheduledService { + private static final int PAGE_SIZE = 30; private Stripchat stripchat; - private boolean showOnline = true; public StripchatFollowedUpdateService(Stripchat stripchat) { this.stripchat = stripchat; @@ -36,22 +36,33 @@ public class StripchatFollowedUpdateService extends PaginatedScheduledService { return new Task>() { @Override public List call() throws IOException { + int startIndex = (getPage() - 1) * PAGE_SIZE; JSONArray favoriteModelIds = loadFavoriteModelIds(); - List models = loadModels(favoriteModelIds); + List modelIdsToLoad = new ArrayList<>(PAGE_SIZE); + List models; + if (startIndex < favoriteModelIds.length()) { + int modelsOnPage = Math.min(PAGE_SIZE, favoriteModelIds.length() - startIndex - 1); + for (int i = 0; i < modelsOnPage; i++) { + modelIdsToLoad.add(favoriteModelIds.getInt(startIndex + i)); + } + models = loadModels(modelIdsToLoad); + } else { + models = Collections.emptyList(); + } return models; } - private List loadModels(JSONArray favoriteModelIds) throws IOException { + private List loadModels(List modelIdsToLoad) throws IOException { List models = new ArrayList<>(); HttpUrl.Builder urlBuilder = HttpUrl.parse(stripchat.getBaseUrl() + "/api/front/models/list").newBuilder(); - for (int i = 0; i < favoriteModelIds.length(); i++) { - urlBuilder.addQueryParameter("modelIds["+i+"]", Integer.toString(favoriteModelIds.getInt(i))); + for (int i = 0; i < modelIdsToLoad.size(); i++) { + urlBuilder.addQueryParameter("modelIds["+i+"]", modelIdsToLoad.get(i).toString()); } Request request = new Request.Builder() .url(urlBuilder.build()) .header(ACCEPT, "*/*") .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) - .header(REFERER, Stripchat.BASE_URI + "/favorites") + .header(REFERER, Stripchat.baseUri + "/favorites") .header(CONTENT_TYPE, MIMETYPE_APPLICATION_JSON) .build(); try (Response response = stripchat.getHttpClient().execute(request)) { @@ -64,10 +75,7 @@ public class StripchatFollowedUpdateService extends PaginatedScheduledService { StripchatModel model = stripchat.createModel(user.optString("username")); model.setDescription(user.optString("description")); model.setPreview(user.optString("previewUrlThumbBig")); - boolean online = Objects.equals(user.optString("status"), "public"); - if (showOnline == online) { - models.add(model); - } + models.add(model); } } } else { @@ -79,21 +87,22 @@ public class StripchatFollowedUpdateService extends PaginatedScheduledService { private JSONArray loadFavoriteModelIds() throws IOException { SiteUiFactory.getUi(stripchat).login(); + stripchat.getHttpClient().login(); long userId = ((StripchatHttpClient) stripchat.getHttpClient()).getUserId(); String url = stripchat.getBaseUrl() + "/api/front/users/" + userId + "/favorites"; Request request = new Request.Builder() .url(url) .header(ACCEPT, MIMETYPE_APPLICATION_JSON) .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) - .header(ORIGIN, Stripchat.BASE_URI) - .header(REFERER, Stripchat.BASE_URI + "/favorites") + .header(ORIGIN, Stripchat.baseUri) + .header(REFERER, Stripchat.baseUri + "/favorites") .header(CONTENT_TYPE, MIMETYPE_APPLICATION_JSON) .build(); try (Response response = stripchat.getHttpClient().execute(request)) { if (response.isSuccessful()) { JSONObject json = new JSONObject(response.body().string()); - if(json.has("userIds")) { - JSONArray userIds = json.getJSONArray("userIds"); + if (json.has("modelIds")) { + JSONArray userIds = json.getJSONArray("modelIds"); return userIds; } else { return new JSONArray(); @@ -105,8 +114,4 @@ public class StripchatFollowedUpdateService extends PaginatedScheduledService { } }; } - - void showOnline(boolean online) { - this.showOnline = online; - } } diff --git a/client/src/main/java/ctbrec/ui/sites/stripchat/StripchatSiteUi.java b/client/src/main/java/ctbrec/ui/sites/stripchat/StripchatSiteUi.java index a7462903..af898505 100644 --- a/client/src/main/java/ctbrec/ui/sites/stripchat/StripchatSiteUi.java +++ b/client/src/main/java/ctbrec/ui/sites/stripchat/StripchatSiteUi.java @@ -1,20 +1,30 @@ package ctbrec.ui.sites.stripchat; import java.io.IOException; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import ctbrec.sites.stripchat.Stripchat; +import ctbrec.sites.stripchat.StripchatHttpClient; +import ctbrec.ui.controls.Dialogs; import ctbrec.ui.sites.AbstractSiteUi; import ctbrec.ui.sites.ConfigUI; import ctbrec.ui.tabs.TabProvider; +import javafx.application.Platform; public class StripchatSiteUi extends AbstractSiteUi { + private static final Logger LOG = LoggerFactory.getLogger(StripchatSiteUi.class); + private StripchatTabProvider tabProvider; private StripchatConfigUI configUi; - private Stripchat stripchat; + private Stripchat site; public StripchatSiteUi(Stripchat stripchat) { - this.stripchat = stripchat; + this.site = stripchat; tabProvider = new StripchatTabProvider(stripchat); configUi = new StripchatConfigUI(stripchat); } @@ -31,7 +41,40 @@ public class StripchatSiteUi extends AbstractSiteUi { @Override public synchronized boolean login() throws IOException { - boolean automaticLogin = stripchat.login(); - return automaticLogin; + boolean automaticLogin = site.login(); + if (automaticLogin) { + return true; + } else { + + BlockingQueue queue = new LinkedBlockingQueue<>(); + + Runnable showDialog = () -> { + // login with external browser + try { + new StripchatElectronLoginDialog(site.getHttpClient().getCookieJar()); + } catch (Exception e1) { + LOG.error("Error logging in with external browser", e1); + Dialogs.showError("Login error", "Couldn't login to " + site.getName(), e1); + } + + try { + queue.put(true); + } catch (InterruptedException e) { + LOG.error("Error while signaling termination", e); + } + }; + + Platform.runLater(showDialog); + try { + queue.take(); + } catch (InterruptedException e) { + LOG.error("Error while waiting for login dialog to close", e); + throw new IOException(e); + } + + StripchatHttpClient httpClient = (StripchatHttpClient) site.getHttpClient(); + boolean loggedIn = httpClient.checkLoginSuccess(); + return loggedIn; + } } } diff --git a/client/src/main/java/ctbrec/ui/tabs/RecordedModelsTab.java b/client/src/main/java/ctbrec/ui/tabs/RecordedModelsTab.java index 48faf307..96721f8c 100644 --- a/client/src/main/java/ctbrec/ui/tabs/RecordedModelsTab.java +++ b/client/src/main/java/ctbrec/ui/tabs/RecordedModelsTab.java @@ -1,9 +1,13 @@ package ctbrec.ui.tabs; +import static ctbrec.SubsequentAction.*; + import java.io.IOException; import java.security.InvalidKeyException; import java.security.NoSuchAlgorithmException; import java.time.Instant; +import java.time.LocalDate; +import java.time.ZoneId; import java.util.ArrayList; import java.util.Collections; import java.util.Iterator; @@ -25,6 +29,7 @@ import ctbrec.Config; import ctbrec.Model; import ctbrec.Recording; import ctbrec.StringUtil; +import ctbrec.SubsequentAction; import ctbrec.recorder.Recorder; import ctbrec.sites.Site; import ctbrec.ui.AutosizeAlert; @@ -59,8 +64,10 @@ import javafx.geometry.Pos; import javafx.scene.control.Alert; import javafx.scene.control.Button; import javafx.scene.control.ContextMenu; +import javafx.scene.control.DatePicker; import javafx.scene.control.Label; import javafx.scene.control.MenuItem; +import javafx.scene.control.RadioButton; import javafx.scene.control.ScrollPane; import javafx.scene.control.SelectionMode; import javafx.scene.control.Tab; @@ -71,6 +78,7 @@ import javafx.scene.control.TableColumn.SortType; import javafx.scene.control.TableRow; import javafx.scene.control.TableView; import javafx.scene.control.TextField; +import javafx.scene.control.ToggleGroup; import javafx.scene.control.Tooltip; import javafx.scene.control.cell.CheckBoxTableCell; import javafx.scene.control.cell.PropertyValueFactory; @@ -84,6 +92,7 @@ import javafx.scene.input.MouseButton; import javafx.scene.input.MouseEvent; import javafx.scene.layout.BorderPane; import javafx.scene.layout.FlowPane; +import javafx.scene.layout.GridPane; import javafx.scene.layout.HBox; import javafx.scene.layout.Priority; import javafx.util.Callback; @@ -612,6 +621,8 @@ public class RecordedModelsTab extends Tab implements TabSelectionListener { pauseRecording.setOnAction(e -> pauseRecording(selectedModels)); MenuItem resumeRecording = new MenuItem("Resume Recording"); resumeRecording.setOnAction(e -> resumeRecording(selectedModels)); + MenuItem stopRecordingAt = new MenuItem("Stop Recording at Date"); + stopRecordingAt.setOnAction(e -> setStopDate(selectedModels.get(0))); MenuItem openInBrowser = new MenuItem("Open in Browser"); openInBrowser.setOnAction(e -> DesktopIntegration.open(selectedModels.get(0).getUrl())); MenuItem openInPlayer = new MenuItem("Open in Player"); @@ -630,6 +641,7 @@ public class RecordedModelsTab extends Tab implements TabSelectionListener { ContextMenu menu = new ContextMenu(stop); if (selectedModels.size() == 1) { menu.getItems().add(selectedModels.get(0).isSuspended() ? resumeRecording : pauseRecording); + menu.getItems().add(stopRecordingAt); } else { menu.getItems().addAll(resumeRecording, pauseRecording); } @@ -646,6 +658,46 @@ public class RecordedModelsTab extends Tab implements TabSelectionListener { return menu; } + private void setStopDate(JavaFxModel model) { + DatePicker datePicker = new DatePicker(); + GridPane grid = new GridPane(); + grid.setHgap(10); + grid.setVgap(10); + grid.setPadding(new Insets(20, 150, 10, 10)); + grid.add(new Label("Stop at"), 0, 0); + grid.add(datePicker, 1, 0); + grid.add(new Label("And then"), 0, 1); + ToggleGroup toggleGroup = new ToggleGroup(); + RadioButton pauseButton = new RadioButton("pause recording"); + pauseButton.setSelected(model.getRecordUntilSubsequentAction() == PAUSE); + pauseButton.setToggleGroup(toggleGroup); + RadioButton removeButton = new RadioButton("remove model"); + removeButton.setSelected(model.getRecordUntilSubsequentAction() == REMOVE); + removeButton.setToggleGroup(toggleGroup); + HBox row = new HBox(); + row.getChildren().addAll(pauseButton, removeButton); + HBox.setMargin(pauseButton, new Insets(5)); + HBox.setMargin(removeButton, new Insets(5)); + grid.add(row, 1, 1); + if (model.getRecordUntil().toEpochMilli() != Model.RECORD_INDEFINITELY) { + LocalDate localDate = LocalDate.ofInstant(model.getRecordUntil(), ZoneId.systemDefault()); + datePicker.setValue(localDate); + } + boolean userClickedOk = Dialogs.showCustomInput(getTabPane().getScene(), "Stop Recording at", grid); + if (userClickedOk) { + SubsequentAction action = pauseButton.isSelected() ? PAUSE : REMOVE; + LOG.info("Stop at {} and {}", datePicker.getValue(), action); + Instant stopAt = Instant.from(datePicker.getValue().atStartOfDay().atZone(ZoneId.systemDefault())); + model.setRecordUntil(stopAt); + model.setRecordUntilSubsequentAction(action); + try { + recorder.stopRecordingAt(model.getDelegate()); + } catch (InvalidKeyException | NoSuchAlgorithmException | IOException e) { + Dialogs.showError(getTabPane().getScene(), "Error", "Couln't set stop date", e); + } + } + } + private void ignore(ObservableList selectedModels) { for (JavaFxModel fxModel : selectedModels) { Model modelToIgnore = fxModel.getDelegate(); diff --git a/client/src/main/java/ctbrec/ui/tabs/RecordingsTab.java b/client/src/main/java/ctbrec/ui/tabs/RecordingsTab.java index 2f4be5c5..5fff704d 100644 --- a/client/src/main/java/ctbrec/ui/tabs/RecordingsTab.java +++ b/client/src/main/java/ctbrec/ui/tabs/RecordingsTab.java @@ -31,6 +31,8 @@ import ctbrec.Config; import ctbrec.Recording; import ctbrec.Recording.State; import ctbrec.StringUtil; +import ctbrec.event.EventBusHolder; +import ctbrec.event.RecordingStateChangedEvent; import ctbrec.recorder.ProgressListener; import ctbrec.recorder.Recorder; import ctbrec.recorder.RecordingPinnedException; @@ -437,15 +439,13 @@ public class RecordingsTab extends Tab implements TabSelectionListener { } MenuItem rerunPostProcessing = new MenuItem("Rerun Post-Processing"); - rerunPostProcessing.setOnAction(e -> triggerPostProcessing(first)); - if (first.getStatus() == FAILED || first.getStatus() == WAITING || first.getStatus() == FINISHED) { - contextMenu.getItems().add(rerunPostProcessing); - } + rerunPostProcessing.setOnAction(e -> triggerPostProcessing(recordings)); + contextMenu.getItems().add(rerunPostProcessing); + rerunPostProcessing.setDisable(!recordings.stream().allMatch(Recording::canBePostProcessed)); if(recordings.size() > 1) { openInPlayer.setDisable(true); openDir.setDisable(true); - rerunPostProcessing.setDisable(true); } return contextMenu; @@ -565,13 +565,15 @@ public class RecordingsTab extends Tab implements TabSelectionListener { new Thread(() -> DesktopIntegration.open(tsFile.getParent())).start(); } - private void triggerPostProcessing(JavaFxRecording first) { + private void triggerPostProcessing(List recs) { new Thread(() -> { - try { - recorder.rerunPostProcessing(first.getDelegate()); - } catch (IOException | InvalidKeyException | NoSuchAlgorithmException | IllegalStateException e1) { - showErrorDialog("Error while starting post-processing", "The post-processing could not be started", e1); - LOG.error("Error while starting post-processing", e1); + for (JavaFxRecording rec : recs) { + try { + recorder.rerunPostProcessing(rec.getDelegate()); + } catch (IOException | InvalidKeyException | NoSuchAlgorithmException | IllegalStateException e1) { + showErrorDialog("Error while starting post-processing", "The post-processing could not be started", e1); + LOG.error("Error while starting post-processing", e1); + } } }).start(); } @@ -633,6 +635,8 @@ public class RecordingsTab extends Tab implements TabSelectionListener { Platform.runLater(() -> { recording.setStatus(FINISHED); recording.setProgress(-1); + RecordingStateChangedEvent evt = new RecordingStateChangedEvent(target, recording.getStatus(), recording.getModel(), recording.getStartDate()); + EventBusHolder.BUS.post(evt); }); } }); diff --git a/client/src/main/java/ctbrec/ui/tabs/ThumbCell.java b/client/src/main/java/ctbrec/ui/tabs/ThumbCell.java index 1f26d841..bb01c4d5 100644 --- a/client/src/main/java/ctbrec/ui/tabs/ThumbCell.java +++ b/client/src/main/java/ctbrec/ui/tabs/ThumbCell.java @@ -644,4 +644,8 @@ public class ThumbCell extends StackPane { return new int[2]; } } + + public void releaseResources() { + iv.setImage(null); + } } diff --git a/client/src/main/java/ctbrec/ui/tabs/ThumbOverviewTab.java b/client/src/main/java/ctbrec/ui/tabs/ThumbOverviewTab.java index 6d052701..7623cf2b 100644 --- a/client/src/main/java/ctbrec/ui/tabs/ThumbOverviewTab.java +++ b/client/src/main/java/ctbrec/ui/tabs/ThumbOverviewTab.java @@ -872,6 +872,15 @@ public class ThumbOverviewTab extends Tab implements TabSelectionListener { updateService.cancel(); } queue.clear(); + + for (Iterator iterator = grid.getChildren().iterator(); iterator.hasNext();) { + Node node = iterator.next(); + if(node instanceof ThumbCell) { + ThumbCell thumbCell = (ThumbCell) node; + thumbCell.releaseResources(); + iterator.remove(); + } + } } void suspendUpdates(boolean suspend) { diff --git a/client/src/main/java/ctbrec/ui/tabs/logging/CtbrecAppender.java b/client/src/main/java/ctbrec/ui/tabs/logging/CtbrecAppender.java new file mode 100644 index 00000000..14ff5b9e --- /dev/null +++ b/client/src/main/java/ctbrec/ui/tabs/logging/CtbrecAppender.java @@ -0,0 +1,13 @@ +package ctbrec.ui.tabs.logging; + +import ch.qos.logback.classic.spi.LoggingEvent; +import ch.qos.logback.core.ConsoleAppender; +import ctbrec.event.EventBusHolder; + +public class CtbrecAppender extends ConsoleAppender { + + @Override + protected void append(LoggingEvent event) { + EventBusHolder.BUS.post(event); + } +} diff --git a/client/src/main/java/ctbrec/ui/tabs/logging/LoggingTab.java b/client/src/main/java/ctbrec/ui/tabs/logging/LoggingTab.java new file mode 100644 index 00000000..b6482b1a --- /dev/null +++ b/client/src/main/java/ctbrec/ui/tabs/logging/LoggingTab.java @@ -0,0 +1,194 @@ +package ctbrec.ui.tabs.logging; + +import java.time.Instant; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.util.Collections; +import java.util.LinkedList; +import java.util.stream.Collectors; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.eventbus.Subscribe; + +import ch.qos.logback.classic.LoggerContext; +import ch.qos.logback.classic.encoder.PatternLayoutEncoder; +import ch.qos.logback.classic.spi.IThrowableProxy; +import ch.qos.logback.classic.spi.LoggingEvent; +import ch.qos.logback.classic.spi.StackTraceElementProxy; +import ctbrec.event.EventBusHolder; +import ctbrec.ui.controls.SearchBox; +import javafx.application.Platform; +import javafx.beans.property.SimpleStringProperty; +import javafx.collections.FXCollections; +import javafx.collections.ObservableList; +import javafx.geometry.Insets; +import javafx.scene.control.ContextMenu; +import javafx.scene.control.MenuItem; +import javafx.scene.control.SelectionMode; +import javafx.scene.control.Tab; +import javafx.scene.control.TableColumn; +import javafx.scene.control.TableView; +import javafx.scene.input.Clipboard; +import javafx.scene.input.ClipboardContent; +import javafx.scene.input.ContextMenuEvent; +import javafx.scene.input.MouseEvent; +import javafx.scene.layout.BorderPane; + +public class LoggingTab extends Tab { + + private static final int HISTORY_LENGTH = 10_000; + private SearchBox filter = new SearchBox(); + private TableView table = new TableView<>(); + private ObservableList history = FXCollections.observableList(Collections.synchronizedList(new LinkedList<>())); + private ObservableList filteredEvents = FXCollections.observableArrayList(); + private DateTimeFormatter timeFormatter = DateTimeFormatter.ISO_LOCAL_DATE_TIME; + private volatile boolean tabClosed = false; + private ContextMenu popup; + private Object eventBustSubscriber = new Object() { + @Subscribe + public void publishLoggingEevent(LoggingEvent event) { + if (!tabClosed) { + Platform.runLater(() -> { + history.add(event); + if (history.size() > HISTORY_LENGTH - 1) { + history.remove(0); + } + filter(); + }); + } + } + }; + + private void filter() { + filteredEvents.clear(); + filteredEvents.addAll(history.stream().filter(evt -> { + String q = filter.getText().toLowerCase(); + return evt.getLevel().toString().toLowerCase().contains(q) + || createLogMessage(evt).toLowerCase().contains(q); + }).collect(Collectors.toList())); + } + + public LoggingTab() { + setText("Logging"); + subscribeToEventBus(); + + table = new TableView<>(filteredEvents); + + int idx = 0; + TableColumn level = createTableColumn("Level", 65, idx++); + level.setCellValueFactory(cdf -> new SimpleStringProperty(cdf.getValue().getLevel().toString())); + table.getColumns().add(level); + + TableColumn time = createTableColumn("Timestamp", 200, idx++); + time.setCellValueFactory(cdf -> { + Instant instant = Instant.ofEpochMilli(cdf.getValue().getTimeStamp()); + return new SimpleStringProperty(instant.atZone(ZoneId.systemDefault()).format(timeFormatter)); + }); + table.getColumns().add(time); + + TableColumn location = createTableColumn("Location", 250, idx++); + location.setCellValueFactory(cdf -> { + StackTraceElement loc = cdf.getValue().getCallerData()[0]; + String l = loc.getFileName() + ":" + loc.getLineNumber(); + return new SimpleStringProperty(l); + }); + table.getColumns().add(location); + + TableColumn msg = createTableColumn("Message", 2000, idx++); + msg.setCellValueFactory(cdf -> new SimpleStringProperty(createLogMessage(cdf.getValue()))); + table.getColumns().add(msg); + + BorderPane layout = new BorderPane(); + BorderPane.setMargin(table, new Insets(10)); + BorderPane.setMargin(filter, new Insets(10, 10, 0, 10)); + layout.setCenter(table); + layout.setTop(filter); + + setContent(layout); + setOnClosed(evt -> { + EventBusHolder.BUS.unregister(eventBustSubscriber); + tabClosed = true; + }); + + filter.setPromptText("Search"); + filter.textProperty().addListener( (observableValue, oldValue, newValue) -> filter()); + + table.getSelectionModel().setSelectionMode(SelectionMode.MULTIPLE); + table.addEventHandler(ContextMenuEvent.CONTEXT_MENU_REQUESTED, event -> { + popup = createContextMenu(); + if (popup != null) { + popup.show(table, event.getScreenX(), event.getScreenY()); + } + event.consume(); + }); + table.addEventHandler(MouseEvent.MOUSE_PRESSED, event -> { + if (popup != null) { + popup.hide(); + } + }); + } + + private ContextMenu createContextMenu() { + final ObservableList selectedEvents = table.getSelectionModel().getSelectedItems(); + if (selectedEvents.isEmpty()) { + return null; + } + MenuItem copy = new MenuItem("Copy"); + copy.setOnAction(e -> { + Platform.runLater(() -> { + String formattedMessages = getFormattedMessages(selectedEvents); + final Clipboard clipboard = Clipboard.getSystemClipboard(); + final ClipboardContent content = new ClipboardContent(); + content.putString(formattedMessages); + clipboard.setContent(content); + }); + }); + + ContextMenu menu = new ContextMenu(copy); + return menu; + } + + private String getFormattedMessages(ObservableList selectedEvents) { + StringBuilder sb = new StringBuilder(); + + ch.qos.logback.classic.Logger rootLogger = (ch.qos.logback.classic.Logger)LoggerFactory.getLogger(Logger.ROOT_LOGGER_NAME); + LoggerContext loggerContext = rootLogger.getLoggerContext(); + loggerContext.reset(); + + PatternLayoutEncoder encoder = new PatternLayoutEncoder(); + encoder.setContext(loggerContext); + encoder.setPattern("%date %level [%thread] %logger{10} [%file:%line] %msg%n"); + encoder.start(); + + for (LoggingEvent evt : selectedEvents) { + byte[] encode = encoder.encode(evt); + sb.append(new String(encode)); + } + return sb.toString(); + } + + private String createLogMessage(LoggingEvent evt) { + StringBuilder sb = new StringBuilder(evt.getFormattedMessage()); + if(evt.getThrowableProxy() != null) { + IThrowableProxy throwableProxy = evt.getThrowableProxy(); + sb.append('\n').append(throwableProxy.getClassName()).append(':').append(' ').append(throwableProxy.getMessage()); + for (StackTraceElementProxy step : throwableProxy.getStackTraceElementProxyArray()) { + sb.append('\n').append('\t').append(step.getSTEAsString()); + } + } + return sb.toString(); + } + + private TableColumn createTableColumn(String text, int width, int idx) { + TableColumn tc = new TableColumn<>(text); + tc.setPrefWidth(width); + tc.setUserData(idx); + return tc; + } + + private void subscribeToEventBus() { + EventBusHolder.BUS.register(eventBustSubscriber); + } +} diff --git a/client/src/main/resources/html/docs/ConfigurationFile.md b/client/src/main/resources/html/docs/ConfigurationFile.md index d1312750..e3ea4bf1 100644 --- a/client/src/main/resources/html/docs/ConfigurationFile.md +++ b/client/src/main/resources/html/docs/ConfigurationFile.md @@ -41,13 +41,17 @@ the port ctbrec tries to connect to, if it is run in remote mode. - **livePreviews** (app only) - Enables the live preview feature in the app. +- **minimumResolution** - [1 - 2147483647]. Sets the minimum video height for a recording. ctbrec tries to find a stream quality, which is higher than or equal to this value. If the only provided stream quality is below this threshold, ctbrec won't record the stream. + - **maximumResolution** - [1 - 2147483647]. Sets the maximum video height for a recording. ctbrec tries to find a stream quality, which is lower than or equal to this value. If the only provided stream quality is above this threshold, ctbrec won't record the stream. - **minimumLengthInSeconds** - [0 - 2147483647] Automatically delete recordings, which are shorter than this amount of seconds. 0 disables this feature. - **minimumSpaceLeftInBytes** - [0 - 9223372036854775807] The space in bytes ctbrec should conserve on the hard drive. 1 GiB = 1024 MiB = 1048576 KiB = 1073741824 bytes -- **onlineCheckIntervalInSecs** - [1 - 2147483647] How often ctbrec checks, if a model is online. This is not a guaranteed interval: If you record many models, the online check for all models can take longer than this interval. A minute is a reasonable value, but you can go lower, if you don't want to miss a anything. But don't go too low, or you risk to do too many requests in a short amount of time and get banned by some sites. +- **onlineCheckIntervalInSecs** - [1 - 2147483647] How often ctbrec checks, if a model is online. This is not a guaranteed interval: If you record many models, the online check for all models can take longer than this interval. A minute is a reasonable value, but you can go lower, if you don't want to miss a anything. But don't go too low, or you risk to do too many requests in a short amount of time and get banned by some sites. + +- **onlineCheckSkipsPausedModels** - [`true`,`false`] Skip the online check for paused models. If you have many models in the recording list, this can reduce the delay when a recording starts after a model came online. - **postProcessing** - Absolute path to a script, which is executed once a recording is finished. See [Post-Processing](/docs/PostProcessing.md). diff --git a/client/src/main/resources/logback.xml b/client/src/main/resources/logback.xml index 4d7ed7bb..fa146bf1 100644 --- a/client/src/main/resources/logback.xml +++ b/client/src/main/resources/logback.xml @@ -8,6 +8,12 @@ + + + + %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n + + @@ -32,6 +38,7 @@ + diff --git a/common/pom.xml b/common/pom.xml index ac6c1fdd..4547a408 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -8,7 +8,7 @@ ctbrec master - 3.8.1 + 3.8.6 ../master diff --git a/common/src/main/java/ctbrec/AbstractModel.java b/common/src/main/java/ctbrec/AbstractModel.java index a9d48b3a..f87bb843 100644 --- a/common/src/main/java/ctbrec/AbstractModel.java +++ b/common/src/main/java/ctbrec/AbstractModel.java @@ -33,6 +33,8 @@ public abstract class AbstractModel implements Model { protected State onlineState = State.UNKNOWN; private Instant lastSeen; private Instant lastRecorded; + private Instant recordUntil; + private SubsequentAction recordUntilSubsequentAction; @Override public boolean isOnline() throws IOException, ExecutionException, InterruptedException { @@ -231,6 +233,26 @@ public abstract class AbstractModel implements Model { this.lastRecorded = lastRecorded; } + @Override + public Instant getRecordUntil() { + return Optional.ofNullable(recordUntil).orElse(Instant.ofEpochMilli(RECORD_INDEFINITELY)); + } + + @Override + public void setRecordUntil(Instant recordUntil) { + this.recordUntil = recordUntil; + } + + @Override + public SubsequentAction getRecordUntilSubsequentAction() { + return Optional.ofNullable(recordUntilSubsequentAction).orElse(SubsequentAction.PAUSE); + } + + @Override + public void setRecordUntilSubsequentAction(SubsequentAction recordUntilSubsequentAction) { + this.recordUntilSubsequentAction = recordUntilSubsequentAction; + } + @Override public Download createDownload() { if (Config.isServerMode() && !Config.getInstance().getSettings().recordSingleFile) { diff --git a/common/src/main/java/ctbrec/Model.java b/common/src/main/java/ctbrec/Model.java index 6fb12ff5..42ccd10e 100644 --- a/common/src/main/java/ctbrec/Model.java +++ b/common/src/main/java/ctbrec/Model.java @@ -20,6 +20,8 @@ import ctbrec.sites.Site; public interface Model extends Comparable, Serializable { + public static final long RECORD_INDEFINITELY = 9000000000000000000l; + public enum State { ONLINE("online"), OFFLINE("offline"), @@ -128,4 +130,10 @@ public interface Model extends Comparable, Serializable { public HttpHeaderFactory getHttpHeaderFactory(); + public Instant getRecordUntil(); + public void setRecordUntil(Instant instant); + + public SubsequentAction getRecordUntilSubsequentAction(); + public void setRecordUntilSubsequentAction(SubsequentAction action); + } \ No newline at end of file diff --git a/common/src/main/java/ctbrec/Recording.java b/common/src/main/java/ctbrec/Recording.java index 9195d801..e6f60bf8 100644 --- a/common/src/main/java/ctbrec/Recording.java +++ b/common/src/main/java/ctbrec/Recording.java @@ -1,5 +1,7 @@ package ctbrec; +import static ctbrec.Recording.State.*; + import java.io.File; import java.io.IOException; import java.io.Serializable; @@ -272,4 +274,8 @@ public class Recording implements Serializable { public void refresh() { sizeInByte = getSize(); } + + public boolean canBePostProcessed() { + return getStatus() == FAILED || getStatus() == WAITING || getStatus() == FINISHED; + } } diff --git a/common/src/main/java/ctbrec/Settings.java b/common/src/main/java/ctbrec/Settings.java index e62e54d6..e6c7e5fc 100644 --- a/common/src/main/java/ctbrec/Settings.java +++ b/common/src/main/java/ctbrec/Settings.java @@ -89,6 +89,7 @@ public class Settings { public List models = new ArrayList<>(); public List modelsIgnored = new ArrayList<>(); public int onlineCheckIntervalInSecs = 60; + public boolean onlineCheckSkipsPausedModels = false; public int overviewUpdateIntervalInSecs = 10; public String password = ""; // chaturbate password TODO maybe rename this onetime public String postProcessing = ""; @@ -122,6 +123,7 @@ public class Settings { public String streamateUsername = ""; public String stripchatUsername = ""; public String stripchatPassword = ""; + public boolean stripchatUseXhamster = false; public boolean transportLayerSecurity = true; public int thumbWidth = 180; public boolean updateThumbnails = true; diff --git a/common/src/main/java/ctbrec/SubsequentAction.java b/common/src/main/java/ctbrec/SubsequentAction.java new file mode 100644 index 00000000..ddfe6228 --- /dev/null +++ b/common/src/main/java/ctbrec/SubsequentAction.java @@ -0,0 +1,6 @@ +package ctbrec; + +public enum SubsequentAction { + PAUSE, + REMOVE +} diff --git a/common/src/main/java/ctbrec/event/EventBusHolder.java b/common/src/main/java/ctbrec/event/EventBusHolder.java index 4e0a7a42..12e265ac 100644 --- a/common/src/main/java/ctbrec/event/EventBusHolder.java +++ b/common/src/main/java/ctbrec/event/EventBusHolder.java @@ -17,7 +17,7 @@ public class EventBusHolder { private EventBusHolder() {} - public static final EventBus BUS = new AsyncEventBus(Executors.newFixedThreadPool(10, r -> { + public static final EventBus BUS = new AsyncEventBus(Executors.newFixedThreadPool(2, r -> { Thread t = new Thread(r); t.setName("EventBus-" + UUID.randomUUID().toString().substring(0, 8)); t.setPriority(Thread.NORM_PRIORITY - 1); diff --git a/common/src/main/java/ctbrec/io/HttpClient.java b/common/src/main/java/ctbrec/io/HttpClient.java index b6ddd76e..50ba3795 100644 --- a/common/src/main/java/ctbrec/io/HttpClient.java +++ b/common/src/main/java/ctbrec/io/HttpClient.java @@ -16,6 +16,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.Objects; import java.util.Set; import java.util.concurrent.TimeUnit; @@ -28,7 +29,6 @@ import javax.net.ssl.X509TrustManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.base.Objects; import com.squareup.moshi.JsonAdapter; import com.squareup.moshi.Moshi; @@ -127,9 +127,8 @@ public abstract class HttpClient { .cookieJar(cookieJar) .connectionPool(GLOBAL_HTTP_CONN_POOL) .connectTimeout(Config.getInstance().getSettings().httpTimeout, TimeUnit.MILLISECONDS) - .readTimeout(Config.getInstance().getSettings().httpTimeout, TimeUnit.MILLISECONDS) - //.addInterceptor(new LoggingInterceptor()) - .connectionPool(new ConnectionPool(50, 10, TimeUnit.MINUTES)); + .readTimeout(Config.getInstance().getSettings().httpTimeout, TimeUnit.MILLISECONDS); + //.addInterceptor(new LoggingInterceptor()); ProxyType proxyType = Config.getInstance().getSettings().proxyType; if (proxyType == ProxyType.HTTP) { @@ -278,7 +277,7 @@ public abstract class HttpClient { for (List cookieList : cookies.values()) { for (Cookie cookie : cookieList) { for (String cookieName : names) { - if (Objects.equal(cookieName, cookie.name())) { + if (Objects.equals(cookieName, cookie.name())) { result.add(cookie); } } diff --git a/common/src/main/java/ctbrec/io/HttpConstants.java b/common/src/main/java/ctbrec/io/HttpConstants.java index 53e131ea..15c95891 100644 --- a/common/src/main/java/ctbrec/io/HttpConstants.java +++ b/common/src/main/java/ctbrec/io/HttpConstants.java @@ -4,9 +4,9 @@ public class HttpConstants { public static final String ACCEPT = "Accept"; public static final String ACCEPT_LANGUAGE = "Accept-Language"; - public static final String COOKIE = "Cookie"; public static final String CONNECTION = "Connection"; public static final String CONTENT_TYPE = "Content-Type"; + public static final String COOKIE = "Cookie"; public static final String KEEP_ALIVE = "keep-alive"; public static final String MIMETYPE_APPLICATION_JSON = "application/json"; public static final String ORIGIN = "Origin"; diff --git a/common/src/main/java/ctbrec/io/HttpException.java b/common/src/main/java/ctbrec/io/HttpException.java index e9664bf5..f58e8cd2 100644 --- a/common/src/main/java/ctbrec/io/HttpException.java +++ b/common/src/main/java/ctbrec/io/HttpException.java @@ -4,15 +4,24 @@ import java.io.IOException; public class HttpException extends IOException { - private int code; - private String msg; + private final String url; + private final int code; + private final String msg; public HttpException(int code, String msg) { super(code + " - " + msg); + this.url = ""; this.code = code; this.msg = msg; } + public HttpException(String url, int code, String msg) { + super(code + " - " + msg + " - " + url); + this.code = code; + this.msg = msg; + this.url = url; + } + public int getResponseCode() { return code; } @@ -20,4 +29,8 @@ public class HttpException extends IOException { public String getResponseMessage() { return msg; } + + public String getUrl() { + return url; + } } diff --git a/common/src/main/java/ctbrec/io/ModelJsonAdapter.java b/common/src/main/java/ctbrec/io/ModelJsonAdapter.java index 6551b94e..a15172ff 100644 --- a/common/src/main/java/ctbrec/io/ModelJsonAdapter.java +++ b/common/src/main/java/ctbrec/io/ModelJsonAdapter.java @@ -15,6 +15,7 @@ import com.squareup.moshi.JsonReader.Token; import com.squareup.moshi.JsonWriter; import ctbrec.Model; +import ctbrec.SubsequentAction; import ctbrec.sites.Site; import ctbrec.sites.chaturbate.ChaturbateModel; @@ -74,6 +75,10 @@ public class ModelJsonAdapter extends JsonAdapter { model.setLastSeen(Instant.ofEpochMilli(reader.nextLong())); } else if(key.equals("lastRecorded")) { model.setLastRecorded(Instant.ofEpochMilli(reader.nextLong())); + } else if(key.equals("recordUntil")) { + model.setRecordUntil(Instant.ofEpochMilli(reader.nextLong())); + } else if(key.equals("recordUntilSubsequentAction")) { + model.setRecordUntilSubsequentAction(SubsequentAction.valueOf(reader.nextString())); } else if(key.equals("siteSpecific")) { reader.beginObject(); try { @@ -115,6 +120,8 @@ public class ModelJsonAdapter extends JsonAdapter { writer.name("suspended").value(model.isSuspended()); writer.name("lastSeen").value(model.getLastSeen().toEpochMilli()); writer.name("lastRecorded").value(model.getLastRecorded().toEpochMilli()); + writer.name("recordUntil").value(model.getRecordUntil().toEpochMilli()); + writer.name("recordUntilSubsequentAction").value(model.getRecordUntilSubsequentAction().name()); writer.name("siteSpecific"); writer.beginObject(); model.writeSiteSpecificData(writer); diff --git a/common/src/main/java/ctbrec/recorder/NextGenLocalRecorder.java b/common/src/main/java/ctbrec/recorder/NextGenLocalRecorder.java index 7d2122c9..c0c93f57 100644 --- a/common/src/main/java/ctbrec/recorder/NextGenLocalRecorder.java +++ b/common/src/main/java/ctbrec/recorder/NextGenLocalRecorder.java @@ -1,5 +1,6 @@ package ctbrec.recorder; +import static ctbrec.SubsequentAction.*; import static ctbrec.event.Event.Type.*; import java.io.File; @@ -10,6 +11,7 @@ import java.security.InvalidKeyException; import java.security.NoSuchAlgorithmException; import java.time.Duration; import java.time.Instant; +import java.time.ZoneId; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -20,8 +22,10 @@ import java.util.Objects; import java.util.Optional; import java.util.UUID; import java.util.concurrent.BlockingQueue; +import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorCompletionService; +import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.LinkedBlockingQueue; @@ -59,7 +63,7 @@ public class NextGenLocalRecorder implements Recorder { private volatile boolean recording = true; private ReentrantLock recorderLock = new ReentrantLock(); private RecorderHttpClient client = new RecorderHttpClient(); - private long lastSpaceMessage = 0; + private long lastPreconditionMessage = 0; private Map recordingProcesses = Collections.synchronizedMap(new HashMap<>()); private RecordingManager recordingManager; @@ -213,92 +217,145 @@ public class NextGenLocalRecorder implements Recorder { private void startRecordingProcess(Model model) throws IOException { recorderLock.lock(); try { - if (!recording) { - // recorder is not in recording mode - return; - } - - if (model.isSuspended()) { - LOG.info("Recording for model {} is suspended.", model); - return; - } - - if (recordingProcesses.containsKey(model)) { - LOG.error("A recording for model {} is already running", model); - return; - } - - if (!models.contains(model)) { - LOG.info("Model {} has been removed. Restarting of recording cancelled.", model); - return; - } - - if (!enoughSpaceForRecording()) { - long now = System.currentTimeMillis(); - if ((now - lastSpaceMessage) > TimeUnit.MINUTES.toMillis(1)) { - LOG.info("Not enough space for recording, not starting recording for {}", model); - lastSpaceMessage = now; - } - return; - } - - if (!downloadSlotAvailable()) { - long now = System.currentTimeMillis(); - if ((now - lastSpaceMessage) > TimeUnit.MINUTES.toMillis(1)) { - LOG.info("The number of downloads is maxed out"); - } - // check, if we can stop a recording for a model with lower priority - Optional lowerPrioRecordingProcess = recordingProcessWithLowerPrio(model.getPriority()); - if (lowerPrioRecordingProcess.isPresent()) { - Download download = lowerPrioRecordingProcess.get().getDownload(); - Model lowerPrioModel = download.getModel(); - LOG.info("Stopping recording for {}. Prio {} < {}", lowerPrioModel.getName(), lowerPrioModel.getPriority(), model.getPriority()); - stopRecordingProcess(lowerPrioModel); - } else { - if ((now - lastSpaceMessage) > TimeUnit.MINUTES.toMillis(1)) { - LOG.info("Other models have higher prio, not starting recording for {}", model.getName()); - } - return; - } - } - + checkRecordingPreconditions(model); LOG.info("Starting recording for model {}", model.getName()); - Download download = model.createDownload(); - download.init(config, model, Instant.now()); - Objects.requireNonNull(download.getStartTime(), - "At this point the download should have set a startTime. Make sure to set a startTime in " + download.getClass() + ".init()"); - LOG.debug("Downloading with {}", download.getClass().getSimpleName()); - - Recording rec = new Recording(); - rec.setDownload(download); - rec.setPath(download.getPath(model).replaceAll("\\\\", "/")); - rec.setModel(model); - rec.setStartDate(download.getStartTime()); - rec.setSingleFile(download.isSingleFile()); - recordingProcesses.put(model, rec); - recordingManager.add(rec); - completionService.submit(() -> { - try { - setRecordingStatus(rec, State.RECORDING); - model.setLastRecorded(rec.getStartDate()); - recordingManager.saveRecording(rec); - download.start(); - } catch (Exception e) { - LOG.error("Download for {} failed. Download state: {}", model.getName(), rec.getStatus(), e); - } - boolean deleted = deleteIfEmpty(rec); - setRecordingStatus(rec, deleted ? State.DELETED : State.WAITING); - if (!deleted) { - // only save the status, if the recording has not been deleted, otherwise we recreate the metadata file - recordingManager.saveRecording(rec); - } - return rec; - }); + Download download = createDownload(model); + Recording rec = createRecording(download); + completionService.submit(createDownloadJob(rec)); + } catch (RecordUntilExpiredException e) { + LOG.info("Precondition not met to record {}: {}", model, e.getLocalizedMessage()); + executeRecordUntilSubsequentAction(model); + } catch (PreconditionNotMetException e) { + LOG.info("Precondition not met to record {}: {}", model, e.getLocalizedMessage()); + return; } finally { recorderLock.unlock(); } } + private Download createDownload(Model model) { + Download download = model.createDownload(); + download.init(config, model, Instant.now()); + Objects.requireNonNull(download.getStartTime(), + "At this point the download should have set a startTime. Make sure to set a startTime in " + download.getClass() + ".init()"); + LOG.debug("Downloading with {}", download.getClass().getSimpleName()); + return download; + } + + private Callable createDownloadJob(Recording rec) { + return () -> { + try { + setRecordingStatus(rec, State.RECORDING); + rec.getModel().setLastRecorded(rec.getStartDate()); + recordingManager.saveRecording(rec); + rec.getDownload().start(); + } catch (Exception e) { + LOG.error("Download for {} failed. Download state: {}", rec.getModel().getName(), rec.getStatus(), e); + } + boolean deleted = deleteIfEmpty(rec); + setRecordingStatus(rec, deleted ? State.DELETED : State.WAITING); + if (!deleted) { + // only save the status, if the recording has not been deleted, otherwise we recreate the metadata file + recordingManager.saveRecording(rec); + } + return rec; + }; + } + + private void executeRecordUntilSubsequentAction(Model model) throws IOException { + if (model.getRecordUntilSubsequentAction() == PAUSE) { + model.setSuspended(true); + } else if (model.getRecordUntilSubsequentAction() == REMOVE) { + try { + LOG.info("Removing {} because the recording timeframe ended at {}", model, model.getRecordUntil().atZone(ZoneId.systemDefault())); + stopRecording(model); + } catch (InvalidKeyException | NoSuchAlgorithmException e1) { + LOG.error("Error while stopping recording", e1); + } + } + // reset values, so that model can be recorded again + model.setRecordUntil(null); + model.setRecordUntilSubsequentAction(PAUSE); + } + + private Recording createRecording(Download download) throws IOException { + Model model = download.getModel(); + Recording rec = new Recording(); + rec.setDownload(download); + rec.setPath(download.getPath(model).replaceAll("\\\\", "/")); + rec.setModel(model); + rec.setStartDate(download.getStartTime()); + rec.setSingleFile(download.isSingleFile()); + recordingProcesses.put(model, rec); + recordingManager.add(rec); + return rec; + } + + private void checkRecordingPreconditions(Model model) throws IOException { + ensureRecorderIsActive(); + ensureModelIsNotSuspended(model); + ensureRecordUntilIsInFuture(model); + ensureNoRecordingRunningForModel(model); + ensureModelShouldBeRecorded(model); + ensureEnoughSpaceForRecording(); + ensureDownloadSlotAvailable(model); + } + + private void ensureRecordUntilIsInFuture(Model model) { + if (Instant.now().isAfter(model.getRecordUntil())) { + throw new RecordUntilExpiredException(model.getRecordUntil()); + } + } + + private void ensureEnoughSpaceForRecording() throws IOException { + if (!enoughSpaceForRecording()) { + throw new PreconditionNotMetException("Not enough disk space for recording"); + } + } + + private void ensureDownloadSlotAvailable(Model model) { + if (!downloadSlotAvailable()) { + long now = System.currentTimeMillis(); + if ((now - lastPreconditionMessage) > TimeUnit.MINUTES.toMillis(1)) { + LOG.info("The number of downloads is maxed out"); + } + // check, if we can stop a recording for a model with lower priority + Optional lowerPrioRecordingProcess = recordingProcessWithLowerPrio(model.getPriority()); + if (lowerPrioRecordingProcess.isPresent()) { + Download download = lowerPrioRecordingProcess.get().getDownload(); + Model lowerPrioModel = download.getModel(); + LOG.info("Stopping recording for {}. Prio {} < {}", lowerPrioModel.getName(), lowerPrioModel.getPriority(), model.getPriority()); + stopRecordingProcess(lowerPrioModel); + } else { + throw new PreconditionNotMetException("Other models have higher prio, not starting recording for " + model.getName()); + } + } + } + + private void ensureModelShouldBeRecorded(Model model) { + if (!models.contains(model)) { + throw new PreconditionNotMetException("Model " + model + " has been removed. Restarting of recording cancelled."); + } + } + + private void ensureNoRecordingRunningForModel(Model model) { + if (recordingProcesses.containsKey(model)) { + throw new PreconditionNotMetException("A recording for model " + model + " is already running"); + } + } + + private void ensureModelIsNotSuspended(Model model) { + if (model.isSuspended()) { + throw new PreconditionNotMetException("Recording for model " + model + " is suspended"); + } + } + + private void ensureRecorderIsActive() { + if (!recording) { + throw new PreconditionNotMetException("Recorder is not in recording mode"); + } + } + private Optional recordingProcessWithLowerPrio(int priority) { Model lowest = null; for (Model m : recordingProcesses.keySet()) { @@ -449,14 +506,26 @@ public class NextGenLocalRecorder implements Recorder { try { // make a copy to avoid ConcurrentModificationException List toStop = new ArrayList<>(recordingProcesses.values()); - for (Recording rec : toStop) { - Optional.ofNullable(rec.getDownload()).ifPresent(Download::stop); + if (!toStop.isEmpty()) { + ExecutorService shutdownPool = Executors.newFixedThreadPool(toStop.size()); + List> shutdownFutures = new ArrayList<>(toStop.size()); + for (Recording rec : toStop) { + Optional.ofNullable(rec.getDownload()).ifPresent(d -> { + shutdownFutures.add(shutdownPool.submit(() -> d.stop())); + }); + } + shutdownPool.shutdown(); + try { + shutdownPool.awaitTermination(10, TimeUnit.MINUTES); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } } } finally { recorderLock.unlock(); } - // wait for post-processing to finish + // wait for downloads to finish LOG.info("Waiting for downloads to finish"); for (int i = 0; i < 60; i++) { if (!recordingProcesses.isEmpty()) { @@ -471,11 +540,12 @@ public class NextGenLocalRecorder implements Recorder { // shutdown threadpools try { - LOG.info("Shutting down pools"); + LOG.info("Shutting down download pool"); downloadPool.shutdown(); - ppPool.shutdown(); client.shutdown(); downloadPool.awaitTermination(1, TimeUnit.MINUTES); + LOG.info("Shutting down post-processing pool"); + ppPool.shutdown(); int minutesToWait = 10; LOG.info("Waiting {} minutes (max) for post-processing to finish", minutesToWait); ppPool.awaitTermination(minutesToWait, TimeUnit.MINUTES); @@ -555,11 +625,11 @@ public class NextGenLocalRecorder implements Recorder { return getModels().stream().filter(m -> { try { return m.isOnline(); - } catch (IOException | ExecutionException e) { - return false; } catch (InterruptedException e) { Thread.currentThread().interrupt(); return false; + } catch (Exception e) { + return false; } }).collect(Collectors.toList()); } @@ -700,4 +770,31 @@ public class NextGenLocalRecorder implements Recorder { rec.setNote(note); recordingManager.saveRecording(rec); } + + @Override + public void stopRecordingAt(Model model) throws IOException, InvalidKeyException, NoSuchAlgorithmException { + recorderLock.lock(); + try { + int index = models.indexOf(model); + if (index >= 0) { + Model m = models.get(index); + m.setRecordUntil(model.getRecordUntil()); + m.setRecordUntilSubsequentAction(model.getRecordUntilSubsequentAction()); + config.save(); + } else { + throw new NoSuchElementException("Model " + model.getName() + " [" + model.getUrl() + "] not found in list of recorded models"); + } + + if (recordingProcesses.containsKey(model)) { + Recording rec = recordingProcesses.get(model); + rec.getDownload().stop(); + } + } finally { + recorderLock.unlock(); + } + + if (Instant.now().isAfter(model.getRecordUntil())) { + executeRecordUntilSubsequentAction(model); + } + } } diff --git a/common/src/main/java/ctbrec/recorder/OnlineMonitor.java b/common/src/main/java/ctbrec/recorder/OnlineMonitor.java index 6d633d61..d3f10744 100644 --- a/common/src/main/java/ctbrec/recorder/OnlineMonitor.java +++ b/common/src/main/java/ctbrec/recorder/OnlineMonitor.java @@ -38,9 +38,11 @@ public class OnlineMonitor extends Thread { private Map states = new HashMap<>(); private Map executors = new HashMap<>(); + private Config config; - public OnlineMonitor(Recorder recorder) { + public OnlineMonitor(Recorder recorder, Config config) { this.recorder = recorder; + this.config = config; setName("OnlineMonitor"); setDaemon(true); } @@ -80,7 +82,11 @@ public class OnlineMonitor extends Thread { // submit online check jobs to the executor for the model's site List> futures = new LinkedList<>(); for (Model model : models) { - futures.add(updateModel(model)); + if (config.getSettings().onlineCheckSkipsPausedModels && model.isSuspended()) { + continue; + } else { + futures.add(updateModel(model)); + } } // wait for all jobs to finish for (Future future : futures) { @@ -134,7 +140,7 @@ public class OnlineMonitor extends Thread { private void suspendUntilNextIteration(List models, Duration timeCheckTook) { LOG.trace("Online check for {} models took {} seconds", models.size(), timeCheckTook.getSeconds()); - long sleepTime = Config.getInstance().getSettings().onlineCheckIntervalInSecs; + long sleepTime = config.getSettings().onlineCheckIntervalInSecs; if(timeCheckTook.getSeconds() < sleepTime) { try { if (running) { diff --git a/common/src/main/java/ctbrec/recorder/PreconditionNotMetException.java b/common/src/main/java/ctbrec/recorder/PreconditionNotMetException.java new file mode 100644 index 00000000..89ae5f45 --- /dev/null +++ b/common/src/main/java/ctbrec/recorder/PreconditionNotMetException.java @@ -0,0 +1,15 @@ +package ctbrec.recorder; + +public class PreconditionNotMetException extends RuntimeException { + public PreconditionNotMetException() { + super("Precondition not met"); + } + + public PreconditionNotMetException(String message) { + super(message); + } + + public PreconditionNotMetException(String message, Throwable t) { + super(message, t); + } +} diff --git a/common/src/main/java/ctbrec/recorder/RecordUntilExpiredException.java b/common/src/main/java/ctbrec/recorder/RecordUntilExpiredException.java new file mode 100644 index 00000000..69434860 --- /dev/null +++ b/common/src/main/java/ctbrec/recorder/RecordUntilExpiredException.java @@ -0,0 +1,21 @@ +package ctbrec.recorder; + +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; + +public class RecordUntilExpiredException extends PreconditionNotMetException { + private Instant until; + + public RecordUntilExpiredException(Instant until) { + this.until = until; + } + + @Override + public String getMessage() { + LocalDateTime dateTime = LocalDateTime.ofInstant(until, ZoneId.systemDefault()); + String date = DateTimeFormatter.ISO_LOCAL_DATE.format(dateTime); + return "Recording expired at " + date; + } +} diff --git a/common/src/main/java/ctbrec/recorder/Recorder.java b/common/src/main/java/ctbrec/recorder/Recorder.java index 7dee8c4d..dd8238a0 100644 --- a/common/src/main/java/ctbrec/recorder/Recorder.java +++ b/common/src/main/java/ctbrec/recorder/Recorder.java @@ -14,6 +14,7 @@ public interface Recorder { public void startRecording(Model model) throws IOException, InvalidKeyException, NoSuchAlgorithmException; public void stopRecording(Model model) throws IOException, InvalidKeyException, NoSuchAlgorithmException; + public void stopRecordingAt(Model model) throws IOException, InvalidKeyException, NoSuchAlgorithmException; public void switchStreamSource(Model model) throws IOException, InvalidKeyException, NoSuchAlgorithmException; diff --git a/common/src/main/java/ctbrec/recorder/RemoteRecorder.java b/common/src/main/java/ctbrec/recorder/RemoteRecorder.java index 84a4b036..10fae6b5 100644 --- a/common/src/main/java/ctbrec/recorder/RemoteRecorder.java +++ b/common/src/main/java/ctbrec/recorder/RemoteRecorder.java @@ -86,6 +86,11 @@ public class RemoteRecorder implements Recorder { sendRequest("stop", model); } + @Override + public void stopRecordingAt(Model model) throws IOException, InvalidKeyException, NoSuchAlgorithmException { + sendRequest("stopAt", model); + } + private void sendRequest(String action, Model model) throws IOException, InvalidKeyException, NoSuchAlgorithmException { String payload = modelRequestAdapter.toJson(new ModelRequest(action, model)); LOG.debug("Sending request to recording server: {}", payload); diff --git a/common/src/main/java/ctbrec/recorder/download/StreamSource.java b/common/src/main/java/ctbrec/recorder/download/StreamSource.java index 5bd01a0c..c9ed2211 100644 --- a/common/src/main/java/ctbrec/recorder/download/StreamSource.java +++ b/common/src/main/java/ctbrec/recorder/download/StreamSource.java @@ -4,6 +4,7 @@ import java.text.DecimalFormat; public class StreamSource implements Comparable { public static final int ORIGIN = Integer.MAX_VALUE - 1; + public static final int UNKNOWN = Integer.MAX_VALUE; public int bandwidth; public int width; public int height; @@ -45,7 +46,7 @@ public class StreamSource implements Comparable { public String toString() { DecimalFormat df = new DecimalFormat("0.00"); float mbit = bandwidth / 1024.0f / 1024.0f; - if (height == Integer.MAX_VALUE) { + if (height == UNKNOWN) { return "unknown resolution (" + df.format(mbit) + " Mbit/s)"; } else if (height == ORIGIN) { return "Origin"; @@ -61,7 +62,7 @@ public class StreamSource implements Comparable { @Override public int compareTo(StreamSource o) { int heightDiff = height - o.height; - if(heightDiff != 0 && height != Integer.MAX_VALUE && o.height != Integer.MAX_VALUE) { + if(heightDiff != 0 && height != UNKNOWN && o.height != UNKNOWN) { return heightDiff; } else { return bandwidth - o.bandwidth; diff --git a/common/src/main/java/ctbrec/recorder/download/hls/AbstractHlsDownload.java b/common/src/main/java/ctbrec/recorder/download/hls/AbstractHlsDownload.java index fee4c801..33b68abd 100644 --- a/common/src/main/java/ctbrec/recorder/download/hls/AbstractHlsDownload.java +++ b/common/src/main/java/ctbrec/recorder/download/hls/AbstractHlsDownload.java @@ -1,6 +1,8 @@ package ctbrec.recorder.download.hls; import static ctbrec.io.HttpConstants.*; +import static ctbrec.io.HttpConstants.ORIGIN; +import static ctbrec.recorder.download.StreamSource.*; import java.io.ByteArrayInputStream; import java.io.IOException; @@ -64,7 +66,7 @@ public abstract class AbstractHlsDownload extends AbstractDownload { protected volatile boolean running = false; protected Model model = new UnknownModel(); protected transient LinkedBlockingQueue downloadQueue = new LinkedBlockingQueue<>(50); - protected transient ExecutorService downloadThreadPool = new ThreadPoolExecutor(5, 5, 2, TimeUnit.MINUTES, downloadQueue, createThreadFactory()); + protected transient ExecutorService downloadThreadPool = new ThreadPoolExecutor(0, 5, 20, TimeUnit.SECONDS, downloadQueue, createThreadFactory()); protected State state = State.UNKNOWN; private int playlistEmptyCount = 0; @@ -160,12 +162,12 @@ public abstract class AbstractHlsDownload extends AbstractDownload { LOG.debug("{} selected {}", model.getName(), streamSources.get(model.getStreamUrlIndex())); url = streamSources.get(model.getStreamUrlIndex()).getMediaPlaylistUrl(); } else { - // filter out stream resolutions, which are too high + // filter out stream resolutions, which are out of range of the configured min and max int minRes = Config.getInstance().getSettings().minimumResolution; int maxRes = Config.getInstance().getSettings().maximumResolution; List filteredStreamSources = streamSources.stream() - .filter(src -> src.height == 0 || minRes <= src.height) - .filter(src -> src.height == 0 || maxRes >= src.height) + .filter(src -> src.height == 0 || src.height == UNKNOWN || minRes <= src.height) + .filter(src -> src.height == 0 || src.height == UNKNOWN || maxRes >= src.height) .collect(Collectors.toList()); if (filteredStreamSources.isEmpty()) { diff --git a/common/src/main/java/ctbrec/recorder/download/hls/MergedFfmpegHlsDownload.java b/common/src/main/java/ctbrec/recorder/download/hls/MergedFfmpegHlsDownload.java index 27e1c242..b90028fe 100644 --- a/common/src/main/java/ctbrec/recorder/download/hls/MergedFfmpegHlsDownload.java +++ b/common/src/main/java/ctbrec/recorder/download/hls/MergedFfmpegHlsDownload.java @@ -1,11 +1,14 @@ package ctbrec.recorder.download.hls; +import static java.util.Optional.*; + import java.io.EOFException; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.net.MalformedURLException; import java.net.URL; import java.nio.file.Files; import java.time.Duration; @@ -14,12 +17,13 @@ import java.time.ZonedDateTime; import java.util.Arrays; import java.util.HashMap; import java.util.LinkedList; -import java.util.Optional; import java.util.Queue; import java.util.concurrent.Callable; +import java.util.concurrent.CancellationException; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.regex.Pattern; import org.slf4j.Logger; @@ -56,6 +60,7 @@ public class MergedFfmpegHlsDownload extends AbstractHlsDownload { private transient OutputStream ffmpegStdIn; protected transient Thread ffmpegThread; private transient Object ffmpegStartMonitor = new Object(); + private Queue> downloads = new LinkedList<>(); public MergedFfmpegHlsDownload(HttpClient client) { super(client); @@ -104,6 +109,7 @@ public class MergedFfmpegHlsDownload extends AbstractHlsDownload { LOG.debug("Starting to download segments"); downloadSegments(segments, true); ffmpegThread.join(); + LOG.debug("FFmpeg thread terminated"); } } catch (ParseException e) { throw new IOException("Couldn't parse stream information", e); @@ -178,7 +184,8 @@ public class MergedFfmpegHlsDownload extends AbstractHlsDownload { } } }); - ffmpegThread.setName("FFmpeg"); + String name = "FFmpeg " + ofNullable(model).map(Model::getName).orElse("").trim(); + ffmpegThread.setName(name); ffmpegThread.start(); } @@ -229,6 +236,9 @@ public class MergedFfmpegHlsDownload extends AbstractHlsDownload { LOG.info("Unexpected error while downloading {}", model, e); } running = false; + } catch (MalformedURLException e) { + LOG.info("Malformed URL {} - {}", model, segmentPlaylistUri, e); + running = false; } catch (Exception e) { LOG.info("Unexpected error while downloading {}", model, e); running = false; @@ -250,7 +260,7 @@ public class MergedFfmpegHlsDownload extends AbstractHlsDownload { int skip = nextSegment - lsp.seq; // add segments to download threadpool - Queue> downloads = new LinkedList<>(); + downloads.clear(); if (downloadQueue.remainingCapacity() == 0) { LOG.warn("Download to slow for this stream. Download queue is full. Skipping segment"); } else { @@ -274,11 +284,15 @@ public class MergedFfmpegHlsDownload extends AbstractHlsDownload { private void writeFinishedSegments(Queue> downloads) throws ExecutionException, IOException { for (Future downloadFuture : downloads) { try { - byte[] segmentData = downloadFuture.get(); + byte[] segmentData = downloadFuture.get(30, TimeUnit.SECONDS); writeSegment(segmentData); } catch (InterruptedException e) { Thread.currentThread().interrupt(); LOG.error("Error while downloading segment", e); + } catch (TimeoutException e) { + LOG.info("Segment download took too long for {}. Not waiting for it any longer", getModel()); + } catch (CancellationException e) { + LOG.info("Segment download cancelled"); } catch (ExecutionException e) { Throwable cause = e.getCause(); if (cause instanceof MissingSegmentException) { @@ -286,7 +300,7 @@ public class MergedFfmpegHlsDownload extends AbstractHlsDownload { LOG.debug("Error while downloading segment, because model {} is offline. Stopping now", model.getName()); running = false; } else { - LOG.debug("Segment not available, but model {} still online. Going on", Optional.ofNullable(model).map(Model::getName).orElse("n/a")); + LOG.debug("Segment not available, but model {} still online. Going on", ofNullable(model).map(Model::getName).orElse("n/a")); } } else if (cause instanceof HttpException) { HttpException he = (HttpException) cause; @@ -295,10 +309,10 @@ public class MergedFfmpegHlsDownload extends AbstractHlsDownload { running = false; } else { if (he.getResponseCode() == 404) { - LOG.info("Playlist for {} not found [HTTP 404]. Stopping now", Optional.ofNullable(model).map(Model::getName).orElse("n/a")); + LOG.info("Playlist for {} not found [HTTP 404]. Stopping now", ofNullable(model).map(Model::getName).orElse("n/a")); running = false; } else if (he.getResponseCode() == 403) { - LOG.info("Playlist for {} not accessible [HTTP 403]. Stopping now", Optional.ofNullable(model).map(Model::getName).orElse("n/a")); + LOG.info("Playlist for {} not accessible [HTTP 403]. Stopping now", ofNullable(model).map(Model::getName).orElse("n/a")); running = false; } else { throw he; @@ -374,9 +388,23 @@ public class MergedFfmpegHlsDownload extends AbstractHlsDownload { @Override synchronized void internalStop() { running = false; + + try { + downloadQueue.clear(); + for (Future future : downloads) { + future.cancel(true); + } + downloadThreadPool.shutdownNow(); + LOG.debug("Waiting for segment download thread pool to terminate for model {}", getModel()); + downloadThreadPool.awaitTermination(60, TimeUnit.SECONDS); + LOG.debug("Segment download thread pool terminated for model {}", getModel()); + } catch (InterruptedException e) { + LOG.error("Interrupted while waiting for segment pool to shutdown"); + Thread.currentThread().interrupt(); + } + if (ffmpegStdIn != null) { try { - downloadQueue.clear(); ffmpegStdIn.close(); } catch (IOException e) { LOG.error("Couldn't terminate FFmpeg by closing stdin", e); @@ -385,7 +413,7 @@ public class MergedFfmpegHlsDownload extends AbstractHlsDownload { if (ffmpeg != null) { try { - boolean waitFor = ffmpeg.waitFor(5, TimeUnit.MINUTES); + boolean waitFor = ffmpeg.waitFor(45, TimeUnit.SECONDS); if (!waitFor && ffmpeg.isAlive()) { LOG.info("FFmpeg didn't terminate. Destroying the process with force!"); ffmpeg.destroyForcibly(); @@ -415,7 +443,7 @@ public class MergedFfmpegHlsDownload extends AbstractHlsDownload { int maxTries = 3; for (int i = 1; i <= maxTries && running; i++) { Builder builder = new Request.Builder().url(url); - addHeaders(builder, Optional.ofNullable(model).map(Model::getHttpHeaderFactory).map(HttpHeaderFactory::createSegmentHeaders).orElse(new HashMap<>())); + addHeaders(builder, ofNullable(model).map(Model::getHttpHeaderFactory).map(HttpHeaderFactory::createSegmentHeaders).orElse(new HashMap<>())); Request request = builder.build(); try (Response response = client.execute(request)) { if (response.isSuccessful()) { diff --git a/common/src/main/java/ctbrec/sites/cam4/Cam4.java b/common/src/main/java/ctbrec/sites/cam4/Cam4.java index f099bfcd..6a8b7562 100644 --- a/common/src/main/java/ctbrec/sites/cam4/Cam4.java +++ b/common/src/main/java/ctbrec/sites/cam4/Cam4.java @@ -44,7 +44,7 @@ public class Cam4 extends AbstractSite { } @Override - public Model createModel(String name) { + public Cam4Model createModel(String name) { Cam4Model m = new Cam4Model(); m.setSite(this); m.setName(name); diff --git a/common/src/main/java/ctbrec/sites/cam4/Cam4Model.java b/common/src/main/java/ctbrec/sites/cam4/Cam4Model.java index 5272b78b..63cf1afc 100644 --- a/common/src/main/java/ctbrec/sites/cam4/Cam4Model.java +++ b/common/src/main/java/ctbrec/sites/cam4/Cam4Model.java @@ -103,7 +103,7 @@ public class Cam4Model extends AbstractModel { onlineState = OFFLINE; break; default: - LOG.debug("Unknown show type {}", showType); + LOG.debug("Unknown show type [{}]", showType); onlineState = UNKNOWN; } diff --git a/common/src/main/java/ctbrec/sites/camsoda/CamsodaModel.java b/common/src/main/java/ctbrec/sites/camsoda/CamsodaModel.java index 9b182202..e980eb78 100644 --- a/common/src/main/java/ctbrec/sites/camsoda/CamsodaModel.java +++ b/common/src/main/java/ctbrec/sites/camsoda/CamsodaModel.java @@ -10,6 +10,7 @@ import java.util.Collections; import java.util.List; import java.util.Locale; import java.util.Objects; +import java.util.Optional; import java.util.Random; import java.util.concurrent.ExecutionException; @@ -44,99 +45,124 @@ public class CamsodaModel extends AbstractModel { private static final String EDGE_SERVERS = "edge_servers"; private static final String STATUS = "status"; private static final Logger LOG = LoggerFactory.getLogger(CamsodaModel.class); - private String streamUrl; private transient List streamSources = null; private transient boolean isNew; private float sortOrder = 0; private Random random = new Random(); int[] resolution = new int[2]; - boolean oldStreamUrl = true; + public String getStreamUrl() throws IOException { - if (streamUrl == null) { - if(oldStreamUrl) { - loadModel(); - } else { - getNewStreamUrl(); - } + Request req = createJsonRequest(getTokenInfoUrl()); + JSONObject response = executeJsonRequest(req); + if (response.optInt(STATUS) == 1 && response.optJSONArray(EDGE_SERVERS) != null && response.optJSONArray(EDGE_SERVERS).length() > 0) { + String edgeServer = response.getJSONArray(EDGE_SERVERS).getString(0); + String streamName = response.getString(STREAM_NAME); + String token = response.getString("token"); + return constructStreamUrl(edgeServer, streamName, token); + } else { + throw new JSONException("JSON response has not the expected structure"); } - return streamUrl; } - public String getNewStreamUrl() throws IOException { + private String getTokenInfoUrl() { String guestUsername = "guest_" + 10_000 + random.nextInt(50_000); - String url = site.getBaseUrl() + "/api/v1/video/vtoken/" + getName() + "?username=" + guestUsername; - Request req = new Request.Builder() - .url(url) + String tokenInfoUrl = site.getBaseUrl() + "/api/v1/video/vtoken/" + getName() + "?username=" + guestUsername; + return tokenInfoUrl; + } + + private String constructStreamUrl(String edgeServer, String streamName, String token) { + StringBuilder url = new StringBuilder("https://"); + url.append(edgeServer).append('/'); + if (streamName.contains("-flu")) { + url.append(streamName); + url.append("_h264_aac"); + url.append(streamName.contains("-flu-hd") ? "_720p" : "_480p"); + url.append("/index.m3u8"); + if (!isPublic(streamName)) { + url.append("?token=").append(token); + } + } else { + // https://vide7-ord.camsoda.com/cam/mp4:maxandtokio-enc10-ord_h264_aac_480p/playlist.m3u8 + url.append("cam/mp4:"); + url.append(streamName); + url.append("_h264_aac_480p/playlist.m3u8"); + } + LOG.trace("Stream URL: {}", url); + return url.toString(); + } + + private Request createJsonRequest(String tokenInfoUrl) { + return new Request.Builder() + .url(tokenInfoUrl) .header(ACCEPT, MIMETYPE_APPLICATION_JSON) .header(ACCEPT_LANGUAGE, Locale.ENGLISH.getLanguage()) .header(X_REQUESTED_WITH, XML_HTTP_REQUEST) .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) .build(); - try (Response response = site.getHttpClient().execute(req)) { + } + + private JSONObject executeJsonRequest(Request request) throws IOException { + try (Response response = site.getHttpClient().execute(request)) { if (response.isSuccessful()) { JSONObject jsonResponse = new JSONObject(response.body().string()); - if (jsonResponse.optInt(STATUS) == 1) { - String edgeServer = jsonResponse.getJSONArray(EDGE_SERVERS).getString(0); - String streamName = jsonResponse.getString(STREAM_NAME); - String token = jsonResponse.getString("token"); - streamUrl = "https://" + edgeServer + '/' + streamName + "_h264_aac_480p/index.m3u8?token=" + token; - } else { - throw new JSONException("Response does not contain a token"); - } + return jsonResponse; } else { throw new HttpException(response.code(), response.message()); } } - return streamUrl; + } + + private boolean isPublic(String streamName) { + return Optional.ofNullable(streamName).orElse("").contains("_public"); } @Override public List getStreamSources() throws IOException, ExecutionException, ParseException, PlaylistException { - String playlistUrl = getStreamUrl(); - if (playlistUrl == null) { - return Collections.emptyList(); - } - LOG.trace("Loading playlist {}", playlistUrl); - Request req = new Request.Builder() - .url(playlistUrl) - .header(ACCEPT_LANGUAGE, Locale.ENGLISH.getLanguage()) - .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) - .build(); - try (Response response = site.getHttpClient().execute(req)) { - if (response.isSuccessful()) { - InputStream inputStream = response.body().byteStream(); - PlaylistParser parser = new PlaylistParser(inputStream, Format.EXT_M3U, Encoding.UTF_8, ParsingMode.LENIENT); - Playlist playlist = parser.parse(); - MasterPlaylist master = playlist.getMasterPlaylist(); - PlaylistData playlistData = master.getPlaylists().get(0); - StreamSource streamsource = new StreamSource(); - if (oldStreamUrl) { - streamsource.mediaPlaylistUrl = streamUrl.replace("playlist.m3u8", playlistData.getUri()); - } else { - int cutOffAt = playlistUrl.indexOf("index.m3u8"); + try { + String playlistUrl = getStreamUrl(); + if (playlistUrl == null) { + return Collections.emptyList(); + } + LOG.trace("Loading playlist {}", playlistUrl); + Request req = new Request.Builder() + .url(playlistUrl) + .header(ACCEPT_LANGUAGE, Locale.ENGLISH.getLanguage()) + .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) + .build(); + try (Response response = site.getHttpClient().execute(req)) { + if (response.isSuccessful()) { + InputStream inputStream = response.body().byteStream(); + PlaylistParser parser = new PlaylistParser(inputStream, Format.EXT_M3U, Encoding.UTF_8, ParsingMode.LENIENT); + Playlist playlist = parser.parse(); + MasterPlaylist master = playlist.getMasterPlaylist(); + PlaylistData playlistData = master.getPlaylists().get(0); + StreamSource streamsource = new StreamSource(); + int cutOffAt = Math.max(playlistUrl.indexOf("index.m3u8"), playlistUrl.indexOf("playlist.m3u8")); String segmentPlaylistUrl = playlistUrl.substring(0, cutOffAt) + playlistData.getUri(); streamsource.mediaPlaylistUrl = segmentPlaylistUrl; - } - if (playlistData.hasStreamInfo()) { - StreamInfo info = playlistData.getStreamInfo(); - streamsource.bandwidth = info.getBandwidth(); - streamsource.width = info.hasResolution() ? info.getResolution().width : 0; - streamsource.height = info.hasResolution() ? info.getResolution().height : 0; + if (playlistData.hasStreamInfo()) { + StreamInfo info = playlistData.getStreamInfo(); + streamsource.bandwidth = info.getBandwidth(); + streamsource.width = info.hasResolution() ? info.getResolution().width : 0; + streamsource.height = info.hasResolution() ? info.getResolution().height : 0; + } else { + streamsource.bandwidth = 0; + streamsource.width = 0; + streamsource.height = 0; + } + streamSources = new ArrayList<>(); + streamSources.add(streamsource); } else { - streamsource.bandwidth = 0; - streamsource.width = 0; - streamsource.height = 0; + LOG.trace("Response: {}", response.body().string()); + throw new HttpException(playlistUrl, response.code(), response.message()); } - streamSources = new ArrayList<>(); - streamSources.add(streamsource); - } else { - LOG.trace("Response: {}", response.body().string()); - throw new HttpException(response.code(), response.message()); } + return streamSources; + } catch (JSONException e) { + return Collections.emptyList(); } - return streamSources; } private void loadModel() throws IOException { @@ -151,15 +177,9 @@ public class CamsodaModel extends AbstractModel { try (Response response = site.getHttpClient().execute(req)) { if (response.isSuccessful()) { JSONObject result = new JSONObject(response.body().string()); - if (result.getBoolean(STATUS)) { + if (result.optBoolean(STATUS)) { JSONObject chat = result.getJSONObject("user").getJSONObject("chat"); String status = chat.getString(STATUS); - oldStreamUrl = !chat.getString(STREAM_NAME).contains("/"); - if (oldStreamUrl && chat.has(EDGE_SERVERS)) { - String edgeServer = chat.getJSONArray(EDGE_SERVERS).getString(0); - String streamName = chat.getString(STREAM_NAME); - streamUrl = "https://" + edgeServer + "/cam/mp4:" + streamName + "_h264_aac_480p/playlist.m3u8"; - } setOnlineStateByStatus(status); } else { throw new IOException("Result was not ok"); @@ -222,11 +242,11 @@ public class CamsodaModel extends AbstractModel { return resolution; } else { try { - List streamSources = getStreamSources(); - if (streamSources.isEmpty()) { + List sources = getStreamSources(); + if (sources.isEmpty()) { return new int[] { 0, 0 }; } else { - StreamSource src = streamSources.get(0); + StreamSource src = sources.get(0); resolution = new int[] { src.width, src.height }; return resolution; } @@ -309,10 +329,6 @@ public class CamsodaModel extends AbstractModel { } } - public void setStreamUrl(String streamUrl) { - this.streamUrl = streamUrl; - } - public float getSortOrder() { return sortOrder; } diff --git a/common/src/main/java/ctbrec/sites/flirt4free/Flirt4FreeModel.java b/common/src/main/java/ctbrec/sites/flirt4free/Flirt4FreeModel.java index 13357872..22a6e831 100644 --- a/common/src/main/java/ctbrec/sites/flirt4free/Flirt4FreeModel.java +++ b/common/src/main/java/ctbrec/sites/flirt4free/Flirt4FreeModel.java @@ -86,12 +86,11 @@ public class Flirt4FreeModel extends AbstractModel { return false; } JSONObject json = new JSONObject(body); - //LOG.debug("check model status: {}", json.toString(2)); - online = Objects.equals(json.optString("status"), "online"); - id = String.valueOf(json.get("model_id")); + online = Objects.equals(json.optString("status"), "online"); // online is true, even if the model is in private or away + updateModelId(json); if (online) { try { - loadStreamUrl(); + loadModelInfo(); } catch (Exception e) { online = false; onlineState = Model.State.OFFLINE; @@ -109,6 +108,18 @@ public class Flirt4FreeModel extends AbstractModel { return online; } + private void updateModelId(JSONObject json) { + if (json.has("model_id")) { + Object modelId = json.get("model_id"); + if (modelId instanceof Number) { + Number n = (Number) modelId; + if (n.intValue() > 0) { + id = String.valueOf(json.get("model_id")); + } + } + } + } + private void loadModelInfo() throws IOException, InterruptedException { String url = getSite().getBaseUrl() + "/webservices/chat-room-interface.php?a=login_room&model_id=" + id; LOG.trace("Loading url {}", url); @@ -127,13 +138,15 @@ public class Flirt4FreeModel extends AbstractModel { // LOG.debug("chat-room-interface {}", json.toString(2)); JSONObject config = json.getJSONObject("config"); JSONObject performer = config.getJSONObject("performer"); - setName(performer.optString("name_seo", "n/a")); - setDisplayName(performer.optString("name", "n/a")); setUrl(getSite().getBaseUrl() + "/rooms/" + getName() + '/'); + setDisplayName(performer.optString("name", getName())); JSONObject room = config.getJSONObject("room"); chatHost = room.getString("host"); chatPort = room.getString("port_to_be"); chatToken = json.getString("token_enc"); + String status = room.optString("status"); + setOnlineState(mapStatus(status)); + online = onlineState == State.ONLINE; JSONObject user = config.getJSONObject("user"); userIp = user.getString("ip"); } else { @@ -147,6 +160,19 @@ public class Flirt4FreeModel extends AbstractModel { } } + private State mapStatus(String status) { + switch (status) { + case "P": + case "F": + return State.PRIVATE; + case "A": + return State.AWAY; + case "O": + default: + return State.ONLINE; + } + } + @Override public List getStreamSources() throws IOException, ExecutionException, ParseException, PlaylistException { return getStreamSources(true); @@ -238,10 +264,9 @@ public class Flirt4FreeModel extends AbstractModel { streamHost = data.getString("stream_host"); // TODO look, if the stream_host is equal to the one encoded in base64 in some of the ajax requests (parameters) online = true; isInteractiveShow = data.optString("devices").equals("1"); - if(data.optString("room_state").equals("P")) { - onlineState = Model.State.PRIVATE; - online = false; - } + String roomState = data.optString("room_state"); + onlineState = mapStatus(roomState); + online = onlineState == State.ONLINE; if(data.optString("room_state").equals("0") && data.optString("login_group_id").equals("14")) { onlineState = Model.State.GROUP; online = false; @@ -262,6 +287,7 @@ public class Flirt4FreeModel extends AbstractModel { synchronized (monitor) { monitor.notify(); } + response.close(); } @Override @@ -455,8 +481,10 @@ public class Flirt4FreeModel extends AbstractModel { @Override public void readSiteSpecificData(JsonReader reader) throws IOException { - reader.nextName(); - id = reader.nextString(); + if (reader.hasNext()) { + reader.nextName(); + id = reader.nextString(); + } } @Override @@ -481,7 +509,7 @@ public class Flirt4FreeModel extends AbstractModel { } private void acquireSlot() throws InterruptedException { - //LOG.debug("Acquire: {}", requestThrottle.availablePermits()); + //LOG.debug("Acquire: {} - Queue: {}", requestThrottle.availablePermits(), requestThrottle.getQueueLength()); requestThrottle.acquire(); long now = System.currentTimeMillis(); long millisSinceLastRequest = now - lastRequest; @@ -494,6 +522,6 @@ public class Flirt4FreeModel extends AbstractModel { private void releaseSlot() { lastRequest = System.currentTimeMillis(); requestThrottle.release(); - //LOG.debug("Release: {}", requestThrottle.availablePermits()); + // LOG.debug("Release: {}", requestThrottle.availablePermits()); } } diff --git a/common/src/main/java/ctbrec/sites/jasmin/LiveJasmin.java b/common/src/main/java/ctbrec/sites/jasmin/LiveJasmin.java index 3f534cb7..e6cfd9dd 100644 --- a/common/src/main/java/ctbrec/sites/jasmin/LiveJasmin.java +++ b/common/src/main/java/ctbrec/sites/jasmin/LiveJasmin.java @@ -5,6 +5,7 @@ import static ctbrec.io.HttpConstants.*; import java.io.IOException; import java.net.URLEncoder; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Locale; import java.util.regex.Matcher; @@ -13,6 +14,8 @@ import java.util.regex.Pattern; import org.json.JSONObject; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import ctbrec.Config; import ctbrec.Model; @@ -27,6 +30,7 @@ import okhttp3.Response; public class LiveJasmin extends AbstractSite { + private static final Logger LOG = LoggerFactory.getLogger(LiveJasmin.class); public static String baseUrl = ""; public static String baseDomain = ""; private HttpClient httpClient; @@ -169,7 +173,8 @@ public class LiveJasmin extends AbstractSite { } return models; } else { - throw new IOException("Response was not successful: " + url + "\n" + body); + LOG.debug("Response was not successful: {}\n{}", url, body); + return Collections.emptyList(); } } else { throw new HttpException(response.code(), response.message()); @@ -198,7 +203,7 @@ public class LiveJasmin extends AbstractSite { String name = m.group(1); return createModel(name); } - m = Pattern.compile("http.*?livejasmin\\.com.*?/chat-html5/(.*)").matcher(url); + m = Pattern.compile("http.*?livejasmin\\.com.*?/chat(?:-html5)?/(.*)").matcher(url); if(m.find()) { String name = m.group(1); return createModel(name); diff --git a/common/src/main/java/ctbrec/sites/jasmin/LiveJasminModel.java b/common/src/main/java/ctbrec/sites/jasmin/LiveJasminModel.java index c7aa3d18..5a4f566d 100644 --- a/common/src/main/java/ctbrec/sites/jasmin/LiveJasminModel.java +++ b/common/src/main/java/ctbrec/sites/jasmin/LiveJasminModel.java @@ -72,6 +72,8 @@ public class LiveJasminModel extends AbstractModel { JSONObject config = data.getJSONObject("config"); JSONObject chatRoom = config.getJSONObject("chatRoom"); setId(chatRoom.getString("p_id")); + setName(chatRoom.getString("performer_id")); + setDisplayName(chatRoom.getString("display_name")); if (chatRoom.has("profile_picture_url")) { setPreview(chatRoom.getString("profile_picture_url")); } @@ -80,11 +82,14 @@ public class LiveJasminModel extends AbstractModel { if (chatRoom.optInt("is_on_private", 0) == 1) { onlineState = State.PRIVATE; } + if (chatRoom.optInt("is_video_call_enabled", 0) == 1) { + onlineState = State.PRIVATE; + } resolution = new int[2]; resolution[0] = config.optInt("streamWidth"); resolution[1] = config.optInt("streamHeight"); online = onlineState == State.ONLINE; - LOG.trace("{} - status:{} {} {} {}", getName(), online, onlineState, Arrays.toString(resolution), getUrl()); + LOG.trace("{} - status:{} {} {} {} {}", getName(), online, onlineState, Arrays.toString(resolution), getUrl(), id); } else { throw new IOException("Response was not successful: " + body); } diff --git a/common/src/main/java/ctbrec/sites/mfc/HlsStreamSourceProvider.java b/common/src/main/java/ctbrec/sites/mfc/HlsStreamSourceProvider.java index d8a3efc5..3b1ff192 100644 --- a/common/src/main/java/ctbrec/sites/mfc/HlsStreamSourceProvider.java +++ b/common/src/main/java/ctbrec/sites/mfc/HlsStreamSourceProvider.java @@ -50,8 +50,8 @@ public class HlsStreamSourceProvider implements StreamSourceProvider { src.width = playlist.getStreamInfo().getResolution().width; src.height = playlist.getStreamInfo().getResolution().height; } else { - src.width = Integer.MAX_VALUE; - src.height = Integer.MAX_VALUE; + src.width = StreamSource.UNKNOWN; + src.height = StreamSource.UNKNOWN; } String masterUrl = streamUrl; String baseUrl = masterUrl.substring(0, masterUrl.lastIndexOf('/') + 1); diff --git a/common/src/main/java/ctbrec/sites/mfc/MyFreeCamsClient.java b/common/src/main/java/ctbrec/sites/mfc/MyFreeCamsClient.java index ade01494..70b7ad8a 100644 --- a/common/src/main/java/ctbrec/sites/mfc/MyFreeCamsClient.java +++ b/common/src/main/java/ctbrec/sites/mfc/MyFreeCamsClient.java @@ -40,6 +40,7 @@ import com.squareup.moshi.Moshi; import ctbrec.Config; import ctbrec.StringUtil; +import ctbrec.io.HttpException; import okhttp3.Cookie; import okhttp3.Request; import okhttp3.Response; @@ -56,6 +57,7 @@ public class MyFreeCamsClient { private static MyFreeCamsClient instance; private MyFreeCams mfc; private WebSocket ws; + private Thread keepAlive; private Moshi moshi; private volatile boolean running = false; @@ -92,6 +94,7 @@ public class MyFreeCamsClient { } public void start() throws IOException { + requestLandingPage(); // to get some cookies running = true; serverConfig = new ServerConfig(mfc); List websocketServers = new ArrayList<>(serverConfig.wsServers.size()); @@ -133,6 +136,21 @@ public class MyFreeCamsClient { watchDog.start(); } + private void requestLandingPage() throws IOException { + Request req = new Request.Builder() + .url(MyFreeCams.baseUrl) + .header(ACCEPT, "*/*") + .header(ACCEPT_LANGUAGE, Locale.ENGLISH.getLanguage()) + .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) + .header(CONNECTION, KEEP_ALIVE) + .build(); + try(Response resp = mfc.getHttpClient().execute(req)) { + if(!resp.isSuccessful()) { + throw new HttpException(resp.code(), resp.message()); + } + } + } + public void stop() { running = false; ws.close(1000, "Good Bye"); // terminate normally (1000) @@ -454,54 +472,6 @@ public class MyFreeCamsClient { model.update(state, getStreamUrl(state)); } - private Message parseMessage(StringBuilder msgBuffer) throws UnsupportedEncodingException { - int packetLengthBytes = 6; - if (msgBuffer.length() < packetLengthBytes) { - // packet size not transmitted completely - return null; - } else { - try { - int packetLength = Integer.parseInt(msgBuffer.substring(0, packetLengthBytes)); - if (packetLength > msgBuffer.length() - packetLengthBytes) { - // packet not complete - return null; - } else { - LOG.trace("<-- {}", msgBuffer); - msgBuffer.delete(0, packetLengthBytes); - StringBuilder rawMessage = new StringBuilder(msgBuffer.substring(0, packetLength)); - int type = parseNextInt(rawMessage); - int sender = parseNextInt(rawMessage); - int receiver = parseNextInt(rawMessage); - int arg1 = parseNextInt(rawMessage); - int arg2 = parseNextInt(rawMessage); - Message message = new Message(type, sender, receiver, arg1, arg2, URLDecoder.decode(rawMessage.toString(), "utf-8")); - msgBuffer.delete(0, packetLength); - return message; - } - } catch (Exception e) { - LOG.error("StringBuilder contains invalid data {}", msgBuffer.toString(), e); - String logfile = "mfc_messages.log"; - try (FileOutputStream fout = new FileOutputStream(logfile)) { - for (String string : receivedTextHistory) { - fout.write(string.getBytes()); - fout.write(10); - } - } catch (Exception e1) { - LOG.error("Couldn't write mfc message history to {}", logfile, e1); - } - msgBuffer.setLength(0); - return null; - } - } - } - - private int parseNextInt(StringBuilder s) { - int nextSpace = s.indexOf(" "); - int i = Integer.parseInt(s.substring(0, nextSpace)); - s.delete(0, nextSpace + 1); - return i; - } - @Override public void onMessage(WebSocket webSocket, ByteString bytes) { super.onMessage(webSocket, bytes); @@ -511,6 +481,54 @@ public class MyFreeCamsClient { return websocket; } + private Message parseMessage(StringBuilder msgBuffer) throws UnsupportedEncodingException { + int packetLengthBytes = 6; + if (msgBuffer.length() < packetLengthBytes) { + // packet size not transmitted completely + return null; + } else { + try { + int packetLength = Integer.parseInt(msgBuffer.substring(0, packetLengthBytes)); + if (packetLength > msgBuffer.length() - packetLengthBytes) { + // packet not complete + return null; + } else { + LOG.trace("<-- {}", msgBuffer); + msgBuffer.delete(0, packetLengthBytes); + StringBuilder rawMessage = new StringBuilder(msgBuffer.substring(0, packetLength)); + int type = parseNextInt(rawMessage); + int sender = parseNextInt(rawMessage); + int receiver = parseNextInt(rawMessage); + int arg1 = parseNextInt(rawMessage); + int arg2 = parseNextInt(rawMessage); + Message message = new Message(type, sender, receiver, arg1, arg2, URLDecoder.decode(rawMessage.toString(), "utf-8")); + msgBuffer.delete(0, packetLength); + return message; + } + } catch (Exception e) { + LOG.error("StringBuilder contains invalid data {}", msgBuffer.toString(), e); + String logfile = "mfc_messages.log"; + try (FileOutputStream fout = new FileOutputStream(logfile)) { + for (String string : receivedTextHistory) { + fout.write(string.getBytes()); + fout.write(10); + } + } catch (Exception e1) { + LOG.error("Couldn't write mfc message history to {}", logfile, e1); + } + msgBuffer.setLength(0); + return null; + } + } + } + + private int parseNextInt(StringBuilder s) { + int nextSpace = s.indexOf(" "); + int i = Integer.parseInt(s.substring(0, nextSpace)); + s.delete(0, nextSpace + 1); + return i; + } + protected boolean follow(int uid) { if (ws != null) { return ws.send(ADDFRIENDREQ + " " + sessionId + " 0 " + uid + " 1\n"); @@ -562,8 +580,11 @@ public class MyFreeCamsClient { } private void startKeepAlive(WebSocket ws) { - Thread keepAlive = new Thread(() -> { - while (running) { + if (keepAlive != null) { + keepAlive.interrupt(); + } + keepAlive = new Thread(() -> { + while (running && !Thread.currentThread().isInterrupted()) { try { if (!connecting) { LOG.trace("--> NULL to keep the connection alive"); @@ -699,4 +720,18 @@ public class MyFreeCamsClient { public Collection getSessionStates() { return Collections.unmodifiableCollection(sessionStates.asMap().values()); } + + public void joinChannel(MyFreeCamsModel model) { + SessionState state = getSessionState(model); + int userChannel = 100000000 + state.getUid(); + LOG.debug("Joining chat channel for model {}", model.getDisplayName()); + try { + search(model.getName()); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + ws.send(MessageTypes.ROOMDATA + " " + sessionId + " 0 1 0\n"); + ws.send(MessageTypes.UEOPT + " " + sessionId + " 0 66 1 111111\n"); + ws.send(MessageTypes.JOINCHAN + " " + sessionId + " 0 " + userChannel + " 9\n"); + } } diff --git a/common/src/main/java/ctbrec/sites/mfc/ServerConfig.java b/common/src/main/java/ctbrec/sites/mfc/ServerConfig.java index 96c7fd65..469e0488 100644 --- a/common/src/main/java/ctbrec/sites/mfc/ServerConfig.java +++ b/common/src/main/java/ctbrec/sites/mfc/ServerConfig.java @@ -7,7 +7,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Objects; +import java.util.Optional; import org.json.JSONArray; import org.json.JSONObject; @@ -74,7 +74,7 @@ public class ServerConfig { } public boolean isOnHtml5VideoServer(SessionState state) { - int camserv = Objects.requireNonNull(Objects.requireNonNull(state.getU()).getCamserv()); + int camserv = getCamServ(state); return isOnObsServer(state) || h5Servers.containsKey(Integer.toString(camserv)) || (camserv >= 904 && camserv <= 915 @@ -86,12 +86,17 @@ public class ServerConfig { } public boolean isOnWzObsVideoServer(SessionState state) { - int camserv = Objects.requireNonNull(Objects.requireNonNull(state.getU()).getCamserv()); + int camserv = getCamServ(state); return wzobsServers.containsKey(Integer.toString(camserv)); } public boolean isOnNgServer(SessionState state) { - int camserv = Objects.requireNonNull(Objects.requireNonNull(state.getU()).getCamserv()); + int camserv = getCamServ(state); return ngVideoServers.containsKey(Integer.toString(camserv)); } + + private static int getCamServ(SessionState state) { + int camserv = Optional.ofNullable(state).map(SessionState::getU).map(User::getCamserv).orElse(-1); + return camserv; + } } diff --git a/common/src/main/java/ctbrec/sites/streamate/StreamateHttpClient.java b/common/src/main/java/ctbrec/sites/streamate/StreamateHttpClient.java index 2184abc2..497ee5d5 100644 --- a/common/src/main/java/ctbrec/sites/streamate/StreamateHttpClient.java +++ b/common/src/main/java/ctbrec/sites/streamate/StreamateHttpClient.java @@ -6,6 +6,7 @@ import java.io.IOException; import java.util.Collections; import java.util.Locale; import java.util.NoSuchElementException; +import java.util.UUID; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -15,7 +16,6 @@ import org.slf4j.LoggerFactory; import ctbrec.Config; import ctbrec.io.HttpClient; -import ctbrec.io.HttpConstants; import ctbrec.io.HttpException; import okhttp3.Cookie; import okhttp3.HttpUrl; @@ -60,8 +60,10 @@ public class StreamateHttpClient extends HttpClient { private void loadXsrfToken() { // do a first request to get cookies and stuff Request req = new Request.Builder() // - .url(Streamate.BASE_URL) // - .header(HttpConstants.USER_AGENT, Config.getInstance().getSettings().httpUserAgent) // + .url(Streamate.BASE_URL + "/initialData.js") // + .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) // + .header(COOKIE, "smtid="+UUID.randomUUID().toString()+"; Xld_rct=1;") // + .header(REFERER, Streamate.BASE_URL) .build(); try (Response resp = execute(req)) { if (resp.code() == 200) { diff --git a/common/src/main/java/ctbrec/sites/stripchat/Stripchat.java b/common/src/main/java/ctbrec/sites/stripchat/Stripchat.java index 7919ef23..7aba56c6 100644 --- a/common/src/main/java/ctbrec/sites/stripchat/Stripchat.java +++ b/common/src/main/java/ctbrec/sites/stripchat/Stripchat.java @@ -23,9 +23,19 @@ import okhttp3.Response; public class Stripchat extends AbstractSite { - public static final String BASE_URI = "https://stripchat.com"; + public static String domain = "stripchat.com"; + public static String baseUri = "https://stripchat.com"; private HttpClient httpClient; + @Override + public void init() throws IOException { + boolean hamster = Config.getInstance().getSettings().stripchatUseXhamster; + if (hamster) { + domain = "xhamsterlive.com"; + baseUri = "https://" + domain; + } + } + @Override public String getName() { return "Stripchat"; @@ -33,7 +43,7 @@ public class Stripchat extends AbstractSite { @Override public String getBaseUrl() { - return BASE_URI; + return baseUri; } @Override @@ -61,15 +71,15 @@ public class Stripchat extends AbstractSite { throw new IOException("Account settings not available"); } - String username = Config.getInstance().getSettings().camsodaUsername; - String url = BASE_URI + "/api/v1/user/" + username; + String username = Config.getInstance().getSettings().stripchatPassword; + String url = baseUri + "/api/v1/user/" + username; Request request = new Request.Builder().url(url).build(); - try(Response response = getHttpClient().execute(request)) { - if(response.isSuccessful()) { + try (Response response = getHttpClient().execute(request)) { + if (response.isSuccessful()) { JSONObject json = new JSONObject(response.body().string()); - if(json.has("user")) { + if (json.has("user")) { JSONObject user = json.getJSONObject("user"); - if(user.has("tokens")) { + if (user.has("tokens")) { return (double) user.getInt("tokens"); } } @@ -93,11 +103,6 @@ public class Stripchat extends AbstractSite { return httpClient; } - @Override - public void init() throws IOException { - // noop - } - @Override public void shutdown() { if (httpClient != null) { @@ -122,7 +127,7 @@ public class Stripchat extends AbstractSite { @Override public List search(String q) throws IOException, InterruptedException { - String url = BASE_URI + "/api/front/v2/models/search?limit=20&query=" + URLEncoder.encode(q, "utf-8"); + String url = baseUri + "/api/front/v2/models/search?limit=20&query=" + URLEncoder.encode(q, "utf-8"); Request req = new Request.Builder() .url(url) .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) @@ -162,7 +167,7 @@ public class Stripchat extends AbstractSite { @Override public Model createModelFromUrl(String url) { - Matcher m = Pattern.compile("https?://(?:.*?\\.)?stripchat.com/([^/]*?)/?").matcher(url); + Matcher m = Pattern.compile("https?://(?:.*?\\.)?(?:stripchat.com|xhamsterlive.com)/([^/]*?)/?").matcher(url); if (m.matches()) { String modelName = m.group(1); return createModel(modelName); diff --git a/common/src/main/java/ctbrec/sites/stripchat/StripchatHttpClient.java b/common/src/main/java/ctbrec/sites/stripchat/StripchatHttpClient.java index 71561a15..5702459b 100644 --- a/common/src/main/java/ctbrec/sites/stripchat/StripchatHttpClient.java +++ b/common/src/main/java/ctbrec/sites/stripchat/StripchatHttpClient.java @@ -4,6 +4,7 @@ import static ctbrec.io.HttpConstants.*; import java.io.IOException; +import org.json.JSONException; import org.json.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -31,14 +32,20 @@ public class StripchatHttpClient extends HttpClient { @Override public boolean login() throws IOException { - if(loggedIn) { + if (loggedIn) { + if (csrfToken == null) { + loadCsrfToken(); + } return true; } // persisted cookies might let us log in - if(checkLoginSuccess()) { + if (checkLoginSuccess()) { loggedIn = true; LOG.debug("Logged in with cookies"); + if (csrfToken == null) { + loadCsrfToken(); + } return true; } @@ -46,7 +53,7 @@ public class StripchatHttpClient extends HttpClient { loadCsrfToken(); } - String url = Stripchat.BASE_URI + "/api/front/auth/login"; + String url = Stripchat.baseUri + "/api/front/auth/login"; JSONObject requestParams = new JSONObject(); requestParams.put("loginOrEmail", Config.getInstance().getSettings().stripchatUsername); requestParams.put("password", Config.getInstance().getSettings().stripchatPassword); @@ -59,8 +66,8 @@ public class StripchatHttpClient extends HttpClient { .url(url) .header(ACCEPT, MIMETYPE_APPLICATION_JSON) .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) - .header(ORIGIN, Stripchat.BASE_URI) - .header(REFERER, Stripchat.BASE_URI) + .header(ORIGIN, Stripchat.baseUri) + .header(REFERER, Stripchat.baseUri) .header(CONTENT_TYPE, MIMETYPE_APPLICATION_JSON) .post(body) .build(); @@ -75,19 +82,20 @@ public class StripchatHttpClient extends HttpClient { return false; } } else { - throw new HttpException(response.code(), response.message()); + LOG.info("Auto-Login failed: {} {} {}", response.code(), response.message(), url); + return false; } } } private void loadCsrfToken() throws IOException { - String url = Stripchat.BASE_URI + "/api/front/v2/config/data?requestPath=%2F&timezoneOffset=0"; + String url = Stripchat.baseUri + "/api/front/v2/config/data?requestPath=%2F&timezoneOffset=0"; Request request = new Request.Builder() .url(url) .header(ACCEPT, MIMETYPE_APPLICATION_JSON) .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) - .header(ORIGIN, Stripchat.BASE_URI) - .header(REFERER, Stripchat.BASE_URI) + .header(ORIGIN, Stripchat.baseUri) + .header(REFERER, Stripchat.baseUri) .header(CONTENT_TYPE, MIMETYPE_APPLICATION_JSON) .build(); try (Response response = execute(request)) { @@ -107,11 +115,48 @@ public class StripchatHttpClient extends HttpClient { * check, if the login worked * @throws IOException */ - public boolean checkLoginSuccess() { - return userId > 0; + public boolean checkLoginSuccess() throws IOException { + long userId = getUserId(); + String url = Stripchat.baseUri + "/api/front/users/" + userId + "/favorites"; + Request request = new Request.Builder() + .url(url) + .header(ACCEPT, MIMETYPE_APPLICATION_JSON) + .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) + .header(ORIGIN, Stripchat.baseUri) + .header(REFERER, Stripchat.baseUri + "/favorites") + .header(CONTENT_TYPE, MIMETYPE_APPLICATION_JSON) + .build(); + try (Response response = execute(request)) { + if (response.isSuccessful()) { + return true; + } + } catch (Exception e) { + LOG.info("Login check returned unsuccessful: {}", e.getLocalizedMessage()); + } + return false; } - public long getUserId() { + public long getUserId() throws JSONException, IOException { + if (userId == 0) { + String url = Stripchat.baseUri + "/api/front/users/username/" + Config.getInstance().getSettings().stripchatUsername; + Request request = new Request.Builder() + .url(url) + .header(ACCEPT, MIMETYPE_APPLICATION_JSON) + .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) + .header(ORIGIN, Stripchat.baseUri) + .header(REFERER, Stripchat.baseUri) + .header(CONTENT_TYPE, MIMETYPE_APPLICATION_JSON) + .build(); + try (Response response = execute(request)) { + if (response.isSuccessful()) { + JSONObject resp = new JSONObject(response.body().string()); + JSONObject user = resp.getJSONObject("user"); + userId = user.optLong("id"); + } else { + throw new HttpException(url, response.code(), response.message()); + } + } + } return userId; } diff --git a/common/src/main/java/ctbrec/sites/stripchat/StripchatModel.java b/common/src/main/java/ctbrec/sites/stripchat/StripchatModel.java index 174fa7ed..c07b5b1c 100644 --- a/common/src/main/java/ctbrec/sites/stripchat/StripchatModel.java +++ b/common/src/main/java/ctbrec/sites/stripchat/StripchatModel.java @@ -95,9 +95,8 @@ public class StripchatModel extends AbstractModel { best.width = broadcastSettings.optInt("width"); best.mediaPlaylistUrl = "https://b-" + serverName + ".stripst.com/hls/" + streamName + "/" + streamName + ".m3u8"; sources.add(best); - Object resolutionObject = broadcastSettings.get("resolutions"); - if (resolutionObject instanceof JSONObject) { - JSONObject resolutions = (JSONObject) resolutionObject; + JSONObject resolutions = broadcastSettings.optJSONObject("resolutions"); + if (resolutions instanceof JSONObject) { JSONArray heights = resolutions.names(); for (int i = 0; i < heights.length(); i++) { String h = heights.getString(i); @@ -145,12 +144,13 @@ public class StripchatModel extends AbstractModel { @Override public boolean follow() throws IOException { + getSite().getHttpClient().login(); JSONObject modelInfo = loadModelInfo(); JSONObject user = modelInfo.getJSONObject("user"); long modelId = user.optLong("id"); StripchatHttpClient client = (StripchatHttpClient) getSite().getHttpClient(); - String url = Stripchat.BASE_URI + "/api/front/users/" + client.getUserId() + "/favorites/" + modelId; + String url = Stripchat.baseUri + "/api/front/users/" + client.getUserId() + "/favorites/" + modelId; JSONObject requestParams = new JSONObject(); requestParams.put("csrfToken", client.getCsrfToken()); requestParams.put("csrfTimestamp", client.getCsrfTimestamp()); @@ -160,8 +160,8 @@ public class StripchatModel extends AbstractModel { .url(url) .header(ACCEPT, "*/*") .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) - .header(ORIGIN, Stripchat.BASE_URI) - .header(REFERER, Stripchat.BASE_URI + '/' + getName()) + .header(ORIGIN, Stripchat.baseUri) + .header(REFERER, Stripchat.baseUri + '/' + getName()) .header(CONTENT_TYPE, MIMETYPE_APPLICATION_JSON) .put(body) .build(); @@ -176,6 +176,7 @@ public class StripchatModel extends AbstractModel { @Override public boolean unfollow() throws IOException { + getSite().getHttpClient().login(); JSONObject modelInfo = loadModelInfo(); JSONObject user = modelInfo.getJSONObject("user"); long modelId = user.optLong("id"); @@ -183,7 +184,7 @@ public class StripchatModel extends AbstractModel { favoriteIds.put(modelId); StripchatHttpClient client = (StripchatHttpClient) getSite().getHttpClient(); - String url = Stripchat.BASE_URI + "/api/front/users/" + client.getUserId() + "/favorites"; + String url = Stripchat.baseUri + "/api/front/users/" + client.getUserId() + "/favorites"; JSONObject requestParams = new JSONObject(); requestParams.put("favoriteIds", favoriteIds); requestParams.put("csrfToken", client.getCsrfToken()); @@ -194,8 +195,8 @@ public class StripchatModel extends AbstractModel { .url(url) .header(ACCEPT, MIMETYPE_APPLICATION_JSON) .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) - .header(ORIGIN, Stripchat.BASE_URI) - .header(REFERER, Stripchat.BASE_URI) + .header(ORIGIN, Stripchat.baseUri) + .header(REFERER, Stripchat.baseUri) .header(CONTENT_TYPE, MIMETYPE_APPLICATION_JSON) .delete(body) .build(); diff --git a/common/src/test/java/ctbrec/ReflectionUtil.java b/common/src/test/java/ctbrec/ReflectionUtil.java new file mode 100644 index 00000000..37bbba44 --- /dev/null +++ b/common/src/test/java/ctbrec/ReflectionUtil.java @@ -0,0 +1,21 @@ +package ctbrec; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +public class ReflectionUtil { + + private ReflectionUtil () {} + + @SuppressWarnings("unchecked") + public static T call(Object target, String methodName, Object...args) throws NoSuchMethodException, SecurityException, IllegalAccessException, IllegalArgumentException, InvocationTargetException { + List> argTypes = Arrays.stream(args).map(arg -> arg.getClass()).collect(Collectors.toList()); + Class[] argTypeArray = argTypes.toArray(new Class[0]); + Method method = target.getClass().getDeclaredMethod(methodName, argTypeArray); + method.setAccessible(true); + return (T) method.invoke(target, args); + } +} diff --git a/common/src/test/java/ctbrec/sites/mfc/MyFreeCamsClientTest.java b/common/src/test/java/ctbrec/sites/mfc/MyFreeCamsClientTest.java new file mode 100644 index 00000000..7e171ffa --- /dev/null +++ b/common/src/test/java/ctbrec/sites/mfc/MyFreeCamsClientTest.java @@ -0,0 +1,36 @@ +package ctbrec.sites.mfc; + +import static org.junit.Assert.*; + +import java.lang.reflect.InvocationTargetException; + +import org.json.JSONObject; +import org.junit.Test; + +import ctbrec.ReflectionUtil; + +public class MyFreeCamsClientTest { + + + @Test + public void testMessageParsing() throws NoSuchMethodException, SecurityException, IllegalAccessException, IllegalArgumentException, InvocationTargetException { + MyFreeCamsClient client = MyFreeCamsClient.getInstance(); + StringBuilder input = new StringBuilder("00001933 0 439895060 1 0 00001933 0 439895060 1 0 0000208 0 439895060 0 112 00038..."); + Message msg = ReflectionUtil.call(client, "parseMessage", input); + assertEquals(33, msg.getType()); + assertEquals(0, msg.getSender()); + assertEquals(439895060, msg.getReceiver()); + assertEquals(1, msg.getArg1()); + assertEquals(0, msg.getArg2()); + assertTrue(msg.getMessage().isBlank()); + + input = new StringBuilder("00207820 439461784 439895060 12 507930 %7B%22lv%22%3A4%2C%22nm%22%3A%22Nivea%22%2C%22pid%22%3A1%2C%22sid%22%3A439461784%2C%22uid%22%3A507930%2C%22vs%22%3A12%2C%22u%22%3A%7B%22age%22%3A33%2C%22avatar%22%3A1%2C%22blurb%22%3A%22I%20love%20when%20my%20nose%20touch%20your%20belly%20when%20I%20do%20you%20a%20blowjob!%20When%20I%20look%20into%20your%20horny%20eyes%20when%22%2C%22camserv%22%3A1367%2C%22chat_color%22%3A%22FF0000%22%2C%22chat_font%22%3A0%2C%22chat_opt%22%3A1%2C%22ethnic%22%3A%22Caucasian%22%2C%22photos%22%3A74%2C%22profile%22%3A1%2C%22status%22%3A%22%22%7D%2C%22m%22%3A%7B%22camscore%22%3A7505.700%2C%22continent%22%3A%22EU%22%2C%22flags%22%3A605224%2C%22hidecs%22%3Atrue%2C%22kbit%22%3A0%2C%22lastnews%22%3A0%2C%22mg%22%3A0%2C%22missmfc%22%3A2%2C%22new_model%22%3A0%2C%22rank%22%3A0%2C%22rc%22%3A20%2C%22sfw%22%3A0%2C%22topic%22%3A%22hi%253A)%255Bnone%255D-Topless%252C500-snap4life%252C50-spanks%252C180-flash%252C700-10%2520mins%2520of%2520Nora%2520fun%252C666-shot%252C27%252C270%2520%253C3%22%7D%2C%22x%22%3A%7B%22fcext%22%3A%7B%22sm%22%3A%22%22%2C%22sfw%22%3A0%7D%2C%22share%22%3A%7B%22follows%22%3A11%2C%22albums%22%3A0%2C%22clubs%22%3A0%2C%22tm_album%22%3A0%2C%22collections%22%3A0%2C%22stores%22%3A0%2C%22goals%22%3A0%2C%22polls%22%3A0%2C%22things%22%3A0%2C%22recent_album_tm%22%3A0%2C%22recent_club_tm%22%3A0%2C%22recent_collection_tm%22%3A0%2C%22recent_goal_tm%22%3A0%2C%22recent_item_tm%22%3A0%2C%22recent_poll_tm%22%3A0%2C%22recent_story_tm%22%3A0%2C%22recent_album_thumb%22%3A%22%22%2C%22recent_club_thumb%22%3A%22%22%2C%22recent_collection_thumb%22%3A%22%22%2C%22recent_goal_thumb%22%3A%22%22%2C%22recent_item_thumb%22%3A%22%22%2C%22recent_poll_thumb%22%3A%22%22%2C%22recent_story_thumb%22%3A%22%22%2C%22recent_album_title%22%3A%22%22%2C%22recent_club_title%22%3A%22%22%2C%22recent_collection_title%22%3A%22%22%2C%22recent_goal_title%22%3A%22%22%2C%22recent_item_title%22%3A%22%22%2C%22recent_poll_title%22%3A%22%22%2C%22recent_story_title%22%3A%22%22%2C%22recent_album_slug%22%3A%22%22%2C%22recent_collection_slug%22%3A%22%22%2C%22tipmenus%22%3A0%7D%7D%7D"); + msg = ReflectionUtil.call(client, "parseMessage", input); + assertEquals(20, msg.getType()); + assertEquals(439461784, msg.getSender()); + assertEquals(439895060, msg.getReceiver()); + assertEquals(12, msg.getArg1()); + assertEquals(507930, msg.getArg2()); + assertEquals(JSONObject.class, new JSONObject(msg.getMessage()).getClass()); // make sure we have a parsable JSON message + } +} diff --git a/master/pom.xml b/master/pom.xml index ebbb31be..d332594f 100644 --- a/master/pom.xml +++ b/master/pom.xml @@ -6,7 +6,7 @@ ctbrec master pom - 3.8.1 + 3.8.6 ../common @@ -16,7 +16,7 @@ UTF-8 - 14-ea+4 + 14.0.2.1 diff --git a/server/pom.xml b/server/pom.xml index 26cf509e..4d56941e 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -8,7 +8,7 @@ ctbrec master - 3.8.1 + 3.8.6 ../master diff --git a/server/src/main/java/ctbrec/recorder/server/HttpServer.java b/server/src/main/java/ctbrec/recorder/server/HttpServer.java index f97d49b2..fb179fbe 100644 --- a/server/src/main/java/ctbrec/recorder/server/HttpServer.java +++ b/server/src/main/java/ctbrec/recorder/server/HttpServer.java @@ -106,7 +106,7 @@ public class HttpServer { safeLogin(site); } } - onlineMonitor = new OnlineMonitor(recorder); + onlineMonitor = new OnlineMonitor(recorder, config); onlineMonitor.start(); startHttpServer(); } diff --git a/server/src/main/java/ctbrec/recorder/server/RecorderServlet.java b/server/src/main/java/ctbrec/recorder/server/RecorderServlet.java index 5c71abec..fb42c4a0 100644 --- a/server/src/main/java/ctbrec/recorder/server/RecorderServlet.java +++ b/server/src/main/java/ctbrec/recorder/server/RecorderServlet.java @@ -96,6 +96,17 @@ public class RecorderServlet extends AbstractCtbrecServlet { response = "{\"status\": \"success\", \"msg\": \"Stopping recording\"}"; resp.getWriter().write(response); break; + case "stopAt": + new Thread(() -> { + try { + recorder.stopRecordingAt(request.model); + } catch (InvalidKeyException | NoSuchAlgorithmException | IllegalStateException | IOException e) { + LOG.error("Couldn't stop recording for model {}", request.model, e); + } + }).start(); + response = "{\"status\": \"success\", \"msg\": \"Stopping recording\"}"; + resp.getWriter().write(response); + break; case "list": resp.getWriter().write("{\"status\": \"success\", \"msg\": \"List of models\", \"models\": ["); JsonAdapter modelAdapter = new ModelJsonAdapter(); diff --git a/server/src/main/resources/html/static/custom.css b/server/src/main/resources/html/static/custom.css index 820f8651..a1df2311 100644 --- a/server/src/main/resources/html/static/custom.css +++ b/server/src/main/resources/html/static/custom.css @@ -58,6 +58,10 @@ th a:hover { text-decoration: none; } +.checkmark-green { + color: #28a745; +} - - +.red { + color: #dc4444; +} \ No newline at end of file diff --git a/server/src/main/resources/html/static/index.html b/server/src/main/resources/html/static/index.html index dd0d9aeb..e9cc978b 100644 --- a/server/src/main/resources/html/static/index.html +++ b/server/src/main/resources/html/static/index.html @@ -96,19 +96,22 @@ Model - Paused Online Recording - Actions + + - - - - + + + + + + + diff --git a/server/src/main/resources/html/static/models.js b/server/src/main/resources/html/static/models.js index 55ac777a..4534b531 100644 --- a/server/src/main/resources/html/static/models.js +++ b/server/src/main/resources/html/static/models.js @@ -69,6 +69,7 @@ function syncModels(models) { } } model.ko_recording = ko.observable(model.online && !model.suspended); + //model.ko_recording_class = ko.observable( (model.online && !model.suspended) ? 'fa fa-circle red' : '' ); model.ko_suspended = ko.observable(model.suspended); model.swallowEvents = false; model.ko_suspended.subscribe(function(checked) { @@ -102,6 +103,7 @@ function syncModels(models) { } } m.ko_online(onlineState); + //m.ko_recording_class( (model.online && !model.suspended) ? 'fa fa-circle red' : ''); m.swallowEvents = true; m.ko_suspended(model.suspended); m.swallowEvents = false; diff --git a/server/src/main/resources/html/static/recordings.js b/server/src/main/resources/html/static/recordings.js index 9380eb82..6c446355 100644 --- a/server/src/main/resources/html/static/recordings.js +++ b/server/src/main/resources/html/static/recordings.js @@ -195,7 +195,6 @@ function updateDiskSpace() { throughput.bytes(data.throughput); throughput.timeframe(data.throughputTimeframe); let bytesPerSecond = data.throughput / data.throughputTimeframe; - console.log(data.throughput, data.throughputTimeframe, bytesPerSecond, calculateSize(bytesPerSecond) + '/s'); throughput.text(calculateSize(bytesPerSecond) + '/s'); } else { if (console)