From 1c8d9bf67890569033904dfa1dfdc373d95e8bf2 Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Sun, 29 Oct 2023 14:23:33 +0100 Subject: [PATCH 1/3] Use the thumbnail for the online check by @winkru In case a model is not online, CB returns a default image. We do a HEAD request and check the size of the response. If it is 21971 bytes, we assume that it is the default image and also that the model is offline. The request hits the CDN instead of the main site, so request throttling should not kick in. --- .../sites/chaturbate/ChaturbateModel.java | 42 +++++++++++++++---- 1 file changed, 35 insertions(+), 7 deletions(-) diff --git a/common/src/main/java/ctbrec/sites/chaturbate/ChaturbateModel.java b/common/src/main/java/ctbrec/sites/chaturbate/ChaturbateModel.java index e01d3920..3a50e3d2 100644 --- a/common/src/main/java/ctbrec/sites/chaturbate/ChaturbateModel.java +++ b/common/src/main/java/ctbrec/sites/chaturbate/ChaturbateModel.java @@ -55,9 +55,15 @@ public class ChaturbateModel extends AbstractModel { // NOSONAR public boolean isOnline(boolean ignoreCache) throws IOException, ExecutionException, InterruptedException { String roomStatus; if (ignoreCache) { - StreamInfo info = loadStreamInfo(); - roomStatus = Optional.ofNullable(info).map(i -> i.room_status).orElse(""); - LOG.trace("Model {} room status: {}", getName(), Optional.ofNullable(info).map(i -> i.room_status).orElse("unknown")); + if (isOffline()) { + roomStatus = "offline"; + onlineState = State.OFFLINE; + LOG.trace("Model {} offline", getName()); + } else { + StreamInfo info = getStreamInfo(); + roomStatus = Optional.ofNullable(info).map(i -> i.room_status).orElse(""); + LOG.trace("Model {} room status: {}", getName(), Optional.ofNullable(info).map(i -> i.room_status).orElse("unknown")); + } } else { StreamInfo info = getStreamInfo(true); roomStatus = Optional.ofNullable(info).map(i -> i.room_status).orElse(""); @@ -65,6 +71,24 @@ public class ChaturbateModel extends AbstractModel { // NOSONAR return Objects.equals(PUBLIC, roomStatus); } + private boolean isOffline() { + String normalizedName = getName().toLowerCase().trim(); + String previewUrl = "https://roomimg.stream.highwebmedia.com/ri/" + normalizedName + ".jpg?" + Instant.now().getEpochSecond(); + Request req = new Request.Builder() + .url(previewUrl) + .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) + .head() + .build(); + try (Response response = getSite().getHttpClient().execute(req)) { + if (response.isSuccessful()) { + return response.header("Content-Length", "0").equals("21971"); + } + } catch (Exception ex) { + // fail silently + } + return false; + } + @Override public int[] getStreamResolution(boolean failFast) throws ExecutionException { if (failFast) { @@ -97,8 +121,12 @@ public class ChaturbateModel extends AbstractModel { // NOSONAR if (failFast) { setOnlineStateByRoomStatus(Optional.ofNullable(streamInfo).map(si -> si.room_status).orElse("Unknown")); } else { - streamInfo = loadStreamInfo(); - setOnlineStateByRoomStatus(streamInfo.room_status); + if (isOffline()) { + onlineState = OFFLINE; + } else { + streamInfo = loadStreamInfo(); + setOnlineStateByRoomStatus(streamInfo.room_status); + } } return onlineState; } @@ -233,7 +261,7 @@ public class ChaturbateModel extends AbstractModel { // NOSONAR } private StreamInfo loadStreamInfo() throws IOException { - if (Duration.between(lastStreamInfoRequest, Instant.now()).getSeconds() < 2) { + if (streamInfo != null && Duration.between(lastStreamInfoRequest, Instant.now()).getSeconds() < 5) { return streamInfo; } RequestBody body = new FormBody.Builder() @@ -250,7 +278,7 @@ public class ChaturbateModel extends AbstractModel { // NOSONAR lastStreamInfoRequest = Instant.now(); if (response.isSuccessful()) { String content = response.body().string(); - LOG.trace("Raw stream info: {}", content); + LOG.trace("Raw stream info for model {}: {}", getName(), content); Moshi moshi = new Moshi.Builder().build(); JsonAdapter adapter = moshi.adapter(StreamInfo.class); streamInfo = adapter.fromJson(content); From addbeab76e6dad636b523b1bdfffedb438a72d46 Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Sun, 29 Oct 2023 15:05:41 +0100 Subject: [PATCH 2/3] Integrate DramCam, Streamray and WinkTv by @winkru --- .../java/ctbrec/ui/CamrecApplication.java | 10 +- .../main/java/ctbrec/ui/SiteUiFactory.java | 27 +- .../ui/sites/dreamcam/DreamcamConfigUI.java | 66 ++++ .../ui/sites/dreamcam/DreamcamSiteUi.java | 40 ++ .../sites/dreamcam/DreamcamTabProvider.java | 37 ++ .../sites/dreamcam/DreamcamUpdateService.java | 86 +++++ .../ui/sites/streamray/StreamrayConfigUI.java | 67 ++++ .../StreamrayElectronLoginDialog.java | 83 +++++ .../streamray/StreamrayFavoritesService.java | 132 +++++++ .../streamray/StreamrayFavoritesTab.java | 77 ++++ .../ui/sites/streamray/StreamraySiteUi.java | 44 +++ .../sites/streamray/StreamrayTabProvider.java | 49 +++ .../streamray/StreamrayUpdateService.java | 146 ++++++++ .../ui/sites/winktv/WinkTvConfigUI.java | 55 +++ .../ctbrec/ui/sites/winktv/WinkTvSiteUi.java | 40 ++ .../ui/sites/winktv/WinkTvTabProvider.java | 37 ++ .../ui/sites/winktv/WinkTvUpdateService.java | 127 +++++++ .../src/main/java/ctbrec/AbstractModel.java | 34 +- common/src/main/java/ctbrec/Model.java | 31 +- common/src/main/java/ctbrec/Settings.java | 5 + .../recorder/download/AbstractDownload.java | 85 +++-- .../download/hls/FfmpegHlsDownload.java | 325 +++++++++++++++++ .../ctbrec/sites/chaturbate/Chaturbate.java | 2 +- .../chaturbate/ChaturbateHttpClient.java | 21 +- .../java/ctbrec/sites/dreamcam/Dreamcam.java | 127 +++++++ .../sites/dreamcam/DreamcamDownload.java | 341 ++++++++++++++++++ .../sites/dreamcam/DreamcamHttpClient.java | 17 + .../ctbrec/sites/dreamcam/DreamcamModel.java | 205 +++++++++++ .../ctbrec/sites/streamray/Streamray.java | 178 +++++++++ .../sites/streamray/StreamrayHttpClient.java | 79 ++++ .../sites/streamray/StreamrayModel.java | 197 ++++++++++ .../main/java/ctbrec/sites/winktv/WinkTv.java | 182 ++++++++++ .../ctbrec/sites/winktv/WinkTvHttpClient.java | 17 + .../java/ctbrec/sites/winktv/WinkTvModel.java | 270 ++++++++++++++ .../ctbrec/recorder/server/HttpServer.java | 22 +- 35 files changed, 3186 insertions(+), 75 deletions(-) create mode 100644 client/src/main/java/ctbrec/ui/sites/dreamcam/DreamcamConfigUI.java create mode 100644 client/src/main/java/ctbrec/ui/sites/dreamcam/DreamcamSiteUi.java create mode 100644 client/src/main/java/ctbrec/ui/sites/dreamcam/DreamcamTabProvider.java create mode 100644 client/src/main/java/ctbrec/ui/sites/dreamcam/DreamcamUpdateService.java create mode 100644 client/src/main/java/ctbrec/ui/sites/streamray/StreamrayConfigUI.java create mode 100644 client/src/main/java/ctbrec/ui/sites/streamray/StreamrayElectronLoginDialog.java create mode 100644 client/src/main/java/ctbrec/ui/sites/streamray/StreamrayFavoritesService.java create mode 100644 client/src/main/java/ctbrec/ui/sites/streamray/StreamrayFavoritesTab.java create mode 100644 client/src/main/java/ctbrec/ui/sites/streamray/StreamraySiteUi.java create mode 100644 client/src/main/java/ctbrec/ui/sites/streamray/StreamrayTabProvider.java create mode 100644 client/src/main/java/ctbrec/ui/sites/streamray/StreamrayUpdateService.java create mode 100644 client/src/main/java/ctbrec/ui/sites/winktv/WinkTvConfigUI.java create mode 100644 client/src/main/java/ctbrec/ui/sites/winktv/WinkTvSiteUi.java create mode 100644 client/src/main/java/ctbrec/ui/sites/winktv/WinkTvTabProvider.java create mode 100644 client/src/main/java/ctbrec/ui/sites/winktv/WinkTvUpdateService.java create mode 100644 common/src/main/java/ctbrec/recorder/download/hls/FfmpegHlsDownload.java create mode 100644 common/src/main/java/ctbrec/sites/dreamcam/Dreamcam.java create mode 100644 common/src/main/java/ctbrec/sites/dreamcam/DreamcamDownload.java create mode 100644 common/src/main/java/ctbrec/sites/dreamcam/DreamcamHttpClient.java create mode 100644 common/src/main/java/ctbrec/sites/dreamcam/DreamcamModel.java create mode 100644 common/src/main/java/ctbrec/sites/streamray/Streamray.java create mode 100644 common/src/main/java/ctbrec/sites/streamray/StreamrayHttpClient.java create mode 100644 common/src/main/java/ctbrec/sites/streamray/StreamrayModel.java create mode 100644 common/src/main/java/ctbrec/sites/winktv/WinkTv.java create mode 100644 common/src/main/java/ctbrec/sites/winktv/WinkTvHttpClient.java create mode 100644 common/src/main/java/ctbrec/sites/winktv/WinkTvModel.java diff --git a/client/src/main/java/ctbrec/ui/CamrecApplication.java b/client/src/main/java/ctbrec/ui/CamrecApplication.java index d8bf8e23..19e1bf6b 100644 --- a/client/src/main/java/ctbrec/ui/CamrecApplication.java +++ b/client/src/main/java/ctbrec/ui/CamrecApplication.java @@ -27,6 +27,7 @@ import ctbrec.sites.cam4.Cam4; import ctbrec.sites.camsoda.Camsoda; import ctbrec.sites.chaturbate.Chaturbate; import ctbrec.sites.cherrytv.CherryTv; +import ctbrec.sites.dreamcam.Dreamcam; import ctbrec.sites.fc2live.Fc2Live; import ctbrec.sites.flirt4free.Flirt4Free; import ctbrec.sites.jasmin.LiveJasmin; @@ -35,7 +36,9 @@ import ctbrec.sites.mfc.MyFreeCams; import ctbrec.sites.secretfriends.SecretFriends; import ctbrec.sites.showup.Showup; import ctbrec.sites.streamate.Streamate; +import ctbrec.sites.streamray.Streamray; import ctbrec.sites.stripchat.Stripchat; +import ctbrec.sites.winktv.WinkTv; import ctbrec.sites.xlovecam.XloveCam; import ctbrec.ui.controls.Dialogs; import ctbrec.ui.news.NewsTab; @@ -123,7 +126,9 @@ public class CamrecApplication extends Application { initSites(); startOnlineMonitor(); createGui(primaryStage); - checkForUpdates(); + if (config.getSettings().checkForUpdates) { + checkForUpdates(); + } registerClipboardListener(); registerTrayIconListener(); } @@ -162,6 +167,7 @@ public class CamrecApplication extends Application { sites.add(new Camsoda()); sites.add(new Chaturbate()); sites.add(new CherryTv()); + sites.add(new Dreamcam()); sites.add(new Fc2Live()); sites.add(new Flirt4Free()); sites.add(new LiveJasmin()); @@ -171,6 +177,8 @@ public class CamrecApplication extends Application { sites.add(new Showup()); sites.add(new Streamate()); sites.add(new Stripchat()); + sites.add(new Streamray()); + sites.add(new WinkTv()); sites.add(new XloveCam()); } diff --git a/client/src/main/java/ctbrec/ui/SiteUiFactory.java b/client/src/main/java/ctbrec/ui/SiteUiFactory.java index 36b27ed0..187ccb0d 100644 --- a/client/src/main/java/ctbrec/ui/SiteUiFactory.java +++ b/client/src/main/java/ctbrec/ui/SiteUiFactory.java @@ -7,6 +7,7 @@ import ctbrec.sites.cam4.Cam4; import ctbrec.sites.camsoda.Camsoda; import ctbrec.sites.chaturbate.Chaturbate; import ctbrec.sites.cherrytv.CherryTv; +import ctbrec.sites.dreamcam.Dreamcam; import ctbrec.sites.fc2live.Fc2Live; import ctbrec.sites.flirt4free.Flirt4Free; import ctbrec.sites.jasmin.LiveJasmin; @@ -15,7 +16,9 @@ import ctbrec.sites.mfc.MyFreeCams; import ctbrec.sites.secretfriends.SecretFriends; import ctbrec.sites.showup.Showup; import ctbrec.sites.streamate.Streamate; +import ctbrec.sites.streamray.Streamray; import ctbrec.sites.stripchat.Stripchat; +import ctbrec.sites.winktv.WinkTv; import ctbrec.sites.xlovecam.XloveCam; import ctbrec.ui.sites.amateurtv.AmateurTvSiteUi; import ctbrec.ui.sites.bonga.BongaCamsSiteUi; @@ -23,6 +26,7 @@ import ctbrec.ui.sites.cam4.Cam4SiteUi; import ctbrec.ui.sites.camsoda.CamsodaSiteUi; import ctbrec.ui.sites.chaturbate.ChaturbateSiteUi; import ctbrec.ui.sites.cherrytv.CherryTvSiteUi; +import ctbrec.ui.sites.dreamcam.DreamcamSiteUi; import ctbrec.ui.sites.fc2live.Fc2LiveSiteUi; import ctbrec.ui.sites.flirt4free.Flirt4FreeSiteUi; import ctbrec.ui.sites.jasmin.LiveJasminSiteUi; @@ -31,7 +35,9 @@ import ctbrec.ui.sites.myfreecams.MyFreeCamsSiteUi; import ctbrec.ui.sites.secretfriends.SecretFriendsSiteUi; import ctbrec.ui.sites.showup.ShowupSiteUi; import ctbrec.ui.sites.streamate.StreamateSiteUi; +import ctbrec.ui.sites.streamray.StreamraySiteUi; import ctbrec.ui.sites.stripchat.StripchatSiteUi; +import ctbrec.ui.sites.winktv.WinkTvSiteUi; import ctbrec.ui.sites.xlovecam.XloveCamSiteUi; public class SiteUiFactory { @@ -52,8 +58,12 @@ public class SiteUiFactory { private static StreamateSiteUi streamateSiteUi; private static StripchatSiteUi stripchatSiteUi; private static XloveCamSiteUi xloveCamSiteUi; + private static StreamraySiteUi streamraySiteUi; + private static WinkTvSiteUi winkTvSiteUi; + private static DreamcamSiteUi dreamcamSiteUi; - private SiteUiFactory () {} + private SiteUiFactory() { + } public static synchronized SiteUI getUi(Site site) { // NOSONAR if (site instanceof AmateurTv) { @@ -86,6 +96,11 @@ public class SiteUiFactory { cherryTvSiteUi = new CherryTvSiteUi((CherryTv) site); } return cherryTvSiteUi; + } else if (site instanceof Dreamcam) { + if (dreamcamSiteUi == null) { + dreamcamSiteUi = new DreamcamSiteUi((Dreamcam) site); + } + return dreamcamSiteUi; } else if (site instanceof Fc2Live) { if (fc2SiteUi == null) { fc2SiteUi = new Fc2LiveSiteUi((Fc2Live) site); @@ -131,6 +146,16 @@ public class SiteUiFactory { stripchatSiteUi = new StripchatSiteUi((Stripchat) site); } return stripchatSiteUi; + } else if (site instanceof Streamray) { + if (streamraySiteUi == null) { + streamraySiteUi = new StreamraySiteUi((Streamray) site); + } + return streamraySiteUi; + } else if (site instanceof WinkTv) { + if (winkTvSiteUi == null) { + winkTvSiteUi = new WinkTvSiteUi((WinkTv) site); + } + return winkTvSiteUi; } else if (site instanceof XloveCam) { if (xloveCamSiteUi == null) { xloveCamSiteUi = new XloveCamSiteUi((XloveCam) site); diff --git a/client/src/main/java/ctbrec/ui/sites/dreamcam/DreamcamConfigUI.java b/client/src/main/java/ctbrec/ui/sites/dreamcam/DreamcamConfigUI.java new file mode 100644 index 00000000..44fcf407 --- /dev/null +++ b/client/src/main/java/ctbrec/ui/sites/dreamcam/DreamcamConfigUI.java @@ -0,0 +1,66 @@ +package ctbrec.ui.sites.dreamcam; + +import ctbrec.Config; +import ctbrec.sites.dreamcam.Dreamcam; +import ctbrec.ui.settings.SettingsTab; +import ctbrec.ui.sites.AbstractConfigUI; +import javafx.geometry.Insets; +import javafx.scene.Parent; +import javafx.scene.control.ToggleGroup; +import javafx.scene.control.Button; +import javafx.scene.control.CheckBox; +import javafx.scene.control.RadioButton; +import javafx.scene.control.Label; +import javafx.scene.layout.GridPane; +import javafx.scene.layout.HBox; + +public class DreamcamConfigUI extends AbstractConfigUI { + private final Dreamcam site; + + public DreamcamConfigUI(Dreamcam site) { + this.site = site; + } + + @Override + public Parent createConfigPanel() { + GridPane layout = SettingsTab.createGridLayout(); + var settings = Config.getInstance().getSettings(); + + var row = 0; + var l = new Label("Active"); + layout.add(l, 0, row); + var enabled = new CheckBox(); + enabled.setSelected(!settings.disabledSites.contains(site.getName())); + enabled.setOnAction(e -> { + if(enabled.isSelected()) { + settings.disabledSites.remove(site.getName()); + } else { + settings.disabledSites.add(site.getName()); + } + save(); + }); + GridPane.setMargin(enabled, new Insets(0, 0, 0, SettingsTab.CHECKBOX_MARGIN)); + layout.add(enabled, 1, row++); + + row++; + l = new Label("VR Mode"); + layout.add(l, 0, row); + var vr = new CheckBox(); + vr.setSelected(settings.dreamcamVR); + vr.setOnAction(e -> { + settings.dreamcamVR = vr.isSelected(); + save(); + }); + GridPane.setMargin(vr, new Insets(0, 0, 0, SettingsTab.CHECKBOX_MARGIN)); + layout.add(vr, 1, row++); + + var deleteCookies = new Button("Delete Cookies"); + deleteCookies.setOnAction(e -> site.getHttpClient().clearCookies()); + layout.add(deleteCookies, 1, row); + GridPane.setColumnSpan(deleteCookies, 2); + + GridPane.setMargin(deleteCookies, new Insets(0, 0, 0, SettingsTab.CHECKBOX_MARGIN)); + return layout; + } + +} diff --git a/client/src/main/java/ctbrec/ui/sites/dreamcam/DreamcamSiteUi.java b/client/src/main/java/ctbrec/ui/sites/dreamcam/DreamcamSiteUi.java new file mode 100644 index 00000000..cde8f98a --- /dev/null +++ b/client/src/main/java/ctbrec/ui/sites/dreamcam/DreamcamSiteUi.java @@ -0,0 +1,40 @@ +package ctbrec.ui.sites.dreamcam; + +import ctbrec.sites.dreamcam.Dreamcam; +import ctbrec.ui.sites.AbstractSiteUi; +import ctbrec.ui.sites.ConfigUI; +import ctbrec.ui.tabs.TabProvider; + +import java.io.IOException; + +public class DreamcamSiteUi extends AbstractSiteUi { + + private DreamcamTabProvider tabProvider; + private DreamcamConfigUI configUi; + private final Dreamcam site; + + public DreamcamSiteUi(Dreamcam site) { + this.site = site; + } + + @Override + public TabProvider getTabProvider() { + if (tabProvider == null) { + tabProvider = new DreamcamTabProvider(site); + } + return tabProvider; + } + + @Override + public ConfigUI getConfigUI() { + if (configUi == null) { + configUi = new DreamcamConfigUI(site); + } + return configUi; + } + + @Override + public synchronized boolean login() throws IOException { + return site.login(); + } +} diff --git a/client/src/main/java/ctbrec/ui/sites/dreamcam/DreamcamTabProvider.java b/client/src/main/java/ctbrec/ui/sites/dreamcam/DreamcamTabProvider.java new file mode 100644 index 00000000..17226cc0 --- /dev/null +++ b/client/src/main/java/ctbrec/ui/sites/dreamcam/DreamcamTabProvider.java @@ -0,0 +1,37 @@ +package ctbrec.ui.sites.dreamcam; + +import ctbrec.sites.dreamcam.Dreamcam; +import ctbrec.sites.dreamcam.DreamcamModel; +import ctbrec.ui.sites.AbstractTabProvider; +import ctbrec.ui.tabs.ThumbOverviewTab; +import java.util.ArrayList; +import java.util.List; +import javafx.scene.Scene; +import javafx.scene.control.Tab; + +public class DreamcamTabProvider extends AbstractTabProvider { + private final static String API_URL = "https://bss.dreamcamtrue.com/api/clients/v1/broadcasts?partnerId=dreamcam_oauth2&show-offline=false&stream-types=video2D&include-tags=false&include-tip-menu=false"; + + public DreamcamTabProvider(Dreamcam site) { + super(site); + } + + @Override + protected List getSiteTabs(Scene scene) { + List tabs = new ArrayList<>(); + tabs.add(createTab("Girls", API_URL + "&tag-categories=girls")); + tabs.add(createTab("Boys", API_URL + "&tag-categories=men")); + tabs.add(createTab("Couples", API_URL + "&tag-categories=couples")); + tabs.add(createTab("Trans", API_URL + "&tag-categories=trans")); + return tabs; + } + + private Tab createTab(String title, String url) { + var updateService = new DreamcamUpdateService((Dreamcam) site, url); + var tab = new ThumbOverviewTab(title, updateService, site); + tab.setImageAspectRatio(10.0 / 16.0); + tab.setRecorder(recorder); + return tab; + } + +} diff --git a/client/src/main/java/ctbrec/ui/sites/dreamcam/DreamcamUpdateService.java b/client/src/main/java/ctbrec/ui/sites/dreamcam/DreamcamUpdateService.java new file mode 100644 index 00000000..b413b8aa --- /dev/null +++ b/client/src/main/java/ctbrec/ui/sites/dreamcam/DreamcamUpdateService.java @@ -0,0 +1,86 @@ +package ctbrec.ui.sites.dreamcam; + +import static ctbrec.io.HttpConstants.*; + +import ctbrec.Config; +import ctbrec.Model; +import ctbrec.io.HttpException; +import ctbrec.sites.dreamcam.Dreamcam; +import ctbrec.sites.dreamcam.DreamcamModel; +import ctbrec.ui.tabs.PaginatedScheduledService; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import javafx.concurrent.Task; +import okhttp3.Request; +import okhttp3.Response; +import org.json.JSONArray; +import org.json.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class DreamcamUpdateService extends PaginatedScheduledService { + private static final Logger LOG = LoggerFactory.getLogger(DreamcamUpdateService.class); + private static final String API_URL = "https://api.dreamcam.co.kr/v1/live"; + private static final int modelsPerPage = 64; + private Dreamcam site; + private String url; + + public DreamcamUpdateService(Dreamcam site, String url) { + this.site = site; + this.url = url; + } + + @Override + protected Task> createTask() { + return new Task>() { + @Override + public List call() throws IOException { + return loadModelList(); + } + }; + } + + private List loadModelList() throws IOException { + int offset = (getPage() - 1) * modelsPerPage; + int limit = modelsPerPage; + String paginatedUrl = url + "&offset=" + offset + "&limit=" + limit; + LOG.debug("Fetching page {}", paginatedUrl); + Request req = new Request.Builder() + .url(paginatedUrl) + .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) + .header(ACCEPT, MIMETYPE_APPLICATION_JSON) + .header(ACCEPT_LANGUAGE, Locale.ENGLISH.getLanguage()) + .header(X_REQUESTED_WITH, XML_HTTP_REQUEST) + .header(REFERER, site.getBaseUrl() + "/") + .header(ORIGIN, site.getBaseUrl()) + .build(); + try (Response response = site.getHttpClient().execute(req)) { + if (response.isSuccessful()) { + List models = new ArrayList<>(); + String content = response.body().string(); + JSONObject json = new JSONObject(content); + if (json.has("pageItems")) { + JSONArray modelNodes = json.getJSONArray("pageItems"); + parseModels(modelNodes, models); + } + return models; + } else { + throw new HttpException(response.code(), response.message()); + } + } + } + + private void parseModels(JSONArray jsonModels, List models) { + for (int i = 0; i < jsonModels.length(); i++) { + JSONObject m = jsonModels.getJSONObject(i); + String name = m.optString("modelNickname"); + DreamcamModel model = (DreamcamModel) site.createModel(name); + model.setDisplayName(name); + model.setPreview(m.optString("modelProfilePhotoUrl")); + model.setDescription(m.optString("broadcastTextStatus")); + models.add(model); + } + } +} diff --git a/client/src/main/java/ctbrec/ui/sites/streamray/StreamrayConfigUI.java b/client/src/main/java/ctbrec/ui/sites/streamray/StreamrayConfigUI.java new file mode 100644 index 00000000..3cc33e35 --- /dev/null +++ b/client/src/main/java/ctbrec/ui/sites/streamray/StreamrayConfigUI.java @@ -0,0 +1,67 @@ +package ctbrec.ui.sites.streamray; + +import ctbrec.Config; +import ctbrec.sites.streamray.Streamray; +import ctbrec.ui.settings.SettingsTab; +import ctbrec.ui.sites.AbstractConfigUI; +import javafx.geometry.Insets; +import javafx.scene.Parent; +import javafx.scene.control.ToggleGroup; +import javafx.scene.control.Button; +import javafx.scene.control.CheckBox; +import javafx.scene.control.RadioButton; +import javafx.scene.control.Label; +import javafx.scene.layout.GridPane; +import javafx.scene.layout.HBox; + +public class StreamrayConfigUI extends AbstractConfigUI { + private final Streamray site; + + public StreamrayConfigUI(Streamray site) { + this.site = site; + } + + @Override + public Parent createConfigPanel() { + GridPane layout = SettingsTab.createGridLayout(); + var settings = Config.getInstance().getSettings(); + + var row = 0; + var l = new Label("Active"); + layout.add(l, 0, row); + var enabled = new CheckBox(); + enabled.setSelected(!settings.disabledSites.contains(site.getName())); + enabled.setOnAction(e -> { + if(enabled.isSelected()) { + settings.disabledSites.remove(site.getName()); + } else { + settings.disabledSites.add(site.getName()); + } + save(); + }); + GridPane.setMargin(enabled, new Insets(0, 0, 0, SettingsTab.CHECKBOX_MARGIN)); + layout.add(enabled, 1, row++); + row++; + + l = new Label("Record Goal/Tipping shows"); + layout.add(l, 0, row); + var cb = new CheckBox(); + cb.setSelected(settings.streamrayRecordGoalShows); + cb.setOnAction(e -> { + settings.streamrayRecordGoalShows = cb.isSelected(); + save(); + }); + GridPane.setMargin(cb, new Insets(0, 0, 0, SettingsTab.CHECKBOX_MARGIN)); + layout.add(cb, 1, row++); + row++; + + var deleteCookies = new Button("Delete Cookies"); + deleteCookies.setOnAction(e -> site.getHttpClient().clearCookies()); + layout.add(deleteCookies, 1, row); + GridPane.setColumnSpan(deleteCookies, 2); + + GridPane.setMargin(deleteCookies, new Insets(0, 0, 0, SettingsTab.CHECKBOX_MARGIN)); + return layout; + } + +} diff --git a/client/src/main/java/ctbrec/ui/sites/streamray/StreamrayElectronLoginDialog.java b/client/src/main/java/ctbrec/ui/sites/streamray/StreamrayElectronLoginDialog.java new file mode 100644 index 00000000..2d727ee0 --- /dev/null +++ b/client/src/main/java/ctbrec/ui/sites/streamray/StreamrayElectronLoginDialog.java @@ -0,0 +1,83 @@ +package ctbrec.ui.sites.streamray; + +import java.io.IOException; +import java.util.Collections; +import java.util.function.Consumer; + +import org.json.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import ctbrec.Config; +import ctbrec.sites.streamray.Streamray; +import ctbrec.ui.ExternalBrowser; +import okhttp3.Cookie; +import okhttp3.Cookie.Builder; +import okhttp3.CookieJar; +import okhttp3.HttpUrl; + +public class StreamrayElectronLoginDialog { + + private static final Logger LOG = LoggerFactory.getLogger(StreamrayElectronLoginDialog.class); + public static final String DOMAIN = "streamray.com"; + public static final String URL = "https://streamray.com/"; + private CookieJar cookieJar; + private ExternalBrowser browser; + private boolean firstCall = true; + private final static Streamray site = new Streamray(); + + public StreamrayElectronLoginDialog(CookieJar cookieJar) throws IOException { + this.cookieJar = cookieJar; + browser = ExternalBrowser.getInstance(); + try { + var config = new JSONObject(); + config.put("url", URL); + config.put("w", 800); + config.put("h", 600); + config.put("userAgent", Config.getInstance().getSettings().httpUserAgent); + var msg = new JSONObject(); + msg.put("config", config); + browser.run(msg, msgHandler); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IOException("Couldn't wait for login dialog", e); + } finally { + browser.close(); + } + } + + private Consumer msgHandler = line -> { + if (!line.startsWith("{")) return; + JSONObject json = new JSONObject(line); + boolean loginCookie = false; + if (json.has("cookies")) { + var cookies = json.getJSONArray("cookies"); + for (var i = 0; i < cookies.length(); i++) { + var cookie = cookies.getJSONObject(i); + if (cookie.getString("domain").contains(DOMAIN)) { + if (cookie.optString("name").equals("memberToken")) { + loginCookie = true; + } + Builder b = new Cookie.Builder() + .path(cookie.getString("path")) + .domain(DOMAIN) + .name(cookie.optString("name")) + .value(cookie.optString("value")) + .expiresAt(Double.valueOf(cookie.optDouble("expirationDate")).longValue()); // NOSONAR + if (cookie.optBoolean("hostOnly")) { + b.hostOnlyDomain(DOMAIN); + } + if (cookie.optBoolean("httpOnly")) { + b.httpOnly(); + } + if (cookie.optBoolean("secure")) { + b.secure(); + } + Cookie c = b.build(); + LOG.trace("Adding cookie {}={}", c.name(), c.value()); + cookieJar.saveFromResponse(HttpUrl.parse(URL), Collections.singletonList(c)); + } // if + } // for + } + }; +} diff --git a/client/src/main/java/ctbrec/ui/sites/streamray/StreamrayFavoritesService.java b/client/src/main/java/ctbrec/ui/sites/streamray/StreamrayFavoritesService.java new file mode 100644 index 00000000..64ab32aa --- /dev/null +++ b/client/src/main/java/ctbrec/ui/sites/streamray/StreamrayFavoritesService.java @@ -0,0 +1,132 @@ +package ctbrec.ui.sites.streamray; + +import static ctbrec.io.HttpConstants.*; + +import ctbrec.Config; +import ctbrec.Model; +import ctbrec.StringUtil; +import ctbrec.io.HttpException; +import ctbrec.sites.streamray.*; +import ctbrec.ui.SiteUiFactory; +import ctbrec.ui.tabs.PaginatedScheduledService; +import java.io.IOException; +import java.net.URLEncoder; +import java.text.MessageFormat; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Locale; +import java.util.Objects; +import java.util.function.Predicate; +import java.util.stream.Collectors; +import javafx.concurrent.Task; +import okhttp3.*; +import org.json.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class StreamrayFavoritesService extends PaginatedScheduledService { + + private static final Logger LOG = LoggerFactory.getLogger(StreamrayFavoritesService.class); + private static final String API_URL = "https://beta-api.cams.com/won/compressed/"; + + private Streamray site; + private static List modelsList; + private static JSONArray mapping; + protected int modelsPerPage = 48; + public boolean loggedIn = false; + + public StreamrayFavoritesService(Streamray site) { + this.site = site; + } + + @Override + protected Task> createTask() { + return new Task>() { + @Override + public List call() throws IOException { + return getModelList().stream() + .skip((page - 1) * (long) modelsPerPage) + .limit(modelsPerPage) + .collect(Collectors.toList()); // NOSONAR + } + }; + } + + private List getModelList() throws IOException { + modelsList = loadModelList(); + if (modelsList == null) { + modelsList = Collections.emptyList(); + } + return modelsList; + } + + private List loadModelList() throws IOException { + LOG.debug("Fetching page {}", API_URL); + StreamrayHttpClient client = (StreamrayHttpClient) site.getHttpClient(); + String token = ""; + if (site.login()) { + loggedIn = true; + token = client.getUserToken(); + } else { + loggedIn = false; + return Collections.emptyList(); + } + Request req = new Request.Builder() + .url(API_URL) + .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) + .header(ACCEPT, MIMETYPE_APPLICATION_JSON) + .header(ACCEPT_LANGUAGE, Locale.ENGLISH.getLanguage()) + .header(REFERER, site.getBaseUrl() + "/") + .header(ORIGIN, site.getBaseUrl()) + .header(X_REQUESTED_WITH, XML_HTTP_REQUEST) + .header(AUTHORIZATION, "Bearer " + token) + .build(); + try (Response response = client.execute(req)) { + if (response.isSuccessful()) { + List models = new ArrayList<>(); + String content = response.body().string(); + JSONObject json = new JSONObject(content); + JSONArray modelNodes = json.getJSONArray("models"); + mapping = json.getJSONArray("mapping"); + parseModels(modelNodes, models); + return models; + } else { + throw new HttpException(response.code(), response.message()); + } + } + } + + private void parseModels(JSONArray jsonModels, List models) { + int name_idx = mapping_index("stream_name"); + int fav_idx = mapping_index("is_favorite"); + for (int i = 0; i < jsonModels.length(); i++) { + JSONArray m = jsonModels.getJSONArray(i); + String name = m.optString(name_idx); + boolean favorite = m.optBoolean(fav_idx); + if (favorite) { + StreamrayModel model = (StreamrayModel) site.createModel(name); + String preview = getPreviewURL(name); + model.setPreview(preview); + models.add(model); + } + } + } + + private String getPreviewURL(String name) { + String lname = name.toLowerCase(); + String url = MessageFormat.format("https://images4.streamray.com/images/streamray/won/jpg/{0}/{1}/{2}_640.jpg", lname.substring(0,1), lname.substring(lname.length()-1), lname); + try { + return MessageFormat.format("https://dynimages.securedataimages.com/unsigned/rs:fill:640::0/g:no/plain/{0}@jpg", URLEncoder.encode(url, "utf-8")); + } catch (Exception ex) { + return url; + } + } + + private int mapping_index(String s) { + for (var i = 0; i < mapping.length(); i++) { + if (Objects.equals(s, mapping.get(i))) return i; + } + return -1; + } +} diff --git a/client/src/main/java/ctbrec/ui/sites/streamray/StreamrayFavoritesTab.java b/client/src/main/java/ctbrec/ui/sites/streamray/StreamrayFavoritesTab.java new file mode 100644 index 00000000..abb1f59f --- /dev/null +++ b/client/src/main/java/ctbrec/ui/sites/streamray/StreamrayFavoritesTab.java @@ -0,0 +1,77 @@ +package ctbrec.ui.sites.streamray; + +import ctbrec.sites.streamray.Streamray; +import ctbrec.ui.tabs.FollowedTab; +import ctbrec.ui.tabs.ThumbOverviewTab; +import javafx.concurrent.WorkerStateEvent; +import javafx.scene.Scene; +import javafx.scene.control.*; +import javafx.scene.input.KeyCode; +import javafx.scene.input.KeyEvent; +import javafx.geometry.Insets; +import javafx.geometry.Pos; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class StreamrayFavoritesTab extends ThumbOverviewTab implements FollowedTab { + private static final Logger LOG = LoggerFactory.getLogger(StreamrayFavoritesTab.class); + private Label status; + private Button loginButton; + private Streamray site; + private StreamrayFavoritesService updateService; + + public StreamrayFavoritesTab(String title, StreamrayFavoritesService updateService, Streamray site) { + super(title, updateService, site); + this.site = site; + this.updateService = updateService; + + status = new Label("Logging in..."); + grid.getChildren().addAll(status); + + loginButton = new Button("Login"); + loginButton.setPadding(new Insets(20)); + loginButton.setOnAction(e -> { + try { + new StreamrayElectronLoginDialog(site.getHttpClient().getCookieJar()); + updateService.restart(); + } catch (Exception ex) {} + }); + } + + protected void addLoginButton() { + grid.getChildren().clear(); + grid.setAlignment(Pos.CENTER); + grid.getChildren().add(loginButton); + } + + @Override + protected void onSuccess() { + grid.getChildren().removeAll(status, loginButton); + grid.setAlignment(Pos.TOP_LEFT); + if (updateService.loggedIn == false) { + addLoginButton(); + } else { + super.onSuccess(); + } + } + + @Override + protected void onFail(WorkerStateEvent event) { + status.setText("Login failed"); + super.onFail(event); + } + + @Override + public void selected() { + status.setText("Logging in..."); + super.selected(); + } + + public void setScene(Scene scene) { + scene.addEventFilter(KeyEvent.KEY_PRESSED, event -> { + if (this.isSelected() && event.getCode() == KeyCode.DELETE) { + follow(selectedThumbCells, false); + } + }); + } +} diff --git a/client/src/main/java/ctbrec/ui/sites/streamray/StreamraySiteUi.java b/client/src/main/java/ctbrec/ui/sites/streamray/StreamraySiteUi.java new file mode 100644 index 00000000..ff3e25ee --- /dev/null +++ b/client/src/main/java/ctbrec/ui/sites/streamray/StreamraySiteUi.java @@ -0,0 +1,44 @@ +package ctbrec.ui.sites.streamray; + +import ctbrec.sites.streamray.Streamray; +import ctbrec.ui.sites.AbstractSiteUi; +import ctbrec.ui.sites.ConfigUI; +import ctbrec.ui.tabs.TabProvider; + +import java.io.IOException; + +public class StreamraySiteUi extends AbstractSiteUi { + + private StreamrayTabProvider tabProvider; + private StreamrayConfigUI configUi; + private final Streamray site; + + public StreamraySiteUi(Streamray site) { + this.site = site; + } + + @Override + public TabProvider getTabProvider() { + if (tabProvider == null) { + tabProvider = new StreamrayTabProvider(site); + } + return tabProvider; + } + + @Override + public ConfigUI getConfigUI() { + if (configUi == null) { + configUi = new StreamrayConfigUI(site); + } + return configUi; + } + + @Override + public boolean login() throws IOException { + return site.login(); + } + + public synchronized boolean checkLogin() throws IOException { + return site.login(); + } +} diff --git a/client/src/main/java/ctbrec/ui/sites/streamray/StreamrayTabProvider.java b/client/src/main/java/ctbrec/ui/sites/streamray/StreamrayTabProvider.java new file mode 100644 index 00000000..8f2d1b86 --- /dev/null +++ b/client/src/main/java/ctbrec/ui/sites/streamray/StreamrayTabProvider.java @@ -0,0 +1,49 @@ +package ctbrec.ui.sites.streamray; + +import ctbrec.sites.streamray.Streamray; +import ctbrec.sites.streamray.StreamrayModel; + +import ctbrec.ui.sites.AbstractTabProvider; +import ctbrec.ui.tabs.ThumbOverviewTab; +import javafx.scene.Scene; +import javafx.scene.control.Tab; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.function.Predicate; + +public class StreamrayTabProvider extends AbstractTabProvider { + + public StreamrayTabProvider(Streamray site) { + super(site); + } + + @Override + protected List getSiteTabs(Scene scene) { + List tabs = new ArrayList<>(); + tabs.add(createTab("Girls", m -> Objects.equals("F", m.getGender()))); + tabs.add(createTab("Boys", m -> Objects.equals("M", m.getGender()))); + tabs.add(createTab("Trans", m -> Objects.equals("TS", m.getGender()))); + tabs.add(createTab("New", m -> m.isNew())); + tabs.add(favoritesTab()); + return tabs; + } + + private Tab createTab(String title, Predicate filter) { + var updateService = new StreamrayUpdateService((Streamray) site, filter); + var tab = new ThumbOverviewTab(title, updateService, site); + tab.setImageAspectRatio(9.0 / 16.0); + tab.setRecorder(recorder); + return tab; + } + + private Tab favoritesTab() { + var updateService = new StreamrayFavoritesService((Streamray) site); + var tab = new StreamrayFavoritesTab("Favorites", updateService, (Streamray) site); + tab.setImageAspectRatio(9.0 / 16.0); + tab.setRecorder(recorder); + return tab; + } + +} diff --git a/client/src/main/java/ctbrec/ui/sites/streamray/StreamrayUpdateService.java b/client/src/main/java/ctbrec/ui/sites/streamray/StreamrayUpdateService.java new file mode 100644 index 00000000..347913b0 --- /dev/null +++ b/client/src/main/java/ctbrec/ui/sites/streamray/StreamrayUpdateService.java @@ -0,0 +1,146 @@ +package ctbrec.ui.sites.streamray; + +import static ctbrec.io.HttpConstants.*; + +import ctbrec.Config; +import ctbrec.Model; +import ctbrec.StringUtil; +import ctbrec.io.HttpException; +import ctbrec.sites.streamray.*; +import ctbrec.ui.SiteUiFactory; +import ctbrec.ui.tabs.PaginatedScheduledService; +import java.io.IOException; +import java.net.URLEncoder; +import java.text.MessageFormat; +import java.time.Duration; +import java.time.Instant; +import java.time.LocalDate; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Locale; +import java.util.Objects; +import java.util.Optional; +import java.util.function.Predicate; +import java.util.stream.Collectors; +import javafx.concurrent.Task; +import okhttp3.Request; +import okhttp3.Response; +import org.json.JSONArray; +import org.json.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class StreamrayUpdateService extends PaginatedScheduledService { + + private static final Logger LOG = LoggerFactory.getLogger(StreamrayUpdateService.class); + private static final String API_URL = "https://beta-api.cams.com/won/compressed/"; + + private Streamray site; + private static List modelsList; + private static Instant lastListInfoRequest = Instant.EPOCH; + private static JSONArray mapping; + protected int modelsPerPage = 48; + protected Predicate filter; + + public StreamrayUpdateService(Streamray site, Predicate filter) { + this.site = site; + this.filter = filter; + } + + @Override + protected Task> createTask() { + return new Task>() { + @Override + public List call() throws IOException { + return getModelList().stream() + .filter(filter) + .skip((page - 1) * (long) modelsPerPage) + .limit(modelsPerPage) + .collect(Collectors.toList()); // NOSONAR + } + }; + } + + private List getModelList() throws IOException { + if (Duration.between(lastListInfoRequest, Instant.now()).getSeconds() < 30) { + return Optional.ofNullable(modelsList).orElse(loadModelList()); + } + modelsList = loadModelList(); + return Optional.ofNullable(modelsList).orElse(Collections.emptyList()); + } + + private List loadModelList() throws IOException { + LOG.debug("Fetching page {}", API_URL); + lastListInfoRequest = Instant.now(); + StreamrayHttpClient client = (StreamrayHttpClient) site.getHttpClient(); + Request req = new Request.Builder() + .url(API_URL) + .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) + .header(ACCEPT, MIMETYPE_APPLICATION_JSON) + .header(ACCEPT_LANGUAGE, Locale.ENGLISH.getLanguage()) + .header(REFERER, site.getBaseUrl() + "/") + .header(ORIGIN, site.getBaseUrl()) + .header(X_REQUESTED_WITH, XML_HTTP_REQUEST) + .build(); + try (Response response = client.execute(req)) { + if (response.isSuccessful()) { + List models = new ArrayList<>(); + String content = response.body().string(); + JSONObject json = new JSONObject(content); + JSONArray modelNodes = json.getJSONArray("models"); + mapping = json.getJSONArray("mapping"); + parseModels(modelNodes, models); + return models; + } else { + throw new HttpException(response.code(), response.message()); + } + } + } + + private void parseModels(JSONArray jsonModels, List models) { + int name_idx = mapping_index("stream_name"); + int date_idx = mapping_index("create_date"); + int gen_idx = mapping_index("gender"); + for (var i = 0; i < jsonModels.length(); i++) { + var m = jsonModels.getJSONArray(i); + String name = m.optString(name_idx); + String gender = m.optString(gen_idx); + String reg = m.optString(date_idx); + StreamrayModel model = (StreamrayModel) site.createModel(name); + try { + LocalDate regDate = LocalDate.parse(reg, DateTimeFormatter.BASIC_ISO_DATE); + model.setRegDate(regDate); + } catch (DateTimeParseException e) { + model.setRegDate(LocalDate.EPOCH); + } + String preview = getPreviewURL(name); + model.setPreview(preview); + model.setGender(gender); + models.add(model); + } + } + + private String getPreviewURL(String name) { + String lname = name.toLowerCase(); + String url = MessageFormat.format("https://images4.streamray.com/images/streamray/won/jpg/{0}/{1}/{2}_640.jpg", lname.substring(0,1), lname.substring(lname.length()-1), lname); + try { + return MessageFormat.format("https://dynimages.securedataimages.com/unsigned/rs:fill:640::0/g:no/plain/{0}@jpg", URLEncoder.encode(url, "utf-8")); + } catch (Exception ex) { + return url; + } + } + + public void setFilter(Predicate filter) { + this.filter = filter; + } + + private int mapping_index(String s) { + for (var i = 0; i < mapping.length(); i++) { + if (Objects.equals(s, mapping.get(i))) return i; + } + return -1; + } +} diff --git a/client/src/main/java/ctbrec/ui/sites/winktv/WinkTvConfigUI.java b/client/src/main/java/ctbrec/ui/sites/winktv/WinkTvConfigUI.java new file mode 100644 index 00000000..d3c3cc42 --- /dev/null +++ b/client/src/main/java/ctbrec/ui/sites/winktv/WinkTvConfigUI.java @@ -0,0 +1,55 @@ +package ctbrec.ui.sites.winktv; + +import ctbrec.Config; +import ctbrec.sites.winktv.WinkTv; +import ctbrec.ui.settings.SettingsTab; +import ctbrec.ui.sites.AbstractConfigUI; +import javafx.geometry.Insets; +import javafx.scene.Parent; +import javafx.scene.control.ToggleGroup; +import javafx.scene.control.Button; +import javafx.scene.control.CheckBox; +import javafx.scene.control.RadioButton; +import javafx.scene.control.Label; +import javafx.scene.layout.GridPane; +import javafx.scene.layout.HBox; + +public class WinkTvConfigUI extends AbstractConfigUI { + private final WinkTv site; + + public WinkTvConfigUI(WinkTv site) { + this.site = site; + } + + @Override + public Parent createConfigPanel() { + GridPane layout = SettingsTab.createGridLayout(); + var settings = Config.getInstance().getSettings(); + + var row = 0; + var l = new Label("Active"); + layout.add(l, 0, row); + var enabled = new CheckBox(); + enabled.setSelected(!settings.disabledSites.contains(site.getName())); + enabled.setOnAction(e -> { + if(enabled.isSelected()) { + settings.disabledSites.remove(site.getName()); + } else { + settings.disabledSites.add(site.getName()); + } + save(); + }); + GridPane.setMargin(enabled, new Insets(0, 0, 0, SettingsTab.CHECKBOX_MARGIN)); + layout.add(enabled, 1, row++); + row++; + + var deleteCookies = new Button("Delete Cookies"); + deleteCookies.setOnAction(e -> site.getHttpClient().clearCookies()); + layout.add(deleteCookies, 1, row); + GridPane.setColumnSpan(deleteCookies, 2); + + GridPane.setMargin(deleteCookies, new Insets(0, 0, 0, SettingsTab.CHECKBOX_MARGIN)); + return layout; + } + +} diff --git a/client/src/main/java/ctbrec/ui/sites/winktv/WinkTvSiteUi.java b/client/src/main/java/ctbrec/ui/sites/winktv/WinkTvSiteUi.java new file mode 100644 index 00000000..d87d3c99 --- /dev/null +++ b/client/src/main/java/ctbrec/ui/sites/winktv/WinkTvSiteUi.java @@ -0,0 +1,40 @@ +package ctbrec.ui.sites.winktv; + +import ctbrec.sites.winktv.WinkTv; +import ctbrec.ui.sites.AbstractSiteUi; +import ctbrec.ui.sites.ConfigUI; +import ctbrec.ui.tabs.TabProvider; + +import java.io.IOException; + +public class WinkTvSiteUi extends AbstractSiteUi { + + private WinkTvTabProvider tabProvider; + private WinkTvConfigUI configUi; + private final WinkTv site; + + public WinkTvSiteUi(WinkTv site) { + this.site = site; + } + + @Override + public TabProvider getTabProvider() { + if (tabProvider == null) { + tabProvider = new WinkTvTabProvider(site); + } + return tabProvider; + } + + @Override + public ConfigUI getConfigUI() { + if (configUi == null) { + configUi = new WinkTvConfigUI(site); + } + return configUi; + } + + @Override + public synchronized boolean login() throws IOException { + return site.login(); + } +} diff --git a/client/src/main/java/ctbrec/ui/sites/winktv/WinkTvTabProvider.java b/client/src/main/java/ctbrec/ui/sites/winktv/WinkTvTabProvider.java new file mode 100644 index 00000000..8bfdc6d7 --- /dev/null +++ b/client/src/main/java/ctbrec/ui/sites/winktv/WinkTvTabProvider.java @@ -0,0 +1,37 @@ +package ctbrec.ui.sites.winktv; + +import ctbrec.sites.winktv.WinkTv; +import ctbrec.sites.winktv.WinkTvModel; + +import ctbrec.ui.sites.AbstractTabProvider; +import ctbrec.ui.tabs.ThumbOverviewTab; +import javafx.scene.Scene; +import javafx.scene.control.Tab; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.function.Predicate; + +public class WinkTvTabProvider extends AbstractTabProvider { + + public WinkTvTabProvider(WinkTv site) { + super(site); + } + + @Override + protected List getSiteTabs(Scene scene) { + List tabs = new ArrayList<>(); + tabs.add(createTab("Live", m -> !m.isAdult())); + return tabs; + } + + private Tab createTab(String title, Predicate filter) { + var updateService = new WinkTvUpdateService((WinkTv) site, filter); + var tab = new ThumbOverviewTab(title, updateService, site); + tab.setImageAspectRatio(9.0 / 16.0); + tab.setRecorder(recorder); + return tab; + } + +} diff --git a/client/src/main/java/ctbrec/ui/sites/winktv/WinkTvUpdateService.java b/client/src/main/java/ctbrec/ui/sites/winktv/WinkTvUpdateService.java new file mode 100644 index 00000000..6efe43ea --- /dev/null +++ b/client/src/main/java/ctbrec/ui/sites/winktv/WinkTvUpdateService.java @@ -0,0 +1,127 @@ +package ctbrec.ui.sites.winktv; + +import static ctbrec.io.HttpConstants.*; + +import ctbrec.Config; +import ctbrec.Model; +import ctbrec.io.HttpException; +import ctbrec.sites.winktv.WinkTv; +import ctbrec.sites.winktv.WinkTvModel; +import ctbrec.ui.SiteUiFactory; +import ctbrec.ui.tabs.PaginatedScheduledService; +import java.io.IOException; +import java.text.MessageFormat; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Locale; +import java.util.Optional; +import java.util.function.Predicate; +import java.util.stream.Collectors; +import javafx.concurrent.Task; +import okhttp3.FormBody; +import okhttp3.Request; +import okhttp3.Response; +import org.json.JSONArray; +import org.json.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class WinkTvUpdateService extends PaginatedScheduledService { + + private static final Logger LOG = LoggerFactory.getLogger(WinkTvUpdateService.class); + private static final String API_URL = "https://api.winktv.co.kr/v1/live"; + + private WinkTv site; + private String url; + private static List modelsList; + private static Instant lastListInfoRequest = Instant.EPOCH; + protected int modelsPerPage = 48; + protected Predicate filter; + + public WinkTvUpdateService(WinkTv site, Predicate filter) { + this.site = site; + this.filter = filter; + } + + @Override + protected Task> createTask() { + return new Task>() { + @Override + public List call() throws IOException { + return getModelList().stream() + .filter(filter) + .skip((page - 1) * (long) modelsPerPage) + .limit(modelsPerPage) + .collect(Collectors.toList()); // NOSONAR + } + }; + } + + private List getModelList() throws IOException { + if (Duration.between(lastListInfoRequest, Instant.now()).getSeconds() < 30) { + return Optional.ofNullable(modelsList).orElse(loadModelList()); + } + modelsList = loadModelList(); + return Optional.ofNullable(modelsList).orElse(Collections.emptyList()); + } + + private List loadModelList() throws IOException { + LOG.debug("Fetching page {}", API_URL); + lastListInfoRequest = Instant.now(); + FormBody body = new FormBody.Builder() + .add("offset", "0") + .add("limit", "500") + .add("orderBy", "hot") + .build(); + Request req = new Request.Builder() + .url(API_URL) + .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) + .header(ACCEPT, MIMETYPE_APPLICATION_JSON) + .header(ACCEPT_LANGUAGE, Locale.ENGLISH.getLanguage()) + .header(X_REQUESTED_WITH, XML_HTTP_REQUEST) + .header(REFERER, site.getBaseUrl() + "/") + .header(ORIGIN, site.getBaseUrl()) + .post(body) + .build(); + try (var response = site.getHttpClient().execute(req)) { + if (response.isSuccessful()) { + List models = new ArrayList<>(); + var content = response.body().string(); + var json = new JSONObject(content); + if (json.optBoolean("result")) { + var modelNodes = json.getJSONArray("list"); + parseModels(modelNodes, models); + } + return models; + } else { + throw new HttpException(response.code(), response.message()); + } + } + } + + private void parseModels(JSONArray jsonModels, List models) { + for (var i = 0; i < jsonModels.length(); i++) { + var m = jsonModels.getJSONObject(i); + String name = m.optString("userId"); + WinkTvModel model = (WinkTvModel) site.createModel(name); + model.setDisplayName(m.getString("userNick")); + boolean isAdult = m.optBoolean("isAdult"); + model.setAdult(isAdult); + if (isAdult && m.has("ivsThumbnail")) { + model.setPreview(m.optString("ivsThumbnail")); + } else { + model.setPreview(m.optString("thumbUrl")); + } + boolean isLive = m.optBoolean("isLive"); + if (isLive) models.add(model); + } + } + + public void setFilter(Predicate filter) { + this.filter = filter; + } + +} diff --git a/common/src/main/java/ctbrec/AbstractModel.java b/common/src/main/java/ctbrec/AbstractModel.java index c21b3f03..5c7489b0 100644 --- a/common/src/main/java/ctbrec/AbstractModel.java +++ b/common/src/main/java/ctbrec/AbstractModel.java @@ -1,18 +1,7 @@ package ctbrec; -import static ctbrec.io.HttpConstants.*; - -import java.io.IOException; -import java.time.Instant; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Optional; -import java.util.concurrent.ExecutionException; - import com.squareup.moshi.JsonReader; import com.squareup.moshi.JsonWriter; - import ctbrec.recorder.download.Download; import ctbrec.recorder.download.HttpHeaderFactory; import ctbrec.recorder.download.HttpHeaderFactoryImpl; @@ -23,6 +12,16 @@ import ctbrec.sites.Site; import okhttp3.Request; import okhttp3.Response; +import java.io.IOException; +import java.time.Instant; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.ExecutionException; + +import static ctbrec.io.HttpConstants.USER_AGENT; + public abstract class AbstractModel implements Model { private String url; @@ -42,6 +41,7 @@ public abstract class AbstractModel implements Model { private Instant lastRecorded; private Instant recordUntil; private Instant addedTimestamp = Instant.EPOCH; + private transient Instant delayUntil = Instant.EPOCH; private SubsequentAction recordUntilSubsequentAction; @Override @@ -72,7 +72,7 @@ public abstract class AbstractModel implements Model { @Override public String getDisplayName() { - if(displayName != null) { + if (displayName != null) { return displayName; } else { return getName(); @@ -149,6 +149,16 @@ public abstract class AbstractModel implements Model { this.suspended = suspended; } + @Override + public void delay() { + this.delayUntil = Instant.now().plusSeconds(120); + } + + @Override + public boolean isDelayed() { + return this.delayUntil.isAfter(Instant.now()); + } + @Override public boolean isMarkedForLaterRecording() { return markedForLaterRecording; diff --git a/common/src/main/java/ctbrec/Model.java b/common/src/main/java/ctbrec/Model.java index dd13beb6..a0aac827 100644 --- a/common/src/main/java/ctbrec/Model.java +++ b/common/src/main/java/ctbrec/Model.java @@ -1,23 +1,21 @@ package ctbrec; -import java.io.IOException; -import java.io.Serializable; -import java.time.Instant; -import java.util.List; -import java.util.concurrent.ExecutionException; - -import javax.xml.bind.JAXBException; - import com.iheartradio.m3u8.ParseException; import com.iheartradio.m3u8.PlaylistException; import com.squareup.moshi.JsonReader; import com.squareup.moshi.JsonWriter; - import ctbrec.recorder.download.Download; import ctbrec.recorder.download.HttpHeaderFactory; import ctbrec.recorder.download.StreamSource; import ctbrec.sites.Site; +import javax.xml.bind.JAXBException; +import java.io.IOException; +import java.io.Serializable; +import java.time.Instant; +import java.util.List; +import java.util.concurrent.ExecutionException; + public interface Model extends Comparable, Serializable { long RECORD_INDEFINITELY = 9000000000000000000L; @@ -32,6 +30,7 @@ public interface Model extends Comparable, Serializable { UNKNOWN("unknown"); final String display; + State(String display) { this.display = display; } @@ -102,10 +101,8 @@ public interface Model extends Comparable, Serializable { /** * Determines the stream resolution for this model * - * @param failFast - * If set to true, the method returns immediately, even if the resolution is unknown. If - * the resolution is unknown, the array contains 0,0 - * + * @param failFast If set to true, the method returns immediately, even if the resolution is unknown. If + * the resolution is unknown, the array contains 0,0 * @return a tupel of width and height represented by an int[2] * @throws ExecutionException */ @@ -127,6 +124,10 @@ public interface Model extends Comparable, Serializable { void setSuspended(boolean suspended); + void delay(); + + boolean isDelayed(); + boolean isMarkedForLaterRecording(); void setMarkedForLaterRecording(boolean marked); @@ -140,14 +141,18 @@ public interface Model extends Comparable, Serializable { HttpHeaderFactory getHttpHeaderFactory(); boolean isRecordingTimeLimited(); + Instant getRecordUntil(); + void setRecordUntil(Instant instant); SubsequentAction getRecordUntilSubsequentAction(); + void setRecordUntilSubsequentAction(SubsequentAction action); /** * Check, if this model account exists + * * @return true, if it exists, false otherwise * @throws IOException */ diff --git a/common/src/main/java/ctbrec/Settings.java b/common/src/main/java/ctbrec/Settings.java index 28083645..1cab26a2 100644 --- a/common/src/main/java/ctbrec/Settings.java +++ b/common/src/main/java/ctbrec/Settings.java @@ -206,4 +206,9 @@ public class Settings { public String webinterfacePassword = "sucks"; public String xlovecamUsername = ""; public String xlovecamPassword = ""; + public boolean stripchatVR = false; + public boolean streamrayRecordGoalShows = false; + public boolean checkForUpdates = true; + public int thumbCacheSize = 16; + public boolean dreamcamVR = false; } diff --git a/common/src/main/java/ctbrec/recorder/download/AbstractDownload.java b/common/src/main/java/ctbrec/recorder/download/AbstractDownload.java index c260fb7f..f79471b8 100644 --- a/common/src/main/java/ctbrec/recorder/download/AbstractDownload.java +++ b/common/src/main/java/ctbrec/recorder/download/AbstractDownload.java @@ -1,18 +1,21 @@ package ctbrec.recorder.download; -import java.io.IOException; -import java.time.Instant; -import java.util.concurrent.ExecutorService; - import ctbrec.Config; import ctbrec.Model; import ctbrec.Settings; import ctbrec.UnknownModel; -import ctbrec.recorder.download.hls.CombinedSplittingStrategy; -import ctbrec.recorder.download.hls.NoopSplittingStrategy; -import ctbrec.recorder.download.hls.SizeSplittingStrategy; -import ctbrec.recorder.download.hls.TimeSplittingStrategy; +import ctbrec.recorder.download.hls.*; +import lombok.extern.slf4j.Slf4j; +import java.io.IOException; +import java.time.Instant; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; + +import static ctbrec.recorder.download.StreamSource.UNKNOWN; + +@Slf4j public abstract class AbstractDownload implements Download { protected Instant startTime; @@ -22,6 +25,7 @@ public abstract class AbstractDownload implements Download { protected Config config; protected SplittingStrategy splittingStrategy; protected ExecutorService downloadExecutor; + protected int selectedResolution = UNKNOWN; @Override public void init(Config config, Model model, Instant startTime, ExecutorService executorService) throws IOException { @@ -45,21 +49,21 @@ public abstract class AbstractDownload implements Download { protected SplittingStrategy initSplittingStrategy(Settings settings) { SplittingStrategy strategy; switch (settings.splitStrategy) { - case TIME: - strategy = new TimeSplittingStrategy(); - break; - case SIZE: - strategy = new SizeSplittingStrategy(); - break; - case TIME_OR_SIZE: - SplittingStrategy timeSplittingStrategy = new TimeSplittingStrategy(); - SplittingStrategy sizeSplittingStrategy = new SizeSplittingStrategy(); - strategy = new CombinedSplittingStrategy(timeSplittingStrategy, sizeSplittingStrategy); - break; - case DONT: - default: - strategy = new NoopSplittingStrategy(); - break; + case TIME: + strategy = new TimeSplittingStrategy(); + break; + case SIZE: + strategy = new SizeSplittingStrategy(); + break; + case TIME_OR_SIZE: + SplittingStrategy timeSplittingStrategy = new TimeSplittingStrategy(); + SplittingStrategy sizeSplittingStrategy = new SizeSplittingStrategy(); + strategy = new CombinedSplittingStrategy(timeSplittingStrategy, sizeSplittingStrategy); + break; + case DONT: + default: + strategy = new NoopSplittingStrategy(); + break; } strategy.init(settings); return strategy; @@ -72,6 +76,39 @@ public abstract class AbstractDownload implements Download { @Override public int getSelectedResolution() { - return StreamSource.UNKNOWN; + return selectedResolution; + } + + public void awaitEnd() { + // do nothing per default + } + + protected StreamSource selectStreamSource(List streamSources) throws ExecutionException { + if (model.getStreamUrlIndex() >= 0 && model.getStreamUrlIndex() < streamSources.size()) { + // TODO don't use the index, but the bandwidth. if the bandwidth does not match, take the closest one + log.debug("Model stream index: {}", model.getStreamUrlIndex()); + streamSources.forEach(ss -> log.debug(ss.toString())); + StreamSource source = streamSources.get(model.getStreamUrlIndex()); + log.debug("{} selected {}", model.getName(), source); + selectedResolution = source.height; + return source; + } else { + // filter out stream resolutions, which are out of range of the configured min and max + int minRes = Config.getInstance().getSettings().minimumResolution; + int maxRes = Config.getInstance().getSettings().maximumResolution; + List filteredStreamSources = streamSources.stream() + .filter(src -> src.height == 0 || src.height == UNKNOWN || minRes <= src.height) + .filter(src -> src.height == 0 || src.height == UNKNOWN || maxRes >= src.height) + .toList(); + + if (filteredStreamSources.isEmpty()) { + throw new ExecutionException(new NoStreamFoundException("No stream left in playlist")); + } else { + StreamSource source = filteredStreamSources.get(filteredStreamSources.size() - 1); + log.debug("{} selected {}", model.getName(), source); + selectedResolution = source.height; + return source; + } + } } } diff --git a/common/src/main/java/ctbrec/recorder/download/hls/FfmpegHlsDownload.java b/common/src/main/java/ctbrec/recorder/download/hls/FfmpegHlsDownload.java new file mode 100644 index 00000000..e028fd2f --- /dev/null +++ b/common/src/main/java/ctbrec/recorder/download/hls/FfmpegHlsDownload.java @@ -0,0 +1,325 @@ +package ctbrec.recorder.download.hls; + +import com.iheartradio.m3u8.*; +import com.iheartradio.m3u8.data.MediaPlaylist; +import com.iheartradio.m3u8.data.Playlist; +import com.iheartradio.m3u8.data.TrackData; +import ctbrec.Config; +import ctbrec.Model; +import ctbrec.OS; +import ctbrec.Recording; +import ctbrec.io.BandwidthMeter; +import ctbrec.io.HttpClient; +import ctbrec.io.HttpException; +import ctbrec.recorder.FFmpeg; +import ctbrec.recorder.InvalidPlaylistException; +import ctbrec.recorder.download.AbstractDownload; +import ctbrec.recorder.download.Download; +import ctbrec.recorder.download.ProcessExitedUncleanException; +import ctbrec.recorder.download.StreamSource; +import okhttp3.Request; +import okhttp3.Response; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.xml.bind.JAXBException; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.SocketTimeoutException; +import java.net.URL; +import java.nio.file.Files; +import java.time.Duration; +import java.time.Instant; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; +import java.util.regex.Pattern; + +import static ctbrec.io.HttpConstants.*; + +public class FfmpegHlsDownload extends AbstractDownload { + + private static final Logger LOG = LoggerFactory.getLogger(FfmpegHlsDownload.class); + private static final int MAX_SECONDS_WITHOUT_TRANSFER = 20; + + private final HttpClient httpClient; + private Instant timeOfLastTransfer = Instant.MAX; + + protected File targetFile; + protected FFmpeg ffmpeg; + protected Process ffmpegProcess; + protected OutputStream ffmpegStdIn; + protected Lock ffmpegStreamLock = new ReentrantLock(); + protected String mediaUrl = null; + + private volatile boolean running; + private volatile boolean started; + private int selectedResolution = 0; + + public FfmpegHlsDownload(HttpClient httpClient) { + this.httpClient = httpClient; + } + + @Override + public void init(Config config, Model model, Instant startTime, ExecutorService executorService) throws IOException { + super.init(config, model, startTime, executorService); + + timeOfLastTransfer = Instant.now(); + String fileSuffix = config.getSettings().ffmpegFileSuffix; + + targetFile = config.getFileForRecording(model, fileSuffix, startTime); + createTargetDirectory(); + startFfmpegProcess(targetFile); + if (ffmpegProcess == null) { + throw new ProcessExitedUncleanException("Couldn't spawn FFmpeg"); + } + } + + @Override + public int getSelectedResolution() { + return selectedResolution; + } + + @Override + public void stop() { + if (running) { + internalStop(); + } + } + + private synchronized void internalStop() { + running = false; + if (ffmpegStdIn != null) { + try { + ffmpegStdIn.close(); + } catch (IOException e) { + LOG.error("Couldn't terminate FFmpeg by closing stdin", e); + } + } + if (ffmpegProcess != null) { + try { + boolean waitFor = ffmpegProcess.waitFor(45, TimeUnit.SECONDS); + if (!waitFor && ffmpegProcess.isAlive()) { + ffmpegProcess.destroy(); + if (ffmpegProcess.isAlive()) { + LOG.info("FFmpeg didn't terminate. Destroying the process with force!"); + ffmpegProcess.destroyForcibly(); + ffmpegProcess = null; + } + } + } catch (InterruptedException e) { + LOG.error("Interrupted while waiting for FFmpeg to terminate"); + Thread.currentThread().interrupt(); + } + } + } + + private void startFfmpegProcess(File target) { + try { + String[] cmdline = prepareCommandLine(target); + ffmpeg = new FFmpeg.Builder() + .logOutput(config.getSettings().logFFmpegOutput) + .onStarted(p -> { + ffmpegProcess = p; + ffmpegStdIn = ffmpegProcess.getOutputStream(); + }) + .build(); + ffmpeg.exec(cmdline, new String[0], target.getParentFile()); + } catch (IOException | ProcessExitedUncleanException e) { + LOG.error("Error in FFmpeg thread", e); + } + } + + private String[] prepareCommandLine(File target) { + String[] args = config.getSettings().ffmpegMergedDownloadArgs.split(" "); + String[] argsPlusFile = new String[args.length + 3]; + int i = 0; + argsPlusFile[i++] = "-i"; + argsPlusFile[i++] = "-"; + System.arraycopy(args, 0, argsPlusFile, i, args.length); + argsPlusFile[argsPlusFile.length - 1] = target.getAbsolutePath(); + return OS.getFFmpegCommand(argsPlusFile); + } + + @Override + public void finalizeDownload() { + internalStop(); + } + + @Override + public boolean isRunning() { + return running; + } + + @Override + public void postprocess(Recording recording) { + // nothing to do + } + + @Override + public File getTarget() { + return targetFile; + } + + @Override + public String getPath(Model model) { + String absolutePath = targetFile.getAbsolutePath(); + String recordingsDir = Config.getInstance().getSettings().recordingsDir; + String relativePath = absolutePath.replaceFirst(Pattern.quote(recordingsDir), ""); + return relativePath; + } + + @Override + public boolean isSingleFile() { + return true; + } + + @Override + public long getSizeInByte() { + return getTarget().length(); + } + + @Override + public Download call() throws Exception { + try { + if (!ffmpegProcess.isAlive()) { + running = false; + int exitValue = ffmpegProcess.exitValue(); + ffmpeg.shutdown(exitValue); + } + } catch (ProcessExitedUncleanException e) { + LOG.error("FFmpeg exited unclean", e); + internalStop(); + } + try { + if (!started) { + started = true; + startDownload(); + } + } catch (Exception e) { + LOG.error("Error while downloading MP4", e); + stop(); + } + if (!model.isOnline()) { + LOG.debug("Model {} not online. Stop recording.", model); + stop(); + } + if (splittingStrategy.splitNecessary(this)) { + LOG.debug("Split necessary for model {}. Stop recording.", model); + internalStop(); + } else { + rescheduleTime = Instant.now().plusSeconds(5); + } + if (Duration.between(timeOfLastTransfer, Instant.now()).getSeconds() > MAX_SECONDS_WITHOUT_TRANSFER) { + LOG.debug("No video data received for {} seconds. Stopping recording for model {}", MAX_SECONDS_WITHOUT_TRANSFER, model); + stop(); + } + return this; + } + + private String getMediaUrl(Model model) throws IOException, ExecutionException, ParseException, PlaylistException, InvalidPlaylistException, JAXBException { + List streamSources = model.getStreamSources(); + Collections.sort(streamSources); + for (StreamSource streamSource : streamSources) { + LOG.debug("{}:{} src {}", model.getSite().getName(), model.getName(), streamSource); + } + StreamSource selectedStreamSource = selectStreamSource(streamSources); + String playlistUrl = selectedStreamSource.getMediaPlaylistUrl(); + selectedResolution = selectedStreamSource.height; + + Request req = new Request.Builder() + .url(playlistUrl) + .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) + .header(ACCEPT, "*/*") + .header(ACCEPT_LANGUAGE, "en") + .header(ORIGIN, model.getSite().getBaseUrl()) + .header(REFERER, model.getSite().getBaseUrl()) + .build(); + try (Response response = model.getSite().getHttpClient().execute(req)) { + if (response.isSuccessful()) { + InputStream inputStream = Objects.requireNonNull(response.body()).byteStream(); + PlaylistParser parser = new PlaylistParser(inputStream, Format.EXT_M3U, Encoding.UTF_8, ParsingMode.LENIENT); + Playlist playlist = parser.parse(); + MediaPlaylist media = playlist.getMediaPlaylist(); + if (media.hasTracks()) { + TrackData firstTrack = media.getTracks().get(0); + if (firstTrack.isEncrypted()) { + LOG.warn("Video track is encrypted. Playlist URL: {}", playlistUrl); + } + String uri = firstTrack.getUri(); + if (!uri.startsWith("http")) { + URL context = new URL(playlistUrl); + uri = new URL(context, uri).toExternalForm(); + } + LOG.debug("Media url {}", uri); + return uri; + } else { + throw new InvalidPlaylistException("Playlist has no media"); + } + } else { + throw new HttpException(response.code(), response.message()); + } + } + } + + private void startDownload() { + downloadExecutor.submit(() -> { + running = true; + ffmpegStreamLock.lock(); + try { + if (mediaUrl == null) { + mediaUrl = getMediaUrl(model); + } + Request request = new Request.Builder() + .url(mediaUrl) + .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) + .header(ACCEPT, "*/*") + .header(ACCEPT_LANGUAGE, "en") + .header(ORIGIN, model.getSite().getBaseUrl()) + .header(REFERER, model.getSite().getBaseUrl()) + .build(); + try (Response resp = httpClient.execute(request)) { + if (resp.isSuccessful()) { + LOG.debug("Recording video stream to {}", targetFile); + InputStream in = Objects.requireNonNull(resp.body()).byteStream(); + byte[] b = new byte[1024 * 4]; + int len; + while (running && !Thread.currentThread().isInterrupted() && (len = in.read(b)) >= 0) { + ffmpegStdIn.write(b, 0, len); + timeOfLastTransfer = Instant.now(); + BandwidthMeter.add(len); + } + } else { + throw new HttpException(resp.code(), resp.message()); + } + } + } catch (SocketTimeoutException e) { + LOG.debug("Socket timeout while downloading MP4 for {}. Stop recording.", model.getName()); + model.delay(); + stop(); + } catch (IOException e) { + LOG.debug("IO error while downloading MP4 for {}. Stop recording.", model.getName()); + model.delay(); + stop(); + } catch (Exception e) { + LOG.error("Error while downloading MP4", e); + stop(); + } finally { + ffmpegStreamLock.unlock(); + } + LOG.debug("Record finished for model {}", model); + running = false; + }); + } + + protected void createTargetDirectory() throws IOException { + Files.createDirectories(targetFile.getParentFile().toPath()); + } +} diff --git a/common/src/main/java/ctbrec/sites/chaturbate/Chaturbate.java b/common/src/main/java/ctbrec/sites/chaturbate/Chaturbate.java index 929cea72..a6bfaf62 100644 --- a/common/src/main/java/ctbrec/sites/chaturbate/Chaturbate.java +++ b/common/src/main/java/ctbrec/sites/chaturbate/Chaturbate.java @@ -115,7 +115,7 @@ public class Chaturbate extends AbstractSite { @Override public boolean supportsTips() { - return true; + return false; } @Override diff --git a/common/src/main/java/ctbrec/sites/chaturbate/ChaturbateHttpClient.java b/common/src/main/java/ctbrec/sites/chaturbate/ChaturbateHttpClient.java index bcc54505..3ea2559d 100644 --- a/common/src/main/java/ctbrec/sites/chaturbate/ChaturbateHttpClient.java +++ b/common/src/main/java/ctbrec/sites/chaturbate/ChaturbateHttpClient.java @@ -4,14 +4,12 @@ import ctbrec.Config; import ctbrec.io.HtmlParser; import ctbrec.io.HttpClient; import okhttp3.*; -import org.jsoup.nodes.Element; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; import java.io.InterruptedIOException; import java.util.NoSuchElementException; -import java.util.Objects; import java.util.concurrent.Semaphore; import static ctbrec.io.HttpConstants.REFERER; @@ -98,6 +96,8 @@ public class ChaturbateHttpClient extends HttpClient { } } response.close(); + } catch (Exception ex) { + LOG.debug("Login failed: {}", ex.getMessage()); } finally { loginTries = 0; } @@ -105,24 +105,15 @@ public class ChaturbateHttpClient extends HttpClient { } public boolean checkLogin() throws IOException { - String url = "https://chaturbate.com/p/" + Config.getInstance().getSettings().chaturbateUsername + "/"; + String url = "https://chaturbate.com/api/ts/chatmessages/pm_users/?offset=0"; Request req = new Request.Builder() .url(url) .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) .build(); try (Response resp = execute(req)) { - if (resp.isSuccessful()) { - String profilePage = resp.body().string(); - try { - Element userIcon = HtmlParser.getTag(profilePage, "img.user_information_header_icon"); - return !Objects.equals("Anonymous Icon", userIcon.attr("alt")); - } catch (Exception e) { - LOG.debug("Token tag not found. Login failed"); - return false; - } - } else { - throw new IOException("HTTP response: " + resp.code() + " - " + resp.message()); - } + return (resp.isSuccessful() && !resp.isRedirect()); + } catch (Exception ex) { + return false; } } diff --git a/common/src/main/java/ctbrec/sites/dreamcam/Dreamcam.java b/common/src/main/java/ctbrec/sites/dreamcam/Dreamcam.java new file mode 100644 index 00000000..93717dd7 --- /dev/null +++ b/common/src/main/java/ctbrec/sites/dreamcam/Dreamcam.java @@ -0,0 +1,127 @@ +package ctbrec.sites.dreamcam; + +import static ctbrec.io.HttpConstants.*; + +import ctbrec.Model; +import ctbrec.io.HttpClient; +import ctbrec.sites.AbstractSite; +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class Dreamcam extends AbstractSite { + + public static String domain = "dreamcam.com"; + public static String baseUri = "https://dreamcam.com"; + private HttpClient httpClient; + + @Override + public void init() throws IOException { + } + + @Override + public String getName() { + return "DreamCam"; + } + + @Override + public String getBaseUrl() { + return baseUri; + } + + @Override + public String getAffiliateLink() { + return baseUri; + } + + @Override + public String getBuyTokensLink() { + return getAffiliateLink(); + } + + @Override + public DreamcamModel createModel(String name) { + DreamcamModel model = new DreamcamModel(); + model.setName(name); + model.setUrl(getBaseUrl() + "/models/" + name); + model.setSite(this); + return model; + } + + @Override + public Double getTokenBalance() throws IOException { + return 0d; + } + + @Override + public synchronized boolean login() throws IOException { + return credentialsAvailable() && getHttpClient().login(); + } + + @Override + public HttpClient getHttpClient() { + if (httpClient == null) { + httpClient = new DreamcamHttpClient(getConfig()); + } + return httpClient; + } + + @Override + public void shutdown() { + if (httpClient != null) { + httpClient.shutdown(); + } + } + + @Override + public boolean supportsTips() { + return false; + } + + @Override + public boolean supportsFollow() { + return false; + } + + @Override + public boolean supportsSearch() { + return false; + } + + @Override + public boolean searchRequiresLogin() { + return false; + } + + @Override + public List search(String q) throws IOException, InterruptedException { + return Collections.emptyList(); + } + + @Override + public boolean isSiteForModel(Model m) { + return m instanceof DreamcamModel; + } + + @Override + public boolean credentialsAvailable() { + return false; + } + + @Override + public Model createModelFromUrl(String url) { + String[] patterns = { + "https://.*?dreamcam.com/models/([_a-zA-Z0-9]+)", + }; + for (String p : patterns) { + Matcher m = Pattern.compile(p).matcher(url); + if (m.matches()) { + String modelName = m.group(1); + return createModel(modelName); + } + } + return super.createModelFromUrl(url); + } +} diff --git a/common/src/main/java/ctbrec/sites/dreamcam/DreamcamDownload.java b/common/src/main/java/ctbrec/sites/dreamcam/DreamcamDownload.java new file mode 100644 index 00000000..2558bd09 --- /dev/null +++ b/common/src/main/java/ctbrec/sites/dreamcam/DreamcamDownload.java @@ -0,0 +1,341 @@ +package ctbrec.sites.dreamcam; + +import ctbrec.Config; +import ctbrec.StringUtil; +import ctbrec.Model; +import ctbrec.OS; +import ctbrec.Recording; +import ctbrec.io.BandwidthMeter; +import ctbrec.io.HttpClient; +import ctbrec.io.HttpException; +import ctbrec.recorder.FFmpeg; +import ctbrec.recorder.InvalidPlaylistException; +import ctbrec.recorder.download.AbstractDownload; +import ctbrec.recorder.download.Download; +import ctbrec.recorder.download.ProcessExitedUncleanException; +import java.io.File; +import java.io.IOException; +import java.io.OutputStream; +import java.io.EOFException; +import java.nio.file.Files; +import java.net.SocketTimeoutException; +import java.time.Duration; +import java.time.Instant; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; +import java.util.regex.Pattern; +import okhttp3.Request; +import okhttp3.Response; +import okhttp3.WebSocket; +import okhttp3.WebSocketListener; +import okio.ByteString; +import org.json.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static ctbrec.io.HttpConstants.*; + +public class DreamcamDownload extends AbstractDownload { + + private static final Logger LOG = LoggerFactory.getLogger(DreamcamDownload.class); + private static final int MAX_SECONDS_WITHOUT_TRANSFER = 30; + + private final HttpClient httpClient; + private Instant timeOfLastTransfer = Instant.MAX; + + protected File targetFile; + protected FFmpeg ffmpeg; + protected Process ffmpegProcess; + protected OutputStream ffmpegStdIn; + protected Lock ffmpegStreamLock = new ReentrantLock(); + protected String wsUrl; + + private volatile boolean running; + private volatile boolean started; + private final transient Object monitor = new Object(); + private WebSocket ws; + private DreamcamModel model; + + public DreamcamDownload(HttpClient httpClient) { + this.httpClient = httpClient; + } + + @Override + public void init(Config config, Model model, Instant startTime, ExecutorService executorService) throws IOException { + super.init(config, model, startTime, executorService); + this.model = (DreamcamModel) model; + timeOfLastTransfer = startTime; + String fileSuffix = config.getSettings().ffmpegFileSuffix; + targetFile = config.getFileForRecording(model, fileSuffix, startTime); + createTargetDirectory(); + startFfmpegProcess(targetFile); + if (ffmpegProcess == null) { + throw new ProcessExitedUncleanException("Couldn't spawn FFmpeg"); + } + } + + @Override + public int getSelectedResolution() { + return 0; + } + + @Override + public void stop() { + if (running) { + internalStop(); + } + } + + private synchronized void internalStop() { + running = false; + if (ws != null) { + ws.close(1000, null); + ws = null; + } + if (ffmpegStdIn != null) { + try { + ffmpegStdIn.close(); + } catch (IOException e) { + LOG.error("Couldn't terminate FFmpeg by closing stdin", e); + } + } + if (ffmpegProcess != null) { + try { + boolean waitFor = ffmpegProcess.waitFor(45, TimeUnit.SECONDS); + if (!waitFor && ffmpegProcess.isAlive()) { + ffmpegProcess.destroy(); + if (ffmpegProcess.isAlive()) { + LOG.info("FFmpeg didn't terminate. Destroying the process with force!"); + ffmpegProcess.destroyForcibly(); + ffmpegProcess = null; + } + } + } catch (InterruptedException e) { + LOG.error("Interrupted while waiting for FFmpeg to terminate"); + Thread.currentThread().interrupt(); + } + } + } + + private void startFfmpegProcess(File target) { + try { + String[] cmdline = prepareCommandLine(target); + ffmpeg = new FFmpeg.Builder() + .logOutput(config.getSettings().logFFmpegOutput) + .onStarted(p -> { + ffmpegProcess = p; + ffmpegStdIn = ffmpegProcess.getOutputStream(); + }) + .build(); + ffmpeg.exec(cmdline, new String[0], target.getParentFile()); + } catch (IOException | ProcessExitedUncleanException e) { + LOG.error("Error in FFmpeg thread", e); + } + } + + private String[] prepareCommandLine(File target) { + String[] args = config.getSettings().ffmpegMergedDownloadArgs.split(" "); + String[] argsPlusFile = new String[args.length + 3]; + int i = 0; + argsPlusFile[i++] = "-i"; + argsPlusFile[i++] = "-"; + System.arraycopy(args, 0, argsPlusFile, i, args.length); + argsPlusFile[argsPlusFile.length - 1] = target.getAbsolutePath(); + return OS.getFFmpegCommand(argsPlusFile); + } + + @Override + public void finalizeDownload() { + internalStop(); + } + + @Override + public boolean isRunning() { + return running; + } + + @Override + public void postprocess(Recording recording) { + // nothing to do + } + + @Override + public File getTarget() { + return targetFile; + } + + @Override + public String getPath(Model model) { + String absolutePath = targetFile.getAbsolutePath(); + String recordingsDir = Config.getInstance().getSettings().recordingsDir; + String relativePath = absolutePath.replaceFirst(Pattern.quote(recordingsDir), ""); + return relativePath; + } + + @Override + public boolean isSingleFile() { + return true; + } + + @Override + public long getSizeInByte() { + return getTarget().length(); + } + + @Override + public Download call() throws Exception { + try { + if (!ffmpegProcess.isAlive()) { + running = false; + int exitValue = ffmpegProcess.exitValue(); + ffmpeg.shutdown(exitValue); + } + } catch (ProcessExitedUncleanException e) { + LOG.error("FFmpeg exited unclean", e); + internalStop(); + } + try { + if (!started) { + started = true; + startDownload(); + } + } catch (Exception e) { + LOG.error("Error while downloading", e); + stop(); + } + if (!model.isOnline()) { + LOG.debug("Model {} not online. Stop recording.", model); + stop(); + } + if (splittingStrategy.splitNecessary(this)) { + LOG.debug("Split necessary for model {}. Stop recording.", model); + internalStop(); + rescheduleTime = Instant.now(); + } else { + rescheduleTime = Instant.now().plusSeconds(5); + } + if (Duration.between(timeOfLastTransfer, Instant.now()).getSeconds() > MAX_SECONDS_WITHOUT_TRANSFER) { + LOG.debug("No video data received for {} seconds. Stopping recording for model {}", MAX_SECONDS_WITHOUT_TRANSFER, model); + stop(); + } + return this; + } + + private void startDownload() { + downloadExecutor.submit(() -> { + running = true; + ffmpegStreamLock.lock(); + try { + wsUrl = model.getWsUrl(); + LOG.debug("{} ws url: {}", model.getName(), wsUrl); + if (StringUtil.isBlank(wsUrl)) { + LOG.error("{}: Stream URL not found", model); + stop(); + return; + } + Request request = new Request.Builder() + .url(wsUrl) + .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) + .header(ACCEPT, "*/*") + .header(ACCEPT_LANGUAGE, "en") + .header(ORIGIN, model.getSite().getBaseUrl()) + .header(REFERER, model.getSite().getBaseUrl() + "/") + .build(); + + ws = httpClient.newWebSocket(request, new WebSocketListener() { + @Override + public void onOpen(WebSocket webSocket, Response response) { + super.onOpen(webSocket, response); + LOG.debug("{}: Websocket open", model); + if (response != null) { + response.close(); + } + JSONObject msg = new JSONObject(); + msg.put("url", "stream/hello"); + msg.put("version", "0.0.1"); + webSocket.send(msg.toString()); + } // onOpen + + @Override + public void onClosed(WebSocket webSocket, int code, String reason) { + super.onClosed(webSocket, code, reason); + LOG.trace("{}: Websocket closed", model); + stop(); + synchronized (monitor) { + monitor.notifyAll(); + } + } // onClosed + + @Override + public void onFailure(WebSocket webSocket, Throwable t, Response response) { + super.onFailure(webSocket, t, response); + LOG.debug("{}: Websocket failed: {}", model, t.getMessage()); + if (response != null) { + response.close(); + } + stop(); + synchronized (monitor) { + monitor.notifyAll(); + } + } // onFailure + + @Override + public void onMessage(WebSocket webSocket, String text) { + super.onMessage(webSocket, text); + LOG.trace("{} ws message: {}", model, text); + JSONObject message = new JSONObject(text); + if (message.optString("url").equals("stream/qual")) { + JSONObject msg = new JSONObject(); + msg.put("quality", "test"); + msg.put("url", "stream/play"); + msg.put("version", "0.0.1"); + webSocket.send(msg.toString()); + } + } // onMessage + + @Override + public void onMessage(WebSocket webSocket, ByteString bytes) { + super.onMessage(webSocket, bytes); + timeOfLastTransfer = Instant.now(); + try { + if (running) { + byte[] videoData = bytes.toByteArray(); + ffmpegStdIn.write(videoData); + BandwidthMeter.add(videoData.length); + } + } catch (IOException e) { + if (running) { + LOG.error("Couldn't write video stream to file", e); + stop(); + } + } + } // onMessage + + }); // websocket + + synchronized (monitor) { + try { + monitor.wait(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + LOG.warn("Interrupted while waiting for the download to terminate"); + } + } + } catch (IOException ex) { + if (running) { + LOG.error("Error while downloading: {}", ex.getMessage()); + stop(); + } + } finally { + ffmpegStreamLock.unlock(); + running = false; + } + }); // submit + } + + protected void createTargetDirectory() throws IOException { + Files.createDirectories(targetFile.getParentFile().toPath()); + } +} diff --git a/common/src/main/java/ctbrec/sites/dreamcam/DreamcamHttpClient.java b/common/src/main/java/ctbrec/sites/dreamcam/DreamcamHttpClient.java new file mode 100644 index 00000000..b67e329b --- /dev/null +++ b/common/src/main/java/ctbrec/sites/dreamcam/DreamcamHttpClient.java @@ -0,0 +1,17 @@ +package ctbrec.sites.dreamcam; + +import ctbrec.Config; +import ctbrec.io.HttpClient; +import java.io.IOException; + +public class DreamcamHttpClient extends HttpClient { + + public DreamcamHttpClient(Config config) { + super("dreamcam", config); + } + + @Override + public boolean login() throws IOException { + return false; + } +} diff --git a/common/src/main/java/ctbrec/sites/dreamcam/DreamcamModel.java b/common/src/main/java/ctbrec/sites/dreamcam/DreamcamModel.java new file mode 100644 index 00000000..cf07878f --- /dev/null +++ b/common/src/main/java/ctbrec/sites/dreamcam/DreamcamModel.java @@ -0,0 +1,205 @@ +package ctbrec.sites.dreamcam; + +import com.iheartradio.m3u8.*; +import com.iheartradio.m3u8.data.MasterPlaylist; +import com.iheartradio.m3u8.data.Playlist; +import com.iheartradio.m3u8.data.PlaylistData; +import ctbrec.AbstractModel; +import ctbrec.Config; +import ctbrec.StringUtil; +import ctbrec.io.HttpClient; +import ctbrec.io.HttpException; +import ctbrec.recorder.InvalidPlaylistException; +import ctbrec.recorder.download.Download; +import ctbrec.recorder.download.StreamSource; +import ctbrec.recorder.download.hls.MergedFfmpegHlsDownload; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.text.MessageFormat; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.ExecutionException; +import javax.xml.bind.JAXBException; +import okhttp3.FormBody; +import okhttp3.Request; +import okhttp3.Response; +import org.json.JSONArray; +import org.json.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static ctbrec.Model.State.*; +import static ctbrec.io.HttpConstants.*; +import static java.nio.charset.StandardCharsets.UTF_8; + +public class DreamcamModel extends AbstractModel { + + private static final Logger LOG = LoggerFactory.getLogger(DreamcamModel.class); + private static final String API_URL = "https://bss.dreamcamtrue.com"; + private int[] resolution = new int[2]; + private JSONObject modelInfo; + private boolean VRMode = false; + + private transient Instant lastInfoRequest = Instant.EPOCH; + + @Override + public boolean isOnline(boolean ignoreCache) throws IOException, ExecutionException, InterruptedException { + if (ignoreCache) { + try { + JSONObject json = getModelInfo(); + mapOnlineState(json.optString("broadcastStatus")); + } catch (Exception e) { + setOnlineState(OFFLINE); + } + } + return onlineState == ONLINE; + } + + private void mapOnlineState(String status) { + switch (status) { + case "public" -> setOnlineState(ONLINE); + case "private" -> setOnlineState(PRIVATE); + case "offline" -> setOnlineState(OFFLINE); + default -> setOnlineState(OFFLINE); + } + } + + @Override + public State getOnlineState(boolean failFast) throws IOException, ExecutionException { + if (failFast && onlineState != UNKNOWN) { + return onlineState; + } else { + try { + JSONObject json = getModelInfo(); + mapOnlineState(json.optString("broadcastStatus")); + } catch (Exception ex) { + setOnlineState(OFFLINE); + } + return onlineState; + } + } + + @Override + public List getStreamSources() throws InvalidPlaylistException { + List sources = new ArrayList<>(); + try { + StreamSource src = new StreamSource(); + src.mediaPlaylistUrl = getPlaylistUrl(); + sources.add(src); + } catch (Exception e) { + LOG.error("Can not get stream sources for {}: {}", getName(), e.getMessage()); + throw new InvalidPlaylistException(e.getMessage()); + } + return sources; + } + + private String getPlaylistUrl() throws IOException, InvalidPlaylistException { + JSONObject json = getModelInfo(); + String mediaUrl = ""; + if (json.has("streams")) { + JSONArray streams = json.getJSONArray("streams"); + for (int i=0; i < streams.length(); i++) { + JSONObject s = streams.getJSONObject(i); + if (s.has("streamType") && s.has("url")) { + String streamType = s.getString("streamType"); + if (streamType.equals("video2D")) { + mediaUrl = s.optString("url"); + LOG.trace("PlaylistUrl for {}: {}", getName(), mediaUrl); + } + } + } + } + if (StringUtil.isBlank(mediaUrl)) { + throw new InvalidPlaylistException("Playlist has no media"); + } + return mediaUrl; + } + + public String getWsUrl() throws IOException { + JSONObject json = getModelInfo(); + return json.optString("streamUrl").replace("fmp4s://", "wss://"); + } + + public String getChatId() throws IOException { + JSONObject json = getModelInfo(); + return json.optString("roomChatId"); + } + + @Override + public int[] getStreamResolution(boolean failFast) throws ExecutionException { + return new int[2]; + } + + private JSONObject getModelInfo() throws IOException { + if (Duration.between(lastInfoRequest, Instant.now()).getSeconds() < 5) { + modelInfo = Optional.ofNullable(modelInfo).orElse(loadModelInfo()); + } else { + modelInfo = loadModelInfo(); + } + return modelInfo; + } + + private JSONObject loadModelInfo() throws IOException { + lastInfoRequest = Instant.now(); + String url = MessageFormat.format(API_URL + "/api/clients/v1/broadcasts/models/{0}?partnerId=dreamcam_oauth2&show-hidden=true&stream-types=video2D,video3D", getName()); + Request req = new Request.Builder() + .url(url) + .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) + .header(ACCEPT, MIMETYPE_APPLICATION_JSON) + .header(ACCEPT_LANGUAGE, Locale.ENGLISH.getLanguage()) + .header(X_REQUESTED_WITH, XML_HTTP_REQUEST) + .header(REFERER, getUrl()) + .header(ORIGIN, getSite().getBaseUrl()) + .build(); + try (Response response = getSite().getHttpClient().execute(req)) { + if (response.isSuccessful()) { + JSONObject json = new JSONObject(response.body().string()); + return json; + } else { + throw new HttpException(response.code(), response.message()); + } + } + } + + public String getPreviewURL() throws IOException { + JSONObject json = getModelInfo(); + return json.optString("modelProfilePhotoUrl"); + } + + @Override + public boolean follow() throws IOException { + return false; + } + + @Override + public boolean unfollow() throws IOException { + return false; + } + + @Override + public void receiveTip(Double tokens) throws IOException { + // not implemented + } + + @Override + public void invalidateCacheEntries() { + resolution = new int[]{0, 0}; + lastInfoRequest = Instant.EPOCH; + modelInfo = null; + } + + @Override + public Download createDownload() { + if (Config.getInstance().getSettings().dreamcamVR) { + return new DreamcamDownload(getSite().getHttpClient()); + } else { + return new MergedFfmpegHlsDownload(getSite().getHttpClient()); + } + } +} diff --git a/common/src/main/java/ctbrec/sites/streamray/Streamray.java b/common/src/main/java/ctbrec/sites/streamray/Streamray.java new file mode 100644 index 00000000..8f0ed763 --- /dev/null +++ b/common/src/main/java/ctbrec/sites/streamray/Streamray.java @@ -0,0 +1,178 @@ +package ctbrec.sites.streamray; + +import static ctbrec.io.HttpConstants.*; + +import java.io.IOException; +import java.net.URLEncoder; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import org.json.JSONArray; +import org.json.JSONObject; +import okhttp3.Request; +import okhttp3.Response; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import ctbrec.Model; +import ctbrec.StringUtil; +import ctbrec.io.HttpClient; +import ctbrec.io.HttpException; +import ctbrec.sites.AbstractSite; + + +public class Streamray extends AbstractSite { + + private static final Logger LOG = LoggerFactory.getLogger(Streamray.class); + + private StreamrayHttpClient httpClient; + public static String domain = "streamray.com"; + public static String baseUri = "https://streamray.com"; + public static String apiURL = "https://beta-api.cams.com"; + + @Override + public void init() throws IOException {} + + @Override + public String getName() { + return "Streamray"; + } + + @Override + public String getBaseUrl() { + return baseUri; + } + + public String getApiUrl() { + return apiURL; + } + + @Override + public StreamrayModel createModel(String name) { + StreamrayModel model = new StreamrayModel(); + model.setName(name); + model.setUrl(getBaseUrl() + '/' + name); + model.setDescription(""); + model.setSite(this); + return model; + } + + @Override + public Double getTokenBalance() throws IOException { + return 0d; + } + + @Override + public String getBuyTokensLink() { + return getBaseUrl(); + } + + @Override + public synchronized boolean login() throws IOException { + boolean result = getHttpClient().login(); + LOG.debug("Streamray site login call result: {}", result); + return result; + } + + @Override + public HttpClient getHttpClient() { + if (httpClient == null) { + httpClient = new StreamrayHttpClient(getConfig()); + } + return httpClient; + } + + @Override + public void shutdown() { + if (httpClient != null) { + httpClient.shutdown(); + } + } + + @Override + public boolean supportsTips() { + return false; + } + + @Override + public boolean supportsFollow() { + return false; + } + + @Override + public boolean supportsSearch() { + return true; + } + + @Override + public boolean searchRequiresLogin() { + return false; + } + + @Override + public List search(String q) throws IOException, InterruptedException { + if (StringUtil.isBlank(q)) { + return Collections.emptyList(); + } + String url = getApiUrl() + "/models/new/?limit=30&search=" + URLEncoder.encode(q, "utf-8") + "&order=is_online"; + Request req = new Request.Builder() + .url(url) + .header(USER_AGENT, getConfig().getSettings().httpUserAgent) + .build(); + try (Response response = getHttpClient().execute(req)) { + if (response.isSuccessful()) { + JSONObject json = new JSONObject(response.body().string()); + if (json.has("results")) { + List models = new ArrayList<>(); + JSONArray results = json.getJSONArray("results"); + if (results.length() == 0) { + return Collections.emptyList(); + } + for (int i = 0; i < results.length(); i++) { + JSONObject result = results.getJSONObject(i); + StreamrayModel model = createModel(result.getString("stream_name")); + String image = result.optString("profile_image"); + if (StringUtil.isBlank(image)) { + image = model.getPreviewURL(); + } + model.setPreview(image); + models.add(model); + } + return models; + } else { + return Collections.emptyList(); + } + } else { + throw new HttpException(response.code(), response.message()); + } + } + } + + @Override + public boolean isSiteForModel(Model m) { + return m instanceof StreamrayModel; + } + + @Override + public boolean credentialsAvailable() { + return true; + } + + @Override + public Model createModelFromUrl(String url) { + Matcher m = Pattern.compile("https://(streamray|cams).com/([_a-zA-Z0-9]+)").matcher(url); + if (m.matches()) { + String modelName = m.group(2); + return createModel(modelName); + } else { + return super.createModelFromUrl(url); + } + } + + @Override + public String getAffiliateLink() { + return getBaseUrl(); + } +} diff --git a/common/src/main/java/ctbrec/sites/streamray/StreamrayHttpClient.java b/common/src/main/java/ctbrec/sites/streamray/StreamrayHttpClient.java new file mode 100644 index 00000000..de20170a --- /dev/null +++ b/common/src/main/java/ctbrec/sites/streamray/StreamrayHttpClient.java @@ -0,0 +1,79 @@ +package ctbrec.sites.streamray; + +import static ctbrec.io.HttpConstants.*; + +import ctbrec.Config; +import ctbrec.io.HttpClient; +import ctbrec.StringUtil; +import java.io.IOException; +import java.util.Locale; +import org.json.JSONObject; +import okhttp3.*; + + +public class StreamrayHttpClient extends HttpClient { + + public StreamrayHttpClient(Config config) { + super("streamray", config); + } + + @Override + public boolean login() throws IOException { + String token = getUserToken(); + if (StringUtil.isBlank(token)) { + return false; + } else { + boolean isSuccess = checkLoginSuccess(); + if (isSuccess) { + return true; + } else { + updateToken(); + return checkLoginSuccess(); + } + } + } + + private void updateToken() { + Request req = new Request.Builder() + .url(Streamray.baseUri) + .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) + .header(ACCEPT, "*/*") + .header(ACCEPT_LANGUAGE, Locale.ENGLISH.getLanguage()) + .build(); + try (Response response = execute(req)) {} catch (Exception ex) {} + } + + private boolean checkLoginSuccess() { + String token = getUserToken(); + Request req = new Request.Builder() + .url(Streamray.apiURL + "/members/me/balance/") + .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) + .header(ACCEPT, MIMETYPE_APPLICATION_JSON) + .header(ACCEPT_LANGUAGE, Locale.ENGLISH.getLanguage()) + .header(AUTHORIZATION, "Bearer " + token) + .header(REFERER, Streamray.baseUri + "/") + .header(ORIGIN, Streamray.baseUri) + .build(); + try (Response response = execute(req)) { + if (response.isSuccessful()) { + String content = response.body().string(); + JSONObject json = new JSONObject(content); + return json.has("balance"); + } else { + return false; + } + } catch (Exception ex) { + return false; + } + } + + public String getUserToken() { + try { + Cookie cookie = getCookieJar().getCookie(HttpUrl.parse(Streamray.baseUri), "memberToken"); + String token = cookie.value(); + return token; + } catch (Exception e) { + return ""; + } + } +} diff --git a/common/src/main/java/ctbrec/sites/streamray/StreamrayModel.java b/common/src/main/java/ctbrec/sites/streamray/StreamrayModel.java new file mode 100644 index 00000000..60c73869 --- /dev/null +++ b/common/src/main/java/ctbrec/sites/streamray/StreamrayModel.java @@ -0,0 +1,197 @@ +package ctbrec.sites.streamray; + +import com.iheartradio.m3u8.ParseException; +import com.iheartradio.m3u8.PlaylistException; +import ctbrec.AbstractModel; +import ctbrec.Config; +import ctbrec.io.HttpException; +import ctbrec.recorder.download.Download; +import ctbrec.recorder.download.StreamSource; +import ctbrec.recorder.download.hls.FfmpegHlsDownload; +import okhttp3.Request; +import okhttp3.Response; +import org.json.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.xml.bind.JAXBException; +import java.io.IOException; +import java.net.URLEncoder; +import java.text.MessageFormat; +import java.time.Duration; +import java.time.Instant; +import java.time.LocalDate; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.ExecutionException; + +import static ctbrec.Model.State.*; +import static ctbrec.io.HttpConstants.*; + +public class StreamrayModel extends AbstractModel { + + private static final Logger LOG = LoggerFactory.getLogger(StreamrayModel.class); + private String status = null; + private String gender = null; + private LocalDate regDate = LocalDate.EPOCH; + private JSONObject modelInfo; + + private transient Instant lastInfoRequest = Instant.EPOCH; + + @Override + public boolean isOnline(boolean ignoreCache) throws IOException, ExecutionException, InterruptedException { + if (ignoreCache) { + try { + JSONObject json = getModelInfo(); + if (json.has("online")) { + status = json.optString("online"); + mapOnlineState(status); + } + } catch (Exception e) { + setOnlineState(UNKNOWN); + } + } + return onlineState == ONLINE; + } + + private void mapOnlineState(String status) { + boolean goalShows = Config.getInstance().getSettings().streamrayRecordGoalShows; + switch (status) { + case "0" -> setOnlineState(OFFLINE); + case "1" -> setOnlineState(ONLINE); + case "6" -> setOnlineState(goalShows ? ONLINE : PRIVATE); + case "2", "3", "4", "7", "10", "11", "12", "13", "14" -> setOnlineState(PRIVATE); + default -> setOnlineState(OFFLINE); + } + } + + @Override + public State getOnlineState(boolean failFast) throws IOException, ExecutionException { + if (failFast && onlineState != UNKNOWN) { + return onlineState; + } else { + try { + onlineState = isOnline(true) ? ONLINE : OFFLINE; + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + onlineState = OFFLINE; + } catch (IOException | ExecutionException e) { + onlineState = OFFLINE; + } + return onlineState; + } + } + + @Override + public List getStreamSources() throws IOException, ExecutionException, ParseException, PlaylistException, JAXBException { + List sources = new ArrayList<>(); + try { + String url = getMasterPlaylistUrl(); + StreamSource src = new StreamSource(); + src.mediaPlaylistUrl = url; + src.height = 0; + src.width = 0; + src.bandwidth = 0; + sources.add(src); + } catch (IOException e) { + LOG.error("Can not get stream sources for {}", getName()); + } + return sources; + } + + private String getMasterPlaylistUrl() throws IOException { + JSONObject json = getModelInfo(); + String mpp = json.getString("mpp"); + String lname = getName().toLowerCase(); + return MessageFormat.format("https://stream14.cams.com/h5live/http/playlist.m3u8?url=rtmp%3A%2F%2F{0}%3A1935%2Fcams%2F{1}%3Fcams%2F{1}_720p&stream={2}", mpp, lname, getName()); + } + + @Override + public int[] getStreamResolution(boolean failFast) throws ExecutionException { + return new int[]{0, 0}; + } + + private JSONObject getModelInfo() throws IOException { + if (Duration.between(lastInfoRequest, Instant.now()).getSeconds() < 5) { + modelInfo = Optional.ofNullable(modelInfo).orElse(loadModelInfo()); + } else { + modelInfo = loadModelInfo(); + } + return modelInfo; + } + + private JSONObject loadModelInfo() throws IOException { + lastInfoRequest = Instant.now(); + String url = "https://beta-api.cams.com/models/stream/" + getName() + "/"; + Request req = new Request.Builder().url(url) + .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) + .header(ACCEPT, MIMETYPE_APPLICATION_JSON) + .header(ACCEPT_LANGUAGE, "en") + .header(REFERER, getSite().getBaseUrl() + '/' + getName()) + .build(); + try (Response response = site.getHttpClient().execute(req)) { + if (response.isSuccessful()) { + JSONObject jsonResponse = new JSONObject(response.body().string()); + return jsonResponse; + } else { + throw new HttpException(response.code(), response.message()); + } + } + } + + public String getPreviewURL() { + String lname = getName().toLowerCase(); + String url = MessageFormat.format("https://images4.streamray.com/images/streamray/won/jpg/{0}/{1}/{2}_640.jpg", lname.substring(0, 1), lname.substring(lname.length() - 1), lname); + try { + return MessageFormat.format("https://dynimages.securedataimages.com/unsigned/rs:fill:320::0/g:no/plain/{0}@jpg", URLEncoder.encode(url, "utf-8")); + } catch (Exception ex) { + return url; + } + } + + @Override + public Download createDownload() { + return new FfmpegHlsDownload(getSite().getHttpClient()); + } + + @Override + public boolean follow() throws IOException { + return false; + } + + @Override + public boolean unfollow() throws IOException { + return false; + } + + @Override + public void receiveTip(Double tokens) throws IOException { + // not implemented + } + + @Override + public void invalidateCacheEntries() { + status = null; + lastInfoRequest = Instant.EPOCH; + modelInfo = null; + } + + public String getGender() { + return gender; + } + + public void setGender(String gender) { + this.gender = gender; + } + + public void setRegDate(LocalDate reg) { + this.regDate = reg; + } + + public boolean isNew() { + return ChronoUnit.DAYS.between(this.regDate, LocalDate.now()) < 30; + } + +} diff --git a/common/src/main/java/ctbrec/sites/winktv/WinkTv.java b/common/src/main/java/ctbrec/sites/winktv/WinkTv.java new file mode 100644 index 00000000..29200bad --- /dev/null +++ b/common/src/main/java/ctbrec/sites/winktv/WinkTv.java @@ -0,0 +1,182 @@ +package ctbrec.sites.winktv; + +import static ctbrec.io.HttpConstants.*; + +import java.io.IOException; +import java.net.URLEncoder; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.Locale; +import org.json.JSONArray; +import org.json.JSONObject; +import okhttp3.Request; +import okhttp3.Response; +import okhttp3.FormBody; +import ctbrec.Model; +import ctbrec.StringUtil; +import ctbrec.io.HttpClient; +import ctbrec.io.HttpException; +import ctbrec.sites.AbstractSite; + +public class WinkTv extends AbstractSite { + + public static String domain = "www.winktv.co.kr"; + public static String baseUri = "https://www.winktv.co.kr"; + private HttpClient httpClient; + + @Override + public void init() throws IOException { + } + + @Override + public String getName() { + return "WinkTv"; + } + + @Override + public String getBaseUrl() { + return baseUri; + } + + @Override + public String getAffiliateLink() { + return baseUri; + } + + @Override + public String getBuyTokensLink() { + return getAffiliateLink(); + } + + @Override + public WinkTvModel createModel(String name) { + WinkTvModel model = new WinkTvModel(); + model.setName(name); + model.setUrl(getBaseUrl() + "/live/play/" + name); + model.setSite(this); + return model; + } + + @Override + public Double getTokenBalance() throws IOException { + return 0d; + } + + @Override + public synchronized boolean login() throws IOException { + return credentialsAvailable() && getHttpClient().login(); + } + + @Override + public HttpClient getHttpClient() { + if (httpClient == null) { + httpClient = new WinkTvHttpClient(getConfig()); + } + return httpClient; + } + + @Override + public void shutdown() { + if (httpClient != null) { + httpClient.shutdown(); + } + } + + @Override + public boolean supportsTips() { + return false; + } + + @Override + public boolean supportsFollow() { + return false; + } + + @Override + public boolean supportsSearch() { + return true; + } + + @Override + public boolean searchRequiresLogin() { + return false; + } + + @Override + public List search(String q) throws IOException, InterruptedException { + if (StringUtil.isBlank(q)) { + return Collections.emptyList(); + } + String url = "https://api.winktv.co.kr/v1/live"; + FormBody body = new FormBody.Builder() + .add("offset", "0") + .add("limit", "30") + .add("orderBy", "user") + .add("searchVal", URLEncoder.encode(q, "utf-8")) + .build(); + Request req = new Request.Builder() + .url(url) + .header(USER_AGENT, getConfig().getSettings().httpUserAgent) + .header(ACCEPT, MIMETYPE_APPLICATION_JSON) + .header(ACCEPT_LANGUAGE, Locale.ENGLISH.getLanguage()) + .header(X_REQUESTED_WITH, XML_HTTP_REQUEST) + .header(REFERER, getBaseUrl() + "/") + .header(ORIGIN, getBaseUrl()) + .post(body) + .build(); + try (Response response = getHttpClient().execute(req)) { + if (response.isSuccessful()) { + JSONObject json = new JSONObject(response.body().string()); + if (json.optBoolean("result") && json.has("list")) { + List models = new ArrayList<>(); + JSONArray results = json.getJSONArray("list"); + if (results.length() == 0) { + return Collections.emptyList(); + } + for (int i = 0; i < results.length(); i++) { + JSONObject result = results.getJSONObject(i); + WinkTvModel model = createModel(result.optString("userId")); + model.setPreview(result.optString("thumbUrl")); + models.add(model); + } + return models; + } else { + return Collections.emptyList(); + } + } else { + throw new HttpException(response.code(), response.message()); + } + } + } + + @Override + public boolean isSiteForModel(Model m) { + return m instanceof WinkTvModel; + } + + @Override + public boolean credentialsAvailable() { + return false; + } + + @Override + public Model createModelFromUrl(String url) { + String[] patterns = { + "https://.*?winktv.co.kr/live/play/([_a-zA-Z0-9]+)", + "https://.*?winktv.co.kr/channel/([_a-zA-Z0-9]+)", + "https://.*?pandalive.co.kr/live/play/([_a-zA-Z0-9]+)", + "https://.*?pandalive.co.kr/channel/([_a-zA-Z0-9]+)" + }; + for (String p : patterns) { + Matcher m = Pattern.compile(p).matcher(url); + if (m.matches()) { + String modelName = m.group(1); + return createModel(modelName); + } + } + return super.createModelFromUrl(url); + } +} diff --git a/common/src/main/java/ctbrec/sites/winktv/WinkTvHttpClient.java b/common/src/main/java/ctbrec/sites/winktv/WinkTvHttpClient.java new file mode 100644 index 00000000..c0d8c1bf --- /dev/null +++ b/common/src/main/java/ctbrec/sites/winktv/WinkTvHttpClient.java @@ -0,0 +1,17 @@ +package ctbrec.sites.winktv; + +import ctbrec.Config; +import ctbrec.io.HttpClient; +import java.io.IOException; + +public class WinkTvHttpClient extends HttpClient { + + public WinkTvHttpClient(Config config) { + super("winktv", config); + } + + @Override + public boolean login() throws IOException { + return false; + } +} diff --git a/common/src/main/java/ctbrec/sites/winktv/WinkTvModel.java b/common/src/main/java/ctbrec/sites/winktv/WinkTvModel.java new file mode 100644 index 00000000..049abc5c --- /dev/null +++ b/common/src/main/java/ctbrec/sites/winktv/WinkTvModel.java @@ -0,0 +1,270 @@ +package ctbrec.sites.winktv; + +import okhttp3.FormBody; +import okhttp3.Request; +import okhttp3.Response; +import org.json.JSONObject; +import org.json.JSONArray; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import javax.xml.bind.JAXBException; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.ExecutionException; +import java.time.Duration; +import java.time.Instant; + +import com.iheartradio.m3u8.*; +import com.iheartradio.m3u8.data.MasterPlaylist; +import com.iheartradio.m3u8.data.Playlist; +import com.iheartradio.m3u8.data.PlaylistData; + +import ctbrec.AbstractModel; +import ctbrec.Config; +import ctbrec.io.HttpClient; +import ctbrec.io.HttpException; +import ctbrec.recorder.download.Download; +import ctbrec.recorder.download.StreamSource; +import ctbrec.recorder.download.hls.HlsdlDownload; +import ctbrec.recorder.download.hls.MergedFfmpegHlsDownload; + +import static ctbrec.Model.State.*; +import static ctbrec.io.HttpConstants.*; +import static java.nio.charset.StandardCharsets.UTF_8; + +public class WinkTvModel extends AbstractModel { + + private static final Logger LOG = LoggerFactory.getLogger(WinkTvModel.class); + private int[] resolution = new int[]{0, 0}; + private boolean adult = false; + private JSONObject modelInfo; + + private transient Instant lastInfoRequest = Instant.EPOCH; + + @Override + public boolean isOnline(boolean ignoreCache) throws IOException, ExecutionException, InterruptedException { + if (ignoreCache) { + try { + JSONObject json = getModelInfo(); + if (json.has("media")) { + JSONObject media = json.getJSONObject("media"); + boolean isLive = media.optBoolean("isLive"); + String meType = media.optString("type"); + if (isLive && meType.equals("free")) { + setOnlineState(ONLINE); + } else { + setOnlineState(PRIVATE); + } + } else { + setOnlineState(OFFLINE); + } + } catch (Exception e) { + setOnlineState(UNKNOWN); + } + } + return onlineState == ONLINE; + } + + @Override + public State getOnlineState(boolean failFast) throws IOException, ExecutionException { + if (failFast && onlineState != UNKNOWN) { + return onlineState; + } else { + try { + onlineState = isOnline(true) ? ONLINE : OFFLINE; + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + onlineState = OFFLINE; + } catch (IOException | ExecutionException e) { + onlineState = OFFLINE; + } + return onlineState; + } + } + + @Override + public List getStreamSources() throws IOException, ExecutionException, ParseException, PlaylistException { + String url = getMasterPlaylistUrl(); + MasterPlaylist masterPlaylist = getMasterPlaylist(url); + List streamSources = extractStreamSources(masterPlaylist); + return streamSources; + } + + private List extractStreamSources(MasterPlaylist masterPlaylist) { + List sources = new ArrayList<>(); + for (PlaylistData playlist : masterPlaylist.getPlaylists()) { + if (playlist.hasStreamInfo()) { + StreamSource src = new StreamSource(); + src.bandwidth = playlist.getStreamInfo().getBandwidth(); + src.height = playlist.getStreamInfo().getResolution().height; + src.mediaPlaylistUrl = playlist.getUri(); + LOG.trace("Media playlist {}", src.mediaPlaylistUrl); + sources.add(src); + } + } + return sources; + } + + private MasterPlaylist getMasterPlaylist(String url) throws IOException, ParseException, PlaylistException { + LOG.trace("Loading master playlist {}", url); + Request req = new Request.Builder() + .url(url) + .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) + .build(); + try (Response response = getSite().getHttpClient().execute(req)) { + if (response.isSuccessful()) { + String body = response.body().string(); + InputStream inputStream = new ByteArrayInputStream(body.getBytes(UTF_8)); + PlaylistParser parser = new PlaylistParser(inputStream, Format.EXT_M3U, Encoding.UTF_8, ParsingMode.LENIENT); + Playlist playlist = parser.parse(); + MasterPlaylist master = playlist.getMasterPlaylist(); + return master; + } else { + throw new HttpException(response.code(), response.message()); + } + } + } + + private String getMasterPlaylistUrl() throws IOException { + JSONObject json = getModelInfo(); + JSONObject info = json.getJSONObject("bjInfo"); + long userIdx = info.optLong("idx"); + String url = "https://api.winktv.co.kr/v1/live/play"; + FormBody body = new FormBody.Builder() + .add("action", "watch") + .add("userIdx", String.valueOf(userIdx)) + .add("password", "") + .build(); + Request req = new Request.Builder() + .url(url) + .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) + .header(ACCEPT, MIMETYPE_APPLICATION_JSON) + .header(ACCEPT_LANGUAGE, Locale.ENGLISH.getLanguage()) + .header(X_REQUESTED_WITH, XML_HTTP_REQUEST) + .header(REFERER, getUrl()) + .header(ORIGIN, getSite().getBaseUrl()) + .post(body) + .build(); + try (Response response = getSite().getHttpClient().execute(req)) { + if (response.isSuccessful()) { + JSONObject jsonResponse = new JSONObject(response.body().string()); + JSONObject playlist = jsonResponse.getJSONObject("PlayList"); + JSONObject hls = playlist.getJSONArray("hls").getJSONObject(0); + String hlsUrl = hls.optString("url"); + return hlsUrl; + } else { + LOG.debug("Error while get master playlist url for {}: {}", getName(), response.body().string()); + throw new HttpException(response.code(), response.message()); + } + } + } + + @Override + public int[] getStreamResolution(boolean failFast) throws ExecutionException { + if (!failFast) { + try { + List sources = getStreamSources(); + if (!sources.isEmpty()) { + StreamSource best = sources.get(sources.size() - 1); + resolution = new int[]{best.getWidth(), best.getHeight()}; + } + } catch (IOException | ParseException | PlaylistException e) { + throw new ExecutionException(e); + } + } + return resolution; + } + + private JSONObject getModelInfo() throws IOException { + if (Duration.between(lastInfoRequest, Instant.now()).getSeconds() < 5) { + return Optional.ofNullable(modelInfo).orElse(new JSONObject()) ; + } + lastInfoRequest = Instant.now(); + modelInfo = loadModelInfo(); + return modelInfo; + } + + private JSONObject loadModelInfo() throws IOException { + String url = "https://api.winktv.co.kr/v1/member/bj"; + FormBody body = new FormBody.Builder() + .add("userId", getName()) + .add("info", "media") + .build(); + Request req = new Request.Builder() + .url(url) + .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) + .header(ACCEPT, MIMETYPE_APPLICATION_JSON) + .header(ACCEPT_LANGUAGE, Locale.ENGLISH.getLanguage()) + .header(X_REQUESTED_WITH, XML_HTTP_REQUEST) + .header(REFERER, getUrl()) + .header(ORIGIN, getSite().getBaseUrl()) + .post(body) + .build(); + try (Response response = getSite().getHttpClient().execute(req)) { + if (response.isSuccessful()) { + JSONObject jsonResponse = new JSONObject(response.body().string()); + return jsonResponse; + } else { + throw new HttpException(response.code(), response.message()); + } + } + } + + public String getPreviewURL() throws IOException { + JSONObject json = getModelInfo(); + if (json.has("media")) { + JSONObject media = json.getJSONObject("media"); + return media.optString("ivsThumbnail"); + } + if (json.has("bjInfo")) { + JSONObject info = json.getJSONObject("bjInfo"); + return info.optString("thumbUrl"); + } + return ""; + } + + @Override + public boolean follow() throws IOException { + return false; + } + + @Override + public boolean unfollow() throws IOException { + return false; + } + + public boolean isAdult() { + return adult; + } + + public void setAdult(boolean a) { + this.adult = a; + } + + @Override + public void receiveTip(Double tokens) throws IOException { + // not implemented + } + + @Override + public void invalidateCacheEntries() { + resolution = new int[]{0, 0}; + lastInfoRequest = Instant.EPOCH; + modelInfo = null; + } + + @Override + public Download createDownload() { + if (Config.getInstance().getSettings().useHlsdl) { + return new HlsdlDownload(); + } else { + return new MergedFfmpegHlsDownload(getSite().getHttpClient()); + } + } +} diff --git a/server/src/main/java/ctbrec/recorder/server/HttpServer.java b/server/src/main/java/ctbrec/recorder/server/HttpServer.java index df325a4d..df1f1ec3 100644 --- a/server/src/main/java/ctbrec/recorder/server/HttpServer.java +++ b/server/src/main/java/ctbrec/recorder/server/HttpServer.java @@ -18,6 +18,7 @@ import ctbrec.sites.cam4.Cam4; import ctbrec.sites.camsoda.Camsoda; import ctbrec.sites.chaturbate.Chaturbate; import ctbrec.sites.cherrytv.CherryTv; +import ctbrec.sites.dreamcam.Dreamcam; import ctbrec.sites.fc2live.Fc2Live; import ctbrec.sites.flirt4free.Flirt4Free; import ctbrec.sites.jasmin.LiveJasmin; @@ -26,7 +27,9 @@ import ctbrec.sites.mfc.MyFreeCams; import ctbrec.sites.secretfriends.SecretFriends; import ctbrec.sites.showup.Showup; import ctbrec.sites.streamate.Streamate; +import ctbrec.sites.streamray.Streamray; import ctbrec.sites.stripchat.Stripchat; +import ctbrec.sites.winktv.WinkTv; import ctbrec.sites.xlovecam.XloveCam; import org.eclipse.jetty.security.*; import org.eclipse.jetty.security.authentication.BasicAuthenticator; @@ -134,6 +137,7 @@ public class HttpServer { sites.add(new Camsoda()); sites.add(new Chaturbate()); sites.add(new CherryTv()); + sites.add(new Dreamcam()); sites.add(new Fc2Live()); sites.add(new Flirt4Free()); sites.add(new LiveJasmin()); @@ -143,7 +147,9 @@ public class HttpServer { sites.add(new Showup()); sites.add(new Streamate()); sites.add(new Stripchat()); + sites.add(new Streamray()); sites.add(new XloveCam()); + sites.add(new WinkTv()); } private void addShutdownHook() { @@ -188,7 +194,7 @@ public class HttpServer { sslContextFactory.setTrustStorePassword(keyStorePassword); try (ServerConnector http = new ServerConnector(server, httpConnectionFactory); - ServerConnector https = new ServerConnector(server, sslContextFactory, httpConnectionFactory)) { + ServerConnector https = new ServerConnector(server, sslContextFactory, httpConnectionFactory)) { // connector for http http.setPort(this.config.getSettings().httpPort); @@ -227,9 +233,9 @@ public class HttpServer { HandlerList handlers = new HandlerList(); if (this.config.getSettings().transportLayerSecurity) { server.addConnector(https); - handlers.setHandlers(new Handler[] { new SecuredRedirectHandler(), basicAuthContext, defaultContext }); + handlers.setHandlers(new Handler[]{new SecuredRedirectHandler(), basicAuthContext, defaultContext}); } else { - handlers.setHandlers(new Handler[] { basicAuthContext, defaultContext }); + handlers.setHandlers(new Handler[]{basicAuthContext, defaultContext}); } server.setHandler(handlers); @@ -255,7 +261,7 @@ public class HttpServer { ServletHolder holder = new ServletHolder(staticFileServlet); String staticFileContext = "/static/*"; defaultContext.addServlet(holder, staticFileContext); - LOG.info("Register static file servlet under {}", defaultContext.getContextPath()+staticFileContext); + LOG.info("Register static file servlet under {}", defaultContext.getContextPath() + staticFileContext); // servlet to retrieve the HMAC (secured by basic auth if a hmac key is set in the config) String username = this.config.getSettings().webinterfaceUsername; @@ -298,7 +304,7 @@ public class HttpServer { @Override protected void handleErrorPage(HttpServletRequest request, Writer writer, int code, String message) throws IOException { if (code == 404) { - writer.write("404

404

Looking for CTB Recorder?

"); + writer.write("404

404

Looking for CTB Recorder?

"); } else { super.handleErrorPage(request, writer, code, message); } @@ -315,7 +321,7 @@ public class HttpServer { @Override public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) throws IOException, ServletException { - ((HttpServletResponse)response).addHeader("Server", "CTB Recorder/" + getVersion()); + ((HttpServletResponse) response).addHeader("Server", "CTB Recorder/" + getVersion()); chain.doFilter(request, response); } @@ -330,14 +336,14 @@ public class HttpServer { private static SecurityHandler basicAuth(String username, String password) { String realm = "CTB Recorder"; UserStore userStore = new UserStore(); - userStore.addUser(username, Credential.getCredential(password), new String[] { "user" }); + userStore.addUser(username, Credential.getCredential(password), new String[]{"user"}); HashLoginService l = new HashLoginService(); l.setUserStore(userStore); l.setName(realm); Constraint constraint = new Constraint(); constraint.setName(Constraint.__BASIC_AUTH); - constraint.setRoles(new String[] { "user" }); + constraint.setRoles(new String[]{"user"}); constraint.setAuthenticate(true); ConstraintMapping cm = new ConstraintMapping(); From 224bb270032b7060e65727b5ac4a4fd5ae6c9485 Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Sun, 29 Oct 2023 19:24:16 +0100 Subject: [PATCH 3/3] Merge almost all changes by @winkru --- .../java/ctbrec/ui/DesktopIntegration.java | 75 ++++-- .../src/main/java/ctbrec/ui/JavaFxModel.java | 24 +- client/src/main/java/ctbrec/ui/Player.java | 2 +- .../ctbrec/ui/action/LaterGroupAction.java | 51 ++++ .../ui/controls/SearchPopoverTreeList.java | 34 +-- .../ctbrec/ui/menu/ModelGroupMenuBuilder.java | 25 +- .../ctbrec/ui/menu/ModelMenuContributor.java | 89 ++++--- .../ctbrec/ui/settings/CacheSettingsPane.java | 53 ++++ .../java/ctbrec/ui/settings/SettingsTab.java | 19 +- .../sites/amateurtv/AmateurTvTabProvider.java | 12 +- .../amateurtv/AmateurTvUpdateService.java | 100 ++++--- .../sites/bonga/BongaCamsUpdateService.java | 11 +- .../ChaturbateElectronLoginDialog.java | 2 +- .../chaturbate/ChaturbateFollowedTab.java | 15 +- .../chaturbate/ChaturbateTabProvider.java | 29 +- .../chaturbate/ChaturbateUpdateService.java | 85 ++++-- .../flirt4free/Flirt4FreeTabProvider.java | 10 +- .../flirt4free/Flirt4FreeUpdateService.java | 2 +- .../sites/jasmin/LiveJasminTabProvider.java | 22 +- .../sites/jasmin/LiveJasminUpdateService.java | 219 +++++++++++----- .../sites/manyvids/MVLiveUpdateService.java | 45 +++- .../SecretFriendsUpdateService.java | 16 +- .../ui/sites/showup/ShowupUpdateService.java | 26 +- .../ui/sites/stripchat/StripchatConfigUI.java | 30 ++- .../StripchatFollowedUpdateService.java | 4 +- .../sites/stripchat/StripchatTabProvider.java | 8 +- .../stripchat/StripchatUpdateService.java | 2 +- .../sites/xlovecam/XloveCamTabProvider.java | 5 + .../java/ctbrec/ui/tabs/RecordingsTab.java | 21 +- client/src/main/resources/logback.xml | 2 - common/src/main/java/ctbrec/Config.java | 9 +- common/src/main/java/ctbrec/Recording.java | 11 +- common/src/main/java/ctbrec/Settings.java | 4 + .../src/main/java/ctbrec/io/HttpClient.java | 120 +++++---- .../ctbrec/recorder/RecordingManager.java | 1 + .../recorder/RecordingPreconditions.java | 9 + .../download/hls/AbstractHlsDownload.java | 36 +-- .../ctbrec/sites/amateurtv/AmateurTv.java | 59 ++++- .../sites/amateurtv/AmateurTvModel.java | 99 +++---- .../java/ctbrec/sites/bonga/BongaCams.java | 34 +-- .../sites/bonga/BongaCamsHttpClient.java | 1 - .../java/ctbrec/sites/cam4/Cam4Model.java | 63 +++-- .../ctbrec/sites/camsoda/CamsodaModel.java | 6 +- .../chaturbate/ChaturbateHttpClient.java | 11 - .../sites/chaturbate/ChaturbateModel.java | 1 - .../sites/flirt4free/Flirt4FreeModel.java | 28 +- .../java/ctbrec/sites/jasmin/LiveJasmin.java | 14 +- .../ctbrec/sites/jasmin/LiveJasminModel.java | 102 ++++---- .../sites/jasmin/LiveJasminModelInfo.java | 2 + .../jasmin/LiveJasminStreamRegistration.java | 100 +++++-- .../sites/jasmin/LiveJasminStreamSource.java | 6 + .../jasmin/LiveJasminWebrtcDownload.java | 247 ++++++++++++++++++ .../sites/streamate/StreamateModel.java | 30 ++- .../sites/streamray/StreamrayModel.java | 3 +- .../ctbrec/sites/stripchat/Stripchat.java | 26 +- .../sites/stripchat/StripchatModel.java | 92 ++++++- master/pom.xml | 2 +- 57 files changed, 1513 insertions(+), 641 deletions(-) create mode 100644 client/src/main/java/ctbrec/ui/action/LaterGroupAction.java create mode 100644 client/src/main/java/ctbrec/ui/settings/CacheSettingsPane.java create mode 100644 common/src/main/java/ctbrec/sites/jasmin/LiveJasminWebrtcDownload.java diff --git a/client/src/main/java/ctbrec/ui/DesktopIntegration.java b/client/src/main/java/ctbrec/ui/DesktopIntegration.java index d9f1af4b..7673458b 100644 --- a/client/src/main/java/ctbrec/ui/DesktopIntegration.java +++ b/client/src/main/java/ctbrec/ui/DesktopIntegration.java @@ -1,6 +1,8 @@ package ctbrec.ui; +import ctbrec.Config; import ctbrec.OS; +import ctbrec.StringUtil; import ctbrec.io.StreamRedirector; import ctbrec.recorder.Recorder; import javafx.application.Platform; @@ -31,30 +33,44 @@ public class DesktopIntegration { private static TrayIcon trayIcon; public static void open(String uri) { - try { - CamrecApplication.hostServices.showDocument(uri); - return; - } catch (Exception e) { - LOG.debug("Couldn't open URL with host services {}", uri); - } + Config cfg = Config.getInstance(); + Runtime rt = Runtime.getRuntime(); + String[] cmdline = createCmdline(uri); - // opening with HostServices failed, now try Desktop - try { - Desktop.getDesktop().browse(new URI(uri)); - return; - } catch (Exception e) { - LOG.debug("Couldn't open URL with Desktop {}", uri); - } - - // try external helpers - var externalHelpers = new String[]{"kde-open5", "kde-open", "gnome-open", "xdg-open"}; - var rt = Runtime.getRuntime(); - for (String helper : externalHelpers) { + if (!cfg.getSettings().browserOverride.isEmpty()) { try { - rt.exec(helper + " " + uri); + rt.exec(cmdline); return; - } catch (IOException e) { - LOG.debug("Couldn't open URL with {} {}", helper, uri); + } catch (Exception e) { + LOG.debug("Couldn't open URL with user-defined {} {}", cmdline, uri); + } + } + + if (!cfg.getSettings().forceBrowserOverride) { + try { + CamrecApplication.hostServices.showDocument(uri); + return; + } catch (Exception e) { + LOG.debug("Couldn't open URL with host services {}", uri); + } + + // opening with HostServices failed, now try Desktop + try { + Desktop.getDesktop().browse(new URI(uri)); + return; + } catch (Exception e) { + LOG.debug("Couldn't open URL with Desktop {}", uri); + } + + // try external helpers + var externalHelpers = new String[]{"kde-open5", "kde-open", "gnome-open", "xdg-open"}; + for (String helper : externalHelpers) { + try { + rt.exec(helper + " " + uri); + return; + } catch (IOException e) { + LOG.debug("Couldn't open URL with {} {}", helper, uri); + } } } @@ -73,6 +89,23 @@ public class DesktopIntegration { info.show(); } + private static String[] createCmdline(String streamUrl) { + Config cfg = Config.getInstance(); + String params = cfg.getSettings().browserParams.trim(); + + String[] cmdline; + if (params.isEmpty()) { + cmdline = new String[2]; + } else { + String[] playerArgs = StringUtil.splitParams(params); + cmdline = new String[playerArgs.length + 2]; + System.arraycopy(playerArgs, 0, cmdline, 1, playerArgs.length); + } + cmdline[0] = cfg.getSettings().browserOverride; + cmdline[cmdline.length - 1] = streamUrl; + return cmdline; + } + public static void open(File f) { try { Desktop.getDesktop().open(f); diff --git a/client/src/main/java/ctbrec/ui/JavaFxModel.java b/client/src/main/java/ctbrec/ui/JavaFxModel.java index c7573adc..092bda18 100644 --- a/client/src/main/java/ctbrec/ui/JavaFxModel.java +++ b/client/src/main/java/ctbrec/ui/JavaFxModel.java @@ -1,17 +1,9 @@ package ctbrec.ui; -import java.io.IOException; -import java.time.Instant; -import java.util.List; -import java.util.concurrent.ExecutionException; - -import javax.xml.bind.JAXBException; - import com.iheartradio.m3u8.ParseException; import com.iheartradio.m3u8.PlaylistException; import com.squareup.moshi.JsonReader; import com.squareup.moshi.JsonWriter; - import ctbrec.Model; import ctbrec.SubsequentAction; import ctbrec.recorder.download.Download; @@ -23,6 +15,12 @@ import javafx.beans.property.SimpleBooleanProperty; import javafx.beans.property.SimpleIntegerProperty; import javafx.beans.property.SimpleObjectProperty; +import javax.xml.bind.JAXBException; +import java.io.IOException; +import java.time.Instant; +import java.util.List; +import java.util.concurrent.ExecutionException; + /** * Just a wrapper for Model, which augments it with JavaFX value binding properties, so that UI widgets get updated proeprly */ @@ -230,6 +228,16 @@ public class JavaFxModel implements Model { pausedProperty.set(suspended); } + @Override + public void delay() { + delegate.delay(); + } + + @Override + public boolean isDelayed() { + return delegate.isDelayed(); + } + @Override public String getDisplayName() { return delegate.getDisplayName(); diff --git a/client/src/main/java/ctbrec/ui/Player.java b/client/src/main/java/ctbrec/ui/Player.java index 371a1f76..28ab4f0f 100644 --- a/client/src/main/java/ctbrec/ui/Player.java +++ b/client/src/main/java/ctbrec/ui/Player.java @@ -189,7 +189,7 @@ public class Player { private void expandPlaceHolders(String[] cmdline) { ModelVariableExpander expander = new ModelVariableExpander(model, Config.getInstance(), null, null); - for (int i = 0; i < cmdline.length; i++) { + for (int i = 1; i < cmdline.length; i++) { var param = cmdline[i]; param = expander.expand(param); cmdline[i] = param; diff --git a/client/src/main/java/ctbrec/ui/action/LaterGroupAction.java b/client/src/main/java/ctbrec/ui/action/LaterGroupAction.java new file mode 100644 index 00000000..2e1751cf --- /dev/null +++ b/client/src/main/java/ctbrec/ui/action/LaterGroupAction.java @@ -0,0 +1,51 @@ +package ctbrec.ui.action; + +import java.io.IOException; +import java.security.InvalidKeyException; +import java.security.NoSuchAlgorithmException; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +import ctbrec.Model; +import ctbrec.ModelGroup; +import ctbrec.recorder.Recorder; +import ctbrec.ui.controls.Dialogs; +import javafx.application.Platform; +import javafx.scene.Node; + +public class LaterGroupAction extends ModelMassEditAction { + + private Recorder recorder; + private Model model; + + public LaterGroupAction(Node source, Recorder recorder, Model model) { + super.source = source; + this.recorder = recorder; + this.model = model; + + action = m -> { + try { + if (recorder.isMarkedForLaterRecording(m) == false) { + recorder.markForLaterRecording(m, true); + } + } catch (InvalidKeyException | NoSuchAlgorithmException | IOException e) { + Platform.runLater(() -> Dialogs.showError(source.getScene(), "Couldn't change model state", "Mark for later recording of " + m.getName() + " failed", e)); + } + }; + } + + @Override + protected List getModels() { + Optional optionalGroup = recorder.getModelGroup(model); + if (optionalGroup.isPresent()) { + ModelGroup group = optionalGroup.get(); + return recorder.getModels().stream() // + .filter(m -> group.getModelUrls().contains(m.getUrl())) // + .collect(Collectors.toList()); + } else { + return Collections.emptyList(); + } + } +} diff --git a/client/src/main/java/ctbrec/ui/controls/SearchPopoverTreeList.java b/client/src/main/java/ctbrec/ui/controls/SearchPopoverTreeList.java index 01d421e7..4f05ad2a 100644 --- a/client/src/main/java/ctbrec/ui/controls/SearchPopoverTreeList.java +++ b/client/src/main/java/ctbrec/ui/controls/SearchPopoverTreeList.java @@ -31,17 +31,10 @@ */ package ctbrec.ui.controls; -import java.net.URL; -import java.util.List; -import java.util.Objects; -import java.util.Optional; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import ctbrec.Config; import ctbrec.GlobalThreadPool; import ctbrec.Model; +import ctbrec.StringUtil; import ctbrec.recorder.Recorder; import ctbrec.ui.action.PlayAction; import ctbrec.ui.action.SetThumbAsPortraitAction; @@ -52,19 +45,20 @@ import javafx.event.EventHandler; import javafx.geometry.Insets; import javafx.scene.Cursor; import javafx.scene.Node; -import javafx.scene.control.Button; -import javafx.scene.control.ContextMenu; -import javafx.scene.control.Label; -import javafx.scene.control.ListCell; -import javafx.scene.control.ListView; -import javafx.scene.control.MenuItem; -import javafx.scene.control.Skin; +import javafx.scene.control.*; import javafx.scene.image.Image; import javafx.scene.image.ImageView; import javafx.scene.input.ContextMenuEvent; import javafx.scene.input.MouseButton; import javafx.scene.input.MouseEvent; import javafx.scene.shape.Rectangle; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.net.URL; +import java.util.List; +import java.util.Objects; +import java.util.Optional; /** * Popover page that displays a list of samples and sample categories for a given SampleCategory. @@ -85,7 +79,7 @@ public class SearchPopoverTreeList extends PopoverTreeList implements Pop @Override protected void itemClicked(Model model) { - if(model == null) { + if (model == null) { return; } @@ -163,8 +157,8 @@ public class SearchPopoverTreeList extends PopoverTreeList implements Pop addEventHandler(ContextMenuEvent.CONTEXT_MENU_REQUESTED, event -> { popup = new CustomMouseBehaviorContextMenu(); ModelMenuContributor.newContributor(this, Config.getInstance(), recorder) // - .withStartStopCallback(m -> setCursor(Cursor.DEFAULT)) // - .contributeToMenu(List.of(model), popup); + .withStartStopCallback(m -> setCursor(Cursor.DEFAULT)) // + .contributeToMenu(List.of(model), popup); var useImageAsPortrait = new MenuItem("Use As Portrait"); useImageAsPortrait.setOnAction(e -> new SetThumbAsPortraitAction(this, model, thumb.getImage()).execute()); popup.getItems().add(useImageAsPortrait); @@ -253,7 +247,7 @@ public class SearchPopoverTreeList extends PopoverTreeList implements Pop this.model = model; URL anonymousPng = getClass().getResource("/anonymous.png"); String previewUrl = Optional.ofNullable(model.getPreview()).orElse(anonymousPng.toString()); - if(!Objects.equals(System.getenv("CTBREC_DEV"), "1")) { + if (!Objects.equals(System.getenv("CTBREC_DEV"), "1") && StringUtil.isNotBlank(previewUrl)) { Image img = new Image(previewUrl, true); thumb.setImage(img); } else { @@ -349,4 +343,4 @@ public class SearchPopoverTreeList extends PopoverTreeList implements Pop public void setRecorder(Recorder recorder) { this.recorder = recorder; } -} \ No newline at end of file +} diff --git a/client/src/main/java/ctbrec/ui/menu/ModelGroupMenuBuilder.java b/client/src/main/java/ctbrec/ui/menu/ModelGroupMenuBuilder.java index 24de7509..7a18b72a 100644 --- a/client/src/main/java/ctbrec/ui/menu/ModelGroupMenuBuilder.java +++ b/client/src/main/java/ctbrec/ui/menu/ModelGroupMenuBuilder.java @@ -1,19 +1,16 @@ package ctbrec.ui.menu; +import ctbrec.Model; +import ctbrec.recorder.Recorder; +import ctbrec.ui.action.*; +import javafx.scene.Node; +import javafx.scene.control.Menu; +import javafx.scene.control.MenuItem; + import java.util.Objects; import java.util.Optional; import java.util.function.Consumer; -import ctbrec.Model; -import ctbrec.recorder.Recorder; -import ctbrec.ui.action.EditGroupAction; -import ctbrec.ui.action.PauseGroupAction; -import ctbrec.ui.action.ResumeGroupAction; -import ctbrec.ui.action.StopGroupAction; -import javafx.scene.Node; -import javafx.scene.control.Menu; -import javafx.scene.control.MenuItem; - public class ModelGroupMenuBuilder { private Model model; @@ -45,7 +42,8 @@ public class ModelGroupMenuBuilder { Objects.requireNonNull(model, "Model has to be set"); Objects.requireNonNull(recorder, "Recorder has to be set"); Objects.requireNonNull(source, "Node has to be set"); - callback = Optional.ofNullable(callback).orElse(m -> {}); + callback = Optional.ofNullable(callback).orElse(m -> { + }); var menu = new Menu("Group"); @@ -61,7 +59,10 @@ public class ModelGroupMenuBuilder { var stopAllOfGroup = new MenuItem("Stop all in group"); stopAllOfGroup.setOnAction(e -> new StopGroupAction(source, recorder, model).execute(callback)); - menu.getItems().addAll(editGroup, resumeAllOfGroup, pauseAllOfGroup, stopAllOfGroup); + var laterAllOfGroup = new MenuItem("Record later all in group"); + laterAllOfGroup.setOnAction(e -> new LaterGroupAction(source, recorder, model).execute(callback)); + + menu.getItems().addAll(editGroup, resumeAllOfGroup, pauseAllOfGroup, stopAllOfGroup, laterAllOfGroup); return menu; } } diff --git a/client/src/main/java/ctbrec/ui/menu/ModelMenuContributor.java b/client/src/main/java/ctbrec/ui/menu/ModelMenuContributor.java index fbc8588a..7fbdad24 100644 --- a/client/src/main/java/ctbrec/ui/menu/ModelMenuContributor.java +++ b/client/src/main/java/ctbrec/ui/menu/ModelMenuContributor.java @@ -1,33 +1,13 @@ package ctbrec.ui.menu; -import java.util.List; -import java.util.Optional; -import java.util.function.Consumer; - -import ctbrec.ui.controls.Dialogs; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import ctbrec.Config; import ctbrec.Model; import ctbrec.ModelGroup; import ctbrec.recorder.Recorder; import ctbrec.ui.DesktopIntegration; import ctbrec.ui.action.AbstractModelAction.Result; -import ctbrec.ui.action.AddToGroupAction; -import ctbrec.ui.action.EditNotesAction; -import ctbrec.ui.action.IgnoreModelsAction; -import ctbrec.ui.action.MarkForLaterRecordingAction; -import ctbrec.ui.action.OpenRecordingsDir; -import ctbrec.ui.action.PlayAction; -import ctbrec.ui.action.RemoveTimeLimitAction; -import ctbrec.ui.action.SetPortraitAction; -import ctbrec.ui.action.SetStopDateAction; -import ctbrec.ui.action.StartRecordingAction; -import ctbrec.ui.action.StopRecordingAction; -import ctbrec.ui.action.SwitchStreamResolutionAction; -import ctbrec.ui.action.TipAction; -import ctbrec.ui.action.TriConsumer; +import ctbrec.ui.action.*; +import ctbrec.ui.controls.Dialogs; import ctbrec.ui.tabs.FollowedTab; import javafx.event.ActionEvent; import javafx.event.EventHandler; @@ -38,6 +18,15 @@ import javafx.scene.control.SeparatorMenuItem; import javafx.scene.control.TabPane; import javafx.scene.input.Clipboard; import javafx.scene.input.ClipboardContent; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.net.URLEncoder; +import java.util.List; +import java.util.Optional; +import java.util.function.Consumer; + +import static java.nio.charset.StandardCharsets.UTF_8; public class ModelMenuContributor { @@ -90,11 +79,16 @@ public class ModelMenuContributor { } public void contributeToMenu(List selectedModels, ContextMenu menu) { - startStopCallback = Optional.ofNullable(startStopCallback).orElse(m -> {}); - followCallback = Optional.ofNullable(followCallback).orElse((m, f, s) -> {}); - ignoreCallback = Optional.ofNullable(ignoreCallback).orElse(m -> {}); - portraitCallback = Optional.ofNullable(portraitCallback).orElse(m -> {}); - callback = Optional.ofNullable(callback).orElse(() -> {}); + startStopCallback = Optional.ofNullable(startStopCallback).orElse(m -> { + }); + followCallback = Optional.ofNullable(followCallback).orElse((m, f, s) -> { + }); + ignoreCallback = Optional.ofNullable(ignoreCallback).orElse(m -> { + }); + portraitCallback = Optional.ofNullable(portraitCallback).orElse(m -> { + }); + callback = Optional.ofNullable(callback).orElse(() -> { + }); addOpenInPlayer(menu, selectedModels); addOpenInBrowser(menu, selectedModels); addCopyUrl(menu, selectedModels); @@ -116,6 +110,7 @@ public class ModelMenuContributor { addOpenRecDir(menu, selectedModels); addNotes(menu, selectedModels); addPortrait(menu, selectedModels); + addOpenOnCamGirlFinder(menu, selectedModels); } public ModelMenuContributor afterwards(Runnable callback) { @@ -157,6 +152,23 @@ public class ModelMenuContributor { menu.getItems().add(openInBrowser); } + private void addOpenOnCamGirlFinder(ContextMenu menu, List selectedModels) { + var openOnCamGirlFinder = new MenuItem("Search on CamGirlFinder"); + openOnCamGirlFinder.setOnAction(e -> { + for (Model model : selectedModels) { + String preview = model.getPreview(); + if (preview != null && !preview.isEmpty()) { + String query = URLEncoder.encode(preview, UTF_8); + DesktopIntegration.open("https://camgirlfinder.net/search?url=" + query); + } else { + String query = URLEncoder.encode(model.getName(), UTF_8); + DesktopIntegration.open("https://camgirlfinder.net/models?m=" + query + "&p=a&g=a"); + } + } + }); + menu.getItems().add(openOnCamGirlFinder); + } + private void addCopyUrl(ContextMenu menu, List selectedModels) { if (selectedModels == null || selectedModels.isEmpty()) { return; @@ -213,8 +225,7 @@ public class ModelMenuContributor { } private boolean isFollowedTab() { - if (source instanceof TabPane) { - var tabPane = (TabPane) source; + if (source instanceof TabPane tabPane) { return tabPane.getSelectionModel().getSelectedItem() instanceof FollowedTab; } return false; @@ -309,8 +320,8 @@ public class ModelMenuContributor { eventHandler = e -> { for (Model selectedModel : selectedModels) { new SetStopDateAction(source, selectedModel, recorder) - .execute() - .thenAccept(r -> executeCallback()); + .execute() + .thenAccept(r -> executeCallback()); } }; } else { @@ -338,8 +349,8 @@ public class ModelMenuContributor { private void removeTimeLimit(Model selectedModel) { new RemoveTimeLimitAction(source, selectedModel, recorder) // - .execute() // - .whenComplete((result, exception) -> executeCallback()); + .execute() // + .whenComplete((result, exception) -> executeCallback()); } private void addOpenInPlayer(ContextMenu menu, List selectedModels) { @@ -377,12 +388,12 @@ public class ModelMenuContributor { private void startRecording(List models) { new StartRecordingAction(source, models, recorder).execute() - .whenComplete((r, ex) -> { - if (ex != null) { - LOG.error("Error while starting recordings", ex); - } - r.stream().map(Result::getModel).forEach(startStopCallback); - }); + .whenComplete((r, ex) -> { + if (ex != null) { + LOG.error("Error while starting recordings", ex); + } + r.stream().map(Result::getModel).forEach(startStopCallback); + }); } private void stopRecording(List models) { diff --git a/client/src/main/java/ctbrec/ui/settings/CacheSettingsPane.java b/client/src/main/java/ctbrec/ui/settings/CacheSettingsPane.java new file mode 100644 index 00000000..9295b374 --- /dev/null +++ b/client/src/main/java/ctbrec/ui/settings/CacheSettingsPane.java @@ -0,0 +1,53 @@ +package ctbrec.ui.settings; + +import ctbrec.Config; +import javafx.collections.FXCollections; +import javafx.collections.ObservableList; +import javafx.scene.control.ComboBox; +import javafx.scene.layout.HBox; +import lombok.extern.slf4j.Slf4j; + +import java.io.IOException; +import java.util.List; + +@Slf4j +public class CacheSettingsPane extends HBox { + + private ComboBox cacheSizeCombo; + private final SettingsTab settingsTab; + private final Config config; + private static final List names = List.of("disabled", "16 MiB", "64 MiB", "128 MiB", "256 MiB", "512 MiB"); + private static final List values = List.of(0, 16, 64, 128, 256, 512); + + public CacheSettingsPane(SettingsTab settingsTab, Config config) { + this.settingsTab = settingsTab; + this.config = config; + setSpacing(5); + getChildren().addAll(buildCacheSizeCombo()); + } + + private ComboBox buildCacheSizeCombo() { + ObservableList lst = FXCollections.observableList(names); + cacheSizeCombo = new ComboBox<>(lst); + cacheSizeCombo.setOnAction(evt -> saveCacheConfig()); + int size = config.getSettings().thumbCacheSize; + int selectedIndex = values.indexOf(size); + if (selectedIndex < 0) { + selectedIndex = 1; + } + cacheSizeCombo.getSelectionModel().select(selectedIndex); + return cacheSizeCombo; + } + + private void saveCacheConfig() { + int index = cacheSizeCombo.getSelectionModel().getSelectedIndex(); + int size = values.get(index); + config.getSettings().thumbCacheSize = size; + try { + config.save(); + settingsTab.showRestartRequired(); + } catch (IOException e) { + log.error("Can't save config", e); + } + } +} diff --git a/client/src/main/java/ctbrec/ui/settings/SettingsTab.java b/client/src/main/java/ctbrec/ui/settings/SettingsTab.java index 4957abab..a65f6e21 100644 --- a/client/src/main/java/ctbrec/ui/settings/SettingsTab.java +++ b/client/src/main/java/ctbrec/ui/settings/SettingsTab.java @@ -71,6 +71,9 @@ public class SettingsTab extends Tab implements TabSelectionListener { private SimpleListProperty startTab; private SimpleFileProperty mediaPlayer; private SimpleStringProperty mediaPlayerParams; + private SimpleFileProperty browserOverride; + private SimpleStringProperty browserParams; + private SimpleBooleanProperty forceBrowserOverride; private SimpleIntegerProperty maximumResolutionPlayer; private SimpleBooleanProperty showPlayerStarting; private SimpleBooleanProperty singlePlayer; @@ -124,6 +127,7 @@ public class SettingsTab extends Tab implements TabSelectionListener { private SimpleLongProperty recordUntilDefaultDurationInMinutes; private SimpleStringProperty dateTimeFormat; private final VariablePlayGroundDialogFactory variablePlayGroundDialogFactory = new VariablePlayGroundDialogFactory(); + private SimpleBooleanProperty checkForUpdates; public SettingsTab(List sites, Recorder recorder) { this.sites = sites; @@ -146,6 +150,9 @@ public class SettingsTab extends Tab implements TabSelectionListener { startTab = new SimpleListProperty<>(null, "startTab", FXCollections.observableList(getTabNames())); mediaPlayer = new SimpleFileProperty(null, "mediaPlayer", settings.mediaPlayer); mediaPlayerParams = new SimpleStringProperty(null, "mediaPlayerParams", settings.mediaPlayerParams); + browserOverride = new SimpleFileProperty(null, "browserOverride", settings.browserOverride); + browserParams = new SimpleStringProperty(null, "browserParams", settings.browserParams); + forceBrowserOverride = new SimpleBooleanProperty(null, "forceBrowserOverride", settings.forceBrowserOverride); maximumResolutionPlayer = new SimpleIntegerProperty(null, "maximumResolutionPlayer", settings.maximumResolutionPlayer); showPlayerStarting = new SimpleBooleanProperty(null, "showPlayerStarting", settings.showPlayerStarting); singlePlayer = new SimpleBooleanProperty(null, "singlePlayer", settings.singlePlayer); @@ -156,7 +163,7 @@ public class SettingsTab extends Tab implements TabSelectionListener { proxyPassword = new SimpleStringProperty(null, "proxyPassword", settings.proxyPassword); recordingsDir = new SimpleDirectoryProperty(null, "recordingsDir", settings.recordingsDir); directoryStructure = new SimpleListProperty<>(null, "recordingsDirStructure", - FXCollections.observableList(List.of(FLAT, ONE_PER_MODEL, ONE_PER_RECORDING))); + FXCollections.observableList(List.of(FLAT, ONE_PER_MODEL, ONE_PER_GROUP, ONE_PER_RECORDING))); splitAfter = new SimpleListProperty<>(null, "splitRecordingsAfterSecs", FXCollections.observableList(getSplitAfterSecsOptions())); splitBiggerThan = new SimpleListProperty<>(null, "splitRecordingsBiggerThanBytes", FXCollections.observableList(getSplitBiggerThanOptions())); resolutionRange = new SimpleRangeProperty<>(rangeValues, "minimumResolution", "maximumResolution", settings.minimumResolution, @@ -198,6 +205,7 @@ public class SettingsTab extends Tab implements TabSelectionListener { recordUntilDefaultDurationInMinutes = new SimpleLongProperty(null, "recordUntilDefaultDurationInMinutes", settings.recordUntilDefaultDurationInMinutes); dateTimeFormat = new SimpleStringProperty(null, "dateTimeFormat", settings.dateTimeFormat); tabsSortable = new SimpleBooleanProperty(null, "tabsSortable", settings.tabsSortable); + checkForUpdates = new SimpleBooleanProperty(null, "checkForUpdates", settings.checkForUpdates); } private void createGui() { @@ -219,6 +227,7 @@ public class SettingsTab extends Tab implements TabSelectionListener { Setting.of("Update overview interval (seconds)", overviewUpdateIntervalInSecs, "Update the thumbnail overviews every x seconds").needsRestart(), Setting.of("Update thumbnails", updateThumbnails, "The overviews will still be updated, but the thumbnails won't be changed. This is useful for less powerful systems."), + Setting.of("Cache size", new CacheSettingsPane(this, config)).needsRestart(), Setting.of("Manually select stream quality", chooseStreamQuality, "Opens a dialog to select the video resolution before recording"), Setting.of("Enable live previews (experimental)", livePreviews), Setting.of("Enable recently watched tab", recentlyWatched).needsRestart(), @@ -227,6 +236,7 @@ public class SettingsTab extends Tab implements TabSelectionListener { Setting.of("Add models from clipboard", monitorClipboard, "Monitor clipboard for model URLs and automatically add them to the recorder").needsRestart(), Setting.of("Show confirmation dialogs", confirmationDialogs, "Show confirmation dialogs for irreversible actions"), Setting.of("Recording tab per site", recordedModelsPerSite, "Add a Recording tab for each site").needsRestart(), + Setting.of("Check for new versions at startup", checkForUpdates, "Search for updates every startup"), Setting.of("Start Tab", startTab)), Group.of("Player", @@ -234,7 +244,12 @@ public class SettingsTab extends Tab implements TabSelectionListener { Setting.of("Start parameters", mediaPlayerParams), Setting.of("Maximum resolution (0 = unlimited)", maximumResolutionPlayer, "video height, e.g. 720 or 1080"), Setting.of("Show \"Player Starting\" Message", showPlayerStarting), - Setting.of("Start only one player at a time", singlePlayer))), + Setting.of("Start only one player at a time", singlePlayer)), + + Group.of("Browser", + Setting.of("Browser", browserOverride), + Setting.of("Start parameters", browserParams), + Setting.of("Force use (ignore default browser)", forceBrowserOverride, "Default behaviour will fallback to OS default if the above browser fails"))), Category.of("Look & Feel", Group.of("Look & Feel", Setting.of("Colors (Base / Accent)", new ColorSettingsPane(Config.getInstance())).needsRestart(), diff --git a/client/src/main/java/ctbrec/ui/sites/amateurtv/AmateurTvTabProvider.java b/client/src/main/java/ctbrec/ui/sites/amateurtv/AmateurTvTabProvider.java index e1bf1d0c..5f0f6e99 100644 --- a/client/src/main/java/ctbrec/ui/sites/amateurtv/AmateurTvTabProvider.java +++ b/client/src/main/java/ctbrec/ui/sites/amateurtv/AmateurTvTabProvider.java @@ -24,32 +24,32 @@ public class AmateurTvTabProvider extends AbstractTabProvider { List tabs = new ArrayList<>(); // all - var url = AmateurTv.BASE_URL + "/v3/readmodel/cache/cams/A"; + var url = AmateurTv.BASE_URL + "/v3/readmodel/cache/onlinecamlist"; var updateService = new AmateurTvUpdateService((AmateurTv) site, url); tabs.add(createTab("All", updateService)); // female - url = AmateurTv.BASE_URL + "/v3/readmodel/cache/cams/W"; + url = AmateurTv.BASE_URL + "/v3/readmodel/cache/sectioncamlist?genre=[%22w%22]"; updateService = new AmateurTvUpdateService((AmateurTv) site, url); tabs.add(createTab("Female", updateService)); // male - url = AmateurTv.BASE_URL + "/v3/readmodel/cache/cams/M"; + url = AmateurTv.BASE_URL + "/v3/readmodel/cache/sectioncamlist?genre=[%22m%22]"; updateService = new AmateurTvUpdateService((AmateurTv) site, url); tabs.add(createTab("Male", updateService)); // couples - url = AmateurTv.BASE_URL + "/v3/readmodel/cache/cams/C"; + url = AmateurTv.BASE_URL + "/v3/readmodel/cache/sectioncamlist?genre=[%22c%22]"; updateService = new AmateurTvUpdateService((AmateurTv) site, url); tabs.add(createTab("Couples", updateService)); // trans - url = AmateurTv.BASE_URL + "/v3/readmodel/cache/cams/T"; + url = AmateurTv.BASE_URL + "/v3/readmodel/cache/sectioncamlist?genre=[%22t%22]"; updateService = new AmateurTvUpdateService((AmateurTv) site, url); tabs.add(createTab("Trans", updateService)); // followed - url = AmateurTv.BASE_URL + "/v3/readmodel/cache/cams/F"; + url = AmateurTv.BASE_URL + "/v3/readmodel/cache/favorites"; updateService = new AmateurTvUpdateService((AmateurTv) site, url); updateService.requiresLogin(true); followedTab = new AmateurTvFollowedTab("Followed", updateService, site); diff --git a/client/src/main/java/ctbrec/ui/sites/amateurtv/AmateurTvUpdateService.java b/client/src/main/java/ctbrec/ui/sites/amateurtv/AmateurTvUpdateService.java index 0e23b7dc..d419278a 100644 --- a/client/src/main/java/ctbrec/ui/sites/amateurtv/AmateurTvUpdateService.java +++ b/client/src/main/java/ctbrec/ui/sites/amateurtv/AmateurTvUpdateService.java @@ -1,17 +1,5 @@ package ctbrec.ui.sites.amateurtv; -import static ctbrec.io.HttpConstants.*; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.Locale; - -import org.json.JSONArray; -import org.json.JSONObject; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import ctbrec.Config; import ctbrec.Model; import ctbrec.sites.amateurtv.AmateurTv; @@ -20,15 +8,33 @@ import ctbrec.ui.SiteUiFactory; import ctbrec.ui.tabs.PaginatedScheduledService; import javafx.concurrent.Task; import okhttp3.Request; +import okhttp3.Response; +import org.json.JSONArray; +import org.json.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Locale; +import java.util.stream.Collectors; + +import static ctbrec.io.HttpConstants.*; public class AmateurTvUpdateService extends PaginatedScheduledService { private static final Logger LOG = LoggerFactory.getLogger(AmateurTvUpdateService.class); - private static final int ITEMS_PER_PAGE = 50; + private static final int ITEMS_PER_PAGE = 48; private AmateurTv site; private String url; private boolean requiresLogin = false; + private List modelsList; + private Instant lastListInfoRequest = Instant.EPOCH; public AmateurTvUpdateService(AmateurTv site, String url) { this.site = site; @@ -41,31 +47,62 @@ public class AmateurTvUpdateService extends PaginatedScheduledService { @Override public List call() throws IOException { if (requiresLogin) { - SiteUiFactory.getUi(site).login(); + if (!SiteUiFactory.getUi(site).login()) { + throw new IOException("- Login is required"); + } + ; } - return loadModelList(); + return getModelList().stream() + .skip((page - 1) * (long) ITEMS_PER_PAGE) + .limit(ITEMS_PER_PAGE) + .collect(Collectors.toList()); // NOSONAR } }; } + private List getModelList() throws IOException { + if (Duration.between(lastListInfoRequest, Instant.now()).getSeconds() < 30) { + return modelsList; + } + lastListInfoRequest = Instant.now(); + modelsList = loadModelList(); + if (modelsList == null) { + modelsList = Collections.emptyList(); + } + return modelsList; + } + private List loadModelList() throws IOException { - int offset = page - 1; - String pageUrl = url + '/' + offset * ITEMS_PER_PAGE + '/' + ITEMS_PER_PAGE + "/en"; - LOG.debug("Fetching page {}", pageUrl); - var request = new Request.Builder() - .url(pageUrl) + LOG.debug("Fetching page {}", url); + Request request = new Request.Builder() + .url(url) .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) .header(ACCEPT, MIMETYPE_APPLICATION_JSON) .header(ACCEPT, Locale.ENGLISH.getLanguage()) .header(REFERER, site.getBaseUrl() + "/following") .build(); - try (var response = site.getHttpClient().execute(request)) { + try (Response response = site.getHttpClient().execute(request)) { if (response.isSuccessful()) { - var content = response.body().string(); + String content = response.body().string(); List models = new ArrayList<>(); - var json = new JSONObject(content); - var modelNodes = json.getJSONObject("cams").getJSONArray("nodes"); - parseModels(modelNodes, models); + JSONObject json = new JSONObject(content); + if (json.has("body")) { + JSONObject body = json.getJSONObject("body"); + if (body.has("cams")) { + JSONArray cams = body.getJSONArray("cams"); + parseModels(cams, models); + } + if (body.has("list") && body.has("total")) { + if (body.optInt("total") > 0) { + JSONArray list = body.getJSONArray("list"); + parseModels(list, models); + } + } + } + if (json.has("cams")) { + JSONArray cams = json.getJSONArray("cams"); + parseModels(cams, models); + } return models; } else { int code = response.code(); @@ -76,12 +113,15 @@ public class AmateurTvUpdateService extends PaginatedScheduledService { private void parseModels(JSONArray jsonModels, List models) { for (var i = 0; i < jsonModels.length(); i++) { - var m = jsonModels.getJSONObject(i); - var user = m.getJSONObject("user"); - var name = user.optString("username"); + JSONObject m = jsonModels.getJSONObject(i); + String name = m.optString("username"); AmateurTvModel model = (AmateurTvModel) site.createModel(name); - model.setPreview(m.optString("imageURL")); - model.setDescription(m.optJSONObject("topic").optString("text")); + if (m.optBoolean("capturesEnabled", true) && m.has("capture")) { + model.setPreview(m.optString("capture")); + } else { + model.setPreview(site.getBaseUrl() + m.optString("avatar")); + } + model.setDescription(m.optString("topic")); models.add(model); } } diff --git a/client/src/main/java/ctbrec/ui/sites/bonga/BongaCamsUpdateService.java b/client/src/main/java/ctbrec/ui/sites/bonga/BongaCamsUpdateService.java index c3272588..20bbc808 100644 --- a/client/src/main/java/ctbrec/ui/sites/bonga/BongaCamsUpdateService.java +++ b/client/src/main/java/ctbrec/ui/sites/bonga/BongaCamsUpdateService.java @@ -2,6 +2,7 @@ package ctbrec.ui.sites.bonga; import ctbrec.Config; import ctbrec.Model; +import ctbrec.StringUtil; import ctbrec.sites.bonga.BongaCams; import ctbrec.sites.bonga.BongaCamsModel; import ctbrec.ui.SiteUiFactory; @@ -84,13 +85,19 @@ public class BongaCamsUpdateService extends PaginatedScheduledService { for (var i = 0; i < jsonModels.length(); i++) { var m = jsonModels.getJSONObject(i); var name = m.optString("username"); - if (name.isEmpty()) { + if (StringUtil.isBlank(name)) { continue; } BongaCamsModel model = (BongaCamsModel) bongaCams.createModel(name); model.mapOnlineState(m.optString("room")); model.setOnline(m.optInt("viewers") > 0); - model.setPreview("https:" + m.getString("thumb_image").replace("{ext}", "jpg")); + model.setPreview("https://en.bongacams.com/images/default/thumb_m_female.png"); + if (m.has("thumb_image")) { + String thumb = m.optString("thumb_image"); + if (StringUtil.isNotBlank(thumb)) { + model.setPreview("https:" + thumb.replace("{ext}", "jpg")); + } + } if (m.has("display_name")) { model.setDisplayName(m.getString("display_name")); } diff --git a/client/src/main/java/ctbrec/ui/sites/chaturbate/ChaturbateElectronLoginDialog.java b/client/src/main/java/ctbrec/ui/sites/chaturbate/ChaturbateElectronLoginDialog.java index 2197a6b9..d376edc6 100644 --- a/client/src/main/java/ctbrec/ui/sites/chaturbate/ChaturbateElectronLoginDialog.java +++ b/client/src/main/java/ctbrec/ui/sites/chaturbate/ChaturbateElectronLoginDialog.java @@ -56,7 +56,7 @@ public class ChaturbateElectronLoginDialog { var url = json.getString("url"); if (url.endsWith("/auth/login/")) { try { - Thread.sleep(500); + Thread.sleep(2000); String username = Config.getInstance().getSettings().chaturbateUsername; if (username != null && !username.trim().isEmpty()) { browser.executeJavaScript("document.getElementById('id_username').value = '" + username + "'"); diff --git a/client/src/main/java/ctbrec/ui/sites/chaturbate/ChaturbateFollowedTab.java b/client/src/main/java/ctbrec/ui/sites/chaturbate/ChaturbateFollowedTab.java index 212aa601..4ddea884 100644 --- a/client/src/main/java/ctbrec/ui/sites/chaturbate/ChaturbateFollowedTab.java +++ b/client/src/main/java/ctbrec/ui/sites/chaturbate/ChaturbateFollowedTab.java @@ -20,9 +20,8 @@ public class ChaturbateFollowedTab extends ThumbOverviewTab implements FollowedT public ChaturbateFollowedTab(String title, String url, Chaturbate chaturbate) { super(title, new ChaturbateUpdateService(url, true, chaturbate), chaturbate); - onlineUrl = url; - offlineUrl = url + "offline/"; - + onlineUrl = url.replace("offline=true", "offline=false"); + offlineUrl = url.replace("offline=false", "offline=true"); status = new Label("Logging in..."); grid.getChildren().add(status); } @@ -41,14 +40,14 @@ public class ChaturbateFollowedTab extends ThumbOverviewTab implements FollowedT offline.setToggleGroup(group); pagination.getChildren().add(online); pagination.getChildren().add(offline); - HBox.setMargin(online, new Insets(5,5,5,40)); - HBox.setMargin(offline, new Insets(5,5,5,5)); + HBox.setMargin(online, new Insets(5, 5, 5, 40)); + HBox.setMargin(offline, new Insets(5, 5, 5, 5)); online.setSelected(true); group.selectedToggleProperty().addListener(e -> { - if(online.isSelected()) { - ((ChaturbateUpdateService)updateService).setUrl(onlineUrl); + if (online.isSelected()) { + ((ChaturbateUpdateService) updateService).setUrl(onlineUrl); } else { - ((ChaturbateUpdateService)updateService).setUrl(offlineUrl); + ((ChaturbateUpdateService) updateService).setUrl(offlineUrl); } queue.clear(); updateService.restart(); diff --git a/client/src/main/java/ctbrec/ui/sites/chaturbate/ChaturbateTabProvider.java b/client/src/main/java/ctbrec/ui/sites/chaturbate/ChaturbateTabProvider.java index 116c3990..f77797c5 100644 --- a/client/src/main/java/ctbrec/ui/sites/chaturbate/ChaturbateTabProvider.java +++ b/client/src/main/java/ctbrec/ui/sites/chaturbate/ChaturbateTabProvider.java @@ -12,29 +12,36 @@ import java.util.List; public class ChaturbateTabProvider extends AbstractTabProvider { - private final ChaturbateFollowedTab followedTab; + private ChaturbateFollowedTab followedTab; + private Chaturbate site; + private String API_URL; public ChaturbateTabProvider(Chaturbate chaturbate) { super(chaturbate); - this.followedTab = new ChaturbateFollowedTab("Followed", chaturbate.getBaseUrl() + "/followed-cams/", chaturbate); + this.site = chaturbate; + API_URL = site.getBaseUrl() + "/api/ts"; } @Override protected List getSiteTabs(Scene scene) { List tabs = new ArrayList<>(); - tabs.add(createTab("Featured", site.getBaseUrl() + "/")); - tabs.add(createTab("Female", site.getBaseUrl() + "/female-cams/")); - tabs.add(createTab("New Female", site.getBaseUrl() + "/new-cams/female/")); - tabs.add(createTab("Male", site.getBaseUrl() + "/male-cams/")); - tabs.add(createTab("Couples", site.getBaseUrl() + "/couple-cams/")); - tabs.add(createTab("Trans", site.getBaseUrl() + "/trans-cams/")); + tabs.add(createTab("All", API_URL + "/roomlist/room-list/?enable_recommendations=false")); + tabs.add(createTab("Girls", API_URL + "/roomlist/room-list/?enable_recommendations=false&genders=f")); + tabs.add(createTab("New Girls", API_URL + "/roomlist/room-list/?enable_recommendations=false&genders=f&new_cams=true")); + tabs.add(createTab("Boys", API_URL + "/roomlist/room-list/?enable_recommendations=false&genders=m")); + tabs.add(createTab("New Boys", API_URL + "/roomlist/room-list/?enable_recommendations=false&genders=m&new_cams=true")); + tabs.add(createTab("Couples", API_URL + "/roomlist/room-list/?enable_recommendations=false&genders=c")); + tabs.add(createTab("Trans", API_URL + "/roomlist/room-list/?enable_recommendations=false&genders=t")); + tabs.add(createTab("Private", API_URL + "/roomlist/room-list/?enable_recommendations=false&private=true")); + tabs.add(createTab("Hidden", API_URL + "/roomlist/room-list/?enable_recommendations=false&hidden=true")); + followedTab = new ChaturbateFollowedTab("Followed", API_URL + "/roomlist/room-list/?enable_recommendations=false&follow=true&offline=false", site); followedTab.setScene(scene); followedTab.setRecorder(recorder); followedTab.setImageAspectRatio(9.0 / 16.0); tabs.add(followedTab); - tabs.add(createApiTab("Top Rated", site.getBaseUrl() + "/api/ts/discover/carousels/top-rated/")); - tabs.add(createApiTab("Trending", site.getBaseUrl() + "/api/ts/discover/carousels/trending/")); - tabs.add(createApiTab("Recommended", site.getBaseUrl() + "/api/ts/discover/carousels/recommended/")); + tabs.add(createApiTab("Top Rated", API_URL + "/discover/carousels/top-rated/")); + tabs.add(createApiTab("Trending", API_URL + "/ts/discover/carousels/trending/")); + tabs.add(createApiTab("Recommended", API_URL + "/discover/carousels/recommended/")); return tabs; } diff --git a/client/src/main/java/ctbrec/ui/sites/chaturbate/ChaturbateUpdateService.java b/client/src/main/java/ctbrec/ui/sites/chaturbate/ChaturbateUpdateService.java index ea8ad4f3..3aa964b8 100644 --- a/client/src/main/java/ctbrec/ui/sites/chaturbate/ChaturbateUpdateService.java +++ b/client/src/main/java/ctbrec/ui/sites/chaturbate/ChaturbateUpdateService.java @@ -2,20 +2,24 @@ package ctbrec.ui.sites.chaturbate; import ctbrec.Config; import ctbrec.Model; +import ctbrec.io.HttpException; import ctbrec.sites.chaturbate.Chaturbate; -import ctbrec.sites.chaturbate.ChaturbateModelParser; import ctbrec.ui.SiteUiFactory; import ctbrec.ui.tabs.PaginatedScheduledService; import javafx.concurrent.Task; import okhttp3.Request; +import okhttp3.Response; +import org.json.JSONArray; +import org.json.JSONObject; +import org.jsoup.Jsoup; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; +import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; +import java.util.Locale; import static ctbrec.io.HttpConstants.*; @@ -25,19 +29,12 @@ public class ChaturbateUpdateService extends PaginatedScheduledService { private String url; private final boolean loginRequired; private final Chaturbate chaturbate; + private final int modelsPerPage = 90; public ChaturbateUpdateService(String url, boolean loginRequired, Chaturbate chaturbate) { this.url = url; this.loginRequired = loginRequired; this.chaturbate = chaturbate; - - ExecutorService executor = Executors.newSingleThreadExecutor(r -> { - var t = new Thread(r); - t.setDaemon(true); - t.setName("ThumbOverviewTab UpdateService"); - return t; - }); - setExecutor(executor); } @Override @@ -48,28 +45,72 @@ public class ChaturbateUpdateService extends PaginatedScheduledService { if (loginRequired && !chaturbate.credentialsAvailable()) { return Collections.emptyList(); } else { - String pageUrl = ChaturbateUpdateService.this.url + "?page=" + page + "&keywords=&_=" + System.currentTimeMillis(); - LOG.debug("Fetching page {}", pageUrl); + int offset = (getPage() - 1) * modelsPerPage; + int limit = modelsPerPage; + String paginatedUrl = url + "&offset=" + offset + "&limit=" + limit; + LOG.debug("Fetching page {}", paginatedUrl); if (loginRequired) { SiteUiFactory.getUi(chaturbate).login(); } - var request = new Request.Builder() - .url(pageUrl) + Request request = new Request.Builder() + .url(paginatedUrl) + .header(ACCEPT, MIMETYPE_APPLICATION_JSON) + .header(ACCEPT_LANGUAGE, Locale.ENGLISH.getLanguage()) + .header(X_REQUESTED_WITH, XML_HTTP_REQUEST) .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) - .header(ACCEPT, MIMETYPE_TEXT_HTML) .build(); - try (var response = chaturbate.getHttpClient().execute(request)) { + try (Response response = chaturbate.getHttpClient().execute(request)) { if (response.isSuccessful()) { - List models = ChaturbateModelParser.parseModels(chaturbate, response.body().string()); - return models; + return parseModels(response.body().string()); } else { - int code = response.code(); - throw new IOException("HTTP status " + code); + throw new HttpException(response.code(), response.message()); + } + } // try + } // if + } // call + }; + } + + private List parseModels(String body) { + List models = new ArrayList<>(); + JSONObject json = new JSONObject(body); + if (json.has("rooms")) { + JSONArray jsonModels = json.getJSONArray("rooms"); + for (int i = 0; i < jsonModels.length(); i++) { + var jsonModel = jsonModels.getJSONObject(i); + try { + String name = jsonModel.getString("username"); + Model model = chaturbate.createModel(name); + model.setDisplayName(name); + model.setPreview(jsonModel.optString("img")); + if (jsonModel.has("tags")) { + JSONArray tags = jsonModel.getJSONArray("tags"); + for (int j = 0; j < tags.length(); j++) { + model.getTags().add(tags.optString(j)); } } + if (jsonModel.has("subject")) { + String html = jsonModel.optString("subject"); + model.setDescription(html2text(html)); + } + models.add(model); + } catch (Exception e) { + LOG.warn("Couldn't parse one of the models: {}", jsonModel, e); } } - }; + return models; + } else { + LOG.debug("Response was not successful: {}", json); + return Collections.emptyList(); + } + } + + public static String html2text(String html) { + try { + return Jsoup.parse(html).text(); + } catch (Exception ex) { + return ""; + } } public void setUrl(String url) { diff --git a/client/src/main/java/ctbrec/ui/sites/flirt4free/Flirt4FreeTabProvider.java b/client/src/main/java/ctbrec/ui/sites/flirt4free/Flirt4FreeTabProvider.java index 3c3391d6..56b2536b 100644 --- a/client/src/main/java/ctbrec/ui/sites/flirt4free/Flirt4FreeTabProvider.java +++ b/client/src/main/java/ctbrec/ui/sites/flirt4free/Flirt4FreeTabProvider.java @@ -25,11 +25,11 @@ public class Flirt4FreeTabProvider extends AbstractTabProvider { @Override protected List getSiteTabs(Scene scene) { List tabs = new ArrayList<>(); - tabs.add(createTab("Girls", site.getBaseUrl() + "/live/girls/", m -> true)); - tabs.add(createTab("New Girls", site.getBaseUrl() + "/live/girls/", Flirt4FreeModel::isNew)); - tabs.add(createTab("Boys", site.getBaseUrl() + "/live/guys/", m -> true)); - tabs.add(createTab("Couples", site.getBaseUrl() + "/live/couples/", m -> m.getCategories().contains("2"))); - tabs.add(createTab("Trans", site.getBaseUrl() + "/live/trans/", m -> true)); + tabs.add(createTab("Girls", site.getBaseUrl() + "/live/girls/?tpl=index2&model=json", m -> true)); + tabs.add(createTab("New Girls", site.getBaseUrl() + "/live/girls/?tpl=index2&model=json", Flirt4FreeModel::isNew)); + tabs.add(createTab("Boys", site.getBaseUrl() + "/live/guys/?tpl=index2&model=json", m -> true)); + tabs.add(createTab("Couples", site.getBaseUrl() + "/live/couples/?tpl=index2&model=json", m -> m.getCategories().contains("2"))); + tabs.add(createTab("Trans", site.getBaseUrl() + "/live/trans/?tpl=index2&model=json", m -> true)); tabs.add(followedTab); return tabs; } diff --git a/client/src/main/java/ctbrec/ui/sites/flirt4free/Flirt4FreeUpdateService.java b/client/src/main/java/ctbrec/ui/sites/flirt4free/Flirt4FreeUpdateService.java index 0bdc73e4..53a66ad9 100644 --- a/client/src/main/java/ctbrec/ui/sites/flirt4free/Flirt4FreeUpdateService.java +++ b/client/src/main/java/ctbrec/ui/sites/flirt4free/Flirt4FreeUpdateService.java @@ -26,7 +26,7 @@ import static ctbrec.io.HttpConstants.*; public class Flirt4FreeUpdateService extends PaginatedScheduledService { private static final Logger LOG = LoggerFactory.getLogger(Flirt4FreeUpdateService.class); - private static final int MODELS_PER_PAGE = 40; + private static final int MODELS_PER_PAGE = 50; private final String url; private final Flirt4Free flirt4Free; private final Predicate filter; diff --git a/client/src/main/java/ctbrec/ui/sites/jasmin/LiveJasminTabProvider.java b/client/src/main/java/ctbrec/ui/sites/jasmin/LiveJasminTabProvider.java index d94ef626..cd1a5e9b 100644 --- a/client/src/main/java/ctbrec/ui/sites/jasmin/LiveJasminTabProvider.java +++ b/client/src/main/java/ctbrec/ui/sites/jasmin/LiveJasminTabProvider.java @@ -15,22 +15,22 @@ public class LiveJasminTabProvider extends AbstractTabProvider { private final LiveJasminFollowedTab followedTab; - public LiveJasminTabProvider(LiveJasmin liveJasmin) { - super(liveJasmin); - followedTab = new LiveJasminFollowedTab(liveJasmin); - followedTab.setRecorder(liveJasmin.getRecorder()); + public LiveJasminTabProvider(LiveJasmin site) { + super(site); + followedTab = new LiveJasminFollowedTab(site); + followedTab.setRecorder(recorder); followedTab.setImageAspectRatio(9.0 / 16.0); } @Override protected List getSiteTabs(Scene scene) { List tabs = new ArrayList<>(); - tabs.add(createTab("Girls", site.getBaseUrl() + "/en/girl/?listPageOrderType=most_popular")); - tabs.add(createTab("Girls HD", site.getBaseUrl() + "/en/girl/hd/?listPageOrderType=most_popular")); - tabs.add(createTab("New Girls", site.getBaseUrl() + "/en/girls/newbie/?listPageOrderType=most_popular")); + tabs.add(createTab("Girls", site.getBaseUrl() + "/en/girls/?listPageOrderType=most_popular")); + tabs.add(createTab("New Girls", site.getBaseUrl() + "/en/girls/new-models/?listPageOrderType=most_popular")); + tabs.add(createTab("Boys", site.getBaseUrl() + "/en/boys/?listPageOrderType=most_popular")); + tabs.add(createTab("New Boys", site.getBaseUrl() + "/en/boys/new-models/?listPageOrderType=most_popular")); tabs.add(createTab("Couples", site.getBaseUrl() + "/en/girls/couple/?listPageOrderType=most_popular")); - tabs.add(createTab("Boys", site.getBaseUrl() + "/en/boy/?listPageOrderType=most_popular")); - tabs.add(createTab("Boys HD", site.getBaseUrl() + "/en/boy/hd/?listPageOrderType=most_popular")); + tabs.add(createTab("Trans", site.getBaseUrl() + "/en/boys/transboy/?listPageOrderType=most_popular")); tabs.add(followedTab); return tabs; } @@ -43,8 +43,8 @@ public class LiveJasminTabProvider extends AbstractTabProvider { private ThumbOverviewTab createTab(String title, String url) { var s = new LiveJasminUpdateService((LiveJasmin) site, url); s.setPeriod(Duration.seconds(60)); - ThumbOverviewTab tab = new LiveJasminTab(title, s, site); - tab.setRecorder(site.getRecorder()); + ThumbOverviewTab tab = new ThumbOverviewTab(title, s, site); + tab.setRecorder(recorder); tab.setImageAspectRatio(9.0 / 16.0); return tab; } diff --git a/client/src/main/java/ctbrec/ui/sites/jasmin/LiveJasminUpdateService.java b/client/src/main/java/ctbrec/ui/sites/jasmin/LiveJasminUpdateService.java index eb16f5c0..30cedd1e 100644 --- a/client/src/main/java/ctbrec/ui/sites/jasmin/LiveJasminUpdateService.java +++ b/client/src/main/java/ctbrec/ui/sites/jasmin/LiveJasminUpdateService.java @@ -1,17 +1,5 @@ package ctbrec.ui.sites.jasmin; -import static ctbrec.io.HttpConstants.*; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Locale; - -import org.json.JSONObject; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import ctbrec.Config; import ctbrec.Model; import ctbrec.io.HttpException; @@ -23,16 +11,40 @@ import javafx.concurrent.Task; import okhttp3.Cookie; import okhttp3.HttpUrl; import okhttp3.Request; +import okhttp3.Response; +import org.json.JSONArray; +import org.json.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.text.MessageFormat; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Locale; +import java.util.stream.Collectors; + +import static ctbrec.io.HttpConstants.*; public class LiveJasminUpdateService extends PaginatedScheduledService { private static final Logger LOG = LoggerFactory.getLogger(LiveJasminUpdateService.class); private String url; + private String listPageId = ""; private LiveJasmin liveJasmin; + private List modelsList; + private int modelsPerPage = 60; + private int lastPageLoaded = 0; + + private transient Instant lastListInfoRequest = Instant.EPOCH; public LiveJasminUpdateService(LiveJasmin liveJasmin, String url) { this.liveJasmin = liveJasmin; this.url = url; + this.lastPageLoaded = 0; } @Override @@ -40,69 +52,136 @@ public class LiveJasminUpdateService extends PaginatedScheduledService { return new Task>() { @Override public List call() throws IOException { - // sort by popularity - var cookieJar = liveJasmin.getHttpClient().getCookieJar(); - var sortCookie = new Cookie.Builder() - .domain(LiveJasmin.baseDomain) - .name("listPageOrderType") - .value("most_popular") - .build(); - cookieJar.saveFromResponse(HttpUrl.parse("https://" + LiveJasmin.baseDomain), Collections.singletonList(sortCookie)); - - // TODO find out how to switch pages - LOG.debug("Fetching page {}", url); - var request = new Request.Builder() - .url(url) - .addHeader(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) - .addHeader(ACCEPT, MIMETYPE_APPLICATION_JSON) - .addHeader(ACCEPT_LANGUAGE, Locale.ENGLISH.getLanguage()) - .addHeader(REFERER, liveJasmin.getBaseUrl()) - .addHeader(X_REQUESTED_WITH, XML_HTTP_REQUEST) - .build(); - try (var response = liveJasmin.getHttpClient().execute(request)) { - LOG.debug("Response {} {}", response.code(), response.message()); - if (response.isSuccessful()) { - var body = response.body().string(); - List models = new ArrayList<>(); - var json = new JSONObject(body); - if(json.optBoolean("success")) { - parseModels(models, json); - } else if(json.optString("error").equals("Please login.")) { - var siteUI = SiteUiFactory.getUi(liveJasmin); - if(siteUI.login()) { - return call(); - } else { - LOG.error("Request failed:\n{}", body); - throw new IOException("Response was not successful"); - } - } else { - LOG.error("Request failed:\n{}", body); - throw new IOException("Response was not successful"); - } - return models; - } else { - throw new HttpException(response.code(), response.message()); - } - } + return getModelList().stream() + .skip((page - 1) * (long) modelsPerPage) + .limit(modelsPerPage) + .collect(Collectors.toList()); // NOSONAR } }; } - private void parseModels(List models, JSONObject json) { - var data = json.getJSONObject("data"); - var content = data.getJSONObject("content"); - var performers = content.getJSONArray("performers"); - for (var i = 0; i < performers.length(); i++) { - var m = performers.getJSONObject(i); - var name = m.optString("pid"); - if(name.isEmpty()) { - continue; + private List getModelList() throws IOException { + page = Math.min(page, 99); + if ((lastPageLoaded > 0) && Duration.between(lastListInfoRequest, Instant.now()).getSeconds() < 60) { + while (page > lastPageLoaded) { + lastPageLoaded++; + modelsList.addAll(loadMore()); + } + return modelsList; + } + lastPageLoaded = 1; + modelsList = loadModelList(); + while (page > lastPageLoaded) { + lastPageLoaded++; + modelsList.addAll(loadMore()); + } + if (modelsList == null) { + return Collections.emptyList(); + } + return modelsList; + } + + private List loadModelList() throws IOException { + lastListInfoRequest = Instant.now(); + var cookieJar = liveJasmin.getHttpClient().getCookieJar(); + + var sortCookie = new Cookie.Builder().domain(LiveJasmin.baseDomain).name("listPageOrderType").value("most_popular").build(); + cookieJar.saveFromResponse(HttpUrl.parse(liveJasmin.getBaseUrl()), Collections.singletonList(sortCookie)); + + String category = (url.indexOf("boys") > -1) ? "boys" : "girls"; + var categoryCookie = new Cookie.Builder().domain(LiveJasmin.baseDomain).name("category").value(category).build(); + cookieJar.saveFromResponse(HttpUrl.parse(liveJasmin.getBaseUrl()), Collections.singletonList(categoryCookie)); + + LOG.debug("Fetching page {}", url); + Request req = new Request.Builder() + .url(url) + .addHeader(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) + .addHeader(ACCEPT, MIMETYPE_APPLICATION_JSON) + .addHeader(ACCEPT_LANGUAGE, Locale.ENGLISH.getLanguage()) + .addHeader(REFERER, liveJasmin.getBaseUrl()) + .addHeader(X_REQUESTED_WITH, XML_HTTP_REQUEST) + .build(); + try (Response response = liveJasmin.getHttpClient().execute(req)) { + LOG.debug("Response {} {}", response.code(), response.message()); + if (response.isSuccessful()) { + String body = response.body().string(); + List models = new ArrayList<>(); + JSONObject json = new JSONObject(body); + if (json.optBoolean("success")) { + parseModels(models, json); + } else if (json.optString("error").equals("Please login.")) { + var siteUI = SiteUiFactory.getUi(liveJasmin); + if (siteUI.login()) { + return loadModelList(); + } else { + LOG.error("Request failed:\n{}", body); + throw new IOException("Response was not successful"); + } + } else { + LOG.error("Request failed:\n{}", body); + throw new IOException("Response was not successful"); + } + return models; + } else { + throw new HttpException(response.code(), response.message()); } - LiveJasminModel model = (LiveJasminModel) liveJasmin.createModel(name); - model.setId(m.getString("id")); - model.setPreview(m.getString("profilePictureUrl")); - model.setOnlineState(LiveJasminModel.mapStatus(m.optInt("status"))); - models.add(model); } } + + private List loadMore() throws IOException { + lastListInfoRequest = Instant.now(); + String moreURL = liveJasmin.getBaseUrl() + MessageFormat.format("/en/list-page-ajax/show-more-json/{0}?wide=true&layout=layout-big&_dc={1}", listPageId, String.valueOf(System.currentTimeMillis())); + LOG.debug("Fetching page {}", moreURL); + Request req = new Request.Builder() + .url(moreURL) + .addHeader(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) + .addHeader(ACCEPT, MIMETYPE_APPLICATION_JSON) + .addHeader(ACCEPT_LANGUAGE, Locale.ENGLISH.getLanguage()) + .addHeader(REFERER, liveJasmin.getBaseUrl()) + .addHeader(X_REQUESTED_WITH, XML_HTTP_REQUEST) + .build(); + try (Response response = liveJasmin.getHttpClient().execute(req)) { + if (response.isSuccessful()) { + String body = response.body().string(); + List models = new ArrayList<>(); + JSONObject json = new JSONObject(body); + if (json.optBoolean("success")) { + parseModels(models, json); + } + return models; + } else { + throw new HttpException(response.code(), response.message()); + } + } + } + + private void parseModels(List models, JSONObject json) { + if (json.has("data")) { + JSONObject data = json.getJSONObject("data"); + if (data.optInt("isLast") > 0) { + lastPageLoaded = 999; + } + if (data.has("content")) { + JSONObject content = data.getJSONObject("content"); + if (content.optInt("isLastPage") > 0) { + lastPageLoaded = 999; + } + listPageId = content.optString("listPageId"); + JSONArray performers = content.getJSONArray("performers"); + for (var i = 0; i < performers.length(); i++) { + var m = performers.getJSONObject(i); + var name = m.optString("pid"); + if (name.isEmpty()) { + continue; + } + LiveJasminModel model = (LiveJasminModel) liveJasmin.createModel(name); + model.setId(m.getString("id")); + model.setPreview(m.optString("profilePictureUrl")); + model.setOnlineState(LiveJasminModel.mapStatus(m.optInt("status"))); + model.setDisplayName(m.optString("display_name", null)); + models.add(model); + } + } // if content + } // if data + } } diff --git a/client/src/main/java/ctbrec/ui/sites/manyvids/MVLiveUpdateService.java b/client/src/main/java/ctbrec/ui/sites/manyvids/MVLiveUpdateService.java index ff0be2fc..2881c0c6 100644 --- a/client/src/main/java/ctbrec/ui/sites/manyvids/MVLiveUpdateService.java +++ b/client/src/main/java/ctbrec/ui/sites/manyvids/MVLiveUpdateService.java @@ -6,42 +6,67 @@ import ctbrec.sites.manyvids.MVLive; import ctbrec.sites.manyvids.MVLiveModel; import ctbrec.ui.tabs.PaginatedScheduledService; import javafx.concurrent.Task; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; import okhttp3.Request; import okhttp3.Response; import org.json.JSONArray; import org.json.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.IOException; +import java.time.Duration; +import java.time.Instant; import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Locale; +import java.util.stream.Collectors; import static ctbrec.io.HttpConstants.*; -@Slf4j -@RequiredArgsConstructor public class MVLiveUpdateService extends PaginatedScheduledService { private final MVLive mvlive; private final String url; + private final int modelsPerPage = 48; + private static List modelsList; + private static Instant lastListInfoRequest = Instant.EPOCH; + + private static final Logger LOG = LoggerFactory.getLogger(MVLiveUpdateService.class); + + public MVLiveUpdateService(MVLive site, String url) { + this.mvlive = site; + this.url = url; + } @Override protected Task> createTask() { - return new Task<>() { + return new Task>() { @Override public List call() throws IOException { - List models = loadModels(url); - return models; + return getModelList().stream() + .skip((page - 1) * (long) modelsPerPage) + .limit(modelsPerPage) + .collect(Collectors.toList()); // NOSONAR } }; } + private List getModelList() throws IOException { + if (Duration.between(lastListInfoRequest, Instant.now()).getSeconds() < 30) { + return modelsList; + } + lastListInfoRequest = Instant.now(); + modelsList = loadModels(url); + if (modelsList == null) { + return Collections.emptyList(); + } + return modelsList; + } + protected List loadModels(String url) throws IOException { List models = new ArrayList<>(); - log.debug("Loading live models from {}", url); + LOG.debug("Loading live models from {}", url); Request req = new Request.Builder() .url(url) .header(ACCEPT, "*/*") @@ -52,11 +77,11 @@ public class MVLiveUpdateService extends PaginatedScheduledService { .build(); try (Response response = mvlive.getHttpClient().execute(req)) { String body = response.body().string(); - log.trace("response body: {}", body); + LOG.trace("response body: {}", body); if (response.isSuccessful()) { JSONObject json = new JSONObject(body); if (!json.has("live_creators")) { - log.debug("Unexpected response:\n{}", json.toString(2)); + LOG.debug("Unexpected response:\n{}", json.toString(2)); return Collections.emptyList(); } JSONArray creators = json.getJSONArray("live_creators"); diff --git a/client/src/main/java/ctbrec/ui/sites/secretfriends/SecretFriendsUpdateService.java b/client/src/main/java/ctbrec/ui/sites/secretfriends/SecretFriendsUpdateService.java index 39c2801d..8a1b7e26 100644 --- a/client/src/main/java/ctbrec/ui/sites/secretfriends/SecretFriendsUpdateService.java +++ b/client/src/main/java/ctbrec/ui/sites/secretfriends/SecretFriendsUpdateService.java @@ -9,7 +9,9 @@ import ctbrec.sites.secretfriends.SecretFriendsModelParser; import ctbrec.ui.SiteUiFactory; import ctbrec.ui.tabs.PaginatedScheduledService; import javafx.concurrent.Task; +import okhttp3.HttpUrl; import okhttp3.Request; +import okhttp3.Response; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; import org.slf4j.Logger; @@ -43,19 +45,23 @@ public class SecretFriendsUpdateService extends PaginatedScheduledService { if (loginRequired && !site.credentialsAvailable()) { return Collections.emptyList(); } else { - String paginatedUrl = url + (url.indexOf('?') >= 0 ? '&' : '?') + "Friend_page=" + page; + String paginatedUrl = url; + if (page > 1) { + String pager = (url.indexOf("/users") > 0) ? "Friend_page" : "AModel_page"; + paginatedUrl = HttpUrl.parse(url).newBuilder().addQueryParameter(pager, String.valueOf(page)).build().toString(); + } LOG.debug("Fetching page {}", paginatedUrl); if (loginRequired) { SiteUiFactory.getUi(site).login(); } - var request = new Request.Builder() + Request request = new Request.Builder() .url(paginatedUrl) - .header(ACCEPT, MIMETYPE_APPLICATION_JSON) + .header(ACCEPT, "*/*") .header(ACCEPT_LANGUAGE, Locale.ENGLISH.getLanguage()) - .header(X_REQUESTED_WITH, XML_HTTP_REQUEST) .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) + .header(REFERER, SecretFriends.BASE_URI) .build(); - try (var response = site.getHttpClient().execute(request)) { + try (Response response = site.getHttpClient().execute(request)) { if (response.isSuccessful()) { return parseModels(Objects.requireNonNull(response.body(), HTTP_RESPONSE_BODY_IS_NULL).string()); } else { diff --git a/client/src/main/java/ctbrec/ui/sites/showup/ShowupUpdateService.java b/client/src/main/java/ctbrec/ui/sites/showup/ShowupUpdateService.java index 0f38a492..29800d58 100644 --- a/client/src/main/java/ctbrec/ui/sites/showup/ShowupUpdateService.java +++ b/client/src/main/java/ctbrec/ui/sites/showup/ShowupUpdateService.java @@ -1,18 +1,21 @@ package ctbrec.ui.sites.showup; -import java.io.IOException; -import java.util.List; - import ctbrec.Model; import ctbrec.sites.showup.Showup; import ctbrec.sites.showup.ShowupHttpClient; import ctbrec.ui.tabs.PaginatedScheduledService; import javafx.concurrent.Task; +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + public class ShowupUpdateService extends PaginatedScheduledService { private final Showup showup; private final String category; + protected int modelsPerPage = 48; public ShowupUpdateService(Showup showup, String category) { this.showup = showup; @@ -24,11 +27,22 @@ public class ShowupUpdateService extends PaginatedScheduledService { return new Task<>() { @Override public List call() throws IOException { - ShowupHttpClient httpClient = (ShowupHttpClient) showup.getHttpClient(); - httpClient.setCookie("category", category); - return showup.getModelList(true); + return getModelList().stream() + .skip((page - 1) * (long) modelsPerPage) + .limit(modelsPerPage) + .collect(Collectors.toList()); // NOSONAR } }; } + private List getModelList() throws IOException { + ShowupHttpClient httpClient = (ShowupHttpClient) showup.getHttpClient(); + httpClient.setCookie("category", category); + var modelsList = showup.getModelList(true); + if (modelsList == null) { + return Collections.emptyList(); + } + return modelsList; + } + } diff --git a/client/src/main/java/ctbrec/ui/sites/stripchat/StripchatConfigUI.java b/client/src/main/java/ctbrec/ui/sites/stripchat/StripchatConfigUI.java index 2d2100df..fc3d86fa 100644 --- a/client/src/main/java/ctbrec/ui/sites/stripchat/StripchatConfigUI.java +++ b/client/src/main/java/ctbrec/ui/sites/stripchat/StripchatConfigUI.java @@ -7,19 +7,13 @@ import ctbrec.ui.settings.SettingsTab; import ctbrec.ui.sites.AbstractConfigUI; import javafx.geometry.Insets; import javafx.scene.Parent; -import javafx.scene.control.Button; -import javafx.scene.control.CheckBox; -import javafx.scene.control.Label; -import javafx.scene.control.PasswordField; -import javafx.scene.control.RadioButton; -import javafx.scene.control.TextField; -import javafx.scene.control.ToggleGroup; +import javafx.scene.control.*; import javafx.scene.layout.GridPane; import javafx.scene.layout.HBox; import javafx.scene.layout.Priority; public class StripchatConfigUI extends AbstractConfigUI { - private Stripchat stripchat; + private final Stripchat stripchat; public StripchatConfigUI(Stripchat stripchat) { this.stripchat = stripchat; @@ -36,7 +30,7 @@ public class StripchatConfigUI extends AbstractConfigUI { var enabled = new CheckBox(); enabled.setSelected(!settings.disabledSites.contains(stripchat.getName())); enabled.setOnAction(e -> { - if(enabled.isSelected()) { + if (enabled.isSelected()) { settings.disabledSites.remove(stripchat.getName()); } else { settings.disabledSites.add(stripchat.getName()); @@ -69,7 +63,7 @@ public class StripchatConfigUI extends AbstractConfigUI { layout.add(new Label("Stripchat User"), 0, row); var username = new TextField(Config.getInstance().getSettings().stripchatUsername); username.textProperty().addListener((ob, o, n) -> { - if(!n.equals(Config.getInstance().getSettings().stripchatUsername)) { + if (!n.equals(Config.getInstance().getSettings().stripchatUsername)) { Config.getInstance().getSettings().stripchatUsername = username.getText(); stripchat.getHttpClient().logout(); save(); @@ -84,7 +78,7 @@ public class StripchatConfigUI extends AbstractConfigUI { var password = new PasswordField(); password.setText(Config.getInstance().getSettings().stripchatPassword); password.textProperty().addListener((ob, o, n) -> { - if(!n.equals(Config.getInstance().getSettings().stripchatPassword)) { + if (!n.equals(Config.getInstance().getSettings().stripchatPassword)) { Config.getInstance().getSettings().stripchatPassword = password.getText(); stripchat.getHttpClient().logout(); save(); @@ -102,9 +96,21 @@ public class StripchatConfigUI extends AbstractConfigUI { var deleteCookies = new Button("Delete Cookies"); deleteCookies.setOnAction(e -> stripchat.getHttpClient().clearCookies()); - layout.add(deleteCookies, 1, row); + layout.add(deleteCookies, 1, row++); GridPane.setColumnSpan(deleteCookies, 2); + row++; + l = new Label("Get VR stream if available"); + layout.add(l, 0, row); + var vr = new CheckBox(); + vr.setSelected(settings.stripchatVR); + vr.setOnAction(e -> { + settings.stripchatVR = vr.isSelected(); + save(); + }); + GridPane.setMargin(vr, new Insets(0, 0, 0, SettingsTab.CHECKBOX_MARGIN)); + layout.add(vr, 1, row); + GridPane.setMargin(username, new Insets(0, 0, 0, SettingsTab.CHECKBOX_MARGIN)); GridPane.setMargin(password, new Insets(0, 0, 0, SettingsTab.CHECKBOX_MARGIN)); GridPane.setMargin(createAccount, new Insets(0, 0, 0, SettingsTab.CHECKBOX_MARGIN)); diff --git a/client/src/main/java/ctbrec/ui/sites/stripchat/StripchatFollowedUpdateService.java b/client/src/main/java/ctbrec/ui/sites/stripchat/StripchatFollowedUpdateService.java index 515bf352..26acf930 100644 --- a/client/src/main/java/ctbrec/ui/sites/stripchat/StripchatFollowedUpdateService.java +++ b/client/src/main/java/ctbrec/ui/sites/stripchat/StripchatFollowedUpdateService.java @@ -14,6 +14,7 @@ import org.json.JSONArray; import org.json.JSONObject; import java.io.IOException; +import java.time.Instant; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -22,7 +23,7 @@ import static ctbrec.Model.State.*; import static ctbrec.io.HttpConstants.*; public class StripchatFollowedUpdateService extends AbstractStripchatUpdateService { - private static final int PAGE_SIZE = 30; + private static final int PAGE_SIZE = 48; private static final String FAVORITES = "/favorites"; private final Stripchat stripchat; @@ -81,6 +82,7 @@ public class StripchatFollowedUpdateService extends AbstractStripchatUpdateServi model.setDescription(user.optString("description")); model.setPreview(getPreviewUrl(user)); model.setOnlineState(mapStatus(user.optString("status"))); + model.setLastSeen(Instant.now()); models.add(model); } } diff --git a/client/src/main/java/ctbrec/ui/sites/stripchat/StripchatTabProvider.java b/client/src/main/java/ctbrec/ui/sites/stripchat/StripchatTabProvider.java index 5bd00905..a95ac062 100644 --- a/client/src/main/java/ctbrec/ui/sites/stripchat/StripchatTabProvider.java +++ b/client/src/main/java/ctbrec/ui/sites/stripchat/StripchatTabProvider.java @@ -13,20 +13,24 @@ import java.util.List; public class StripchatTabProvider extends AbstractTabProvider { private final String urlTemplate; + private final String urlFilterTemplate; private final StripchatFollowedTab followedTab; public StripchatTabProvider(Stripchat stripchat) { super(stripchat); followedTab = new StripchatFollowedTab("Followed", stripchat); urlTemplate = stripchat.getBaseUrl() + "/api/front/models?primaryTag={0}&sortBy=viewersRating&withMixedTags=true&parentTag="; + urlFilterTemplate = stripchat.getBaseUrl() + "/api/front/models?primaryTag=girls&filterGroupTags=%5B%5B%22{0}%22%5D%5D&parentTag={0}"; } @Override protected List getSiteTabs(Scene scene) { List tabs = new ArrayList<>(); tabs.add(createTab("Girls", MessageFormat.format(urlTemplate, "girls"))); - tabs.add(createTab("Girls HD", site.getBaseUrl() + "/api/front/models?primaryTag=girls&filterGroupTags=%5B%5B%22autoTagHd%22%5D%5D&parentTag=autoTagHd")); - tabs.add(createTab("New Girls", site.getBaseUrl() + "/api/front/models?primaryTag=girls&filterGroupTags=%5B%5B%22autoTagNew%22%5D%5D&parentTag=autoTagNew")); + tabs.add(createTab("Girls New", MessageFormat.format(urlFilterTemplate, "autoTagNew"))); + tabs.add(createTab("Girls HD", MessageFormat.format(urlFilterTemplate, "autoTagHd"))); + tabs.add(createTab("Girls VR", MessageFormat.format(urlFilterTemplate, "autoTagVr"))); + tabs.add(createTab("Mobile", MessageFormat.format(urlFilterTemplate, "mobile"))); tabs.add(createTab("Couples", MessageFormat.format(urlTemplate, "couples"))); tabs.add(createTab("Boys", MessageFormat.format(urlTemplate, "men"))); tabs.add(createTab("Trans", MessageFormat.format(urlTemplate, "trans"))); diff --git a/client/src/main/java/ctbrec/ui/sites/stripchat/StripchatUpdateService.java b/client/src/main/java/ctbrec/ui/sites/stripchat/StripchatUpdateService.java index 5ac38a90..6334a5bd 100644 --- a/client/src/main/java/ctbrec/ui/sites/stripchat/StripchatUpdateService.java +++ b/client/src/main/java/ctbrec/ui/sites/stripchat/StripchatUpdateService.java @@ -27,7 +27,7 @@ public class StripchatUpdateService extends AbstractStripchatUpdateService { private final String url; private final boolean loginRequired; private final Stripchat stripchat; - int modelsPerPage = 60; + int modelsPerPage = 48; public StripchatUpdateService(String url, boolean loginRequired, Stripchat stripchat) { this.url = url; diff --git a/client/src/main/java/ctbrec/ui/sites/xlovecam/XloveCamTabProvider.java b/client/src/main/java/ctbrec/ui/sites/xlovecam/XloveCamTabProvider.java index ff266ffe..7c033303 100644 --- a/client/src/main/java/ctbrec/ui/sites/xlovecam/XloveCamTabProvider.java +++ b/client/src/main/java/ctbrec/ui/sites/xlovecam/XloveCamTabProvider.java @@ -17,6 +17,7 @@ public class XloveCamTabProvider extends AbstractTabProvider { private final XloveCam xloveCam; private static final String FILTER_PARAM = "config[filter][10][]"; + private static final String FILTER_PARAM_NEW = "config[filter][100522][]"; public XloveCamTabProvider(XloveCam xloveCam) { super(xloveCam); @@ -31,6 +32,10 @@ public class XloveCamTabProvider extends AbstractTabProvider { var updateService = new XloveCamUpdateService(xloveCam, Collections.emptyMap()); tabs.add(createTab("All", updateService)); + // new + updateService = new XloveCamUpdateService(xloveCam, Map.of(FILTER_PARAM_NEW, "3")); + tabs.add(createTab("New", updateService)); + // Young Women updateService = new XloveCamUpdateService(xloveCam, Map.of(FILTER_PARAM, "1")); tabs.add(createTab("Young Women", updateService)); diff --git a/client/src/main/java/ctbrec/ui/tabs/RecordingsTab.java b/client/src/main/java/ctbrec/ui/tabs/RecordingsTab.java index b3061c09..11cc66e3 100644 --- a/client/src/main/java/ctbrec/ui/tabs/RecordingsTab.java +++ b/client/src/main/java/ctbrec/ui/tabs/RecordingsTab.java @@ -149,6 +149,15 @@ public class RecordingsTab extends Tab implements TabSelectionListener, Shutdown resolution.setPrefWidth(100); resolution.setCellValueFactory(cdf -> new SimpleIntegerProperty(cdf.getValue().getSelectedResolution())); resolution.setCellFactory(tc -> createResolutionCell()); + + TableColumn siteName = new TableColumn<>("Site"); + siteName.setId("siteName"); + siteName.setPrefWidth(200); + siteName.setCellValueFactory(cdf -> { + var sname = cdf.getValue().getModel().getSite().getName(); + return new SimpleObjectProperty<>(sname); + }); + TableColumn notes = new TableColumn<>("Notes"); notes.setId("notes"); notes.setPrefWidth(400); @@ -158,7 +167,7 @@ public class RecordingsTab extends Tab implements TabSelectionListener, Shutdown modelNotes.setPrefWidth(400); modelNotes.setCellValueFactory(cdf -> new SimpleStringProperty(config.getModelNotes(cdf.getValue().getModel()))); - table.getColumns().addAll(name, date, status, progress, size, resolution, notes, modelNotes); + table.getColumns().addAll(siteName, name, date, status, progress, size, resolution, notes, modelNotes); table.setItems(observableRecordings); table.addEventHandler(ContextMenuEvent.CONTEXT_MENU_REQUESTED, this::onContextMenuRequested); table.addEventHandler(MouseEvent.MOUSE_PRESSED, this::onMousePressed); @@ -261,7 +270,7 @@ public class RecordingsTab extends Tab implements TabSelectionListener, Shutdown Recording recording = table.getSelectionModel().getSelectedItem(); if (recording != null) { var state = recording.getStatus(); - if(state == FINISHED || state == RECORDING) { + if (state == FINISHED || state == RECORDING) { play(recording); } } @@ -448,10 +457,10 @@ public class RecordingsTab extends Tab implements TabSelectionListener, Shutdown var tmp = new CustomMouseBehaviorContextMenu(); ModelMenuContributor.newContributor(getTabPane(), Config.getInstance(), recorder) // - .withStartStopCallback(m -> getTabPane().setCursor(Cursor.DEFAULT)) // - .removeModelAfterIgnore(true) // - .afterwards(table::refresh) // - .contributeToMenu(List.of(recordings.get(0).getModel()), tmp); + .withStartStopCallback(m -> getTabPane().setCursor(Cursor.DEFAULT)) // + .removeModelAfterIgnore(true) // + .afterwards(table::refresh) // + .contributeToMenu(List.of(recordings.get(0).getModel()), tmp); var modelSubMenu = new Menu("Model"); modelSubMenu.getItems().addAll(tmp.getItems()); contextMenu.getItems().add(modelSubMenu); diff --git a/client/src/main/resources/logback.xml b/client/src/main/resources/logback.xml index 62404622..69b16dfd 100644 --- a/client/src/main/resources/logback.xml +++ b/client/src/main/resources/logback.xml @@ -51,8 +51,6 @@ - - diff --git a/common/src/main/java/ctbrec/Config.java b/common/src/main/java/ctbrec/Config.java index 19f48aab..96922484 100644 --- a/common/src/main/java/ctbrec/Config.java +++ b/common/src/main/java/ctbrec/Config.java @@ -71,11 +71,18 @@ public class Config { if (src.exists()) { File target = new File(src.getParentFile(), src.getName() + "_backup_" + dateTimeFormatter.format(LocalDateTime.now())); LOG.info("Creating a backup of {} the config in {}", src, target); - FileUtils.copyDirectory(src, target, pathname -> !(pathname.toString().contains("minimal-browser") && pathname.toString().contains("Cache")), true); + FileUtils.copyDirectory(src, target, pathname -> includeDir(pathname), true); deleteOldBackups(currentConfigDir); } } + private boolean includeDir(File pathname) { + String name = pathname.getName(); + if (name.contains("minimal-browser") && name.contains("Cache")) return false; + if (name.contains("cache")) return false; + return true; + } + private void deleteOldBackups(File currentConfigDir) { File parent = currentConfigDir.getParentFile(); File[] backupDirectories = parent.listFiles(file -> file.isDirectory() && file.getName().matches(".*?_backup_\\d{4}-\\d{2}-\\d{2}_\\d{2}-\\d{2}-\\d{2}_\\d{3}")); diff --git a/common/src/main/java/ctbrec/Recording.java b/common/src/main/java/ctbrec/Recording.java index 20d05f1e..f880b1af 100644 --- a/common/src/main/java/ctbrec/Recording.java +++ b/common/src/main/java/ctbrec/Recording.java @@ -41,6 +41,7 @@ public class Recording implements Serializable, Callable { private File absoluteFile = null; private File postProcessedFile = null; private int selectedResolution = -1; + private long lastSizeUpdate = 0; /** * Signals, if the recording has been changed and it has to be refreshed @@ -291,9 +292,13 @@ public class Recording implements Serializable, Callable { } public void refresh() { - if ((status != FINISHED && status != FAILED) || dirtyFlag) { - sizeInByte = getSize(); - dirtyFlag = false; + long now = System.currentTimeMillis(); + if (now - lastSizeUpdate > 2500) { + if ((status != FINISHED && status != FAILED) || dirtyFlag) { + sizeInByte = getSize(); + lastSizeUpdate = now; + dirtyFlag = false; + } } } diff --git a/common/src/main/java/ctbrec/Settings.java b/common/src/main/java/ctbrec/Settings.java index 1cab26a2..21d3746e 100644 --- a/common/src/main/java/ctbrec/Settings.java +++ b/common/src/main/java/ctbrec/Settings.java @@ -12,6 +12,7 @@ public class Settings { public enum DirectoryStructure { FLAT("all recordings in one directory"), ONE_PER_MODEL("one directory for each model"), + ONE_PER_GROUP("one directory for each group"), ONE_PER_RECORDING("one directory for each recording"); private final String description; @@ -102,6 +103,9 @@ public class Settings { public int maximumResolutionPlayer = 0; public String mediaPlayer = "/usr/bin/mpv"; public String mediaPlayerParams = ""; + public String browserOverride = ""; + public String browserParams = ""; + public boolean forceBrowserOverride = false; public String mfcBaseUrl = "https://www.myfreecams.com"; public List mfcDisabledModelsTableColumns = new ArrayList<>(); public String[] mfcModelsTableColumnIds = new String[0]; diff --git a/common/src/main/java/ctbrec/io/HttpClient.java b/common/src/main/java/ctbrec/io/HttpClient.java index bbfc4bd2..b9432857 100644 --- a/common/src/main/java/ctbrec/io/HttpClient.java +++ b/common/src/main/java/ctbrec/io/HttpClient.java @@ -5,10 +5,9 @@ import com.squareup.moshi.Moshi; import ctbrec.Config; import ctbrec.LoggingInterceptor; import ctbrec.Settings.ProxyType; +import lombok.extern.slf4j.Slf4j; import okhttp3.*; import okhttp3.OkHttpClient.Builder; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import javax.net.ssl.*; import java.io.ByteArrayOutputStream; @@ -17,12 +16,12 @@ import java.io.FileOutputStream; import java.io.IOException; import java.net.Authenticator; import java.net.PasswordAuthentication; -import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.security.KeyManagementException; import java.security.NoSuchAlgorithmException; import java.security.SecureRandom; import java.security.cert.X509Certificate; +import java.text.NumberFormat; import java.util.*; import java.util.Map.Entry; import java.util.concurrent.TimeUnit; @@ -33,12 +32,12 @@ import static ctbrec.io.HttpConstants.ACCEPT_ENCODING_GZIP; import static ctbrec.io.HttpConstants.CONTENT_ENCODING; import static java.nio.charset.StandardCharsets.UTF_8; +@Slf4j public abstract class HttpClient { - private static final Logger LOG = LoggerFactory.getLogger(HttpClient.class); - private static final ConnectionPool GLOBAL_HTTP_CONN_POOL = new ConnectionPool(10, 2, TimeUnit.MINUTES); protected OkHttpClient client; + protected Cache cache; protected CookieJarImpl cookieJar; protected Config config; protected boolean loggedIn = false; @@ -59,56 +58,58 @@ public abstract class HttpClient { private void loadProxySettings() { ProxyType proxyType = config.getSettings().proxyType; switch (proxyType) { - case HTTP: - System.setProperty("http.proxyHost", config.getSettings().proxyHost); - System.setProperty("http.proxyPort", config.getSettings().proxyPort); - System.setProperty("https.proxyHost", config.getSettings().proxyHost); - System.setProperty("https.proxyPort", config.getSettings().proxyPort); - if(config.getSettings().proxyUser != null && !config.getSettings().proxyUser.isEmpty()) { - String username = config.getSettings().proxyUser; - String password = config.getSettings().proxyPassword; - System.setProperty("http.proxyUser", username); - System.setProperty("http.proxyPassword", password); - } - break; - case SOCKS4: - System.setProperty("socksProxyVersion", "4"); - System.setProperty("socksProxyHost", config.getSettings().proxyHost); - System.setProperty("socksProxyPort", config.getSettings().proxyPort); - break; - case SOCKS5: - System.setProperty("socksProxyVersion", "5"); - System.setProperty("socksProxyHost", config.getSettings().proxyHost); - System.setProperty("socksProxyPort", config.getSettings().proxyPort); - if(config.getSettings().proxyUser != null && !config.getSettings().proxyUser.isEmpty()) { - String username = config.getSettings().proxyUser; - String password = config.getSettings().proxyPassword; - Authenticator.setDefault(new SocksProxyAuth(username, password)); - } - break; - case DIRECT: - default: - System.clearProperty("http.proxyHost"); - System.clearProperty("http.proxyPort"); - System.clearProperty("https.proxyHost"); - System.clearProperty("https.proxyPort"); - System.clearProperty("socksProxyVersion"); - System.clearProperty("socksProxyHost"); - System.clearProperty("socksProxyPort"); - System.clearProperty("java.net.socks.username"); - System.clearProperty("java.net.socks.password"); - System.clearProperty("http.proxyUser"); - System.clearProperty("http.proxyPassword"); - break; + case HTTP: + System.setProperty("http.proxyHost", config.getSettings().proxyHost); + System.setProperty("http.proxyPort", config.getSettings().proxyPort); + System.setProperty("https.proxyHost", config.getSettings().proxyHost); + System.setProperty("https.proxyPort", config.getSettings().proxyPort); + if (config.getSettings().proxyUser != null && !config.getSettings().proxyUser.isEmpty()) { + String username = config.getSettings().proxyUser; + String password = config.getSettings().proxyPassword; + System.setProperty("http.proxyUser", username); + System.setProperty("http.proxyPassword", password); + } + break; + case SOCKS4: + System.setProperty("socksProxyVersion", "4"); + System.setProperty("socksProxyHost", config.getSettings().proxyHost); + System.setProperty("socksProxyPort", config.getSettings().proxyPort); + break; + case SOCKS5: + System.setProperty("socksProxyVersion", "5"); + System.setProperty("socksProxyHost", config.getSettings().proxyHost); + System.setProperty("socksProxyPort", config.getSettings().proxyPort); + if (config.getSettings().proxyUser != null && !config.getSettings().proxyUser.isEmpty()) { + String username = config.getSettings().proxyUser; + String password = config.getSettings().proxyPassword; + Authenticator.setDefault(new SocksProxyAuth(username, password)); + } + break; + case DIRECT: + default: + System.clearProperty("http.proxyHost"); + System.clearProperty("http.proxyPort"); + System.clearProperty("https.proxyHost"); + System.clearProperty("https.proxyPort"); + System.clearProperty("socksProxyVersion"); + System.clearProperty("socksProxyHost"); + System.clearProperty("socksProxyPort"); + System.clearProperty("java.net.socks.username"); + System.clearProperty("java.net.socks.password"); + System.clearProperty("http.proxyUser"); + System.clearProperty("http.proxyPassword"); + break; } } public Response execute(Request req) throws IOException { + log.trace("Cache hit ratio {}/{} = {}", cache.hitCount(), cache.requestCount(), NumberFormat.getPercentInstance().format(cache.hitCount() / (double) cache.requestCount())); Response resp = client.newCall(req).execute(); return resp; } public Response execute(Request request, int timeoutInMillis) throws IOException { + log.trace("Cache hit ratio {}/{} = {}", cache.hitCount(), cache.requestCount(), NumberFormat.getPercentInstance().format(cache.hitCount() / (double) cache.requestCount())); return client.newBuilder() // .connectTimeout(timeoutInMillis, TimeUnit.MILLISECONDS) // .readTimeout(timeoutInMillis, TimeUnit.MILLISECONDS).build() // @@ -120,9 +121,14 @@ public abstract class HttpClient { public void reconfigure() { loadProxySettings(); loadCookies(); + long cacheSize = (long) config.getSettings().thumbCacheSize * 1024 * 1024; + File configDir = config.getConfigDir(); + File cacheDir = new File(configDir, "cache"); + cache = new Cache(cacheDir, cacheSize); Builder builder = new OkHttpClient.Builder() .cookieJar(cookieJar) .connectionPool(GLOBAL_HTTP_CONN_POOL) + .cache(cache) .connectTimeout(config.getSettings().httpTimeout, TimeUnit.MILLISECONDS) .readTimeout(config.getSettings().httpTimeout, TimeUnit.MILLISECONDS) .addNetworkInterceptor(new LoggingInterceptor()); @@ -156,12 +162,16 @@ public abstract class HttpClient { X509Certificate[] x509Certificates = new X509Certificate[0]; return x509Certificates; } - @Override public void checkServerTrusted(final X509Certificate[] chain, final String authType) { /* noop*/ } - @Override public void checkClientTrusted(final X509Certificate[] chain, final String authType) { /* noop*/ } + + @Override + public void checkServerTrusted(final X509Certificate[] chain, final String authType) { /* noop*/ } + + @Override + public void checkClientTrusted(final X509Certificate[] chain, final String authType) { /* noop*/ } }; try { - final TrustManager[] trustManagers = new TrustManager[] { x509TrustManager }; + final TrustManager[] trustManagers = new TrustManager[]{x509TrustManager}; final String PROTOCOL = "TLSv1.2"; SSLContext sslContext = SSLContext.getInstance(PROTOCOL); KeyManager[] keyManagers = null; @@ -171,7 +181,7 @@ public abstract class HttpClient { builder.sslSocketFactory(sslSocketFactory, x509TrustManager); builder.hostnameVerifier((hostname, sslSession) -> true); } catch (KeyManagementException | NoSuchAlgorithmException e) { - LOG.error("Couldn't install trust managers for TLS connections"); + log.error("Couldn't install trust managers for TLS connections"); } } @@ -192,18 +202,18 @@ public abstract class HttpClient { String json = adapter.toJson(cookies); File cookieFile = new File(config.getConfigDir(), "cookies-" + name + ".json"); - try(FileOutputStream fout = new FileOutputStream(cookieFile)) { + try (FileOutputStream fout = new FileOutputStream(cookieFile)) { fout.write(json.getBytes(UTF_8)); } } catch (Exception e) { - LOG.error("Couldn't persist cookies for {}", name, e); + log.error("Couldn't persist cookies for {}", name, e); } } private void loadCookies() { try { File cookieFile = new File(config.getConfigDir(), "cookies-" + name + ".json"); - if(!cookieFile.exists()) { + if (!cookieFile.exists()) { return; } byte[] jsonBytes = Files.readAllBytes(cookieFile.toPath()); @@ -224,7 +234,7 @@ public abstract class HttpClient { } } catch (Exception e) { - LOG.error("Couldn't load cookies for {}", name, e); + log.error("Couldn't load cookies for {}", name, e); } } @@ -310,7 +320,7 @@ public abstract class HttpClient { while ((len = gzipIn.read(b)) >= 0) { bos.write(b, 0, len); } - return bos.toString(StandardCharsets.UTF_8.toString()); + return bos.toString(UTF_8); } else { return Objects.requireNonNull(response.body()).string(); } diff --git a/common/src/main/java/ctbrec/recorder/RecordingManager.java b/common/src/main/java/ctbrec/recorder/RecordingManager.java index d3b305be..ddeebf15 100644 --- a/common/src/main/java/ctbrec/recorder/RecordingManager.java +++ b/common/src/main/java/ctbrec/recorder/RecordingManager.java @@ -78,6 +78,7 @@ public class RecordingManager { String json = Files.readString(file.toPath()); try { Recording recording = adapter.fromJson(json); + recording.setMetaDataFile(file.getCanonicalPath()); loadRecording(recording); } catch (Exception e) { LOG.error("Couldn't load recording {}", file, e); diff --git a/common/src/main/java/ctbrec/recorder/RecordingPreconditions.java b/common/src/main/java/ctbrec/recorder/RecordingPreconditions.java index 3fa1fd1e..13d79da0 100644 --- a/common/src/main/java/ctbrec/recorder/RecordingPreconditions.java +++ b/common/src/main/java/ctbrec/recorder/RecordingPreconditions.java @@ -39,6 +39,7 @@ public class RecordingPreconditions { ensureRecorderIsActive(); ensureNotInTimeoutPeriod(); ensureModelIsNotSuspended(model); + ensureModelIsNotDelayed(model); ensureModelIsNotMarkedForLaterRecording(model); ensureRecordUntilIsInFuture(model); ensureNoRecordingRunningForModel(model); @@ -141,6 +142,13 @@ public class RecordingPreconditions { } } + private void ensureModelIsNotDelayed(Model model) { + Optional modelGroup = recorder.getModelGroup(model); + if (modelGroup.isPresent() && model.isDelayed()) { + throw new PreconditionNotMetException("Recording for model " + model + " is delayed"); + } + } + private void ensureModelIsNotMarkedForLaterRecording(Model model) { if (model.isMarkedForLaterRecording()) { throw new PreconditionNotMetException("Model " + model + " is marked for later recording"); @@ -204,6 +212,7 @@ public class RecordingPreconditions { try { ensureRecorderIsActive(); ensureModelIsNotSuspended(model); + ensureModelIsNotDelayed(model); ensureModelIsNotMarkedForLaterRecording(model); ensureRecordUntilIsInFuture(model); ensureModelShouldBeRecorded(model); diff --git a/common/src/main/java/ctbrec/recorder/download/hls/AbstractHlsDownload.java b/common/src/main/java/ctbrec/recorder/download/hls/AbstractHlsDownload.java index d4670f34..a6512dba 100644 --- a/common/src/main/java/ctbrec/recorder/download/hls/AbstractHlsDownload.java +++ b/common/src/main/java/ctbrec/recorder/download/hls/AbstractHlsDownload.java @@ -136,12 +136,15 @@ public abstract class AbstractHlsDownload extends AbstractDownload { } } catch (ParseException e) { LOG.error("Couldn't parse HLS playlist for model {}\n{}", model, e.getInput(), e); + model.delay(); stop(); } catch (PlaylistException e) { LOG.error("Couldn't parse HLS playlist for model {}", model, e); + model.delay(); stop(); } catch (PlaylistTimeoutException e) { if (consecutivePlaylistTimeouts >= 5) { + model.delay(); stop(); } else { rescheduleTime = beforeLastPlaylistRequest; // try again as fast as possible @@ -151,9 +154,11 @@ public abstract class AbstractHlsDownload extends AbstractDownload { LOG.debug("Reached end of playlist for model {}", model); stop(); } catch (HttpException e) { + consecutivePlaylistErrors++; handleHttpException(e); } catch (Exception e) { LOG.error("Couldn't download segment for model {}", model, e); + model.delay(); stop(); } finally { if (consecutivePlaylistErrors > 0) { @@ -218,6 +223,7 @@ public abstract class AbstractHlsDownload extends AbstractDownload { } if (consecutivePlaylistErrors >= 3) { LOG.info("Playlist could not be downloaded for model {} {} times. Stopping recording", model, consecutivePlaylistErrors, e); + model.delay(); stop(); } else { LOG.info("Playlist could not be downloaded for model {} {} times: {}", model, consecutivePlaylistErrors, e.getLocalizedMessage()); @@ -252,31 +258,9 @@ public abstract class AbstractHlsDownload extends AbstractDownload { for (StreamSource streamSource : streamSources) { LOG.debug("{}:{} src {}", model.getSite().getName(), model.getName(), streamSource); } - String url; - if (model.getStreamUrlIndex() >= 0 && model.getStreamUrlIndex() < streamSources.size()) { - // TODO don't use the index, but the bandwidth. if the bandwidth does not match, take the closest one - StreamSource source = streamSources.get(model.getStreamUrlIndex()); - LOG.debug("{} selected {}", model.getName(), source); - url = source.getMediaPlaylistUrl(); - selectedResolution = source.height; - } else { - // filter out stream resolutions, which are out of range of the configured min and max - int minRes = Config.getInstance().getSettings().minimumResolution; - int maxRes = Config.getInstance().getSettings().maximumResolution; - List filteredStreamSources = streamSources.stream() - .filter(src -> src.height == 0 || src.height == UNKNOWN || minRes <= src.height) - .filter(src -> src.height == 0 || src.height == UNKNOWN || maxRes >= src.height) - .toList(); - - if (filteredStreamSources.isEmpty()) { - throw new ExecutionException(new NoStreamFoundException("No stream left in playlist")); - } else { - StreamSource source = filteredStreamSources.get(filteredStreamSources.size() - 1); - LOG.debug("{} selected {}", model.getName(), source); - url = source.getMediaPlaylistUrl(); - selectedResolution = source.height; - } - } + StreamSource selectedStreamSource = selectStreamSource(streamSources); + String url = selectedStreamSource.getMediaPlaylistUrl(); + selectedResolution = selectedStreamSource.height; LOG.debug("Segment playlist url {}", url); return url; } @@ -297,7 +281,6 @@ public abstract class AbstractHlsDownload extends AbstractDownload { // no segments, empty playlist return new SegmentPlaylist(segmentPlaylistUrl); } - byte[] bytes = body.getBytes(UTF_8); BandwidthMeter.add(bytes.length); InputStream inputStream = new ByteArrayInputStream(bytes); @@ -375,6 +358,7 @@ public abstract class AbstractHlsDownload extends AbstractDownload { } if (playlistEmptyCount == 10) { LOG.info("Last 10 playlists were empty for {}. Stopping recording!", getModel()); + model.delay(); internalStop(); } } diff --git a/common/src/main/java/ctbrec/sites/amateurtv/AmateurTv.java b/common/src/main/java/ctbrec/sites/amateurtv/AmateurTv.java index d28fe669..f6b37752 100644 --- a/common/src/main/java/ctbrec/sites/amateurtv/AmateurTv.java +++ b/common/src/main/java/ctbrec/sites/amateurtv/AmateurTv.java @@ -1,14 +1,26 @@ package ctbrec.sites.amateurtv; +import ctbrec.Model; +import ctbrec.StringUtil; +import ctbrec.io.HttpClient; +import ctbrec.io.HttpException; +import ctbrec.sites.AbstractSite; +import okhttp3.Request; +import okhttp3.Response; +import org.json.JSONArray; +import org.json.JSONObject; + import java.io.IOException; +import java.net.URLEncoder; +import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Locale; import java.util.regex.Matcher; import java.util.regex.Pattern; -import ctbrec.Model; -import ctbrec.io.HttpClient; -import ctbrec.sites.AbstractSite; +import static ctbrec.io.HttpConstants.*; +import static java.nio.charset.StandardCharsets.UTF_8; public class AmateurTv extends AbstractSite { @@ -33,7 +45,7 @@ public class AmateurTv extends AbstractSite { } @Override - public Model createModel(String name) { + public AmateurTvModel createModel(String name) { AmateurTvModel model = new AmateurTvModel(); model.setName(name); model.setUrl(BASE_URL + '/' + name); @@ -84,17 +96,40 @@ public class AmateurTv extends AbstractSite { @Override public boolean supportsSearch() { - return false; - } - - @Override - public boolean searchRequiresLogin() { - return false; + return true; } @Override public List search(String q) throws IOException, InterruptedException { - return Collections.emptyList(); + if (StringUtil.isBlank(q)) { + return Collections.emptyList(); + } + String url = getBaseUrl() + "/v3/readmodel/cache/filterbyusername/" + URLEncoder.encode(q, UTF_8); + var req = new Request.Builder() + .url(url) + .header(USER_AGENT, getConfig().getSettings().httpUserAgent) + .header(ACCEPT, MIMETYPE_APPLICATION_JSON) + .header(ACCEPT, Locale.ENGLISH.getLanguage()) + .header(REFERER, getBaseUrl()) + .build(); + try (Response response = getHttpClient().execute(req)) { + if (response.isSuccessful()) { + JSONObject json = new JSONObject(response.body().string()); + List models = new ArrayList<>(); + JSONArray results = json.getJSONObject("cams").getJSONArray("nodes"); + int maxResults = Math.min(30, results.length()); + for (int i = 0; i < maxResults; i++) { + JSONObject result = results.getJSONObject(i); + var user = result.getJSONObject("user"); + AmateurTvModel model = createModel(user.optString("username")); + model.setPreview(result.optString("imageURL")); + models.add(model); + } + return models; + } else { + throw new HttpException(response.code(), response.message()); + } + } } @Override @@ -111,7 +146,7 @@ public class AmateurTv extends AbstractSite { @Override public Model createModelFromUrl(String url) { Matcher m = Pattern.compile("https?://.*?amateur.tv/(.*)").matcher(url); - if(m.matches()) { + if (m.matches()) { String modelName = m.group(1); return createModel(modelName); } else { diff --git a/common/src/main/java/ctbrec/sites/amateurtv/AmateurTvModel.java b/common/src/main/java/ctbrec/sites/amateurtv/AmateurTvModel.java index bf68c315..5865990c 100644 --- a/common/src/main/java/ctbrec/sites/amateurtv/AmateurTvModel.java +++ b/common/src/main/java/ctbrec/sites/amateurtv/AmateurTvModel.java @@ -1,25 +1,25 @@ package ctbrec.sites.amateurtv; -import com.iheartradio.m3u8.*; -import com.iheartradio.m3u8.data.MediaPlaylist; -import com.iheartradio.m3u8.data.Playlist; +import com.iheartradio.m3u8.ParseException; +import com.iheartradio.m3u8.PlaylistException; import ctbrec.AbstractModel; import ctbrec.Config; -import ctbrec.io.HttpClient; import ctbrec.io.HttpException; import ctbrec.recorder.download.Download; import ctbrec.recorder.download.StreamSource; +import ctbrec.recorder.download.hls.FfmpegHlsDownload; import okhttp3.FormBody; import okhttp3.Request; import okhttp3.RequestBody; import okhttp3.Response; +import org.json.JSONArray; import org.json.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.xml.bind.JAXBException; import java.io.IOException; -import java.io.InputStream; +import java.text.MessageFormat; import java.util.ArrayList; import java.util.List; import java.util.Locale; @@ -32,17 +32,25 @@ import static ctbrec.io.HttpConstants.*; public class AmateurTvModel extends AbstractModel { private static final Logger LOG = LoggerFactory.getLogger(AmateurTvModel.class); - - private boolean online = false; + private JSONArray qualities = new JSONArray(); + private int[] resolution = new int[2]; @Override public boolean isOnline(boolean ignoreCache) throws IOException, ExecutionException, InterruptedException { if (ignoreCache) { JSONObject json = getModelInfo(); - online = json.optString("status").equalsIgnoreCase("online"); - onlineState = online ? ONLINE : OFFLINE; + setOnlineState(OFFLINE); + + boolean online = json.optString("status").equalsIgnoreCase("online"); + if (online) setOnlineState(ONLINE); + + boolean brb = json.optBoolean("brb"); + if (brb) setOnlineState(AWAY); + + boolean privateChat = json.optString("privateChatStatus").equalsIgnoreCase("exclusive_private"); + if (privateChat) setOnlineState(PRIVATE); } - return online; + return onlineState == ONLINE; } @Override @@ -65,46 +73,30 @@ public class AmateurTvModel extends AbstractModel { @Override public List getStreamSources() throws IOException, ExecutionException, ParseException, PlaylistException, JAXBException { List streamSources = new ArrayList<>(); - String streamUrl = getStreamUrl(); - Request req = new Request.Builder().url(streamUrl).build(); - try (Response response = site.getHttpClient().execute(req)) { - if (response.isSuccessful()) { - InputStream inputStream = Objects.requireNonNull(response.body()).byteStream(); - PlaylistParser parser = new PlaylistParser(inputStream, Format.EXT_M3U, Encoding.UTF_8, ParsingMode.LENIENT); - Playlist playlist = parser.parse(); - MediaPlaylist media = playlist.getMediaPlaylist(); - String vodUri; - String trackUri = media.getTracks().get(0).getUri(); - if (trackUri.startsWith("http")) { - vodUri = trackUri; - } else if (trackUri.startsWith("/")) { - String baseUrl = streamUrl.substring(0, streamUrl.indexOf("/", 8)); - vodUri = baseUrl + trackUri; - } else { - String baseUrl = streamUrl.substring(0, streamUrl.lastIndexOf('/') + 1); - vodUri = baseUrl + trackUri; - } - StreamSource streamsource = new StreamSource(); - streamsource.mediaPlaylistUrl = vodUri; - streamsource.width = 0; - streamsource.height = 0; - streamSources.add(streamsource); - } else { - throw new HttpException(response.code(), response.message()); - } - } + String mediaPlaylistUrl = getMasterPlaylistUrl(); + qualities.forEach(item -> { + String value = (String) item; + String[] res = value.split("x"); + StreamSource src = new StreamSource(); + src.mediaPlaylistUrl = MessageFormat.format("{0}&variant={1}", mediaPlaylistUrl, res[1]); + src.width = Integer.parseInt(res[0]); + src.height = Integer.parseInt(res[1]); + src.bandwidth = 0; + streamSources.add(src); + }); return streamSources; } - private String getStreamUrl() throws IOException { + private String getMasterPlaylistUrl() throws IOException { JSONObject json = getModelInfo(); JSONObject videoTech = json.getJSONObject("videoTechnologies"); + qualities = json.getJSONArray("qualities"); return videoTech.getString("fmp4-hls"); } @Override public void invalidateCacheEntries() { - // nothing to do + resolution = new int[2]; } @Override @@ -114,11 +106,18 @@ public class AmateurTvModel extends AbstractModel { @Override public int[] getStreamResolution(boolean failFast) throws ExecutionException { - try { - return new int[]{getStreamSources().get(0).width, getStreamSources().get(0).height}; - } catch (Exception e) { - throw new ExecutionException(e); + if (!failFast) { + try { + List sources = getStreamSources(); + if (!sources.isEmpty()) { + StreamSource best = sources.get(0); + resolution = new int[]{best.getWidth(), best.getHeight()}; + } + } catch (IOException | ParseException | PlaylistException | JAXBException e) { + throw new ExecutionException(e); + } } + return resolution; } @Override @@ -176,14 +175,18 @@ public class AmateurTvModel extends AbstractModel { .header(ACCEPT_LANGUAGE, "en") .header(REFERER, getSite().getBaseUrl() + '/' + getName()) .build(); - try (Response resp = site.getHttpClient().execute(req)) { - JSONObject json = new JSONObject(HttpClient.bodyToJsonObject(resp)); - return json; + try (Response response = site.getHttpClient().execute(req)) { + if (response.isSuccessful()) { + JSONObject jsonResponse = new JSONObject(response.body().string()); + return jsonResponse; + } else { + throw new HttpException(response.code(), response.message()); + } } } @Override public Download createDownload() { - return new AmateurTvDownload(getSite().getHttpClient()); + return new FfmpegHlsDownload(getSite().getHttpClient()); } } diff --git a/common/src/main/java/ctbrec/sites/bonga/BongaCams.java b/common/src/main/java/ctbrec/sites/bonga/BongaCams.java index 69aa1d1e..84c7c6d9 100644 --- a/common/src/main/java/ctbrec/sites/bonga/BongaCams.java +++ b/common/src/main/java/ctbrec/sites/bonga/BongaCams.java @@ -1,6 +1,7 @@ package ctbrec.sites.bonga; import ctbrec.Model; +import ctbrec.StringUtil; import ctbrec.io.HttpClient; import ctbrec.io.HttpException; import ctbrec.sites.AbstractSite; @@ -21,7 +22,6 @@ import java.util.regex.Pattern; import static ctbrec.ErrorMessages.HTTP_RESPONSE_BODY_IS_NULL; import static ctbrec.io.HttpConstants.*; -import static java.nio.charset.StandardCharsets.UTF_8; public class BongaCams extends AbstractSite { @@ -133,19 +133,18 @@ public class BongaCams extends AbstractSite { return true; } - @Override - public boolean searchRequiresLogin() { - return true; - } - @Override public List search(String q) throws IOException, InterruptedException { - String url = baseUrl + "/tools/listing_v3.php?offset=0&model_search[display_name][text]=" + URLEncoder.encode(q, UTF_8); + if (StringUtil.isBlank(q)) { + return Collections.emptyList(); + } + String url = getBaseUrl() + "/tools/listing_v3.php?livetab=all&_suggest=1&model_search[display_name][text]=" + URLEncoder.encode(q, "utf-8"); Request req = new Request.Builder() .url(url) .addHeader(USER_AGENT, getConfig().getSettings().httpUserAgent) - .addHeader(ACCEPT, MIMETYPE_APPLICATION_JSON) - .addHeader(ACCEPT_LANGUAGE, Locale.ENGLISH.getLanguage()) + .addHeader("Accept", "*/*") + .addHeader("Accept-Encoding", "deflate") + .addHeader("Accept-Language", "en,en-US;q=0.9") .addHeader(REFERER, getBaseUrl()) .addHeader(X_REQUESTED_WITH, XML_HTTP_REQUEST) .build(); @@ -153,7 +152,7 @@ public class BongaCams extends AbstractSite { if (response.isSuccessful()) { String body = Objects.requireNonNull(response.body(), HTTP_RESPONSE_BODY_IS_NULL).string(); JSONObject json = new JSONObject(body); - if (json.optString("status").equals("success")) { + if (json.has("models")) { List models = new ArrayList<>(); parseModelList(models, json); return models; @@ -169,16 +168,17 @@ public class BongaCams extends AbstractSite { private void parseModelList(List models, JSONObject json) { JSONArray results = json.getJSONArray("models"); - for (int i = 0; i < results.length(); i++) { + int maxResults = Math.min(30, results.length()); + for (int i = 0; i < maxResults; i++) { JSONObject result = results.getJSONObject(i); if (result.has("username")) { - Model model = createModel(result.getString("username")); - String thumb = result.getString("thumb_image").replace("{ext}", "jpg"); - if (thumb != null) { - model.setPreview("https:" + thumb); + String username = result.getString("username"); + Model model = createModel(username.toLowerCase()); + if (result.has("avatar")) { + model.setPreview("https://i.bcicdn.com" + result.getString("avatar")); } - if (result.has("display_name")) { - model.setDisplayName(result.getString("display_name")); + if (result.has("name")) { + model.setDisplayName(result.getString("name")); } models.add(model); } diff --git a/common/src/main/java/ctbrec/sites/bonga/BongaCamsHttpClient.java b/common/src/main/java/ctbrec/sites/bonga/BongaCamsHttpClient.java index 6e2c240c..80ed477d 100644 --- a/common/src/main/java/ctbrec/sites/bonga/BongaCamsHttpClient.java +++ b/common/src/main/java/ctbrec/sites/bonga/BongaCamsHttpClient.java @@ -42,7 +42,6 @@ public class BongaCamsHttpClient extends HttpClient { .value("%7B%22limit%22%3A20%2C%22c_limit%22%3A10%2C%22th_type%22%3A%22live%22%2C%22sorting%22%3A%22popular%22%2C%22display%22%3A%22auto%22%7D") .build(); - Map> cookies = cookieJar.getCookies(); for (Entry> entry : cookies.entrySet()) { List cookieList = entry.getValue(); diff --git a/common/src/main/java/ctbrec/sites/cam4/Cam4Model.java b/common/src/main/java/ctbrec/sites/cam4/Cam4Model.java index 3a2ffe13..e0fa77e8 100644 --- a/common/src/main/java/ctbrec/sites/cam4/Cam4Model.java +++ b/common/src/main/java/ctbrec/sites/cam4/Cam4Model.java @@ -21,6 +21,7 @@ import org.slf4j.LoggerFactory; import java.io.IOException; import java.io.InputStream; +import java.text.MessageFormat; import java.util.*; import java.util.concurrent.ExecutionException; import java.util.regex.Matcher; @@ -37,28 +38,30 @@ public class Cam4Model extends AbstractModel { private static final Logger LOG = LoggerFactory.getLogger(Cam4Model.class); private String playlistUrl; private int[] resolution = null; - private boolean privateRoom = false; + private JSONObject modelInfo; @Override public boolean isOnline(boolean ignoreCache) throws IOException, ExecutionException, InterruptedException { - if (ignoreCache || onlineState == UNKNOWN) { + if (ignoreCache) { try { - loadModelDetails(); - getPlaylistUrl(); + modelInfo = loadModelInfo(); + if (modelInfo.optBoolean("privateRoom")) { + onlineState = PRIVATE; + } } catch (Exception e) { onlineState = OFFLINE; } } - return onlineState == ONLINE && !privateRoom && StringUtil.isNotBlank(playlistUrl); + return onlineState == ONLINE; } - private void loadModelDetails() throws IOException { + private JSONObject loadModelInfo() throws IOException { JSONObject roomState = new Cam4WsClient(Config.getInstance(), (Cam4) getSite(), this).getRoomState(); - if (LOG.isTraceEnabled()) LOG.trace(roomState.toString(2)); + LOG.trace(roomState.toString(2)); String state = roomState.optString("newShowsState"); setOnlineStateByShowType(state); - privateRoom = roomState.optBoolean("privateRoom"); setDescription(roomState.optString("status")); + return roomState; } public void setOnlineStateByShowType(String showType) { @@ -81,7 +84,7 @@ public class Cam4Model extends AbstractModel { public State getOnlineState(boolean failFast) throws IOException, ExecutionException { if (!failFast && onlineState == UNKNOWN) { try { - loadModelDetails(); + modelInfo = loadModelInfo(); } catch (Exception e) { LOG.warn("Couldn't load model details {}", e.getMessage()); } @@ -90,15 +93,39 @@ public class Cam4Model extends AbstractModel { } private String getPlaylistUrl() throws IOException { + try { + getPlaylistUrlFromStreamUrl(); + if (StringUtil.isNotBlank(playlistUrl)) { + return playlistUrl; + } + } catch (IOException e) { + LOG.debug("Couldn't get playlist url from stream info: {}", e.getMessage()); + } + if (modelInfo != null && modelInfo.has("hls")) { + String hls = modelInfo.optString("hls"); + LOG.debug("Stream hls: {}", hls); + if (StringUtil.isNotBlank(hls) && hls.startsWith("http")) { + playlistUrl = hls; + return playlistUrl; + } + } + if (modelInfo != null && modelInfo.has("streamUUID")) { + String uuid = modelInfo.optString("streamUUID"); + LOG.debug("Stream UUID: {}", uuid); + String[] parts = uuid.split("-"); + if (parts.length > 3) { + String urlTemplate = "https://cam4-hls.xcdnpro.com/{0}/cam4-origin-live/{1}_aac/playlist.m3u8"; + playlistUrl = MessageFormat.format(urlTemplate, parts[1], uuid); + return playlistUrl; + } + } String page = loadModelPage(); Matcher m = Pattern.compile("hlsUrl\\s*:\\s*'(.*?)'", DOTALL | MULTILINE).matcher(page); if (m.find()) { playlistUrl = m.group(1); - } else { - LOG.trace("hlsUrl not in page"); - getPlaylistUrlFromStreamUrl(); + return playlistUrl; } - if (playlistUrl == null) { + if (StringUtil.isBlank(playlistUrl)) { throw new IOException("Couldn't determine playlist url"); } return playlistUrl; @@ -122,9 +149,9 @@ public class Cam4Model extends AbstractModel { if (LOG.isTraceEnabled()) LOG.trace(json.toString(2)); if (json.has("canUseCDN")) { if (json.getBoolean("canUseCDN")) { - playlistUrl = json.getString("cdnURL"); + playlistUrl = json.optString("cdnURL"); } else { - playlistUrl = json.getString("edgeURL"); + playlistUrl = json.optString("edgeURL"); } } } else { @@ -164,8 +191,7 @@ public class Cam4Model extends AbstractModel { if (playlist.getUri().startsWith("http")) { src.mediaPlaylistUrl = playlist.getUri(); } else { - String masterUrl = getPlaylistUrl(); - String baseUrl = masterUrl.substring(0, masterUrl.lastIndexOf('/') + 1); + String baseUrl = playlistUrl.substring(0, playlistUrl.lastIndexOf('/') + 1); src.mediaPlaylistUrl = baseUrl + playlist.getUri(); } LOG.trace("Media playlist {}", src.mediaPlaylistUrl); @@ -177,7 +203,8 @@ public class Cam4Model extends AbstractModel { private MasterPlaylist getMasterPlaylist() throws IOException, ParseException, PlaylistException { String masterPlaylistUrl = getPlaylistUrl(); - LOG.trace("Loading master playlist [{}]", masterPlaylistUrl); + masterPlaylistUrl = masterPlaylistUrl.replace("_sfm4s", ""); + LOG.debug("Loading master playlist [{}]", masterPlaylistUrl); Request.Builder builder = new Request.Builder().url(masterPlaylistUrl); getHttpHeaderFactory().createMasterPlaylistHeaders().forEach(builder::header); Request req = builder.build(); diff --git a/common/src/main/java/ctbrec/sites/camsoda/CamsodaModel.java b/common/src/main/java/ctbrec/sites/camsoda/CamsodaModel.java index bdf6a68e..0a0a9359 100644 --- a/common/src/main/java/ctbrec/sites/camsoda/CamsodaModel.java +++ b/common/src/main/java/ctbrec/sites/camsoda/CamsodaModel.java @@ -44,10 +44,10 @@ public class CamsodaModel extends AbstractModel { public String getStreamUrl() throws IOException { Request req = createJsonRequest(getTokenInfoUrl()); JSONObject response = executeJsonRequest(req); - if (response.optInt(STATUS) == 1 && response.optJSONArray(EDGE_SERVERS) != null && response.optJSONArray(EDGE_SERVERS).length() > 0) { + if (response.optInt(STATUS) == 1 && response.optJSONArray(EDGE_SERVERS) != null && !response.optJSONArray(EDGE_SERVERS).isEmpty()) { String edgeServer = response.getJSONArray(EDGE_SERVERS).getString(0); String streamName = response.getString(STREAM_NAME); - String token = response.getString("token"); + String token = response.optString("token"); return constructStreamUrl(edgeServer, streamName, token); } else { throw new JSONException("JSON response has not the expected structure"); @@ -185,7 +185,7 @@ public class CamsodaModel extends AbstractModel { @Override public boolean isOnline(boolean ignoreCache) throws IOException, ExecutionException, InterruptedException { - if (ignoreCache || onlineState == UNKNOWN) { + if (ignoreCache) { loadModel(); } return onlineState == ONLINE; diff --git a/common/src/main/java/ctbrec/sites/chaturbate/ChaturbateHttpClient.java b/common/src/main/java/ctbrec/sites/chaturbate/ChaturbateHttpClient.java index 3ea2559d..e0aef351 100644 --- a/common/src/main/java/ctbrec/sites/chaturbate/ChaturbateHttpClient.java +++ b/common/src/main/java/ctbrec/sites/chaturbate/ChaturbateHttpClient.java @@ -49,13 +49,11 @@ public class ChaturbateHttpClient extends HttpClient { if (loggedIn) { return true; } - if (checkLogin()) { loggedIn = true; LOG.debug("Logged in with cookies"); return true; } - try { Request login = new Request.Builder() .url(Chaturbate.baseUrl + PATH) @@ -88,18 +86,9 @@ public class ChaturbateHttpClient extends HttpClient { loggedIn = true; extractCsrfToken(login); } - } else { - if (loginTries++ < 3) { - login(); - } else { - throw new IOException("Login failed: " + response.code() + " " + response.message()); - } } - response.close(); } catch (Exception ex) { LOG.debug("Login failed: {}", ex.getMessage()); - } finally { - loginTries = 0; } return loggedIn; } diff --git a/common/src/main/java/ctbrec/sites/chaturbate/ChaturbateModel.java b/common/src/main/java/ctbrec/sites/chaturbate/ChaturbateModel.java index 3a50e3d2..9b342f6a 100644 --- a/common/src/main/java/ctbrec/sites/chaturbate/ChaturbateModel.java +++ b/common/src/main/java/ctbrec/sites/chaturbate/ChaturbateModel.java @@ -94,7 +94,6 @@ public class ChaturbateModel extends AbstractModel { // NOSONAR if (failFast) { return resolution; } - try { resolution = getResolution(); } catch (Exception e) { diff --git a/common/src/main/java/ctbrec/sites/flirt4free/Flirt4FreeModel.java b/common/src/main/java/ctbrec/sites/flirt4free/Flirt4FreeModel.java index 6406f9d9..8fa11ab8 100644 --- a/common/src/main/java/ctbrec/sites/flirt4free/Flirt4FreeModel.java +++ b/common/src/main/java/ctbrec/sites/flirt4free/Flirt4FreeModel.java @@ -4,6 +4,7 @@ import com.iheartradio.m3u8.*; import com.iheartradio.m3u8.data.MasterPlaylist; import com.iheartradio.m3u8.data.Playlist; import com.iheartradio.m3u8.data.PlaylistData; +import com.iheartradio.m3u8.data.StreamInfo; import com.squareup.moshi.JsonReader; import com.squareup.moshi.JsonWriter; import ctbrec.AbstractModel; @@ -92,6 +93,15 @@ public class Flirt4FreeModel extends AbstractModel { return; } JSONObject json = new JSONObject(body); + if (Objects.equals(json.optString("status"), "failed")) { + if (Objects.equals(json.optString("message"), "Model is inactive")) { + LOG.debug("Model inactive or deleted: {}", getName()); + setMarkedForLaterRecording(true); + } + online = false; + onlineState = Model.State.OFFLINE; + return; + } online = Objects.equals(json.optString(STATUS), "online"); // online is true, even if the model is in private or away updateModelId(json); if (online) { @@ -188,8 +198,10 @@ public class Flirt4FreeModel extends AbstractModel { for (PlaylistData playlist : masterPlaylist.getPlaylists()) { if (playlist.hasStreamInfo()) { StreamSource src = new StreamSource(); - src.bandwidth = playlist.getStreamInfo().getBandwidth(); - src.height = playlist.getStreamInfo().getResolution().height; + StreamInfo info = playlist.getStreamInfo(); + src.bandwidth = info.getBandwidth(); + src.height = (info.hasResolution()) ? info.getResolution().height : 0; + src.width = (info.hasResolution()) ? info.getResolution().width : 0; HttpUrl masterPlaylistUrl = HttpUrl.parse(streamUrl); src.mediaPlaylistUrl = "https://" + masterPlaylistUrl.host() + '/' + playlist.getUri(); LOG.trace("Media playlist {}", src.mediaPlaylistUrl); @@ -473,7 +485,7 @@ public class Flirt4FreeModel extends AbstractModel { String url = getSite().getBaseUrl() + "/external.php?a=" + (add ? "add_favorite" : "delete_favorite") + "&id=" + id + - "&name=" + getDisplayName() + + "&name=" + getName() + "&t=" + System.currentTimeMillis(); LOG.debug("Sending follow/unfollow request: {}", url); Request req = new Request.Builder() @@ -527,6 +539,16 @@ public class Flirt4FreeModel extends AbstractModel { this.isNew = isNew; } + @Override + public String getName() { + String original = super.getName(); + String fixed = original.toLowerCase().replace(" ", "-").replace("_", "-"); + if (!fixed.equals(original)) { + setName(fixed); + } + return fixed; + } + private void acquireSlot() throws InterruptedException { requestThrottle.acquire(); long now = System.currentTimeMillis(); diff --git a/common/src/main/java/ctbrec/sites/jasmin/LiveJasmin.java b/common/src/main/java/ctbrec/sites/jasmin/LiveJasmin.java index 9290f1d7..65497a21 100644 --- a/common/src/main/java/ctbrec/sites/jasmin/LiveJasmin.java +++ b/common/src/main/java/ctbrec/sites/jasmin/LiveJasmin.java @@ -15,7 +15,10 @@ import org.slf4j.LoggerFactory; import java.io.IOException; import java.net.URLEncoder; -import java.util.*; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Objects; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -24,8 +27,8 @@ import static ctbrec.io.HttpConstants.*; public class LiveJasmin extends AbstractSite { private static final Logger LOG = LoggerFactory.getLogger(LiveJasmin.class); - public static String baseUrl = ""; - public static String baseDomain = ""; + public static String baseUrl = "https://www.livejasmin.com"; + public static String baseDomain = "www.livejasmin.com"; private HttpClient httpClient; @Override @@ -41,7 +44,6 @@ public class LiveJasmin extends AbstractSite { @Override public String getAffiliateLink() { return "https://awejmp.com/?siteId=jasmin&categoryName=girl&pageName=listpage&performerName=&prm[psid]=0xb00bface&prm[pstool]=205_1&prm[psprogram]=revs&prm[campaign_id]=&subAffId={SUBAFFID}&filters="; - // return "https://awejmp.com/?siteId=jasmin&categoryName=girl&pageName=listpage&performerName=&prm[psid]=0xb00bface&prm[pstool]=205_1&prm[psprogram]=pps&prm[campaign_id]=&subAffId={SUBAFFID}&filters="; } @Override @@ -196,12 +198,12 @@ public class LiveJasmin extends AbstractSite { @Override public Model createModelFromUrl(String url) { Matcher m = Pattern.compile("http.*?livejasmin\\.com.*?#!chat/(.*)").matcher(url); - if(m.find()) { + if (m.find()) { String name = m.group(1); return createModel(name); } m = Pattern.compile("http.*?livejasmin\\.com.*?/chat(?:-html5)?/(.*)").matcher(url); - if(m.find()) { + if (m.find()) { String name = m.group(1); return createModel(name); } diff --git a/common/src/main/java/ctbrec/sites/jasmin/LiveJasminModel.java b/common/src/main/java/ctbrec/sites/jasmin/LiveJasminModel.java index b9908854..ef799c40 100644 --- a/common/src/main/java/ctbrec/sites/jasmin/LiveJasminModel.java +++ b/common/src/main/java/ctbrec/sites/jasmin/LiveJasminModel.java @@ -6,6 +6,7 @@ import com.squareup.moshi.JsonReader; import com.squareup.moshi.JsonWriter; import ctbrec.AbstractModel; import ctbrec.Config; +import ctbrec.StringUtil; import ctbrec.io.HttpException; import ctbrec.recorder.download.Download; import ctbrec.recorder.download.StreamSource; @@ -41,53 +42,54 @@ public class LiveJasminModel extends AbstractModel { } protected void loadModelInfo() throws IOException { - String url = "https://m." + LiveJasmin.baseDomain + "/en/chat-html5/" + getName(); - Request req = new Request.Builder().url(url) - .header(USER_AGENT, - "Mozilla/5.0 (iPhone; CPU OS 10_14 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/11.1.1 Mobile/14E304 Safari/605.1.15") - .header(ACCEPT, MIMETYPE_APPLICATION_JSON) + Request req = new Request.Builder().url(LiveJasmin.baseUrl) + .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) + .header(ACCEPT, "*/*") + .header(ACCEPT_ENCODING, "deflate") .header(ACCEPT_LANGUAGE, Locale.ENGLISH.getLanguage()) - .header(REFERER, getSite().getBaseUrl()) + .header(REFERER, getSite().getBaseUrl() + "/") + .header(ORIGIN, getSite().getBaseUrl()) + .build(); + try (Response response = getSite().getHttpClient().execute(req)) { + // do nothing we just want the cookies + LOG.debug("Initial request succeeded: {} - {}", response.isSuccessful(), response.code()); + } + + String url = LiveJasmin.baseUrl + "/en/flash/get-performer-details/" + getName(); + req = new Request.Builder().url(url) + .header(USER_AGENT, "Mozilla/5.0 (iPhone; CPU OS 10_14 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/11.1.1 Mobile/14E304 Safari/605.1.15") + .header(ACCEPT, MIMETYPE_APPLICATION_JSON) + .header(ACCEPT_ENCODING, "deflate") + .header(ACCEPT_LANGUAGE, Locale.ENGLISH.getLanguage()) + .header(REFERER, getSite().getBaseUrl() + "/") + .header(ORIGIN, getSite().getBaseUrl()) .header(X_REQUESTED_WITH, XML_HTTP_REQUEST) .build(); try (Response response = getSite().getHttpClient().execute(req)) { if (response.isSuccessful()) { String body = response.body().string(); JSONObject json = new JSONObject(body); - //LOG.debug(json.toString(2)); - //Files.writeString(Path.of("/tmp/model.json"), json.toString(2)); + LOG.trace(json.toString(2)); if (json.optBoolean("success")) { JSONObject data = json.getJSONObject("data"); - JSONObject config = data.getJSONObject("config"); - JSONObject chatRoom = config.getJSONObject("chatRoom"); - JSONObject armageddonConfig = config.getJSONObject("armageddonConfig"); - setId(chatRoom.getString("p_id")); - setName(chatRoom.getString("performer_id")); - setDisplayName(chatRoom.getString("display_name")); - if (chatRoom.has("profile_picture_url")) { - setPreview(chatRoom.getString("profile_picture_url")); - } - int status = chatRoom.optInt("status", -1); - onlineState = mapStatus(status); - if (chatRoom.optInt("is_on_private", 0) == 1) { - onlineState = State.PRIVATE; - } - if (chatRoom.optInt("is_video_call_enabled", 0) == 1) { - onlineState = State.PRIVATE; - } - resolution = new int[2]; - resolution[0] = config.optInt("streamWidth"); - resolution[1] = config.optInt("streamHeight"); modelInfo = new LiveJasminModelInfo.LiveJasminModelInfoBuilder() - .sbIp(chatRoom.getString("sb_ip")) - .sbHash(chatRoom.getString("sb_hash")) - .sessionId(armageddonConfig.getString("sessionid")) - .jsm2session(armageddonConfig.getString("jsm2session")) - .performerId(getName()) + .sbIp(data.optString("sb_ip", null)) + .sbHash(data.optString("sb_hash", null)) + .sessionId("m12345678901234567890123456789012") + .jsm2session(getSite().getHttpClient().getCookiesByName("session").get(0).value()) + .performerId(data.optString("performer_id", getName())) + .displayName(data.optString("display_name", getName())) .clientInstanceId(randomClientInstanceId()) + .status(data.optInt("status", -1)) .build(); - online = onlineState == State.ONLINE; - LOG.trace("{} - status:{} {} {} {} {}", getName(), online, onlineState, Arrays.toString(resolution), getUrl(), id); + if (data.has("channelsiteurl")) { + setUrl(LiveJasmin.baseUrl + data.getString("channelsiteurl")); + } + onlineState = mapStatus(modelInfo.getStatus()); + online = onlineState == State.ONLINE + && StringUtil.isNotBlank(modelInfo.getSbIp()) + && StringUtil.isNotBlank(modelInfo.getSbHash()); + LOG.debug("{} - status:{} {} {} {} {}", getName(), online, onlineState, Arrays.toString(resolution), getUrl(), id); } else { throw new IOException("Response was not successful: " + body); } @@ -107,15 +109,19 @@ public class LiveJasminModel extends AbstractModel { public static State mapStatus(int status) { switch (status) { - case 0: + case 0 -> { return State.OFFLINE; - case 1: + } + case 1 -> { return State.ONLINE; - case 2, 3: + } + case 2, 3 -> { return State.PRIVATE; - default: + } + default -> { LOG.debug("Unkown state {}", status); return State.UNKNOWN; + } } } @@ -129,17 +135,17 @@ public class LiveJasminModel extends AbstractModel { public List getStreamSources() throws IOException, ExecutionException, ParseException, PlaylistException { loadModelInfo(); - String websocketUrlTemplate = "wss://dss-relay-{ipWithDashes}.dditscdn.com/?random={clientInstanceId}"; + String websocketUrlTemplate = "wss://dss-relay-{ipWithDashes}.dditscdn.com/memberChat/jasmin{modelName}{sb_hash}?random={clientInstanceId}"; String websocketUrl = websocketUrlTemplate .replace("{ipWithDashes}", modelInfo.getSbIp().replace('.', '-')) + .replace("{modelName}", getName()) + .replace("{sb_hash}", modelInfo.getSbHash()) .replace("{clientInstanceId}", modelInfo.getClientInstanceId()); modelInfo.setWebsocketUrl(websocketUrl); LiveJasminStreamRegistration liveJasminStreamRegistration = new LiveJasminStreamRegistration(site, modelInfo); List streamSources = liveJasminStreamRegistration.getStreamSources(); - streamSources.stream().max(Comparator.naturalOrder()).ifPresent(ss -> { - new LiveJasminStreamStarter().start(site, modelInfo, (LiveJasminStreamSource) ss); - }); + Collections.sort(streamSources); return streamSources; } @@ -174,10 +180,8 @@ public class LiveJasminModel extends AbstractModel { } catch (IOException e) { throw new ExecutionException(e); } - return resolution; - } else { - return resolution; } + return resolution; } @Override @@ -253,10 +257,6 @@ public class LiveJasminModel extends AbstractModel { @Override public Download createDownload() { - if (Config.isServerMode() && !Config.getInstance().getSettings().recordSingleFile) { - return new LiveJasminHlsDownload(getSite().getHttpClient()); - } else { - return new LiveJasminMergedHlsDownload(getSite().getHttpClient()); - } + return new LiveJasminWebrtcDownload(getSite().getHttpClient()); } } diff --git a/common/src/main/java/ctbrec/sites/jasmin/LiveJasminModelInfo.java b/common/src/main/java/ctbrec/sites/jasmin/LiveJasminModelInfo.java index ed0e3eef..e2e29c9d 100644 --- a/common/src/main/java/ctbrec/sites/jasmin/LiveJasminModelInfo.java +++ b/common/src/main/java/ctbrec/sites/jasmin/LiveJasminModelInfo.java @@ -12,5 +12,7 @@ public class LiveJasminModelInfo { private String sessionId; private String jsm2session; private String performerId; + private String displayName; private String clientInstanceId; + private int status; } diff --git a/common/src/main/java/ctbrec/sites/jasmin/LiveJasminStreamRegistration.java b/common/src/main/java/ctbrec/sites/jasmin/LiveJasminStreamRegistration.java index 41a7770b..b6ae44a9 100644 --- a/common/src/main/java/ctbrec/sites/jasmin/LiveJasminStreamRegistration.java +++ b/common/src/main/java/ctbrec/sites/jasmin/LiveJasminStreamRegistration.java @@ -16,12 +16,12 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.net.URLEncoder; -import java.util.LinkedList; -import java.util.List; +import java.util.*; import java.util.concurrent.BrokenBarrierException; import java.util.concurrent.CyclicBarrier; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import java.util.stream.Collectors; import static ctbrec.io.HttpConstants.USER_AGENT; import static java.nio.charset.StandardCharsets.UTF_8; @@ -29,6 +29,7 @@ import static java.nio.charset.StandardCharsets.UTF_8; public class LiveJasminStreamRegistration { private static final Logger LOG = LoggerFactory.getLogger(LiveJasminStreamRegistration.class); + private static final String KEY_EVENT = "event"; private static final String KEY_FUNC_NAME = "funcName"; @@ -36,61 +37,65 @@ public class LiveJasminStreamRegistration { private final LiveJasminModelInfo modelInfo; private final CyclicBarrier barrier = new CyclicBarrier(2); + private int streamCount = 0; + private WebSocket webSocket; + public LiveJasminStreamRegistration(Site site, LiveJasminModelInfo modelInfo) { this.site = site; this.modelInfo = modelInfo; } List getStreamSources() { - var streamSources = new LinkedList(); + var streamSources = new LinkedList(); try { Request webSocketRequest = new Request.Builder() .url(modelInfo.getWebsocketUrl()) .addHeader(USER_AGENT, Config.getInstance().getSettings().httpUserAgentMobile) .build(); LOG.debug("Websocket: {}", modelInfo.getWebsocketUrl()); - site.getHttpClient().newWebSocket(webSocketRequest, new WebSocketListener() { + webSocket = site.getHttpClient().newWebSocket(webSocketRequest, new WebSocketListener() { @Override public void onOpen(@NotNull WebSocket webSocket, @NotNull Response response) { - LOG.debug("onOpen"); + Thread.currentThread().setName("Stream registration for " + modelInfo.getPerformerId()); + LOG.trace("onOpen"); JSONObject register = new JSONObject() .put(KEY_EVENT, "register") .put("applicationId", "memberChat/jasmin" + modelInfo.getPerformerId() + modelInfo.getSbHash()) .put("connectionData", new JSONObject() - .put("jasmin2App", false) - .put("isMobileClient", true) - .put("platform", "mobile") - .put("chatID", "freechat") .put("sessionID", modelInfo.getSessionId()) + .put("jasmin2App", true) + .put("isMobileClient", false) + .put("platform", "desktop") + .put("chatID", "freechat") .put("jsm2SessionId", modelInfo.getJsm2session()) .put("userType", "user") .put("performerId", modelInfo.getPerformerId()) .put("clientRevision", "") - .put("playerVer", "nanoPlayerVersion: 4.12.1 appCodeName: Mozilla appName: Netscape appVersion: 5.0 (iPad; CPU OS 10_14 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/11.1.1 Mobile/15E148 Safari/605.1.15 platform: iPad") .put("livejasminTvmember", false) .put("newApplet", true) .put("livefeedtype", JSONObject.NULL) .put("gravityCookieId", "") .put("passparam", "") - .put("clientInstanceId", modelInfo.getClientInstanceId()) - .put("armaVersion", "39.158.0") - .put("isPassive", false) .put("brandID", "jasmin") - .put("cobrandId", "") + .put("cobrandId", "livejasmin") .put("subbrand", "livejasmin") .put("siteName", "LiveJasmin") - .put("siteUrl", "https://m." + LiveJasmin.baseDomain) - .put("chatHistoryRequired", false) + .put("siteUrl", "https://www.livejasmin.com") + .put("clientInstanceId", modelInfo.getClientInstanceId()) + .put("armaVersion", "38.32.1-LIVEJASMIN-44016-1") + .put("isPassive", false) .put("peekPatternJsm2", true) + .put("chatHistoryRequired", true) ); - webSocket.send(register.toString()); - webSocket.send(new JSONObject().put(KEY_EVENT, "ping").toString()); - webSocket.send(new JSONObject() + LOG.trace("Stream registration\n{}", register.toString(2)); + send(register.toString()); + send(new JSONObject().put(KEY_EVENT, "ping").toString()); + send(new JSONObject() .put(KEY_EVENT, "call") .put(KEY_FUNC_NAME, "makeActive") .put("data", new JSONArray()) .toString()); - webSocket.send(new JSONObject() + send(new JSONObject() .put(KEY_EVENT, "call") .put(KEY_FUNC_NAME, "setVideo") .put("data", new JSONArray() @@ -100,11 +105,10 @@ public class LiveJasminStreamRegistration { .put(modelInfo.getJsm2session()) ) .toString()); - webSocket.send(new JSONObject() + send(new JSONObject() .put(KEY_EVENT, "connectSharedObject") .put("name", "data/chat_so") .toString()); - //webSocket.close(1000, "Good bye"); } @Override @@ -116,6 +120,7 @@ public class LiveJasminStreamRegistration { @Override public void onMessage(@NotNull WebSocket webSocket, @NotNull String text) { + LOG.trace("< {}", text); JSONObject message = new JSONObject(text); if (message.opt(KEY_EVENT).equals("pong")) { new Thread(() -> { @@ -124,7 +129,7 @@ public class LiveJasminStreamRegistration { } catch (InterruptedException e) { Thread.currentThread().interrupt(); } - webSocket.send(new JSONObject().put(KEY_EVENT, "ping").toString()); + send(new JSONObject().put(KEY_EVENT, "ping").toString()); }).start(); } else if (message.optString(KEY_EVENT).equals("updateSharedObject") && message.optString("name").equals("data/chat_so")) { LOG.trace(message.toString(2)); @@ -140,9 +145,34 @@ public class LiveJasminStreamRegistration { JSONObject stream = streams.getJSONObject(j); addStreamSource(streamSources, freePattern, stream); } - webSocket.close(1000, ""); + Collections.sort(streamSources); + Collections.reverse(streamSources); + for (LiveJasminStreamSource src : streamSources) { + JSONObject getVideoData = new JSONObject() + .put(KEY_EVENT, "call") + .put(KEY_FUNC_NAME, "getVideoData") + .put("data", new JSONArray() + .put(new JSONObject() + .put("protocols", new JSONArray() + .put("h5live") + ) + .put("streamId", src.getStreamId()) + .put("correlationId", UUID.randomUUID().toString().replace("-", "").substring(0, 16)) + ) + ); + streamCount++; + send(getVideoData.toString()); + } } } + } else if (message.optString(KEY_FUNC_NAME).equals("setVideoData")) { + JSONObject data = message.getJSONArray("data").getJSONArray(0).getJSONObject(0); + String streamId = data.getString("streamId"); + String wssUrl = data.getJSONObject("protocol").getJSONObject("h5live").getString("wssUrl"); + streamSources.stream().filter(src -> Objects.equals(src.getStreamId(), streamId)).findAny().ifPresent(src -> src.mediaPlaylistUrl = wssUrl); + if (--streamCount == 0) { + awaitBarrier(); + } } else if (!message.optString(KEY_FUNC_NAME).equals("chatHistory")) { LOG.trace("onMessageT {}", new JSONObject(text).toString(2)); } @@ -156,7 +186,7 @@ public class LiveJasminStreamRegistration { @Override public void onClosed(@NotNull WebSocket webSocket, int code, @NotNull String reason) { - LOG.debug("onClosed {} {}", code, reason); + LOG.trace("onClosed {} {}", code, reason); super.onClosed(webSocket, code, reason); } @@ -165,6 +195,11 @@ public class LiveJasminStreamRegistration { LOG.trace("onClosing {} {}", code, reason); awaitBarrier(); } + + private void send(String msg) { + LOG.trace("Send > {}", msg); + webSocket.send(msg); + } }); LOG.debug("Waiting for websocket to return"); @@ -173,15 +208,16 @@ public class LiveJasminStreamRegistration { } catch (Exception e) { LOG.error("Couldn't determine stream sources", e); } - return streamSources; + return streamSources.stream().map(StreamSource.class::cast).collect(Collectors.toList()); // NOSONAR } - private void addStreamSource(LinkedList streamSources, String pattern, JSONObject stream) { + private void addStreamSource(LinkedList streamSources, String pattern, JSONObject stream) { int w = stream.getInt("width"); int h = stream.getInt("height"); int bitrate = stream.getInt("bitrate") * 1024; String name = stream.getString("name"); String streamName = pattern.replace("{$streamname}", name); + String streamId = stream.getString("streamId"); String rtmpUrl = "rtmp://{ip}/memberChat/jasmin{modelName}{sb_hash}?sessionId-{sessionId}|clientInstanceId-{clientInstanceId}" .replace("{ip}", modelInfo.getSbIp()) @@ -200,8 +236,10 @@ public class LiveJasminStreamRegistration { streamSource.width = w; streamSource.height = h; streamSource.bandwidth = bitrate; - streamSource.rtmpUrl = rtmpUrl; - streamSource.streamName = streamName; + streamSource.setRtmpUrl(rtmpUrl); + streamSource.setStreamName(streamName); + streamSource.setStreamId(streamId); + streamSource.setStreamRegistration(this); streamSources.add(streamSource); } @@ -215,4 +253,8 @@ public class LiveJasminStreamRegistration { LOG.error(e.getLocalizedMessage(), e); } } + + void close() { + webSocket.close(1000, ""); + } } diff --git a/common/src/main/java/ctbrec/sites/jasmin/LiveJasminStreamSource.java b/common/src/main/java/ctbrec/sites/jasmin/LiveJasminStreamSource.java index 7f7d5a13..d788026b 100644 --- a/common/src/main/java/ctbrec/sites/jasmin/LiveJasminStreamSource.java +++ b/common/src/main/java/ctbrec/sites/jasmin/LiveJasminStreamSource.java @@ -1,8 +1,14 @@ package ctbrec.sites.jasmin; import ctbrec.recorder.download.StreamSource; +import lombok.Getter; +import lombok.Setter; +@Getter +@Setter public class LiveJasminStreamSource extends StreamSource { public String rtmpUrl; public String streamName; + public String streamId; + public LiveJasminStreamRegistration streamRegistration; } diff --git a/common/src/main/java/ctbrec/sites/jasmin/LiveJasminWebrtcDownload.java b/common/src/main/java/ctbrec/sites/jasmin/LiveJasminWebrtcDownload.java new file mode 100644 index 00000000..8c3881e7 --- /dev/null +++ b/common/src/main/java/ctbrec/sites/jasmin/LiveJasminWebrtcDownload.java @@ -0,0 +1,247 @@ +package ctbrec.sites.jasmin; + +import com.iheartradio.m3u8.ParseException; +import com.iheartradio.m3u8.PlaylistException; +import ctbrec.Config; +import ctbrec.Model; +import ctbrec.Recording; +import ctbrec.io.BandwidthMeter; +import ctbrec.io.HttpClient; +import ctbrec.recorder.download.AbstractDownload; +import ctbrec.recorder.download.Download; +import ctbrec.recorder.download.StreamSource; +import okhttp3.Request; +import okhttp3.Response; +import okhttp3.WebSocket; +import okhttp3.WebSocketListener; +import okio.ByteString; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.EOFException; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.file.Files; +import java.time.Duration; +import java.time.Instant; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.regex.Pattern; + +import static ctbrec.io.HttpConstants.*; + +public class LiveJasminWebrtcDownload extends AbstractDownload { + + private static final Logger LOG = LoggerFactory.getLogger(LiveJasminWebrtcDownload.class); + private static final int MAX_SECONDS_WITHOUT_TRANSFER = 20; + + private final HttpClient httpClient; + private WebSocket ws; + private FileOutputStream fout; + private Instant timeOfLastTransfer = Instant.MAX; + + private volatile boolean running; + private volatile boolean started; + + + private File targetFile; + + public LiveJasminWebrtcDownload(HttpClient httpClient) { + this.httpClient = httpClient; + } + + @Override + public void init(Config config, Model model, Instant startTime, ExecutorService executorService) throws IOException { + this.config = config; + this.model = model; + this.startTime = startTime; + this.downloadExecutor = executorService; + splittingStrategy = initSplittingStrategy(config.getSettings()); + targetFile = config.getFileForRecording(model, "mp4", startTime); + timeOfLastTransfer = Instant.now(); + } + + @Override + public void stop() { + running = false; + if (ws != null) { + ws.close(1000, ""); + ws = null; + } + } + + @Override + public void finalizeDownload() { + if (fout != null) { + try { + LOG.debug("Closing recording file {}", targetFile); + fout.close(); + } catch (IOException e) { + LOG.error("Error while closing recording file {}", targetFile, e); + } + } + } + + @Override + public boolean isRunning() { + return running; + } + + @Override + public void postprocess(Recording recording) { + // nothing to do + } + + @Override + public File getTarget() { + return targetFile; + } + + @Override + public String getPath(Model model) { + String absolutePath = targetFile.getAbsolutePath(); + String recordingsDir = Config.getInstance().getSettings().recordingsDir; + String relativePath = absolutePath.replaceFirst(Pattern.quote(recordingsDir), ""); + return relativePath; + } + + @Override + public boolean isSingleFile() { + return true; + } + + @Override + public long getSizeInByte() { + return getTarget().length(); + } + + @Override + public Download call() throws Exception { + if (!started) { + started = true; + startDownload(); + } + + if (splittingStrategy.splitNecessary(this)) { + stop(); + rescheduleTime = Instant.now(); + } else { + rescheduleTime = Instant.now().plusSeconds(5); + } + if (!model.isOnline(true)) { + stop(); + } + if (Duration.between(timeOfLastTransfer, Instant.now()).getSeconds() > MAX_SECONDS_WITHOUT_TRANSFER) { + LOG.info("No video data received for {} seconds. Stopping recording for model {}", MAX_SECONDS_WITHOUT_TRANSFER, model); + stop(); + } + return this; + } + + private void startDownload() throws IOException, PlaylistException, ParseException, ExecutionException { + LiveJasminModel liveJasminModel = (LiveJasminModel) model; + List streamSources = liveJasminModel.getStreamSources(); + LiveJasminStreamSource streamSource = (LiveJasminStreamSource) selectStreamSource(streamSources); + LiveJasminStreamRegistration streamRegistration = streamSource.getStreamRegistration(); + Request request = new Request.Builder() + .url(streamSource.getMediaPlaylistUrl()) + .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) + .header(ACCEPT, "*/*") + .header(ACCEPT_LANGUAGE, "en") + .header(REFERER, model.getSite().getBaseUrl() + "/") + .header(ORIGIN, model.getSite().getBaseUrl()) + .build(); + + running = true; + LOG.debug("Opening webrtc connection {}", request.url()); + ws = httpClient.newWebSocket(request, new WebSocketListener() { + @Override + public void onOpen(WebSocket webSocket, Response response) { + super.onOpen(webSocket, response); + LOG.trace("onOpen {} {}", webSocket, response); + response.close(); + try { + LOG.trace("Recording video stream to {}", targetFile); + Files.createDirectories(targetFile.getParentFile().toPath()); + fout = new FileOutputStream(targetFile); + } catch (Exception e) { + LOG.error("Couldn't open file {} to save the video stream", targetFile, e); + stop(); + } + } + + @Override + public void onMessage(WebSocket webSocket, ByteString bytes) { + super.onMessage(webSocket, bytes); + LOG.trace("received video data with length {}", bytes.size()); + timeOfLastTransfer = Instant.now(); + try { + byte[] videoData = bytes.toByteArray(); + fout.write(videoData); + BandwidthMeter.add(videoData.length); + } catch (IOException e) { + if (running) { + LOG.error("Couldn't write video stream to file", e); + stop(); + } + } + } + + @Override + public void onMessage(WebSocket webSocket, String text) { + super.onMessage(webSocket, text); + LOG.trace("onMessageT {} {}", webSocket, text); + } + + @Override + public void onFailure(WebSocket webSocket, Throwable t, Response response) { + super.onFailure(webSocket, t, response); + stop(); + if (t instanceof EOFException) { + LOG.info("End of stream detected for model {}", model); + } else { + LOG.error("Websocket failure for model {} {}", model, response, t); + } + if (response != null) { + response.close(); + } + streamRegistration.close(); + } + + @Override + public void onClosing(WebSocket webSocket, int code, String reason) { + super.onClosing(webSocket, code, reason); + LOG.trace("Websocket closing for model {} {} {}", model, code, reason); + } + + @Override + public void onClosed(WebSocket webSocket, int code, String reason) { + super.onClosed(webSocket, code, reason); + LOG.debug("Websocket closed for model {} {} {}", model, code, reason); + stop(); + streamRegistration.close(); + } + }); + } + + @Override + public void awaitEnd() { + long secondsToWait = 30; + for (int i = 0; i < secondsToWait; i++) { + if (ws == null) { + return; + } else { + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + LOG.warn("Interrupted while waiting for the download to terminate"); + } + } + } + LOG.warn("Download didn't finish in {} seconds", secondsToWait); + } + +} diff --git a/common/src/main/java/ctbrec/sites/streamate/StreamateModel.java b/common/src/main/java/ctbrec/sites/streamate/StreamateModel.java index a9c1ba86..5e9a0697 100644 --- a/common/src/main/java/ctbrec/sites/streamate/StreamateModel.java +++ b/common/src/main/java/ctbrec/sites/streamate/StreamateModel.java @@ -6,7 +6,6 @@ import com.squareup.moshi.JsonReader; import com.squareup.moshi.JsonWriter; import ctbrec.AbstractModel; import ctbrec.Config; -import ctbrec.Model; import ctbrec.NotImplementedExcetion; import ctbrec.io.HttpException; import ctbrec.recorder.download.StreamSource; @@ -19,6 +18,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; +import java.net.URLEncoder; import java.util.*; import java.util.concurrent.ExecutionException; @@ -26,6 +26,7 @@ import static ctbrec.ErrorMessages.HTTP_RESPONSE_BODY_IS_NULL; import static ctbrec.Model.State.*; import static ctbrec.io.HttpConstants.*; import static ctbrec.sites.streamate.StreamateHttpClient.JSON; +import static java.nio.charset.StandardCharsets.UTF_8; public class StreamateModel extends AbstractModel { @@ -70,11 +71,6 @@ public class StreamateModel extends AbstractModel { return onlineState; } - @Override - public void setOnlineState(State onlineState) { - this.onlineState = onlineState; - } - @Override public List getStreamSources() throws IOException, ExecutionException, ParseException, PlaylistException { String url = "https://sea1c-ls.naiadsystems.com/sea1c-edge-ls/80/live/s:" + getName() + ".json"; @@ -129,10 +125,22 @@ public class StreamateModel extends AbstractModel { resolution = null; } - void loadModelId() throws IOException, InterruptedException { - List models = getSite().search(getName()); - if (!models.isEmpty()) { - id = ((StreamateModel)models.get(0)).getId(); + void loadModelId() throws IOException { + String url = "https://www.streamate.com/api/performer/lookup?nicknames" + URLEncoder.encode(getName(), UTF_8); + Request req = new Request.Builder().url(url) + .addHeader(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) + .addHeader(ACCEPT, "*/*") + .addHeader(ACCEPT_LANGUAGE, Locale.ENGLISH.getLanguage()) + .addHeader(REFERER, Streamate.BASE_URL + '/' + getName()) + .addHeader(X_REQUESTED_WITH, XML_HTTP_REQUEST) + .build(); + try (Response response = site.getHttpClient().execute(req)) { + if (response.isSuccessful()) { + String body = Objects.requireNonNull(response.body(), HTTP_RESPONSE_BODY_IS_NULL).string(); + id = new JSONObject(body).getJSONObject("result").getLong(getName()); + } else { + throw new HttpException(response.code(), response.message()); + } } } @@ -228,8 +236,6 @@ public class StreamateModel extends AbstractModel { loadModelId(); } catch (IOException e) { LOG.error("Couldn't load model ID for {}. This can cause problems with saving / loading the model", getName(), e); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); } } writer.name("id").value(id); diff --git a/common/src/main/java/ctbrec/sites/streamray/StreamrayModel.java b/common/src/main/java/ctbrec/sites/streamray/StreamrayModel.java index 60c73869..c045bc40 100644 --- a/common/src/main/java/ctbrec/sites/streamray/StreamrayModel.java +++ b/common/src/main/java/ctbrec/sites/streamray/StreamrayModel.java @@ -29,6 +29,7 @@ import java.util.concurrent.ExecutionException; import static ctbrec.Model.State.*; import static ctbrec.io.HttpConstants.*; +import static java.nio.charset.StandardCharsets.UTF_8; public class StreamrayModel extends AbstractModel { @@ -145,7 +146,7 @@ public class StreamrayModel extends AbstractModel { String lname = getName().toLowerCase(); String url = MessageFormat.format("https://images4.streamray.com/images/streamray/won/jpg/{0}/{1}/{2}_640.jpg", lname.substring(0, 1), lname.substring(lname.length() - 1), lname); try { - return MessageFormat.format("https://dynimages.securedataimages.com/unsigned/rs:fill:320::0/g:no/plain/{0}@jpg", URLEncoder.encode(url, "utf-8")); + return MessageFormat.format("https://dynimages.securedataimages.com/unsigned/rs:fill:320::0/g:no/plain/{0}@jpg", URLEncoder.encode(url, UTF_8)); } catch (Exception ex) { return url; } diff --git a/common/src/main/java/ctbrec/sites/stripchat/Stripchat.java b/common/src/main/java/ctbrec/sites/stripchat/Stripchat.java index be1348f8..1b2f21cb 100644 --- a/common/src/main/java/ctbrec/sites/stripchat/Stripchat.java +++ b/common/src/main/java/ctbrec/sites/stripchat/Stripchat.java @@ -1,6 +1,13 @@ package ctbrec.sites.stripchat; -import static ctbrec.io.HttpConstants.*; +import ctbrec.Model; +import ctbrec.io.HttpClient; +import ctbrec.io.HttpException; +import ctbrec.sites.AbstractSite; +import okhttp3.Request; +import okhttp3.Response; +import org.json.JSONArray; +import org.json.JSONObject; import java.io.IOException; import java.net.URLEncoder; @@ -10,15 +17,8 @@ import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; -import org.json.JSONArray; -import org.json.JSONObject; - -import ctbrec.Model; -import ctbrec.io.HttpClient; -import ctbrec.io.HttpException; -import ctbrec.sites.AbstractSite; -import okhttp3.Request; -import okhttp3.Response; +import static ctbrec.io.HttpConstants.USER_AGENT; +import static java.nio.charset.StandardCharsets.UTF_8; public class Stripchat extends AbstractSite { @@ -70,8 +70,8 @@ public class Stripchat extends AbstractSite { throw new IOException("Account settings not available"); } - String username = getConfig().getSettings().stripchatPassword; - String url = baseUri + "/api/v1/user/" + username; + String username = getConfig().getSettings().stripchatUsername; + String url = baseUri + "/api/front/users/username/" + username; Request request = new Request.Builder().url(url).build(); try (Response response = getHttpClient().execute(request)) { if (response.isSuccessful()) { @@ -126,7 +126,7 @@ public class Stripchat extends AbstractSite { @Override public List search(String q) throws IOException, InterruptedException { - String url = baseUri + "/api/front/v2/models/search?limit=20&query=" + URLEncoder.encode(q, "utf-8"); + String url = baseUri + "/api/front/v2/models/search?limit=20&query=" + URLEncoder.encode(q, UTF_8); Request req = new Request.Builder() .url(url) .header(USER_AGENT, getConfig().getSettings().httpUserAgent) diff --git a/common/src/main/java/ctbrec/sites/stripchat/StripchatModel.java b/common/src/main/java/ctbrec/sites/stripchat/StripchatModel.java index c69c9e70..924f3ccf 100644 --- a/common/src/main/java/ctbrec/sites/stripchat/StripchatModel.java +++ b/common/src/main/java/ctbrec/sites/stripchat/StripchatModel.java @@ -23,9 +23,12 @@ import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; import java.text.MessageFormat; +import java.time.Duration; +import java.time.Instant; import java.util.ArrayList; import java.util.List; import java.util.Locale; +import java.util.Optional; import java.util.concurrent.ExecutionException; import static ctbrec.Model.State.*; @@ -36,22 +39,38 @@ import static java.nio.charset.StandardCharsets.UTF_8; public class StripchatModel extends AbstractModel { private static final Logger LOG = LoggerFactory.getLogger(StripchatModel.class); - private String status = null; private int[] resolution = new int[]{0, 0}; + private int modelId = 0; + private boolean isVr = false; + private JSONObject modelInfo; + + private transient Instant lastInfoRequest = Instant.EPOCH; @Override public boolean isOnline(boolean ignoreCache) throws IOException, ExecutionException, InterruptedException { - if (ignoreCache || status == null) { - JSONObject jsonResponse = loadModelInfo(); + if (ignoreCache) { + JSONObject jsonResponse = getModelInfo(); if (jsonResponse.has("user")) { JSONObject user = jsonResponse.getJSONObject("user"); - status = user.optString("status"); + String status = user.optString("status"); mapOnlineState(status); + if (isBanned(user)) { + LOG.debug("Model inactive or deleted: {}", getName()); + setMarkedForLaterRecording(true); + } + modelId = user.optInt("id"); + isVr = user.optBoolean("isVr", false); } } return onlineState == ONLINE; } + private boolean isBanned(JSONObject user) { + boolean isDeleted = user.optBoolean("isDeleted", false); + boolean isApprovedModel = user.optBoolean("isApprovedModel", true); + return (!isApprovedModel || isDeleted); + } + private void mapOnlineState(String status) { switch (status) { case "public" -> setOnlineState(ONLINE); @@ -65,6 +84,15 @@ public class StripchatModel extends AbstractModel { } } + private JSONObject getModelInfo() throws IOException { + if (Duration.between(lastInfoRequest, Instant.now()).getSeconds() < 5) { + return Optional.ofNullable(modelInfo).orElse(new JSONObject()); + } + lastInfoRequest = Instant.now(); + modelInfo = loadModelInfo(); + return modelInfo; + } + private JSONObject loadModelInfo() throws IOException { String name = getName(); String url = getSite().getBaseUrl() + "/api/front/users/username/" + name; @@ -94,8 +122,15 @@ public class StripchatModel extends AbstractModel { try { String originalUrl = url.replace("_auto", ""); masterPlaylist = getMasterPlaylist(originalUrl); - List originalStreamSource = extractStreamSources(masterPlaylist); - streamSources.addAll(originalStreamSource); + for (StreamSource original : extractStreamSources(masterPlaylist)) { + boolean found = false; + for (StreamSource source : streamSources) { + if (source.height == original.height) { + found = true; + } + } + if (!found) streamSources.add(original); + } } catch (Exception e) { LOG.warn("Original stream quality not available", e); } @@ -142,6 +177,12 @@ public class StripchatModel extends AbstractModel { } private String getMasterPlaylistUrl() throws IOException { + boolean VR = Config.getInstance().getSettings().stripchatVR; + String hlsUrlTemplate = "https://edge-hls.doppiocdn.com/hls/{0}{1}/master/{0}{1}_auto.m3u8?playlistType=Standart"; + String vrSuffix = (VR && isVr) ? "_vr" : ""; + if (modelId > 0) { + return MessageFormat.format(hlsUrlTemplate, String.valueOf(modelId), vrSuffix); + } String name = getName(); String url = getSite().getBaseUrl() + "/api/front/models/username/" + name + "/cam?triggerRequest=loadCam"; Request req = new Request.Builder() @@ -157,11 +198,11 @@ public class StripchatModel extends AbstractModel { String body = response.body().string(); LOG.trace(body); JSONObject jsonResponse = new JSONObject(body); - String streamName = jsonResponse.optString("streamName", jsonResponse.optString("")); - JSONObject viewServers = jsonResponse.getJSONObject("viewServers"); - String serverName = viewServers.optString("flashphoner-hls"); - String hslUrlTemplate = "https://b-{0}.doppiocdn.com/hls/{1}/master/{1}_auto.m3u8"; - return MessageFormat.format(hslUrlTemplate, serverName, streamName); + String streamName = jsonResponse.optString("streamName"); + JSONObject broadcastSettings = jsonResponse.getJSONObject("broadcastSettings"); + String vrBroadcastServer = broadcastSettings.optString("vrBroadcastServer"); + vrSuffix = (!VR || vrBroadcastServer.isEmpty()) ? "" : "_vr"; + return MessageFormat.format(hlsUrlTemplate, streamName, vrSuffix); } else { throw new HttpException(response.code(), response.message()); } @@ -171,8 +212,9 @@ public class StripchatModel extends AbstractModel { @Override public void invalidateCacheEntries() { - status = null; resolution = new int[]{0, 0}; + lastInfoRequest = Instant.EPOCH; + modelInfo = null; } @Override @@ -199,7 +241,7 @@ public class StripchatModel extends AbstractModel { @Override public boolean follow() throws IOException { getSite().getHttpClient().login(); - JSONObject modelInfo = loadModelInfo(); + JSONObject modelInfo = getModelInfo(); JSONObject user = modelInfo.getJSONObject("user"); long modelId = user.optLong("id"); @@ -231,7 +273,7 @@ public class StripchatModel extends AbstractModel { @Override public boolean unfollow() throws IOException { getSite().getHttpClient().login(); - JSONObject modelInfo = loadModelInfo(); + JSONObject modelInfo = getModelInfo(); JSONObject user = modelInfo.getJSONObject("user"); long modelId = user.optLong("id"); JSONArray favoriteIds = new JSONArray(); @@ -263,6 +305,28 @@ public class StripchatModel extends AbstractModel { } } + @Override + public boolean exists() throws IOException { + Request req = new Request.Builder() // @formatter:off + .url(getUrl()) + .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) + .build(); // @formatter:on + try (Response response = getSite().getHttpClient().execute(req)) { + if (!response.isSuccessful() && response.code() == 404) { + return false; + } + } + JSONObject jsonResponse = getModelInfo(); + if (jsonResponse.has("user")) { + JSONObject user = jsonResponse.getJSONObject("user"); + if (isBanned(user)) { + LOG.debug("Model inactive or deleted: {}", getName()); + return false; + } + } + return true; + } + @Override public Download createDownload() { if (Config.getInstance().getSettings().useHlsdl) { diff --git a/master/pom.xml b/master/pom.xml index 48a6838e..5dae98ac 100644 --- a/master/pom.xml +++ b/master/pom.xml @@ -18,7 +18,7 @@ UTF-8 17 17 - 20 + 20.0.2 5.7.2