From 531b7b698f8d5c52f68c33df4fcc09869ceb5399 Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Sat, 30 Dec 2023 22:05:57 +0100 Subject: [PATCH] Streamate fixes and new tabs - Fixed "Couldn't load model ID" error while adding models by URL or by nickname - Online/Offline switch on all tabs. Up to 10 000 offline models in each category. How do you like it, Elon Musk? - Added "New Girls" tab and adjusted others. All same models on less tabs. --- CHANGELOG.md | 6 +++ .../streamate/StreamateFollowedService.java | 18 +++++--- .../ui/sites/streamate/StreamateTab.java | 39 ++++++++++++++++ .../sites/streamate/StreamateTabProvider.java | 17 +++---- .../streamate/StreamateUpdateService.java | 27 +++++++---- .../sites/streamate/StreamateHttpClient.java | 45 +++++++++++-------- .../sites/streamate/StreamateModel.java | 4 +- 7 files changed, 113 insertions(+), 43 deletions(-) create mode 100644 client/src/main/java/ctbrec/ui/sites/streamate/StreamateTab.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 01abce69..9ec4cc19 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,12 @@ * Added setting to restrict recording by bit rate * Added setting to use the shortest side to restrict the resolution * Chaturbate: Added "Gaming" tab + * Streamate: + - Fixed "Couldn't load model ID" error while adding models by URL or by + nickname + - Online/Offline switch on all tabs. Up to 10 000 offline models in each + category. How do you like it, Elon Musk? + - Added "New Girls" tab and adjusted others. All same models on less tabs. 5.2.3 ======================== diff --git a/client/src/main/java/ctbrec/ui/sites/streamate/StreamateFollowedService.java b/client/src/main/java/ctbrec/ui/sites/streamate/StreamateFollowedService.java index 611a952d..5a470779 100644 --- a/client/src/main/java/ctbrec/ui/sites/streamate/StreamateFollowedService.java +++ b/client/src/main/java/ctbrec/ui/sites/streamate/StreamateFollowedService.java @@ -34,7 +34,7 @@ public class StreamateFollowedService extends PaginatedScheduledService { public StreamateFollowedService(Streamate streamate) { this.streamate = streamate; - this.httpClient = (StreamateHttpClient) streamate.getHttpClient(); + this.httpClient = streamate.getHttpClient(); this.url = NAIAD_URL + "/favorites?domain=streamate.com&filters="; } @@ -43,11 +43,19 @@ public class StreamateFollowedService extends PaginatedScheduledService { return new Task<>() { @Override public List call() throws IOException { - httpClient.login(); + try { + httpClient.login(); + } catch (Exception e) { + LOG.debug("Login was not successful"); + return Collections.emptyList(); + } String saKey = httpClient.getSaKey(); String pageUrl = url + "&from=" + ((page - 1) * MODELS_PER_PAGE) + "&size=" + MODELS_PER_PAGE; LOG.debug("Fetching page {}", pageUrl); - var smtid = UUID.randomUUID() + "G0211569057409"; + String smxxx = UUID.randomUUID() + "G0211569057409"; + String smtid = Optional.ofNullable(httpClient.getCookieValue("smtid")).orElse(smxxx); + String smeid = Optional.ofNullable(httpClient.getCookieValue("smeid")).orElse(smxxx); + String smvid = Optional.ofNullable(httpClient.getCookieValue("smvid")).orElse(smxxx); var request = new Request.Builder() .url(pageUrl) .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) @@ -58,8 +66,8 @@ public class StreamateFollowedService extends PaginatedScheduledService { .header("sakey", saKey) .header("platform", "SCP") .header("smtid", smtid) - .header("smeid", smtid) - .header("smvid", smtid) + .header("smeid", smeid) + .header("smvid", smvid) .build(); try (var response = streamate.getHttpClient().execute(request)) { if (response.isSuccessful()) { diff --git a/client/src/main/java/ctbrec/ui/sites/streamate/StreamateTab.java b/client/src/main/java/ctbrec/ui/sites/streamate/StreamateTab.java new file mode 100644 index 00000000..f77b0a02 --- /dev/null +++ b/client/src/main/java/ctbrec/ui/sites/streamate/StreamateTab.java @@ -0,0 +1,39 @@ +package ctbrec.ui.sites.streamate; + +import ctbrec.sites.Site; +import ctbrec.ui.tabs.ThumbOverviewTab; +import javafx.geometry.Insets; +import javafx.scene.control.RadioButton; +import javafx.scene.control.ToggleGroup; +import javafx.scene.layout.HBox; + +public class StreamateTab extends ThumbOverviewTab { + + public StreamateTab(String title, StreamateUpdateService updateService, Site site) { + super(title, updateService, site); + } + + @Override + protected void createGui() { + super.createGui(); + addOnlineOfflineSelector(); + } + + private void addOnlineOfflineSelector() { + var group = new ToggleGroup(); + var online = new RadioButton("online"); + online.setToggleGroup(group); + var offline = new RadioButton("offline"); + offline.setToggleGroup(group); + pagination.getChildren().add(online); + pagination.getChildren().add(offline); + HBox.setMargin(online, new Insets(5, 5, 5, 40)); + HBox.setMargin(offline, new Insets(5, 5, 5, 5)); + online.setSelected(true); + group.selectedToggleProperty().addListener(e -> { + ((StreamateUpdateService) updateService).setOnline(online.isSelected()); + queue.clear(); + updateService.restart(); + }); + } +} diff --git a/client/src/main/java/ctbrec/ui/sites/streamate/StreamateTabProvider.java b/client/src/main/java/ctbrec/ui/sites/streamate/StreamateTabProvider.java index 36fbe0d5..d48348d6 100644 --- a/client/src/main/java/ctbrec/ui/sites/streamate/StreamateTabProvider.java +++ b/client/src/main/java/ctbrec/ui/sites/streamate/StreamateTabProvider.java @@ -19,15 +19,12 @@ public class StreamateTabProvider extends AbstractTabProvider { @Override protected List getSiteTabs(Scene scene) { List tabs = new ArrayList<>(); - tabs.add(createTab("Girls", Streamate.NAIAD_URL + "/performers?domain=streamate.com&boostedFilters=&excludedFilters=&useProductScore=false&filters=gender:f;online:true")); - tabs.add(createTab("Guys", Streamate.NAIAD_URL + "/performers?domain=streamate.com&boostedFilters=&excludedFilters=&useProductScore=false&filters=gender:m;online:true")); - tabs.add(createTab("Couples", Streamate.NAIAD_URL + "/performers?domain=streamate.com&boostedFilters=&excludedFilters=&useProductScore=false&filters=gender:mf;online:true")); - tabs.add(createTab("Lesbian", Streamate.NAIAD_URL + "/performers?domain=streamate.com&boostedFilters=&excludedFilters=&useProductScore=false&filters=gender:ff;online:true")); - tabs.add(createTab("Gay", Streamate.NAIAD_URL + "/performers?domain=streamate.com&boostedFilters=&excludedFilters=&useProductScore=false&filters=gender:mm;online:true")); - tabs.add(createTab("Groups", Streamate.NAIAD_URL + "/performers?domain=streamate.com&boostedFilters=&excludedFilters=&useProductScore=false&filters=gender:g;online:true")); - tabs.add(createTab("Trans female", Streamate.NAIAD_URL + "/performers?domain=streamate.com&boostedFilters=&excludedFilters=&useProductScore=false&filters=gender:tm2f;online:true")); - tabs.add(createTab("Trans male", Streamate.NAIAD_URL + "/performers?domain=streamate.com&boostedFilters=&excludedFilters=&useProductScore=false&filters=gender:tf2m;online:true")); - tabs.add(createTab("New", Streamate.NAIAD_URL + "/performers?domain=streamate.com&boostedFilters=&excludedFilters=&useProductScore=false&filters=new:true;online:true")); + tabs.add(createTab("Girls", Streamate.NAIAD_URL + "/performers?domain=streamate.com&loggedInRec=1&boostedFilters=&country=US&language=en&index=default&filters=gender:f,ff%3Bonline:true")); + tabs.add(createTab("Girls New", Streamate.NAIAD_URL + "/performers?domain=streamate.com&loggedInRec=1&boostedFilters=&country=US&language=en&index=default&filters=gender:f,ff%3Bnew:true%3Bonline:true")); + tabs.add(createTab("Guys", Streamate.NAIAD_URL + "/performers?domain=streamate.com&loggedInRec=1&boostedFilters=&country=US&language=en&index=default&filters=gender:m%3Bonline:true")); + tabs.add(createTab("Couples", Streamate.NAIAD_URL + "/performers?domain=streamate.com&loggedInRec=1&boostedFilters=&country=US&language=en&index=default&filters=gender:mf,mm,g%3Bonline:true")); + tabs.add(createTab("Trans", Streamate.NAIAD_URL + "/performers?domain=streamate.com&loggedInRec=1&boostedFilters=&country=US&language=en&index=default&filters=gender:tm2f,tf2m%3Bonline:true")); + tabs.add(createTab("New", Streamate.NAIAD_URL + "/performers?domain=streamate.com&loggedInRec=1&boostedFilters=&country=US&language=en&index=default&filters=new:true%3Bonline:true")); followedTab = new StreamateFollowedTab((Streamate) site); followedTab.setRecorder(recorder); @@ -43,7 +40,7 @@ public class StreamateTabProvider extends AbstractTabProvider { private Tab createTab(String title, String url) { var updateService = new StreamateUpdateService((Streamate) site, url); - var tab = new ThumbOverviewTab(title, updateService, site); + var tab = new StreamateTab(title, updateService, site); tab.setRecorder(recorder); return tab; } diff --git a/client/src/main/java/ctbrec/ui/sites/streamate/StreamateUpdateService.java b/client/src/main/java/ctbrec/ui/sites/streamate/StreamateUpdateService.java index 4e8cc32e..b6ce4ff3 100644 --- a/client/src/main/java/ctbrec/ui/sites/streamate/StreamateUpdateService.java +++ b/client/src/main/java/ctbrec/ui/sites/streamate/StreamateUpdateService.java @@ -12,10 +12,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.Objects; -import java.util.UUID; +import java.util.*; import static ctbrec.ErrorMessages.HTTP_RESPONSE_BODY_IS_NULL; import static ctbrec.Model.State.*; @@ -28,12 +25,12 @@ public class StreamateUpdateService extends PaginatedScheduledService { private static final int MODELS_PER_PAGE = 48; private final Streamate streamate; private final StreamateHttpClient httpClient; - private final String url; + private String url; public StreamateUpdateService(Streamate streamate, String url) { this.streamate = streamate; this.url = url; - this.httpClient = (StreamateHttpClient) streamate.getHttpClient(); + this.httpClient = streamate.getHttpClient(); } @Override @@ -45,15 +42,19 @@ public class StreamateUpdateService extends PaginatedScheduledService { String saKey = httpClient.getSaKey(); String pageUrl = url + "&from=" + from + "&size=" + MODELS_PER_PAGE; LOG.debug("Fetching page {}", pageUrl); - var smtid = UUID.randomUUID() + "G0211569057409"; + String smxxx = UUID.randomUUID() + "G0211569057409"; + String smtid = Optional.ofNullable(httpClient.getCookieValue("smtid")).orElse(smxxx); + String smeid = Optional.ofNullable(httpClient.getCookieValue("smeid")).orElse(smxxx); + String smvid = Optional.ofNullable(httpClient.getCookieValue("smvid")).orElse(smxxx); + var request = httpClient.newRequestBuilder() .url(pageUrl) .header(ORIGIN, streamate.getBaseUrl()) .header("sakey", saKey) .header("platform", "SCP") .header("smtid", smtid) - .header("smeid", smtid) - .header("smvid", smtid) + .header("smeid", smeid) + .header("smvid", smvid) .build(); try (var response = httpClient.execute(request)) { if (response.isSuccessful()) { @@ -98,4 +99,12 @@ public class StreamateUpdateService extends PaginatedScheduledService { } return models; } + + public void setOnline(boolean online) { + if (online) { + url = url.replace("online:false", "online:true"); + } else { + url = url.replace("online:true", "online:false"); + } + } } diff --git a/common/src/main/java/ctbrec/sites/streamate/StreamateHttpClient.java b/common/src/main/java/ctbrec/sites/streamate/StreamateHttpClient.java index e9cfb6f8..14dc5f34 100644 --- a/common/src/main/java/ctbrec/sites/streamate/StreamateHttpClient.java +++ b/common/src/main/java/ctbrec/sites/streamate/StreamateHttpClient.java @@ -1,12 +1,13 @@ package ctbrec.sites.streamate; import ctbrec.Config; +import ctbrec.StringUtil; import ctbrec.io.HttpClient; import ctbrec.io.HttpException; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; import okhttp3.*; import org.json.JSONObject; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.*; @@ -16,15 +17,17 @@ import java.util.regex.Pattern; import static ctbrec.ErrorMessages.HTTP_RESPONSE_BODY_IS_NULL; import static ctbrec.io.HttpConstants.*; +@Slf4j public class StreamateHttpClient extends HttpClient { private static final String SAKEY_KEY = "sakey"; - private static final Logger LOG = LoggerFactory.getLogger(StreamateHttpClient.class); public static final MediaType JSON = MediaType.parse("application/json; charset=utf-8"); private Long userId; + @Getter private String saKey = ""; + @Getter private String userNickname = ""; private String xsrfToken = ""; @@ -59,19 +62,19 @@ public class StreamateHttpClient extends HttpClient { try (Response resp = execute(req)) { if (resp.code() == 200) { String body = Objects.requireNonNull(resp.body(), HTTP_RESPONSE_BODY_IS_NULL).string(); - LOG.info("Initial request was fine, Extracting XSRF token"); + log.info("Initial request was fine, Extracting XSRF token"); Matcher m = Pattern.compile("\"xsrfToken\":\"(.*?)\"", Pattern.DOTALL).matcher(body); if (m.find()) { xsrfToken = m.group(1); - LOG.info("XSRF token is {}", xsrfToken); + log.info("XSRF token is {}", xsrfToken); } else { - LOG.warn("Couldn't find xsrf in initialData.js"); + log.warn("Couldn't find xsrf in initialData.js"); } } else { throw new HttpException(resp.code(), resp.message()); } } catch (IOException e) { - LOG.error("Initial request failed", e); + log.error("Initial request failed", e); } } @@ -84,7 +87,7 @@ public class StreamateHttpClient extends HttpClient { boolean cookiesWorked = checkLoginSuccess(); if (cookiesWorked) { loggedIn = true; - LOG.info("Logged in with cookies"); + log.info("Logged in with cookies"); return true; } @@ -93,10 +96,15 @@ public class StreamateHttpClient extends HttpClient { } private synchronized boolean loginWithoutCookies() throws IOException { + String username = config.getSettings().streamateUsername; + String password = config.getSettings().streamatePassword; + if (StringUtil.isBlank(username) || StringUtil.isBlank(password)) { + return false; + } JSONObject loginRequest = new JSONObject(); loginRequest.put("allowLoginRedirection", true); - loginRequest.put("email", config.getSettings().streamateUsername); - loginRequest.put("password", config.getSettings().streamatePassword); + loginRequest.put("email", username); + loginRequest.put("password", password); loginRequest.put("referrerId", 0); loginRequest.put("siteId", 1); loginRequest.put("siteType", "premium"); @@ -126,7 +134,6 @@ public class StreamateHttpClient extends HttpClient { throw new IOException("Login failed: " + response.code() + " " + response.message()); } } - return loggedIn; } @@ -166,18 +173,20 @@ public class StreamateHttpClient extends HttpClient { } } - public String getSaKey() { - return saKey; - } - public Long getUserId() throws IOException { - if(userId == null) { + if (userId == null) { loginWithoutCookies(); } return userId; } - public String getUserNickname() { - return userNickname; + public String getCookieValue(String name) { + try { + getXsrfToken(); + Cookie cookie = getCookieJar().getCookie(HttpUrl.parse(Streamate.API_URL), name); + return cookie.value(); + } catch (Exception ex) { + return null; + } } } diff --git a/common/src/main/java/ctbrec/sites/streamate/StreamateModel.java b/common/src/main/java/ctbrec/sites/streamate/StreamateModel.java index 1158177f..dede64a0 100644 --- a/common/src/main/java/ctbrec/sites/streamate/StreamateModel.java +++ b/common/src/main/java/ctbrec/sites/streamate/StreamateModel.java @@ -121,13 +121,15 @@ public class StreamateModel extends AbstractModel { } void loadModelId() throws IOException { - String url = "https://www.streamate.com/api/performer/lookup?nicknames" + URLEncoder.encode(getName(), UTF_8); + String url = "https://www.streamate.com/api/performer/lookup?nicknames=" + URLEncoder.encode(getName(), UTF_8); Request req = new Request.Builder().url(url) .addHeader(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) .addHeader(ACCEPT, "*/*") .addHeader(ACCEPT_LANGUAGE, Locale.ENGLISH.getLanguage()) + .addHeader(ORIGIN, Streamate.BASE_URL) .addHeader(REFERER, Streamate.BASE_URL + '/' + getName()) .addHeader(X_REQUESTED_WITH, XML_HTTP_REQUEST) + .addHeader(X_XSRF_TOKEN, ((StreamateHttpClient) site.getHttpClient()).getXsrfToken()) .build(); try (Response response = site.getHttpClient().execute(req)) { if (response.isSuccessful()) {