Streamate fixes and new tabs

- Fixed "Couldn't load model ID" error while adding models by URL or by nickname
- Online/Offline switch on all tabs. Up to 10 000 offline models in each category. How do you like it, Elon Musk?
- Added "New Girls" tab and adjusted others. All same models on less tabs.

CHANGELOG.md

@@ -13,6 +13,12 @@
   * Added setting to restrict recording by bit rate
   * Added setting to use the shortest side to restrict the resolution
   * Chaturbate: Added "Gaming" tab
+  * Streamate:
+    - Fixed "Couldn't load model ID" error while adding models by URL or by
+      nickname
+    - Online/Offline switch on all tabs. Up to 10 000 offline models in each 
+      category. How do you like it, Elon Musk?
+    - Added "New Girls" tab and adjusted others. All same models on less tabs.
 
 5.2.3
 ========================

client/src/main/java/ctbrec/ui/sites/streamate/StreamateFollowedService.java

@@ -34,7 +34,7 @@ public class StreamateFollowedService extends PaginatedScheduledService {
 
     public StreamateFollowedService(Streamate streamate) {
         this.streamate = streamate;
-        this.httpClient = (StreamateHttpClient) streamate.getHttpClient();
+        this.httpClient = streamate.getHttpClient();
         this.url = NAIAD_URL + "/favorites?domain=streamate.com&filters=";
     }
 
@@ -43,11 +43,19 @@ public class StreamateFollowedService extends PaginatedScheduledService {
         return new Task<>() {
             @Override
             public List<Model> call() throws IOException {
+                try {
                     httpClient.login();
+                } catch (Exception e) {
+                    LOG.debug("Login was not successful");
+                    return Collections.emptyList();
+                }
                 String saKey = httpClient.getSaKey();
                 String pageUrl = url + "&from=" + ((page - 1) * MODELS_PER_PAGE) + "&size=" + MODELS_PER_PAGE;
                 LOG.debug("Fetching page {}", pageUrl);
-                var smtid = UUID.randomUUID() + "G0211569057409";
+                String smxxx = UUID.randomUUID() + "G0211569057409";
+                String smtid = Optional.ofNullable(httpClient.getCookieValue("smtid")).orElse(smxxx);
+                String smeid = Optional.ofNullable(httpClient.getCookieValue("smeid")).orElse(smxxx);
+                String smvid = Optional.ofNullable(httpClient.getCookieValue("smvid")).orElse(smxxx);
                 var request = new Request.Builder()
                         .url(pageUrl)
                         .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
@@ -58,8 +66,8 @@ public class StreamateFollowedService extends PaginatedScheduledService {
                         .header("sakey", saKey)
                         .header("platform", "SCP")
                         .header("smtid", smtid)
-                        .header("smeid", smtid)
+                        .header("smeid", smeid)
-                        .header("smvid", smtid)
+                        .header("smvid", smvid)
                         .build();
                 try (var response = streamate.getHttpClient().execute(request)) {
                     if (response.isSuccessful()) {

client/src/main/java/ctbrec/ui/sites/streamate/StreamateTab.java

@@ -0,0 +1,39 @@
+package ctbrec.ui.sites.streamate;
+
+import ctbrec.sites.Site;
+import ctbrec.ui.tabs.ThumbOverviewTab;
+import javafx.geometry.Insets;
+import javafx.scene.control.RadioButton;
+import javafx.scene.control.ToggleGroup;
+import javafx.scene.layout.HBox;
+
+public class StreamateTab extends ThumbOverviewTab {
+
+    public StreamateTab(String title, StreamateUpdateService updateService, Site site) {
+        super(title, updateService, site);
+    }
+
+    @Override
+    protected void createGui() {
+        super.createGui();
+        addOnlineOfflineSelector();
+    }
+
+    private void addOnlineOfflineSelector() {
+        var group = new ToggleGroup();
+        var online = new RadioButton("online");
+        online.setToggleGroup(group);
+        var offline = new RadioButton("offline");
+        offline.setToggleGroup(group);
+        pagination.getChildren().add(online);
+        pagination.getChildren().add(offline);
+        HBox.setMargin(online, new Insets(5, 5, 5, 40));
+        HBox.setMargin(offline, new Insets(5, 5, 5, 5));
+        online.setSelected(true);
+        group.selectedToggleProperty().addListener(e -> {
+            ((StreamateUpdateService) updateService).setOnline(online.isSelected());
+            queue.clear();
+            updateService.restart();
+        });
+    }
+}

client/src/main/java/ctbrec/ui/sites/streamate/StreamateTabProvider.java

@@ -19,15 +19,12 @@ public class StreamateTabProvider extends AbstractTabProvider {
     @Override
     protected List<Tab> getSiteTabs(Scene scene) {
         List<Tab> tabs = new ArrayList<>();
-        tabs.add(createTab("Girls", Streamate.NAIAD_URL + "/performers?domain=streamate.com&boostedFilters=&excludedFilters=&useProductScore=false&filters=gender:f;online:true"));
+        tabs.add(createTab("Girls",     Streamate.NAIAD_URL + "/performers?domain=streamate.com&loggedInRec=1&boostedFilters=&country=US&language=en&index=default&filters=gender:f,ff%3Bonline:true"));
-        tabs.add(createTab("Guys", Streamate.NAIAD_URL + "/performers?domain=streamate.com&boostedFilters=&excludedFilters=&useProductScore=false&filters=gender:m;online:true"));
+        tabs.add(createTab("Girls New", Streamate.NAIAD_URL + "/performers?domain=streamate.com&loggedInRec=1&boostedFilters=&country=US&language=en&index=default&filters=gender:f,ff%3Bnew:true%3Bonline:true"));
-        tabs.add(createTab("Couples", Streamate.NAIAD_URL + "/performers?domain=streamate.com&boostedFilters=&excludedFilters=&useProductScore=false&filters=gender:mf;online:true"));
+        tabs.add(createTab("Guys",      Streamate.NAIAD_URL + "/performers?domain=streamate.com&loggedInRec=1&boostedFilters=&country=US&language=en&index=default&filters=gender:m%3Bonline:true"));
-        tabs.add(createTab("Lesbian", Streamate.NAIAD_URL + "/performers?domain=streamate.com&boostedFilters=&excludedFilters=&useProductScore=false&filters=gender:ff;online:true"));
+        tabs.add(createTab("Couples",   Streamate.NAIAD_URL + "/performers?domain=streamate.com&loggedInRec=1&boostedFilters=&country=US&language=en&index=default&filters=gender:mf,mm,g%3Bonline:true"));
-        tabs.add(createTab("Gay", Streamate.NAIAD_URL + "/performers?domain=streamate.com&boostedFilters=&excludedFilters=&useProductScore=false&filters=gender:mm;online:true"));
+        tabs.add(createTab("Trans",     Streamate.NAIAD_URL + "/performers?domain=streamate.com&loggedInRec=1&boostedFilters=&country=US&language=en&index=default&filters=gender:tm2f,tf2m%3Bonline:true"));
-        tabs.add(createTab("Groups", Streamate.NAIAD_URL + "/performers?domain=streamate.com&boostedFilters=&excludedFilters=&useProductScore=false&filters=gender:g;online:true"));
+        tabs.add(createTab("New",       Streamate.NAIAD_URL + "/performers?domain=streamate.com&loggedInRec=1&boostedFilters=&country=US&language=en&index=default&filters=new:true%3Bonline:true"));
-        tabs.add(createTab("Trans female", Streamate.NAIAD_URL + "/performers?domain=streamate.com&boostedFilters=&excludedFilters=&useProductScore=false&filters=gender:tm2f;online:true"));
-        tabs.add(createTab("Trans male", Streamate.NAIAD_URL + "/performers?domain=streamate.com&boostedFilters=&excludedFilters=&useProductScore=false&filters=gender:tf2m;online:true"));
-        tabs.add(createTab("New", Streamate.NAIAD_URL + "/performers?domain=streamate.com&boostedFilters=&excludedFilters=&useProductScore=false&filters=new:true;online:true"));
 
         followedTab = new StreamateFollowedTab((Streamate) site);
         followedTab.setRecorder(recorder);
@@ -43,7 +40,7 @@ public class StreamateTabProvider extends AbstractTabProvider {
 
     private Tab createTab(String title, String url) {
         var updateService = new StreamateUpdateService((Streamate) site, url);
-        var tab = new ThumbOverviewTab(title, updateService, site);
+        var tab = new StreamateTab(title, updateService, site);
         tab.setRecorder(recorder);
         return tab;
     }

client/src/main/java/ctbrec/ui/sites/streamate/StreamateUpdateService.java

@@ -12,10 +12,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
-import java.util.ArrayList;
+import java.util.*;
-import java.util.List;
-import java.util.Objects;
-import java.util.UUID;
 
 import static ctbrec.ErrorMessages.HTTP_RESPONSE_BODY_IS_NULL;
 import static ctbrec.Model.State.*;
@@ -28,12 +25,12 @@ public class StreamateUpdateService extends PaginatedScheduledService {
     private static final int MODELS_PER_PAGE = 48;
     private final Streamate streamate;
     private final StreamateHttpClient httpClient;
-    private final String url;
+    private String url;
 
     public StreamateUpdateService(Streamate streamate, String url) {
         this.streamate = streamate;
         this.url = url;
-        this.httpClient = (StreamateHttpClient) streamate.getHttpClient();
+        this.httpClient = streamate.getHttpClient();
     }
 
     @Override
@@ -45,15 +42,19 @@ public class StreamateUpdateService extends PaginatedScheduledService {
                 String saKey = httpClient.getSaKey();
                 String pageUrl = url + "&from=" + from + "&size=" + MODELS_PER_PAGE;
                 LOG.debug("Fetching page {}", pageUrl);
-                var smtid = UUID.randomUUID() + "G0211569057409";
+                String smxxx = UUID.randomUUID() + "G0211569057409";
+                String smtid = Optional.ofNullable(httpClient.getCookieValue("smtid")).orElse(smxxx);
+                String smeid = Optional.ofNullable(httpClient.getCookieValue("smeid")).orElse(smxxx);
+                String smvid = Optional.ofNullable(httpClient.getCookieValue("smvid")).orElse(smxxx);
+
                 var request = httpClient.newRequestBuilder()
                         .url(pageUrl)
                         .header(ORIGIN, streamate.getBaseUrl())
                         .header("sakey", saKey)
                         .header("platform", "SCP")
                         .header("smtid", smtid)
-                        .header("smeid", smtid)
+                        .header("smeid", smeid)
-                        .header("smvid", smtid)
+                        .header("smvid", smvid)
                         .build();
                 try (var response = httpClient.execute(request)) {
                     if (response.isSuccessful()) {
@@ -98,4 +99,12 @@ public class StreamateUpdateService extends PaginatedScheduledService {
         }
         return models;
     }
+
+    public void setOnline(boolean online) {
+        if (online) {
+            url = url.replace("online:false", "online:true");
+        } else {
+            url = url.replace("online:true", "online:false");
+        }
+    }
 }

common/src/main/java/ctbrec/sites/streamate/StreamateHttpClient.java

@@ -1,12 +1,13 @@
 package ctbrec.sites.streamate;
 
 import ctbrec.Config;
+import ctbrec.StringUtil;
 import ctbrec.io.HttpClient;
 import ctbrec.io.HttpException;
+import lombok.Getter;
+import lombok.extern.slf4j.Slf4j;
 import okhttp3.*;
 import org.json.JSONObject;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
 import java.util.*;
@@ -16,15 +17,17 @@ import java.util.regex.Pattern;
 import static ctbrec.ErrorMessages.HTTP_RESPONSE_BODY_IS_NULL;
 import static ctbrec.io.HttpConstants.*;
 
+@Slf4j
 public class StreamateHttpClient extends HttpClient {
 
     private static final String SAKEY_KEY = "sakey";
 
-    private static final Logger LOG = LoggerFactory.getLogger(StreamateHttpClient.class);
     public static final MediaType JSON = MediaType.parse("application/json; charset=utf-8");
 
     private Long userId;
+    @Getter
     private String saKey = "";
+    @Getter
     private String userNickname = "";
     private String xsrfToken = "";
 
@@ -59,19 +62,19 @@ public class StreamateHttpClient extends HttpClient {
         try (Response resp = execute(req)) {
             if (resp.code() == 200) {
                 String body = Objects.requireNonNull(resp.body(), HTTP_RESPONSE_BODY_IS_NULL).string();
-                LOG.info("Initial request was fine, Extracting XSRF token");
+                log.info("Initial request was fine, Extracting XSRF token");
                 Matcher m = Pattern.compile("\"xsrfToken\":\"(.*?)\"", Pattern.DOTALL).matcher(body);
                 if (m.find()) {
                     xsrfToken = m.group(1);
-                    LOG.info("XSRF token is {}", xsrfToken);
+                    log.info("XSRF token is {}", xsrfToken);
                 } else {
-                    LOG.warn("Couldn't find xsrf in initialData.js");
+                    log.warn("Couldn't find xsrf in initialData.js");
                 }
             } else {
                 throw new HttpException(resp.code(), resp.message());
             }
         } catch (IOException e) {
-            LOG.error("Initial request failed", e);
+            log.error("Initial request failed", e);
         }
     }
 
@@ -84,7 +87,7 @@ public class StreamateHttpClient extends HttpClient {
         boolean cookiesWorked = checkLoginSuccess();
         if (cookiesWorked) {
             loggedIn = true;
-            LOG.info("Logged in with cookies");
+            log.info("Logged in with cookies");
             return true;
         }
 
@@ -93,10 +96,15 @@ public class StreamateHttpClient extends HttpClient {
     }
 
     private synchronized boolean loginWithoutCookies() throws IOException {
+        String username = config.getSettings().streamateUsername;
+        String password = config.getSettings().streamatePassword;
+        if (StringUtil.isBlank(username) || StringUtil.isBlank(password)) {
+            return false;
+        }
         JSONObject loginRequest = new JSONObject();
         loginRequest.put("allowLoginRedirection", true);
-        loginRequest.put("email", config.getSettings().streamateUsername);
+        loginRequest.put("email", username);
-        loginRequest.put("password", config.getSettings().streamatePassword);
+        loginRequest.put("password", password);
         loginRequest.put("referrerId", 0);
         loginRequest.put("siteId", 1);
         loginRequest.put("siteType", "premium");
@@ -126,7 +134,6 @@ public class StreamateHttpClient extends HttpClient {
                 throw new IOException("Login failed: " + response.code() + " " + response.message());
             }
         }
-
         return loggedIn;
     }
 
@@ -166,10 +173,6 @@ public class StreamateHttpClient extends HttpClient {
         }
     }
 
-    public String getSaKey() {
-        return saKey;
-    }
-
     public Long getUserId() throws IOException {
         if (userId == null) {
             loginWithoutCookies();
@@ -177,7 +180,13 @@ public class StreamateHttpClient extends HttpClient {
         return userId;
     }
 
-    public String getUserNickname() {
+    public String getCookieValue(String name) {
-        return userNickname;
+        try {
+            getXsrfToken();
+            Cookie cookie = getCookieJar().getCookie(HttpUrl.parse(Streamate.API_URL), name);
+            return cookie.value();
+        } catch (Exception ex) {
+            return null;
+        }
     }
 }

common/src/main/java/ctbrec/sites/streamate/StreamateModel.java

@@ -121,13 +121,15 @@ public class StreamateModel extends AbstractModel {
     }
 
     void loadModelId() throws IOException {
-        String url = "https://www.streamate.com/api/performer/lookup?nicknames" + URLEncoder.encode(getName(), UTF_8);
+        String url = "https://www.streamate.com/api/performer/lookup?nicknames=" + URLEncoder.encode(getName(), UTF_8);
         Request req = new Request.Builder().url(url)
                 .addHeader(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
                 .addHeader(ACCEPT, "*/*")
                 .addHeader(ACCEPT_LANGUAGE, Locale.ENGLISH.getLanguage())
+                .addHeader(ORIGIN, Streamate.BASE_URL)
                 .addHeader(REFERER, Streamate.BASE_URL + '/' + getName())
                 .addHeader(X_REQUESTED_WITH, XML_HTTP_REQUEST)
+                .addHeader(X_XSRF_TOKEN, ((StreamateHttpClient) site.getHttpClient()).getXsrfToken())
                 .build();
         try (Response response = site.getHttpClient().execute(req)) {
             if (response.isSuccessful()) {