Switch to much simpler JSON api

client/src/main/java/ctbrec/ui/sites/streamate/StreamateTabProvider.java

@@ -1,8 +1,6 @@
 package ctbrec.ui.sites.streamate;
 
-import java.io.ByteArrayOutputStream;
 import java.io.IOException;
-import java.io.InputStream;
 import java.util.ArrayList;
 import java.util.List;
 
@@ -30,7 +28,15 @@ public class StreamateTabProvider extends TabProvider {
     public List<Tab> getTabs(Scene scene) {
         List<Tab> tabs = new ArrayList<>();
         try {
-            tabs.add(createTab("Girls", "/ctbrec/ui/sites/streamate/girls.sml"));
+            tabs.add(createTab("Girls",         Streamate.BASE_URL + "/api/search/list?domain=streamate.com&index=availperf&filters=gender:f"));
+            tabs.add(createTab("Guys",          Streamate.BASE_URL + "/api/search/list?domain=streamate.com&index=availperf&filters=gender:m"));
+            tabs.add(createTab("Couples",       Streamate.BASE_URL + "/api/search/list?domain=streamate.com&index=availperf&filters=gender:mf"));
+            tabs.add(createTab("Lesbian",       Streamate.BASE_URL + "/api/search/list?domain=streamate.com&index=availperf&filters=gender:ff"));
+            tabs.add(createTab("Gay",           Streamate.BASE_URL + "/api/search/list?domain=streamate.com&index=availperf&filters=gender:mm"));
+            tabs.add(createTab("Groups",        Streamate.BASE_URL + "/api/search/list?domain=streamate.com&index=availperf&filters=gender:g"));
+            tabs.add(createTab("Trans female",  Streamate.BASE_URL + "/api/search/list?domain=streamate.com&index=availperf&filters=gender:tm2f"));
+            tabs.add(createTab("Trans male",    Streamate.BASE_URL + "/api/search/list?domain=streamate.com&index=availperf&filters=gender:tf2m"));
+            tabs.add(createTab("New",           Streamate.BASE_URL + "/api/search/list?domain=streamate.com&index=availperf&filters=new:true"));
         } catch (IOException e) {
             LOG.error("Couldn't create streamate tab", e);
         }
@@ -42,21 +48,10 @@ public class StreamateTabProvider extends TabProvider {
         return null;
     }
 
-    private Tab createTab(String title, String queryFile) throws IOException {
+    private Tab createTab(String title, String url) throws IOException {
-        StreamateUpdateService updateService = new StreamateUpdateService(loadQuery(queryFile), streamate);
+        StreamateUpdateService updateService = new StreamateUpdateService(streamate, url);
         ThumbOverviewTab tab = new ThumbOverviewTab(title, updateService, streamate);
         tab.setRecorder(recorder);
         return tab;
     }
-
-    private String loadQuery(String file) throws IOException {
-        InputStream is = getClass().getResourceAsStream(file);
-        ByteArrayOutputStream bos = new ByteArrayOutputStream();
-        byte[] b = new byte[1024];
-        int len = -1;
-        while( (len = is.read(b)) >= 0) {
-            bos.write(b, 0, len);
-        }
-        return new String(bos.toByteArray(), "utf-8");
-    }
 }

client/src/main/java/ctbrec/ui/sites/streamate/StreamateUpdateService.java

@@ -1,44 +1,39 @@
 package ctbrec.ui.sites.streamate;
 
-import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 
-import javax.xml.parsers.DocumentBuilderFactory;
 import javax.xml.parsers.ParserConfigurationException;
 import javax.xml.xpath.XPathExpressionException;
 
+import org.json.JSONArray;
+import org.json.JSONObject;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import org.w3c.dom.Document;
-import org.w3c.dom.Node;
-import org.w3c.dom.NodeList;
 import org.xml.sax.SAXException;
 
 import ctbrec.Config;
 import ctbrec.Model;
-import ctbrec.io.XmlParserUtils;
+import ctbrec.io.HttpException;
 import ctbrec.sites.streamate.Streamate;
 import ctbrec.sites.streamate.StreamateModel;
 import ctbrec.ui.PaginatedScheduledService;
 import javafx.concurrent.Task;
-import okhttp3.MediaType;
 import okhttp3.Request;
-import okhttp3.RequestBody;
 import okhttp3.Response;
 
 public class StreamateUpdateService extends PaginatedScheduledService {
 
     private static final transient Logger LOG = LoggerFactory.getLogger(StreamateUpdateService.class);
 
-    private static final String URL = "http://affiliate.streamate.com/SMLive/SMLResult.xml";
+    private static final int MODELS_PER_PAGE = 48;
     private Streamate streamate;
-    private String query;
+    private String url;
 
-    public StreamateUpdateService(String query, Streamate streamate) {
+    public StreamateUpdateService(Streamate streamate, String url) {
-        this.query = query;
         this.streamate = streamate;
+        this.url = url;
     }
 
     @Override
@@ -46,47 +41,35 @@ public class StreamateUpdateService extends PaginatedScheduledService {
         return new Task<List<Model>>() {
             @Override
             public List<Model> call() throws IOException, SAXException, ParserConfigurationException, XPathExpressionException {
-                LOG.debug("Fetching page {}", URL);
+                int from = (page - 1) * MODELS_PER_PAGE;
-                String q = query
+                String _url = url + "&from=" + from + "&size=" + MODELS_PER_PAGE;
-                        .replace("{maxresults}", "50")
+                LOG.debug("Fetching page {}", _url);
-                        .replace("{pagenum}", Integer.toString(page));
-                //LOG.debug("Query:\n{}", q);
-                RequestBody body = RequestBody.create(MediaType.parse("text/xml"), q);
                 Request request = new Request.Builder()
-                        .url(URL)
+                        .url(_url)
                         .addHeader("User-Agent", Config.getInstance().getSettings().httpUserAgent)
-                        .addHeader("Accept", "text/xml, */*")
+                        .addHeader("Accept", "application/json, */*")
                         .addHeader("Accept-Language", "en")
                         .addHeader("Referer", streamate.getBaseUrl())
-                        .post(body)
                         .build();
-                Response response = streamate.getHttpClient().execute(request);
+                try(Response response = streamate.getHttpClient().execute(request)) {
-                if (response.isSuccessful()) {
+                    if (response.isSuccessful()) {
-                    List<Model> models = new ArrayList<>();
+                        List<Model> models = new ArrayList<>();
-                    String content = response.body().string();
+                        String content = response.body().string();
-                    ByteArrayInputStream in = new ByteArrayInputStream(content.getBytes("utf-8"));
+                        JSONObject json = new JSONObject(content);
-                    Document doc = DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(in);
+                        JSONArray performers = json.getJSONArray("performers");
-                    NodeList performers = doc.getElementsByTagName("Performer");
+                        for (int i = 0; i < performers.length(); i++) {
-                    for (int i = 0; i < performers.getLength(); i++) {
+                            JSONObject p = performers.getJSONObject(i);
-                        Node performer = performers.item(i);
+                            String nickname = p.getString("nickname");
-                        String name = performer.getAttributes().getNamedItem("Name").getNodeValue();
+                            StreamateModel model = (StreamateModel) streamate.createModel(nickname);
-                        String id = performer.getAttributes().getNamedItem("Id").getNodeValue();
+                            model.setId(Long.toString(p.getLong("id")));
-                        String GoldShow = performer.getAttributes().getNamedItem("GoldShow").getNodeValue();
+                            model.setPreview(p.getString("thumbnail"));
-                        String PreGoldShow = performer.getAttributes().getNamedItem("PreGoldShow").getNodeValue();
+                            model.setOnline(p.optBoolean("online"));
-                        String PartyChat = performer.getAttributes().getNamedItem("PartyChat").getNodeValue();
+                            models.add(model);
-                        StreamateModel model = (StreamateModel) streamate.createModel(name);
+                        }
-                        model.setId(id);
+                        return models;
-                        models.add(model);
+                    } else {
-                        Node pic = XmlParserUtils.getNodeWithXpath(performer, "Media/Pic/Full");
+                        throw new HttpException(response.code(), response.message());
-                        String previewUrl = "https:" + pic.getAttributes().getNamedItem("Src").getNodeValue();
-                        model.setPreview(previewUrl);
-                        //LOG.debug("Name {} - {}{}{}", name, PartyChat, PreGoldShow, GoldShow);
                     }
-                    return models;
-                } else {
-                    int code = response.code();
-                    response.close();
-                    throw new IOException("HTTP status " + code);
                 }
             }
         };

client/src/main/resources/ctbrec/ui/sites/streamate/girls.sml

@@ -1,18 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<SMLQuery>
-	<Options MaxResults="{maxresults}" />
-	<AvailablePerformers Exact="false" PageNum="{pagenum}" CountTotalResults="true" QueryId="Girls">
-		<Include>
-			<Rating />
-			<Descriptions />
-			<Media>biopic, staticbiopic</Media>
-			<HDSort />
-		</Include>
-		<Constraints>
-			<StreamType>live,recorded</StreamType>
-			<!--
-			<Name>Name</Name> 
-			-->
-		</Constraints>
-	</AvailablePerformers>
-</SMLQuery>

common/src/main/java/ctbrec/sites/streamate/StreamateModel.java

@@ -3,8 +3,6 @@ package ctbrec.sites.streamate;
 import static ctbrec.Model.State.*;
 
 import java.io.IOException;
-import java.io.InputStream;
-import java.net.URLEncoder;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
@@ -15,16 +13,8 @@ import org.json.JSONObject;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.iheartradio.m3u8.Encoding;
-import com.iheartradio.m3u8.Format;
 import com.iheartradio.m3u8.ParseException;
-import com.iheartradio.m3u8.ParsingMode;
 import com.iheartradio.m3u8.PlaylistException;
-import com.iheartradio.m3u8.PlaylistParser;
-import com.iheartradio.m3u8.data.MasterPlaylist;
-import com.iheartradio.m3u8.data.Playlist;
-import com.iheartradio.m3u8.data.PlaylistData;
-import com.iheartradio.m3u8.data.StreamInfo;
 
 import ctbrec.AbstractModel;
 import ctbrec.Config;
@@ -45,10 +35,17 @@ public class StreamateModel extends AbstractModel {
     @Override
     public boolean isOnline(boolean ignoreCache) throws IOException, ExecutionException, InterruptedException {
         if(ignoreCache) {
-            JSONObject roomInfo = getRoomInfo();
+            String url = "https://sea1c-ls.naiadsystems.com/sea1c-edge-ls/80/live/s:" + getName() + ".json";
-            JSONObject stream = roomInfo.getJSONObject("stream");
+            Request req = new Request.Builder().url(url)
-            String serverId = stream.optString("serverId");
+                    .addHeader("User-Agent", Config.getInstance().getSettings().httpUserAgent)
-            online = !serverId.equals("0");
+                    .addHeader("Accept", "*/*")
+                    .addHeader("Accept-Language", "en")
+                    .addHeader("Referer", Streamate.BASE_URL + '/' + getName())
+                    .addHeader("X-Requested-With", "XMLHttpRequest")
+                    .build();
+            try(Response response = site.getHttpClient().execute(req)) {
+                online = response.isSuccessful();
+            }
         }
         return online;
     }
@@ -76,137 +73,48 @@ public class StreamateModel extends AbstractModel {
 
     @Override
     public List<StreamSource> getStreamSources() throws IOException, ExecutionException, ParseException, PlaylistException {
-        String streamUrl = getStreamUrl();
+        String url = "https://sea1c-ls.naiadsystems.com/sea1c-edge-ls/80/live/s:" + getName() + ".json";
-        if (streamUrl == null) {
+        Request req = new Request.Builder().url(url)
-            return Collections.emptyList();
-        }
-        LOG.debug(streamUrl);
-        Request req = new Request.Builder().url(streamUrl).build();
-        try(Response response = site.getHttpClient().execute(req)) {
-            if(response.isSuccessful()) {
-                InputStream inputStream = response.body().byteStream();
-                PlaylistParser parser = new PlaylistParser(inputStream, Format.EXT_M3U, Encoding.UTF_8, ParsingMode.LENIENT);
-                Playlist playlist = parser.parse();
-                MasterPlaylist master = playlist.getMasterPlaylist();
-                streamSources.clear();
-                for (PlaylistData playlistData : master.getPlaylists()) {
-                    StreamSource streamsource = new StreamSource();
-                    streamsource.mediaPlaylistUrl = playlistData.getUri();
-                    if (playlistData.hasStreamInfo()) {
-                        StreamInfo info = playlistData.getStreamInfo();
-                        streamsource.bandwidth = info.getBandwidth();
-                        streamsource.width = info.hasResolution() ? info.getResolution().width : 0;
-                        streamsource.height = info.hasResolution() ? info.getResolution().height : 0;
-                    } else {
-                        streamsource.bandwidth = 0;
-                        streamsource.width = 0;
-                        streamsource.height = 0;
-                    }
-                    streamSources.add(streamsource);
-                }
-            } else {
-                throw new HttpException(response.code(), response.message());
-            }
-        }
-        return streamSources;
-    }
-
-    private String getStreamUrl() throws IOException {
-        JSONObject json = getRoomInfo();
-        JSONObject performer = json.getJSONObject("performer");
-        id = Long.toString(performer.getLong("id"));
-        JSONObject stream = json.getJSONObject("stream");
-        String sserver = stream.getString("serverId");
-        String streamId = stream.getString("streamId");
-        String wsHost = stream.getString("nodeHost");
-        JSONObject liveservices = json.getJSONObject("liveservices");
-        String streamHost = liveservices.getString("host").replace("wss", "https");
-
-        String roomId;
-        try {
-            roomId = getRoomId(wsHost, sserver, streamId);
-            LOG.debug("room id: {}", roomId);
-        } catch (InterruptedException e) {
-            throw new IOException("Couldn't get room id", e);
-        }
-
-        String streamFormatUrl = getStreamFormatUrl(streamHost, roomId);
-        return getMasterPlaylistUrl(streamFormatUrl);
-    }
-
-    private String getMasterPlaylistUrl(String url) throws IOException {
-        LOG.debug(url);
-        Request req = new Request.Builder()
                 .addHeader("User-Agent", Config.getInstance().getSettings().httpUserAgent)
                 .addHeader("Accept", "*/*")
                 .addHeader("Accept-Language", "en")
                 .addHeader("Referer", Streamate.BASE_URL + '/' + getName())
                 .addHeader("X-Requested-With", "XMLHttpRequest")
-                .url(url)
                 .build();
         try(Response response = site.getHttpClient().execute(req)) {
             if(response.isSuccessful()) {
                 JSONObject json = new JSONObject(response.body().string());
                 JSONObject formats = json.getJSONObject("formats");
+                JSONObject ws = formats.getJSONObject("mp4-ws");
                 JSONObject hls = formats.getJSONObject("mp4-hls");
-                return hls.getString("manifest");
+
-            } else {
+                // add encodings
-                throw new HttpException(response.code(), response.message());
+                JSONArray encodings = hls.getJSONArray("encodings");
-            }
+                streamSources.clear();
-        }
+                for (int i = 0; i < encodings.length(); i++) {
-    }
+                    JSONObject encoding = encodings.getJSONObject(i);
-
+                    StreamSource src = new StreamSource();
-    private String getStreamFormatUrl(String streamHost, String roomId) throws IOException {
+                    src.mediaPlaylistUrl = encoding.getString("location");
-        String url = streamHost + "/videourl?payload="
+                    src.width = encoding.optInt("videoWidth");
-                + URLEncoder.encode("{\"puserid\":" + id + ",\"roomid\":\"" + roomId + "\",\"showtype\":1,\"nginx\":1}", "utf-8");
+                    src.height = encoding.optInt("videoHeight");
-        LOG.debug(url);
+                    src.bandwidth = (encoding.optInt("videoKbps") + encoding.optInt("audioKbps")) * 1024;
-        Request req = new Request.Builder()
+                    streamSources.add(src);
-                .addHeader("User-Agent", Config.getInstance().getSettings().httpUserAgent)
+                }
-                .addHeader("Accept", "*/*")
+
-                .addHeader("Accept-Language", "en")
+                // add raw source stream
-                .addHeader("Referer", Streamate.BASE_URL + '/' + getName())
+                JSONObject origin = hls.getJSONObject("origin");
-                .addHeader("X-Requested-With", "XMLHttpRequest")
+                StreamSource src = new StreamSource();
-                .url(url)
+                src.mediaPlaylistUrl = origin.getString("location");
-                .build();
+                origin = ws.getJSONObject("origin"); // switch to web socket origin, because it has width, height and bitrates
-        try(Response response = site.getHttpClient().execute(req)) {
+                src.width = origin.optInt("videoWidth");
-            if(response.isSuccessful()) {
+                src.height = origin.optInt("videoHeight");
-                JSONArray streamConfig = new JSONArray(response.body().string());
+                src.bandwidth = (origin.optInt("videoKbps") + origin.optInt("audioKbps")) * 1024;
-                JSONObject obj = streamConfig.getJSONObject(0);
+                streamSources.add(src);
-                return obj.getString("url");
-            } else {
-                throw new HttpException(response.code(), response.message());
-            }
-        }
-    }
-
-    private String getRoomId(String wsHost, String sserver, String streamId) throws InterruptedException {
-        String wsUrl = wsHost + "/socket.io/?"
-                + "performerid=" + id
-                + "&sserver=" + sserver
-                + "&streamid=" + streamId
-                + "&sakey=&sessiontype=preview&perfdiscountid=0&minduration=0&goldshowid=0&version=7&referrer=hybrid.client.6.3.16/avchat.swf&usertype=false&lang=en&EIO=3&transport=websocket";
-
-        StreamateWebsocketClient wsClient = new StreamateWebsocketClient(wsUrl, site.getHttpClient());
-        return wsClient.getRoomId();
-    }
-
-    private JSONObject getRoomInfo() throws IOException {
-        String url = "https://hybridclient.naiadsystems.com/api/v1/config/?sabasic=&sakey=&sk=www.streamate.com&userid=0&version=6.3.16&ajax=1&name=" + getName();
-        Request req = new Request.Builder()
-                .addHeader("User-Agent", Config.getInstance().getSettings().httpUserAgent)
-                .addHeader("Accept", "application/json, text/javascript, */*")
-                .addHeader("Accept-Language", "en")
-                .addHeader("Referer", Streamate.BASE_URL + '/' + getName())
-                .addHeader("X-Requested-With", "XMLHttpRequest")
-                .url(url)
-                .build();
-        try(Response response = site.getHttpClient().execute(req)) {
-            if(response.isSuccessful()) {
-                return new JSONObject(response.body().string());
             } else {
                 throw new HttpException(response.code(), response.message());
             }
         }
+        return streamSources;
     }
 
     @Override