Determine online state of Cam4 models through the chat websocket

common/src/main/java/ctbrec/sites/ModelOfflineException.java

@@ -0,0 +1,10 @@
+package ctbrec.sites;
+
+import ctbrec.Model;
+
+public class ModelOfflineException extends RuntimeException {
+
+    public ModelOfflineException(Model model) {
+        super("Model " + model + " is offline");
+    }
+}

common/src/main/java/ctbrec/sites/cam4/Cam4Model.java

@@ -6,15 +6,18 @@ import static java.util.regex.Pattern.*;
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.time.Duration;
+import java.time.Instant;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
-import org.json.JSONArray;
 import org.json.JSONObject;
 import org.jsoup.nodes.Element;
 import org.slf4j.Logger;
@@ -44,6 +47,7 @@ import okhttp3.Response;
 public class Cam4Model extends AbstractModel {
 
     private static final Logger LOG = LoggerFactory.getLogger(Cam4Model.class);
+    private transient Instant playlistRequestTimestamp  = Instant.EPOCH;
     private String playlistUrl;
     private int[] resolution = null;
     private boolean privateRoom = false;
@@ -53,58 +57,34 @@ public class Cam4Model extends AbstractModel {
         if (ignoreCache || onlineState == UNKNOWN) {
             try {
                 loadModelDetails();
-            } catch (ModelDetailsEmptyException e) {
+                getPlaylistUrl();
-                // nothing to do, keep going
+            } catch (Exception e) {
-            }
+                onlineState = OFFLINE;
-            if (playlistUrl == null || onlineState == OFFLINE) {
-                try {
-                    getPlaylistUrl();
-                    onlineState = ONLINE;
-                } catch (IOException e) {
-                    return false;
-                }
             }
         }
         return onlineState == ONLINE && !privateRoom && playlistUrl != null && !playlistUrl.isEmpty();
     }
 
-    private void loadModelDetails() throws IOException, ModelDetailsEmptyException {
+    private void loadModelDetails() throws IOException {
-        String url = site.getBaseUrl() + "/directoryCams?directoryJson=true&online=true&username=" + getName();
+        JSONObject roomState = new Cam4WsClient(Config.getInstance(), (Cam4)getSite(), this).getRoomState();
-        LOG.trace("Loading model details {}", url);
+        if(LOG.isTraceEnabled()) LOG.trace(roomState.toString(2));
-        Request req = new Request.Builder().url(url).build();
+        String state = roomState.optString("newShowsState");
-        try (Response response = site.getHttpClient().execute(req)) {
+        setOnlineStateByShowType(state);
-            if (response.isSuccessful()) {
+        privateRoom = roomState.optBoolean("privateRoom");
-                JSONArray json = new JSONArray(response.body().string());
+        setDescription(roomState.optString("status"));
-                if (json.length() == 0) {
-                    onlineState = OFFLINE;
-                    throw new ModelDetailsEmptyException("Model details are empty");
-                }
-                JSONObject details = json.getJSONObject(0);
-                String showType = details.getString("showType");
-                setOnlineStateByShowType(showType);
-                playlistUrl = details.getString("hlsPreviewUrl");
-                privateRoom = details.getBoolean("privateRoom");
-                if (privateRoom) {
-                    onlineState = PRIVATE;
-                }
-                if (details.has("resolution")) {
-                    String res = details.getString("resolution");
-                    String[] tokens = res.split(":");
-                    resolution = new int[] { Integer.parseInt(tokens[0]), Integer.parseInt(tokens[1]) };
-                }
-            } else {
-                throw new HttpException(response.code(), response.message());
-            }
-        }
     }
 
     public void setOnlineStateByShowType(String showType) {
         switch(showType) {
         case "NORMAL":
+        case "ACCEPTING":
         case "GROUP_SHOW_SELLING_TICKETS":
+        case "GS_SELLING_TICKETS":
+        case "GS_SELLING_TICKETS_UNSUCCESSFUL":
             onlineState = ONLINE;
             break;
         case "PRIVATE_SHOW":
+        case "INSIDE_PS":
             onlineState = PRIVATE;
             break;
         case "GROUP_SHOW":
@@ -114,7 +94,7 @@ public class Cam4Model extends AbstractModel {
             onlineState = OFFLINE;
             break;
         default:
-            LOG.debug("Unknown show type [{}]", showType);
+            LOG.debug("############################## Unknown show type [{}]", showType);
             onlineState = UNKNOWN;
         }
 
@@ -128,7 +108,7 @@ public class Cam4Model extends AbstractModel {
             if(onlineState == UNKNOWN) {
                 try {
                     loadModelDetails();
-                } catch (ModelDetailsEmptyException e) {
+                } catch (Exception e) {
                     LOG.warn("Couldn't load model details {}", e.getMessage());
                 }
             }
@@ -137,7 +117,7 @@ public class Cam4Model extends AbstractModel {
     }
 
     private String getPlaylistUrl() throws IOException {
-        if (playlistUrl == null || playlistUrl.trim().isEmpty()) {
+        if (playlistUrl == null || playlistUrl.trim().isEmpty() || playlistIsOutdated()) {
             String page = loadModelPage();
             Matcher m = Pattern.compile("hlsUrl\\s*:\\s*'(.*?)'", DOTALL | MULTILINE).matcher(page);
             if (m.find()) {
@@ -145,21 +125,29 @@ public class Cam4Model extends AbstractModel {
             } else {
                 m = Pattern.compile("\"videoPlayUrl\"\\s*:\\s*\"(.*?)\"", DOTALL | MULTILINE).matcher(page);
                 if (m.find()) {
-                    String streamName = m.group(1);
+                    generatePlaylistUrlFromStreamName(m.group(1));
-                    m = Pattern.compile(".*?-(\\d{3,})-.*?").matcher(streamName);
-                    if (m.find()) {
-                        String number = m.group(1);
-                        playlistUrl = "https://cam4-hls.xcdnpro.com/" + number + "/cam4-origin-live/ngrp:" + streamName + "_all/playlist.m3u8";
-                    }
                 }
             }
             if (playlistUrl == null) {
                 throw new IOException("Couldn't determine playlist url");
             }
+            playlistRequestTimestamp = Instant.now();
         }
         return playlistUrl;
     }
 
+    private boolean playlistIsOutdated() {
+        return Duration.between(playlistRequestTimestamp, Instant.now()).getSeconds() > TimeUnit.MINUTES.toSeconds(2);
+    }
+
+    private void generatePlaylistUrlFromStreamName(String streamName) {
+        Matcher m = Pattern.compile(".*?-(\\d{3,})-.*").matcher(streamName);
+        if (m.find()) {
+            String number = m.group(1);
+            playlistUrl = "https://cam4-hls.xcdnpro.com/" + number + "/cam4-origin-live/ngrp:" + streamName + "_all/playlist.m3u8";
+        }
+    }
+
     private String loadModelPage() throws IOException {
         Request req = new Request.Builder().url(getUrl()).build();
         try (Response response = site.getHttpClient().execute(req)) {
@@ -192,7 +180,7 @@ public class Cam4Model extends AbstractModel {
     }
 
     private MasterPlaylist getMasterPlaylist() throws IOException, ParseException, PlaylistException {
-        LOG.debug("Loading master playlist [{}]", getPlaylistUrl());
+        LOG.trace("Loading master playlist [{}]", getPlaylistUrl());
         Request req = new Request.Builder().url(getPlaylistUrl()).build();
 
         try (Response response = site.getHttpClient().execute(req)) {
@@ -224,19 +212,27 @@ public class Cam4Model extends AbstractModel {
         if(resolution == null) {
             if(failFast) {
                 return new int[2];
-            } else {
-                try {
-                    if(onlineState != OFFLINE) {
-                        loadModelDetails();
-                    } else {
-                        resolution = new int[2];
-                    }
-                } catch (Exception e) {
-                    throw new ExecutionException(e);
-                }
             }
+            try {
+                if(!isOnline()) {
+                    return new int[2];
+                }
+                List<StreamSource> sources = getStreamSources();
+                Collections.sort(sources);
+                StreamSource best = sources.get(sources.size()-1);
+                resolution = new int[] {best.width, best.height};
+            } catch (InterruptedException e) {
+                Thread.currentThread().interrupt();
+                LOG.warn("Couldn't determine stream resolution for {} - {}", getName(), e.getMessage());
+                resolution = new int[2];
+            } catch (ExecutionException | IOException | ParseException | PlaylistException e) {
+                LOG.warn("Couldn't determine stream resolution for {} - {}", getName(), e.getMessage());
+                resolution = new int[2];
+            }
+            return resolution;
+        } else {
+            return resolution;
         }
-        return resolution;
     }
 
     @Override

common/src/main/java/ctbrec/sites/cam4/Cam4WsClient.java

@@ -0,0 +1,202 @@
+package ctbrec.sites.cam4;
+
+import static ctbrec.io.HttpConstants.*;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import org.json.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import ctbrec.Config;
+import ctbrec.io.HttpException;
+import ctbrec.sites.ModelOfflineException;
+import okhttp3.Request;
+import okhttp3.Response;
+import okhttp3.WebSocket;
+import okhttp3.WebSocketListener;
+import okio.ByteString;
+
+public class Cam4WsClient {
+
+    private static final Logger LOG = LoggerFactory.getLogger(Cam4WsClient.class);
+
+    private Cam4 site;
+    private Cam4Model model;
+    private Config config;
+    private String shard;
+    private String token;
+    private WebSocket websocket;
+    private int r = 1;
+    private Map<String, CompletableFuture<String>> responseFutures = new HashMap<>();
+
+    public Cam4WsClient(Config config, Cam4 site, Cam4Model model) {
+        this.config = config;
+        this.site = site;
+        this.model = model;
+    }
+
+    public JSONObject getRoomState() throws IOException {
+        requestAccessToken();
+        if (connectAndAuthorize()) {
+            return requestRoomState();
+        } else {
+            throw new IOException("Connect or authorize failed");
+        }
+    }
+
+    private JSONObject requestRoomState() throws IOException {
+        String p = "chatRooms/" + model.getName() + "/roomState";
+        CompletableFuture<String> roomStateFuture = send(p, "{\"t\":\"d\",\"d\":{\"r\":" + (r++) + ",\"a\":\"q\",\"b\":{\"p\":\"" + p + "\",\"h\":\"\"}}}");
+        try {
+            JSONObject roomState = parseRoomStateResponse(roomStateFuture.get(5000, TimeUnit.SECONDS));
+            websocket.close(1000, "");
+            return roomState;
+        } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
+            throw new IOException("Interrupted while getting room state with websocket");
+        } catch (TimeoutException | ExecutionException e) {
+            throw new IOException(e);
+        }
+    }
+
+    private boolean connectAndAuthorize() throws IOException {
+        CompletableFuture<Boolean> connectedAndAuthorized = openWebsocketConnection();
+        try {
+            return connectedAndAuthorized.get(5000, TimeUnit.SECONDS);
+        } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
+            throw new IOException("Interrupted while connecting with websocket");
+        } catch (TimeoutException | ExecutionException e) {
+            throw new IOException(e);
+        }
+    }
+
+    private CompletableFuture<String> send(String p, String msg) {
+        CompletableFuture<String> future = new CompletableFuture<>();
+        LOG.trace("--> {}", msg);
+        boolean sent = websocket.send(msg);
+        if (!sent) {
+            future.completeExceptionally(new IOException("send() returned false"));
+        } else {
+            responseFutures.put(p, future);
+        }
+        return future;
+    }
+
+    private void requestAccessToken() throws IOException {
+        Request req = new Request.Builder() // @formatter:off
+                .url("https://webchat.cam4.com/requestAccess?roomname=" + model.getName())
+                .header(USER_AGENT, config.getSettings().httpUserAgent)
+                .header(REFERER, Cam4.BASE_URI + '/' + model.getName())
+                .header(ORIGIN, Cam4.BASE_URI)
+                .header(ACCEPT, "*/*")
+                .build(); // @formatter:on
+        try (Response response = site.getHttpClient().execute(req)) {
+            if (response.isSuccessful()) {
+                JSONObject body = new JSONObject(response.body().string());
+                if (body.optString("status").equals("success")) {
+                    shard = body.getString("shard").replace("https", "wss");
+                    token = body.getString("token");
+                } else {
+                    throw new ModelOfflineException(model);
+                }
+            } else {
+                throw new HttpException(response.code(), response.message());
+            }
+        }
+    }
+
+    private JSONObject parseRoomStateResponse(String msg) {
+        JSONObject json = new JSONObject(msg);
+        JSONObject d = json.getJSONObject("d");
+        JSONObject b = d.getJSONObject("b");
+        return b.getJSONObject("d");
+    }
+
+    private CompletableFuture<Boolean> openWebsocketConnection() {
+        CompletableFuture<Boolean> connectedAndAuthorized = new CompletableFuture<>();
+
+        String url = shard + ".ws?v=5";
+        LOG.trace("Opening websocket {}", url);
+        Request req = new Request.Builder() // @formatter:off
+                .url(url)
+                .header(USER_AGENT, config.getSettings().httpUserAgent)
+                .header(REFERER, Cam4.BASE_URI + '/' + model.getName())
+                .header(ORIGIN, Cam4.BASE_URI)
+                .header(ACCEPT, "*/*")
+                .build(); // @formatter:on
+
+        websocket = site.getHttpClient().newWebSocket(req, new WebSocketListener() {
+            @Override
+            public void onOpen(WebSocket webSocket, Response response) {
+                super.onOpen(webSocket, response);
+                try {
+                    LOG.trace("open: {}", response.body().string());
+                } catch (IOException e) {
+                    LOG.error("Connection open, but couldn't get the response body", e);
+                }
+                send("", "{\"t\":\"d\",\"d\":{\"r\":" + (r++) + ",\"a\":\"s\",\"b\":{\"c\":{\"sdk.js.2-3-1\":1}}}}");
+                send("", "{\"t\":\"d\",\"d\":{\"r\":" + (r++) + ",\"a\":\"auth\",\"b\":{\"cred\":\"" + token + "\"}}}");
+            }
+
+            @Override
+            public void onClosed(WebSocket webSocket, int code, String reason) {
+                super.onClosed(webSocket, code, reason);
+                LOG.trace("closed: {} {}", code, reason);
+                connectedAndAuthorized.complete(false);
+            }
+
+            @Override
+            public void onFailure(WebSocket webSocket, Throwable t, Response response) {
+                super.onFailure(webSocket, t, response);
+                try {
+                    if(response != null) {
+                        LOG.error("failure: {}", response.body().string(), t);
+                    } else {
+                        LOG.error("failure:", t);
+                    }
+                } catch (IOException e) {
+                    LOG.error("Connection failure and couldn't get the response body", e);
+                }
+                connectedAndAuthorized.completeExceptionally(t);
+            }
+
+            @Override
+            public void onMessage(WebSocket webSocket, String text) {
+                super.onMessage(webSocket, text);
+                LOG.trace("msgt: {}", text);
+                JSONObject response = new JSONObject(text);
+                if (response.has("d")) {
+                    JSONObject d = response.getJSONObject("d");
+                    int responseSequence = d.optInt("r");
+                    if (responseSequence == 2) {
+                        JSONObject body = d.getJSONObject("b");
+                        String status = body.optString("s");
+                        connectedAndAuthorized.complete(status.equals("ok"));
+                    } else if (d.has("b")) {
+                        JSONObject body = d.getJSONObject("b");
+                        String p = body.optString("p", "-");
+                        if (responseFutures.containsKey(p)) {
+                            CompletableFuture<String> future = responseFutures.get(p);
+                            future.complete(text);
+                        }
+                    }
+                }
+            }
+
+            @Override
+            public void onMessage(WebSocket webSocket, ByteString bytes) {
+                super.onMessage(webSocket, bytes);
+                LOG.trace("msgb: {}", bytes.hex());
+            }
+        });
+        return connectedAndAuthorized;
+    }
+}