Add websocket client

client/src/main/java/ctbrec/ui/sites/manyvids/MVLiveTabProvider.java

@@ -21,7 +21,7 @@ public class MVLiveTabProvider extends TabProvider {
     @Override
     public List<Tab> getTabs(Scene scene) {
         List<Tab> tabs = new ArrayList<>();
-        tabs.add(createTab("Online", mvlive.getBaseUrl() + "/MVLive/"));
+        tabs.add(createTab("Online", mvlive.getBaseUrl()));
         return tabs;
     }
 

client/src/main/java/ctbrec/ui/sites/manyvids/MVLiveUpdateService.java

@@ -22,7 +22,7 @@ import okhttp3.Response;
 
 public class MVLiveUpdateService extends PaginatedScheduledService {
 
-    private static final transient Logger LOG = LoggerFactory.getLogger(MVLiveUpdateService.class);
+    private static final Logger LOG = LoggerFactory.getLogger(MVLiveUpdateService.class);
     private String url;
     private MVLive mvlive;
 
@@ -46,24 +46,27 @@ public class MVLiveUpdateService extends PaginatedScheduledService {
                     if (response.isSuccessful()) {
                         List<Model> models = new ArrayList<>();
                         String content = response.body().string();
-                        Elements cards = HtmlParser.getTags(content, "div.live-room");
+                        Elements cards = HtmlParser.getTags(content, "div[class~=mv-live-model]");
                         for (Element card : cards) {
                             try {
                                 String cardHtml = card.html();
-                                Element link = HtmlParser.getTag(cardHtml, "h5 a");
+                                Element link = HtmlParser.getTag(cardHtml, "a");
-                                MVLiveModel model = (MVLiveModel) mvlive.createModel(link.text().trim());
+                                link.setBaseUri(mvlive.getBaseUrl());
-                                model.setUrl(mvlive.getBaseUrl() + link.attr("href"));
+                                String name = HtmlParser.getText(cardHtml, "h4 a");
-                                Element thumb = HtmlParser.getTag(cardHtml, "a img");
+                                MVLiveModel model = (MVLiveModel) mvlive.createModel(name);
-                                model.setPreview(thumb.attr("src"));
+                                model.setUrl(link.absUrl("href"));
-                                Element status = HtmlParser.getTag(cardHtml, "div[class~=model-status]");
+                                Element thumb = HtmlParser.getTag(cardHtml, "a img.b-lazy");
-                                String cssClass = status.attr("class");
+                                thumb.setBaseUri(mvlive.getBaseUrl());
-                                if(cssClass.contains("live")) {
+                                model.setPreview(thumb.absUrl("data-src"));
-                                    model.setOnlineState(Model.State.ONLINE);
+                                //                                Element status = HtmlParser.getTag(cardHtml, "div[class~=model-status]");
-                                } else if(cssClass.contains("private")) {
+                                //                                String cssClass = status.attr("class");
-                                    model.setOnlineState(Model.State.PRIVATE);
+                                //                                if(cssClass.contains("live")) {
-                                } else {
+                                //                                    model.setOnlineState(Model.State.ONLINE);
-                                    model.setOnlineState(Model.State.UNKNOWN);
+                                //                                } else if(cssClass.contains("private")) {
-                                }
+                                //                                    model.setOnlineState(Model.State.PRIVATE);
+                                //                                } else {
+                                //                                    model.setOnlineState(Model.State.UNKNOWN);
+                                //                                }
                                 models.add(model);
                             } catch(RuntimeException e) {
                                 if(e.getMessage().contains("No element selected by")) {

client/src/main/resources/logback.xml

@@ -27,7 +27,7 @@
 		<appender-ref ref="FILE" />
 	</root>
 	
-	<logger name="ctbrec.LoggingInterceptor" level="INFO"/>
+	<logger name="ctbrec.LoggingInterceptor" level="info"/>
 	<logger name="ctbrec.io.CookieJarImpl" level="INFO"/>
 	<logger name="ctbrec.recorder.server.HlsServlet" level="INFO"/>
 	<logger name="ctbrec.recorder.server.RecorderServlet" level="INFO"/>

common/src/main/java/ctbrec/io/HttpClient.java

@@ -11,6 +11,7 @@ import java.security.NoSuchAlgorithmException;
 import java.security.SecureRandom;
 import java.security.cert.CertificateException;
 import java.security.cert.X509Certificate;
+import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -27,6 +28,7 @@ import javax.net.ssl.X509TrustManager;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.google.common.base.Objects;
 import com.squareup.moshi.JsonAdapter;
 import com.squareup.moshi.Moshi;
 
@@ -126,8 +128,8 @@ public abstract class HttpClient {
                 .connectionPool(GLOBAL_HTTP_CONN_POOL)
                 .connectTimeout(Config.getInstance().getSettings().httpTimeout, TimeUnit.MILLISECONDS)
                 .readTimeout(Config.getInstance().getSettings().httpTimeout, TimeUnit.MILLISECONDS)
+                //.addInterceptor(new LoggingInterceptor())
                 .connectionPool(new ConnectionPool(50, 10, TimeUnit.MINUTES));
-        //.addInterceptor(new LoggingInterceptor());
 
         ProxyType proxyType = Config.getInstance().getSettings().proxyType;
         if (proxyType == ProxyType.HTTP) {
@@ -269,4 +271,19 @@ public abstract class HttpClient {
         //Request request = new Request.Builder().url(url).build();
         return client.newWebSocket(request, l);
     }
+
+    public List<Cookie> getCookiesByName(String... names) {
+        List<Cookie> result = new ArrayList<>();
+        Map<String, List<Cookie>> cookies = getCookieJar().getCookies();
+        for (List<Cookie> cookieList : cookies.values()) {
+            for (Cookie cookie : cookieList) {
+                for (String cookieName : names) {
+                    if (Objects.equal(cookieName, cookie.name())) {
+                        result.add(cookie);
+                    }
+                }
+            }
+        }
+        return result;
+    }
 }

common/src/main/java/ctbrec/io/HttpConstants.java

@@ -4,6 +4,7 @@ public class HttpConstants {
 
     public static final String ACCEPT = "Accept";
     public static final String ACCEPT_LANGUAGE = "Accept-Language";
+    public static final String COOKIE = "Cookie";
     public static final String CONNECTION = "Connection";
     public static final String CONTENT_TYPE = "Content-Type";
     public static final String KEEP_ALIVE = "keep-alive";

common/src/main/java/ctbrec/sites/manyvids/MVLive.java

@@ -8,7 +8,8 @@ import ctbrec.sites.AbstractSite;
 
 public class MVLive extends AbstractSite {
 
-    public static final String BASE_URL = "https://www.manyvids.com";
+    public static final String WS_URL = "https://live.manyvids.com";
+    public static final String BASE_URL = "https://www.manyvids.com/MVLive/";
 
     private MVLiveHttpClient httpClient;
 
@@ -61,6 +62,13 @@ public class MVLive extends AbstractSite {
 
     @Override
     public void init() throws IOException {
+        try {
+            MVLiveClient.getInstance().setSite(this);
+            MVLiveClient.getInstance().start();
+        } catch (IOException e) {
+            // TODO Auto-generated catch block
+            e.printStackTrace();
+        }
     }
 
     @Override

common/src/main/java/ctbrec/sites/manyvids/MVLiveClient.java

@@ -0,0 +1,232 @@
+package ctbrec.sites.manyvids;
+
+import static ctbrec.io.HttpConstants.*;
+import static ctbrec.sites.manyvids.MVLive.*;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Random;
+import java.util.UUID;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+import java.util.stream.Collectors;
+
+import org.json.JSONArray;
+import org.json.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Objects;
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
+import com.squareup.moshi.Moshi;
+
+import ctbrec.Config;
+import ctbrec.io.HttpException;
+import ctbrec.sites.manyvids.wsmsg.JoinChat;
+import ctbrec.sites.manyvids.wsmsg.Message;
+import ctbrec.sites.manyvids.wsmsg.Ping;
+import ctbrec.sites.manyvids.wsmsg.SendMessage;
+import okhttp3.Cookie;
+import okhttp3.Request;
+import okhttp3.Response;
+import okhttp3.WebSocket;
+import okhttp3.WebSocketListener;
+import okio.ByteString;
+
+public class MVLiveClient {
+
+    private static final Logger LOG = LoggerFactory.getLogger(MVLiveClient.class);
+
+    private static MVLiveClient instance;
+    private MVLive site;
+    private WebSocket ws;
+    private Moshi moshi;
+    private Random rng = new Random();
+    private volatile boolean running = false;
+
+    private Cache<Integer, MVLiveModel> models = CacheBuilder.newBuilder().maximumSize(4000).build();
+    private Lock lock = new ReentrantLock();
+    private volatile boolean connecting = false;
+
+    private MVLiveClient() {
+        moshi = new Moshi.Builder().build();
+    }
+
+    public static synchronized MVLiveClient getInstance() {
+        if (instance == null) {
+            instance = new MVLiveClient();
+        }
+        return instance;
+    }
+
+    public void setSite(MVLive site) {
+        this.site = site;
+    }
+
+    public void start() throws IOException {
+        running = true;
+
+
+        // make a call to the website to get necessary cookies
+        Request req = new Request.Builder()
+                .url(site.getBaseUrl())
+                .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
+                .build();
+        try (Response response = site.getHttpClient().execute(req)) {
+            if (!response.isSuccessful()) {
+                throw new HttpException(response.code(), response.message());
+            }
+        }
+
+        int randomNumber = 100 + rng.nextInt(800);
+        String randomString = UUID.randomUUID().toString().replace("-", "").substring(0, 8);
+        String roomNumber = "176";
+        String wsUrl = String.format("%s/api/%s/eventbus/%s/%s/websocket", WS_URL, roomNumber, randomNumber, randomString);
+        List<Cookie> cookies = site.getHttpClient().getCookiesByName("PHPSESSID"); //, "XSRF-TOKEN");
+        String cookieHeaderValue = cookies.stream().map(Object::toString).collect(Collectors.joining("; "));
+
+        Thread watchDog = new Thread(() -> {
+            while (running) {
+                if (ws == null && !connecting) {
+                    LOG.info("Websocket is null. Starting a new connection to {}", wsUrl);
+                    Request websocketRequest = new Request.Builder()
+                            .url(wsUrl)
+                            .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
+                            .header(ORIGIN, WS_URL)
+                            .header(COOKIE, cookieHeaderValue)
+                            .build();
+                    ws = createWebSocket(websocketRequest);
+                }
+
+                try {
+                    Thread.sleep(10000);
+                } catch (InterruptedException e) {
+                    Thread.currentThread().interrupt();
+                    LOG.error("WatchDog couldn't sleep", e);
+                    stop();
+                    running = false;
+                }
+            }
+        });
+        watchDog.setDaemon(true);
+        watchDog.setName("MVLive WebSocket WatchDog");
+        watchDog.setPriority(Thread.MIN_PRIORITY);
+        watchDog.start();
+    }
+
+    public void stop() {
+        running = false;
+        ws.close(1000, "Good Bye"); // terminate normally (1000)
+    }
+
+    public List<MVLiveModel> getModels() {
+        lock.lock();
+        try {
+            LOG.trace("Models: {}", models.size());
+            return new ArrayList<>(this.models.asMap().values());
+        } finally {
+            lock.unlock();
+        }
+    }
+
+    private WebSocket createWebSocket(Request req) {
+        connecting = true;
+        WebSocket websocket = site.getHttpClient().newWebSocket(req, new WebSocketListener() {
+            @Override
+            public void onOpen(WebSocket webSocket, Response response) {
+                super.onOpen(webSocket, response);
+                try {
+                    connecting = false;
+                    models.invalidateAll();
+                    LOG.debug("WS open: [{}]", response.body().string());
+                } catch (IOException e) {
+                    LOG.error("Error while processing onOpen event", e);
+                }
+            }
+
+            @Override
+            public void onClosed(WebSocket webSocket, int code, String reason) {
+                super.onClosed(webSocket, code, reason);
+                connecting = false;
+                LOG.info("MVLive websocket closed: {} {}", code, reason);
+                MVLiveClient.this.ws = null;
+                if (!running) {
+                    site.getHttpClient().shutdown();
+                }
+            }
+
+            @Override
+            public void onFailure(WebSocket webSocket, Throwable t, Response response) {
+                super.onFailure(webSocket, t, response);
+                connecting = false;
+                if (response != null) {
+                    int code = response.code();
+                    String message = response.message();
+                    LOG.error("MVLive websocket failure: {} {}", code, message, t);
+                    response.close();
+                } else {
+                    LOG.error("MVLive websocket failure", t);
+                }
+                MVLiveClient.this.ws = null;
+            }
+
+            @Override
+            public void onMessage(WebSocket webSocket, String text) {
+                super.onMessage(webSocket, text);
+                //msgBuffer.append(text);
+                LOG.debug("Message: {}", text);
+                text = Optional.ofNullable(text).orElse("");
+                if (Objects.equal("o", text)) {
+                    sendMessages(new Ping());
+                    String roomId = "3d9b273a-8039-41f7-a5d8-bdd6972d697c";
+                    String showId = "HARIBOO";
+                    sendMessages(new JoinChat(roomId, showId, (m, r) -> {
+                        System.out.println(m);
+                        System.out.println(r);
+                    }));
+                } else if (text.startsWith("a")) {
+                    JSONArray jsonArray = new JSONArray(text.substring(1));
+                    for (int i = 0; i < jsonArray.length(); i++) {
+                        String respJson = jsonArray.getString(i);
+                        JSONObject response = new JSONObject(respJson);
+                        String address = response.optString("address");
+                        if (!address.isBlank()) {
+                            SendMessage sendMessage = futureResponses.get(address);
+                            if (sendMessage != null) {
+                                sendMessage.handleResponse(response);
+                            }
+                        }
+                    }
+                }
+            }
+
+            Map<String, SendMessage> futureResponses = new HashMap<>();
+
+            private void sendMessages(Message... messages) {
+                JSONArray msgs = new JSONArray();
+                for (Message msg : messages) {
+                    msgs.put(msg.toString());
+                    if(msg instanceof SendMessage) {
+                        SendMessage sendMessage = (SendMessage) msg;
+                        futureResponses.put(sendMessage.getReplyAddress(), sendMessage);
+                    }
+                }
+                ws.send(msgs.toString());
+            }
+
+            @Override
+            public void onMessage(WebSocket webSocket, ByteString bytes) {
+                super.onMessage(webSocket, bytes);
+                LOG.debug("Binary Message: {}", bytes.hex());
+            }
+        });
+        return websocket;
+    }
+
+
+}

common/src/main/java/ctbrec/sites/manyvids/wsmsg/JoinChat.java

@@ -0,0 +1,34 @@
+package ctbrec.sites.manyvids.wsmsg;
+
+import java.util.UUID;
+import java.util.function.BiConsumer;
+
+import org.json.JSONObject;
+
+public class JoinChat extends SendMessage {
+
+    private String roomId;
+    private String showId;
+
+    public JoinChat(String roomId, String showId, BiConsumer<Message, JSONObject> responseConsumer) {
+        super(responseConsumer);
+        this.roomId = roomId;
+        this.showId = showId;
+        address = "api/ChatService";
+        action = "join";
+    }
+
+    @Override
+    public String toString() {
+        JSONObject msg = build();
+        JSONObject body = new JSONObject();
+        body.put("connectionId", "");
+        body.put("telemetryId", UUID.randomUUID().toString());
+        body.put("roomId", roomId);
+        body.put("showId", showId);
+        body.put("joinPrivateShow", false);
+        body.put("deviceSourceType", "DESKTOP");
+        msg.put("body", body);
+        return msg.toString();
+    }
+}

common/src/main/java/ctbrec/sites/manyvids/wsmsg/Message.java

@@ -0,0 +1,19 @@
+package ctbrec.sites.manyvids.wsmsg;
+
+import java.util.Optional;
+import java.util.function.BiConsumer;
+
+import org.json.JSONObject;
+
+public abstract class Message extends JSONObject {
+
+    private BiConsumer<Message, JSONObject> responseConsumer;
+
+    public Message(BiConsumer<Message, JSONObject> responseConsumer) {
+        this.responseConsumer = responseConsumer;
+    }
+
+    public void handleResponse(JSONObject response) {
+        Optional.ofNullable(responseConsumer).ifPresent(consumer -> consumer.accept(this, response));
+    }
+}

common/src/main/java/ctbrec/sites/manyvids/wsmsg/Ping.java

@@ -0,0 +1,9 @@
+package ctbrec.sites.manyvids.wsmsg;
+
+public class Ping extends Message {
+
+    public Ping() {
+        super(null);
+        put("type", "ping");
+    }
+}

common/src/main/java/ctbrec/sites/manyvids/wsmsg/Response.java

@@ -0,0 +1,7 @@
+package ctbrec.sites.manyvids.wsmsg;
+
+import org.json.JSONObject;
+
+public class Response extends JSONObject {
+
+}

common/src/main/java/ctbrec/sites/manyvids/wsmsg/SendMessage.java

@@ -0,0 +1,33 @@
+package ctbrec.sites.manyvids.wsmsg;
+
+import java.util.UUID;
+import java.util.function.BiConsumer;
+
+import org.json.JSONObject;
+
+public class SendMessage extends Message {
+
+    protected String address;
+    protected String action;
+    protected String replyAddress;
+
+    public SendMessage(BiConsumer<Message, JSONObject> responseConsumer) {
+        super(responseConsumer);
+        replyAddress = UUID.randomUUID().toString();
+    }
+
+    public String getReplyAddress() {
+        return replyAddress;
+    }
+
+    protected JSONObject build() {
+        JSONObject msg = new JSONObject();
+        msg.put("type", "send");
+        msg.put("address", address);
+        msg.put("replyAddress", replyAddress);
+        JSONObject headers = new JSONObject();
+        headers.put("action", action);
+        msg.put("headers", headers);
+        return msg;
+    }
+}