From d3d9662ec596c5edddfa57a6ca9c5eb73689cd16 Mon Sep 17 00:00:00 2001 From: 0xboobface <0xboobface@gmail.com> Date: Sat, 20 Jun 2020 11:17:19 +0200 Subject: [PATCH] Add websocket client --- .../ui/sites/manyvids/MVLiveTabProvider.java | 2 +- .../sites/manyvids/MVLiveUpdateService.java | 35 +-- client/src/main/resources/logback.xml | 2 +- .../src/main/java/ctbrec/io/HttpClient.java | 21 +- .../main/java/ctbrec/io/HttpConstants.java | 1 + .../java/ctbrec/sites/manyvids/MVLive.java | 10 +- .../ctbrec/sites/manyvids/MVLiveClient.java | 232 ++++++++++++++++++ .../ctbrec/sites/manyvids/wsmsg/JoinChat.java | 34 +++ .../ctbrec/sites/manyvids/wsmsg/Message.java | 19 ++ .../ctbrec/sites/manyvids/wsmsg/Ping.java | 9 + .../ctbrec/sites/manyvids/wsmsg/Response.java | 7 + .../sites/manyvids/wsmsg/SendMessage.java | 33 +++ 12 files changed, 384 insertions(+), 21 deletions(-) create mode 100644 common/src/main/java/ctbrec/sites/manyvids/MVLiveClient.java create mode 100644 common/src/main/java/ctbrec/sites/manyvids/wsmsg/JoinChat.java create mode 100644 common/src/main/java/ctbrec/sites/manyvids/wsmsg/Message.java create mode 100644 common/src/main/java/ctbrec/sites/manyvids/wsmsg/Ping.java create mode 100644 common/src/main/java/ctbrec/sites/manyvids/wsmsg/Response.java create mode 100644 common/src/main/java/ctbrec/sites/manyvids/wsmsg/SendMessage.java diff --git a/client/src/main/java/ctbrec/ui/sites/manyvids/MVLiveTabProvider.java b/client/src/main/java/ctbrec/ui/sites/manyvids/MVLiveTabProvider.java index 66b3a919..51bbd0ae 100644 --- a/client/src/main/java/ctbrec/ui/sites/manyvids/MVLiveTabProvider.java +++ b/client/src/main/java/ctbrec/ui/sites/manyvids/MVLiveTabProvider.java @@ -21,7 +21,7 @@ public class MVLiveTabProvider extends TabProvider { @Override public List getTabs(Scene scene) { List tabs = new ArrayList<>(); - tabs.add(createTab("Online", mvlive.getBaseUrl() + "/MVLive/")); + tabs.add(createTab("Online", mvlive.getBaseUrl())); return tabs; } diff --git a/client/src/main/java/ctbrec/ui/sites/manyvids/MVLiveUpdateService.java b/client/src/main/java/ctbrec/ui/sites/manyvids/MVLiveUpdateService.java index b8134135..ede57711 100644 --- a/client/src/main/java/ctbrec/ui/sites/manyvids/MVLiveUpdateService.java +++ b/client/src/main/java/ctbrec/ui/sites/manyvids/MVLiveUpdateService.java @@ -22,7 +22,7 @@ import okhttp3.Response; public class MVLiveUpdateService extends PaginatedScheduledService { - private static final transient Logger LOG = LoggerFactory.getLogger(MVLiveUpdateService.class); + private static final Logger LOG = LoggerFactory.getLogger(MVLiveUpdateService.class); private String url; private MVLive mvlive; @@ -46,24 +46,27 @@ public class MVLiveUpdateService extends PaginatedScheduledService { if (response.isSuccessful()) { List models = new ArrayList<>(); String content = response.body().string(); - Elements cards = HtmlParser.getTags(content, "div.live-room"); + Elements cards = HtmlParser.getTags(content, "div[class~=mv-live-model]"); for (Element card : cards) { try { String cardHtml = card.html(); - Element link = HtmlParser.getTag(cardHtml, "h5 a"); - MVLiveModel model = (MVLiveModel) mvlive.createModel(link.text().trim()); - model.setUrl(mvlive.getBaseUrl() + link.attr("href")); - Element thumb = HtmlParser.getTag(cardHtml, "a img"); - model.setPreview(thumb.attr("src")); - Element status = HtmlParser.getTag(cardHtml, "div[class~=model-status]"); - String cssClass = status.attr("class"); - if(cssClass.contains("live")) { - model.setOnlineState(Model.State.ONLINE); - } else if(cssClass.contains("private")) { - model.setOnlineState(Model.State.PRIVATE); - } else { - model.setOnlineState(Model.State.UNKNOWN); - } + Element link = HtmlParser.getTag(cardHtml, "a"); + link.setBaseUri(mvlive.getBaseUrl()); + String name = HtmlParser.getText(cardHtml, "h4 a"); + MVLiveModel model = (MVLiveModel) mvlive.createModel(name); + model.setUrl(link.absUrl("href")); + Element thumb = HtmlParser.getTag(cardHtml, "a img.b-lazy"); + thumb.setBaseUri(mvlive.getBaseUrl()); + model.setPreview(thumb.absUrl("data-src")); + // Element status = HtmlParser.getTag(cardHtml, "div[class~=model-status]"); + // String cssClass = status.attr("class"); + // if(cssClass.contains("live")) { + // model.setOnlineState(Model.State.ONLINE); + // } else if(cssClass.contains("private")) { + // model.setOnlineState(Model.State.PRIVATE); + // } else { + // model.setOnlineState(Model.State.UNKNOWN); + // } models.add(model); } catch(RuntimeException e) { if(e.getMessage().contains("No element selected by")) { diff --git a/client/src/main/resources/logback.xml b/client/src/main/resources/logback.xml index 7eb5fbe7..b49b6ff7 100644 --- a/client/src/main/resources/logback.xml +++ b/client/src/main/resources/logback.xml @@ -27,7 +27,7 @@ - + diff --git a/common/src/main/java/ctbrec/io/HttpClient.java b/common/src/main/java/ctbrec/io/HttpClient.java index 16cc2b4f..b6ddd76e 100644 --- a/common/src/main/java/ctbrec/io/HttpClient.java +++ b/common/src/main/java/ctbrec/io/HttpClient.java @@ -11,6 +11,7 @@ import java.security.NoSuchAlgorithmException; import java.security.SecureRandom; import java.security.cert.CertificateException; import java.security.cert.X509Certificate; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -27,6 +28,7 @@ import javax.net.ssl.X509TrustManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.common.base.Objects; import com.squareup.moshi.JsonAdapter; import com.squareup.moshi.Moshi; @@ -47,7 +49,7 @@ public abstract class HttpClient { private static final Logger LOG = LoggerFactory.getLogger(HttpClient.class); private static final ConnectionPool GLOBAL_HTTP_CONN_POOL = new ConnectionPool(10, 2, TimeUnit.MINUTES); - + protected OkHttpClient client; protected CookieJarImpl cookieJar = new CookieJarImpl(); protected boolean loggedIn = false; @@ -126,8 +128,8 @@ public abstract class HttpClient { .connectionPool(GLOBAL_HTTP_CONN_POOL) .connectTimeout(Config.getInstance().getSettings().httpTimeout, TimeUnit.MILLISECONDS) .readTimeout(Config.getInstance().getSettings().httpTimeout, TimeUnit.MILLISECONDS) + //.addInterceptor(new LoggingInterceptor()) .connectionPool(new ConnectionPool(50, 10, TimeUnit.MINUTES)); - //.addInterceptor(new LoggingInterceptor()); ProxyType proxyType = Config.getInstance().getSettings().proxyType; if (proxyType == ProxyType.HTTP) { @@ -269,4 +271,19 @@ public abstract class HttpClient { //Request request = new Request.Builder().url(url).build(); return client.newWebSocket(request, l); } + + public List getCookiesByName(String... names) { + List result = new ArrayList<>(); + Map> cookies = getCookieJar().getCookies(); + for (List cookieList : cookies.values()) { + for (Cookie cookie : cookieList) { + for (String cookieName : names) { + if (Objects.equal(cookieName, cookie.name())) { + result.add(cookie); + } + } + } + } + return result; + } } diff --git a/common/src/main/java/ctbrec/io/HttpConstants.java b/common/src/main/java/ctbrec/io/HttpConstants.java index 92826433..53e131ea 100644 --- a/common/src/main/java/ctbrec/io/HttpConstants.java +++ b/common/src/main/java/ctbrec/io/HttpConstants.java @@ -4,6 +4,7 @@ public class HttpConstants { public static final String ACCEPT = "Accept"; public static final String ACCEPT_LANGUAGE = "Accept-Language"; + public static final String COOKIE = "Cookie"; public static final String CONNECTION = "Connection"; public static final String CONTENT_TYPE = "Content-Type"; public static final String KEEP_ALIVE = "keep-alive"; diff --git a/common/src/main/java/ctbrec/sites/manyvids/MVLive.java b/common/src/main/java/ctbrec/sites/manyvids/MVLive.java index 2a0183c1..9b2a44fb 100644 --- a/common/src/main/java/ctbrec/sites/manyvids/MVLive.java +++ b/common/src/main/java/ctbrec/sites/manyvids/MVLive.java @@ -8,7 +8,8 @@ import ctbrec.sites.AbstractSite; public class MVLive extends AbstractSite { - public static final String BASE_URL = "https://www.manyvids.com"; + public static final String WS_URL = "https://live.manyvids.com"; + public static final String BASE_URL = "https://www.manyvids.com/MVLive/"; private MVLiveHttpClient httpClient; @@ -61,6 +62,13 @@ public class MVLive extends AbstractSite { @Override public void init() throws IOException { + try { + MVLiveClient.getInstance().setSite(this); + MVLiveClient.getInstance().start(); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } } @Override diff --git a/common/src/main/java/ctbrec/sites/manyvids/MVLiveClient.java b/common/src/main/java/ctbrec/sites/manyvids/MVLiveClient.java new file mode 100644 index 00000000..04ef34bb --- /dev/null +++ b/common/src/main/java/ctbrec/sites/manyvids/MVLiveClient.java @@ -0,0 +1,232 @@ +package ctbrec.sites.manyvids; + +import static ctbrec.io.HttpConstants.*; +import static ctbrec.sites.manyvids.MVLive.*; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Random; +import java.util.UUID; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; +import java.util.stream.Collectors; + +import org.json.JSONArray; +import org.json.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.base.Objects; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import com.squareup.moshi.Moshi; + +import ctbrec.Config; +import ctbrec.io.HttpException; +import ctbrec.sites.manyvids.wsmsg.JoinChat; +import ctbrec.sites.manyvids.wsmsg.Message; +import ctbrec.sites.manyvids.wsmsg.Ping; +import ctbrec.sites.manyvids.wsmsg.SendMessage; +import okhttp3.Cookie; +import okhttp3.Request; +import okhttp3.Response; +import okhttp3.WebSocket; +import okhttp3.WebSocketListener; +import okio.ByteString; + +public class MVLiveClient { + + private static final Logger LOG = LoggerFactory.getLogger(MVLiveClient.class); + + private static MVLiveClient instance; + private MVLive site; + private WebSocket ws; + private Moshi moshi; + private Random rng = new Random(); + private volatile boolean running = false; + + private Cache models = CacheBuilder.newBuilder().maximumSize(4000).build(); + private Lock lock = new ReentrantLock(); + private volatile boolean connecting = false; + + private MVLiveClient() { + moshi = new Moshi.Builder().build(); + } + + public static synchronized MVLiveClient getInstance() { + if (instance == null) { + instance = new MVLiveClient(); + } + return instance; + } + + public void setSite(MVLive site) { + this.site = site; + } + + public void start() throws IOException { + running = true; + + + // make a call to the website to get necessary cookies + Request req = new Request.Builder() + .url(site.getBaseUrl()) + .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) + .build(); + try (Response response = site.getHttpClient().execute(req)) { + if (!response.isSuccessful()) { + throw new HttpException(response.code(), response.message()); + } + } + + int randomNumber = 100 + rng.nextInt(800); + String randomString = UUID.randomUUID().toString().replace("-", "").substring(0, 8); + String roomNumber = "176"; + String wsUrl = String.format("%s/api/%s/eventbus/%s/%s/websocket", WS_URL, roomNumber, randomNumber, randomString); + List cookies = site.getHttpClient().getCookiesByName("PHPSESSID"); //, "XSRF-TOKEN"); + String cookieHeaderValue = cookies.stream().map(Object::toString).collect(Collectors.joining("; ")); + + Thread watchDog = new Thread(() -> { + while (running) { + if (ws == null && !connecting) { + LOG.info("Websocket is null. Starting a new connection to {}", wsUrl); + Request websocketRequest = new Request.Builder() + .url(wsUrl) + .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) + .header(ORIGIN, WS_URL) + .header(COOKIE, cookieHeaderValue) + .build(); + ws = createWebSocket(websocketRequest); + } + + try { + Thread.sleep(10000); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + LOG.error("WatchDog couldn't sleep", e); + stop(); + running = false; + } + } + }); + watchDog.setDaemon(true); + watchDog.setName("MVLive WebSocket WatchDog"); + watchDog.setPriority(Thread.MIN_PRIORITY); + watchDog.start(); + } + + public void stop() { + running = false; + ws.close(1000, "Good Bye"); // terminate normally (1000) + } + + public List getModels() { + lock.lock(); + try { + LOG.trace("Models: {}", models.size()); + return new ArrayList<>(this.models.asMap().values()); + } finally { + lock.unlock(); + } + } + + private WebSocket createWebSocket(Request req) { + connecting = true; + WebSocket websocket = site.getHttpClient().newWebSocket(req, new WebSocketListener() { + @Override + public void onOpen(WebSocket webSocket, Response response) { + super.onOpen(webSocket, response); + try { + connecting = false; + models.invalidateAll(); + LOG.debug("WS open: [{}]", response.body().string()); + } catch (IOException e) { + LOG.error("Error while processing onOpen event", e); + } + } + + @Override + public void onClosed(WebSocket webSocket, int code, String reason) { + super.onClosed(webSocket, code, reason); + connecting = false; + LOG.info("MVLive websocket closed: {} {}", code, reason); + MVLiveClient.this.ws = null; + if (!running) { + site.getHttpClient().shutdown(); + } + } + + @Override + public void onFailure(WebSocket webSocket, Throwable t, Response response) { + super.onFailure(webSocket, t, response); + connecting = false; + if (response != null) { + int code = response.code(); + String message = response.message(); + LOG.error("MVLive websocket failure: {} {}", code, message, t); + response.close(); + } else { + LOG.error("MVLive websocket failure", t); + } + MVLiveClient.this.ws = null; + } + + @Override + public void onMessage(WebSocket webSocket, String text) { + super.onMessage(webSocket, text); + //msgBuffer.append(text); + LOG.debug("Message: {}", text); + text = Optional.ofNullable(text).orElse(""); + if (Objects.equal("o", text)) { + sendMessages(new Ping()); + String roomId = "3d9b273a-8039-41f7-a5d8-bdd6972d697c"; + String showId = "HARIBOO"; + sendMessages(new JoinChat(roomId, showId, (m, r) -> { + System.out.println(m); + System.out.println(r); + })); + } else if (text.startsWith("a")) { + JSONArray jsonArray = new JSONArray(text.substring(1)); + for (int i = 0; i < jsonArray.length(); i++) { + String respJson = jsonArray.getString(i); + JSONObject response = new JSONObject(respJson); + String address = response.optString("address"); + if (!address.isBlank()) { + SendMessage sendMessage = futureResponses.get(address); + if (sendMessage != null) { + sendMessage.handleResponse(response); + } + } + } + } + } + + Map futureResponses = new HashMap<>(); + + private void sendMessages(Message... messages) { + JSONArray msgs = new JSONArray(); + for (Message msg : messages) { + msgs.put(msg.toString()); + if(msg instanceof SendMessage) { + SendMessage sendMessage = (SendMessage) msg; + futureResponses.put(sendMessage.getReplyAddress(), sendMessage); + } + } + ws.send(msgs.toString()); + } + + @Override + public void onMessage(WebSocket webSocket, ByteString bytes) { + super.onMessage(webSocket, bytes); + LOG.debug("Binary Message: {}", bytes.hex()); + } + }); + return websocket; + } + + +} diff --git a/common/src/main/java/ctbrec/sites/manyvids/wsmsg/JoinChat.java b/common/src/main/java/ctbrec/sites/manyvids/wsmsg/JoinChat.java new file mode 100644 index 00000000..9ed56c50 --- /dev/null +++ b/common/src/main/java/ctbrec/sites/manyvids/wsmsg/JoinChat.java @@ -0,0 +1,34 @@ +package ctbrec.sites.manyvids.wsmsg; + +import java.util.UUID; +import java.util.function.BiConsumer; + +import org.json.JSONObject; + +public class JoinChat extends SendMessage { + + private String roomId; + private String showId; + + public JoinChat(String roomId, String showId, BiConsumer responseConsumer) { + super(responseConsumer); + this.roomId = roomId; + this.showId = showId; + address = "api/ChatService"; + action = "join"; + } + + @Override + public String toString() { + JSONObject msg = build(); + JSONObject body = new JSONObject(); + body.put("connectionId", ""); + body.put("telemetryId", UUID.randomUUID().toString()); + body.put("roomId", roomId); + body.put("showId", showId); + body.put("joinPrivateShow", false); + body.put("deviceSourceType", "DESKTOP"); + msg.put("body", body); + return msg.toString(); + } +} diff --git a/common/src/main/java/ctbrec/sites/manyvids/wsmsg/Message.java b/common/src/main/java/ctbrec/sites/manyvids/wsmsg/Message.java new file mode 100644 index 00000000..81246f76 --- /dev/null +++ b/common/src/main/java/ctbrec/sites/manyvids/wsmsg/Message.java @@ -0,0 +1,19 @@ +package ctbrec.sites.manyvids.wsmsg; + +import java.util.Optional; +import java.util.function.BiConsumer; + +import org.json.JSONObject; + +public abstract class Message extends JSONObject { + + private BiConsumer responseConsumer; + + public Message(BiConsumer responseConsumer) { + this.responseConsumer = responseConsumer; + } + + public void handleResponse(JSONObject response) { + Optional.ofNullable(responseConsumer).ifPresent(consumer -> consumer.accept(this, response)); + } +} diff --git a/common/src/main/java/ctbrec/sites/manyvids/wsmsg/Ping.java b/common/src/main/java/ctbrec/sites/manyvids/wsmsg/Ping.java new file mode 100644 index 00000000..88cd4c05 --- /dev/null +++ b/common/src/main/java/ctbrec/sites/manyvids/wsmsg/Ping.java @@ -0,0 +1,9 @@ +package ctbrec.sites.manyvids.wsmsg; + +public class Ping extends Message { + + public Ping() { + super(null); + put("type", "ping"); + } +} diff --git a/common/src/main/java/ctbrec/sites/manyvids/wsmsg/Response.java b/common/src/main/java/ctbrec/sites/manyvids/wsmsg/Response.java new file mode 100644 index 00000000..4e48e4cb --- /dev/null +++ b/common/src/main/java/ctbrec/sites/manyvids/wsmsg/Response.java @@ -0,0 +1,7 @@ +package ctbrec.sites.manyvids.wsmsg; + +import org.json.JSONObject; + +public class Response extends JSONObject { + +} diff --git a/common/src/main/java/ctbrec/sites/manyvids/wsmsg/SendMessage.java b/common/src/main/java/ctbrec/sites/manyvids/wsmsg/SendMessage.java new file mode 100644 index 00000000..9db7995f --- /dev/null +++ b/common/src/main/java/ctbrec/sites/manyvids/wsmsg/SendMessage.java @@ -0,0 +1,33 @@ +package ctbrec.sites.manyvids.wsmsg; + +import java.util.UUID; +import java.util.function.BiConsumer; + +import org.json.JSONObject; + +public class SendMessage extends Message { + + protected String address; + protected String action; + protected String replyAddress; + + public SendMessage(BiConsumer responseConsumer) { + super(responseConsumer); + replyAddress = UUID.randomUUID().toString(); + } + + public String getReplyAddress() { + return replyAddress; + } + + protected JSONObject build() { + JSONObject msg = new JSONObject(); + msg.put("type", "send"); + msg.put("address", address); + msg.put("replyAddress", replyAddress); + JSONObject headers = new JSONObject(); + headers.put("action", action); + msg.put("headers", headers); + return msg; + } +}