ctbrecuser
/
ctbrec
forked from j62/ctbrec
1
0
Fork 0
Code Pull Requests Activity

Add websocket client

This commit is contained in:
0xboobface 2020-06-20 11:17:19 +02:00
parent 58663d8e8d
commit d3d9662ec5
12 changed files with 384 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
client/src/main/java/ctbrec/ui/sites/manyvids/MVLiveTabProvider.java
View File

@ -21,7 +21,7 @@ public class MVLiveTabProvider extends TabProvider {
@Override
public List<Tab> getTabs(Scene scene) {
List<Tab> tabs = new ArrayList<>();
tabs.add(createTab("Online", mvlive.getBaseUrl() + "/MVLive/"));
tabs.add(createTab("Online", mvlive.getBaseUrl()));
return tabs;
}

35
client/src/main/java/ctbrec/ui/sites/manyvids/MVLiveUpdateService.java
View File

@ -22,7 +22,7 @@ import okhttp3.Response;
public class MVLiveUpdateService extends PaginatedScheduledService {
private static final transient Logger LOG = LoggerFactory.getLogger(MVLiveUpdateService.class);
private static final Logger LOG = LoggerFactory.getLogger(MVLiveUpdateService.class);
private String url;
private MVLive mvlive;
@ -46,24 +46,27 @@ public class MVLiveUpdateService extends PaginatedScheduledService {
if (response.isSuccessful()) {
List<Model> models = new ArrayList<>();
String content = response.body().string();
Elements cards = HtmlParser.getTags(content, "div.live-room");
Elements cards = HtmlParser.getTags(content, "div[class~=mv-live-model]");
for (Element card : cards) {
try {
String cardHtml = card.html();
Element link = HtmlParser.getTag(cardHtml, "h5 a");
MVLiveModel model = (MVLiveModel) mvlive.createModel(link.text().trim());
model.setUrl(mvlive.getBaseUrl() + link.attr("href"));
Element thumb = HtmlParser.getTag(cardHtml, "a img");
model.setPreview(thumb.attr("src"));
Element status = HtmlParser.getTag(cardHtml, "div[class~=model-status]");
String cssClass = status.attr("class");
if(cssClass.contains("live")) {
model.setOnlineState(Model.State.ONLINE);
} else if(cssClass.contains("private")) {
model.setOnlineState(Model.State.PRIVATE);
} else {
model.setOnlineState(Model.State.UNKNOWN);
}
Element link = HtmlParser.getTag(cardHtml, "a");
link.setBaseUri(mvlive.getBaseUrl());
String name = HtmlParser.getText(cardHtml, "h4 a");
MVLiveModel model = (MVLiveModel) mvlive.createModel(name);
model.setUrl(link.absUrl("href"));
Element thumb = HtmlParser.getTag(cardHtml, "a img.b-lazy");
thumb.setBaseUri(mvlive.getBaseUrl());
model.setPreview(thumb.absUrl("data-src"));
// Element status = HtmlParser.getTag(cardHtml, "div[class~=model-status]");
// String cssClass = status.attr("class");
// if(cssClass.contains("live")) {
// model.setOnlineState(Model.State.ONLINE);
// } else if(cssClass.contains("private")) {
// model.setOnlineState(Model.State.PRIVATE);
// } else {
// model.setOnlineState(Model.State.UNKNOWN);
// }
models.add(model);
} catch(RuntimeException e) {
if(e.getMessage().contains("No element selected by")) {

2
client/src/main/resources/logback.xml
View File

@ -27,7 +27,7 @@
<appender-ref ref="FILE" />
</root>
<logger name="ctbrec.LoggingInterceptor" level="INFO"/>
<logger name="ctbrec.LoggingInterceptor" level="info"/>
<logger name="ctbrec.io.CookieJarImpl" level="INFO"/>
<logger name="ctbrec.recorder.server.HlsServlet" level="INFO"/>
<logger name="ctbrec.recorder.server.RecorderServlet" level="INFO"/>

21
common/src/main/java/ctbrec/io/HttpClient.java
View File

@ -11,6 +11,7 @@ import java.security.NoSuchAlgorithmException;
import java.security.SecureRandom;
import java.security.cert.CertificateException;
import java.security.cert.X509Certificate;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ -27,6 +28,7 @@ import javax.net.ssl.X509TrustManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.base.Objects;
import com.squareup.moshi.JsonAdapter;
import com.squareup.moshi.Moshi;
@ -47,7 +49,7 @@ public abstract class HttpClient {
private static final Logger LOG = LoggerFactory.getLogger(HttpClient.class);
private static final ConnectionPool GLOBAL_HTTP_CONN_POOL = new ConnectionPool(10, 2, TimeUnit.MINUTES);
protected OkHttpClient client;
protected CookieJarImpl cookieJar = new CookieJarImpl();
protected boolean loggedIn = false;
@ -126,8 +128,8 @@ public abstract class HttpClient {
.connectionPool(GLOBAL_HTTP_CONN_POOL)
.connectTimeout(Config.getInstance().getSettings().httpTimeout, TimeUnit.MILLISECONDS)
.readTimeout(Config.getInstance().getSettings().httpTimeout, TimeUnit.MILLISECONDS)
//.addInterceptor(new LoggingInterceptor())
.connectionPool(new ConnectionPool(50, 10, TimeUnit.MINUTES));
//.addInterceptor(new LoggingInterceptor());
ProxyType proxyType = Config.getInstance().getSettings().proxyType;
if (proxyType == ProxyType.HTTP) {
@ -269,4 +271,19 @@ public abstract class HttpClient {
//Request request = new Request.Builder().url(url).build();
return client.newWebSocket(request, l);
}
public List<Cookie> getCookiesByName(String... names) {
List<Cookie> result = new ArrayList<>();
Map<String, List<Cookie>> cookies = getCookieJar().getCookies();
for (List<Cookie> cookieList : cookies.values()) {
for (Cookie cookie : cookieList) {
for (String cookieName : names) {
if (Objects.equal(cookieName, cookie.name())) {
result.add(cookie);
}
}
}
}
return result;
}
}

1
common/src/main/java/ctbrec/io/HttpConstants.java
View File

@ -4,6 +4,7 @@ public class HttpConstants {
public static final String ACCEPT = "Accept";
public static final String ACCEPT_LANGUAGE = "Accept-Language";
public static final String COOKIE = "Cookie";
public static final String CONNECTION = "Connection";
public static final String CONTENT_TYPE = "Content-Type";
public static final String KEEP_ALIVE = "keep-alive";

10
common/src/main/java/ctbrec/sites/manyvids/MVLive.java
View File

@ -8,7 +8,8 @@ import ctbrec.sites.AbstractSite;
public class MVLive extends AbstractSite {
public static final String BASE_URL = "https://www.manyvids.com";
public static final String WS_URL = "https://live.manyvids.com";
public static final String BASE_URL = "https://www.manyvids.com/MVLive/";
private MVLiveHttpClient httpClient;
@ -61,6 +62,13 @@ public class MVLive extends AbstractSite {
@Override
public void init() throws IOException {
try {
MVLiveClient.getInstance().setSite(this);
MVLiveClient.getInstance().start();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
@Override

232
common/src/main/java/ctbrec/sites/manyvids/MVLiveClient.java Normal file
View File

@ -0,0 +1,232 @@
package ctbrec.sites.manyvids;
import static ctbrec.io.HttpConstants.*;
import static ctbrec.sites.manyvids.MVLive.*;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Random;
import java.util.UUID;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import java.util.stream.Collectors;
import org.json.JSONArray;
import org.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.base.Objects;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import com.squareup.moshi.Moshi;
import ctbrec.Config;
import ctbrec.io.HttpException;
import ctbrec.sites.manyvids.wsmsg.JoinChat;
import ctbrec.sites.manyvids.wsmsg.Message;
import ctbrec.sites.manyvids.wsmsg.Ping;
import ctbrec.sites.manyvids.wsmsg.SendMessage;
import okhttp3.Cookie;
import okhttp3.Request;
import okhttp3.Response;
import okhttp3.WebSocket;
import okhttp3.WebSocketListener;
import okio.ByteString;
public class MVLiveClient {
private static final Logger LOG = LoggerFactory.getLogger(MVLiveClient.class);
private static MVLiveClient instance;
private MVLive site;
private WebSocket ws;
private Moshi moshi;
private Random rng = new Random();
private volatile boolean running = false;
private Cache<Integer, MVLiveModel> models = CacheBuilder.newBuilder().maximumSize(4000).build();
private Lock lock = new ReentrantLock();
private volatile boolean connecting = false;
private MVLiveClient() {
moshi = new Moshi.Builder().build();
}
public static synchronized MVLiveClient getInstance() {
if (instance == null) {
instance = new MVLiveClient();
}
return instance;
}
public void setSite(MVLive site) {
this.site = site;
}
public void start() throws IOException {
running = true;
// make a call to the website to get necessary cookies
Request req = new Request.Builder()
.url(site.getBaseUrl())
.header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
.build();
try (Response response = site.getHttpClient().execute(req)) {
if (!response.isSuccessful()) {
throw new HttpException(response.code(), response.message());
}
}
int randomNumber = 100 + rng.nextInt(800);
String randomString = UUID.randomUUID().toString().replace("-", "").substring(0, 8);
String roomNumber = "176";
String wsUrl = String.format("%s/api/%s/eventbus/%s/%s/websocket", WS_URL, roomNumber, randomNumber, randomString);
List<Cookie> cookies = site.getHttpClient().getCookiesByName("PHPSESSID"); //, "XSRF-TOKEN");
String cookieHeaderValue = cookies.stream().map(Object::toString).collect(Collectors.joining("; "));
Thread watchDog = new Thread(() -> {
while (running) {
if (ws == null && !connecting) {
LOG.info("Websocket is null. Starting a new connection to {}", wsUrl);
Request websocketRequest = new Request.Builder()
.url(wsUrl)
.header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
.header(ORIGIN, WS_URL)
.header(COOKIE, cookieHeaderValue)
.build();
ws = createWebSocket(websocketRequest);
}
try {
Thread.sleep(10000);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
LOG.error("WatchDog couldn't sleep", e);
stop();
running = false;
}
}
});
watchDog.setDaemon(true);
watchDog.setName("MVLive WebSocket WatchDog");
watchDog.setPriority(Thread.MIN_PRIORITY);
watchDog.start();
}
public void stop() {
running = false;
ws.close(1000, "Good Bye"); // terminate normally (1000)
}
public List<MVLiveModel> getModels() {
lock.lock();
try {
LOG.trace("Models: {}", models.size());
return new ArrayList<>(this.models.asMap().values());
} finally {
lock.unlock();
}
}
private WebSocket createWebSocket(Request req) {
connecting = true;
WebSocket websocket = site.getHttpClient().newWebSocket(req, new WebSocketListener() {
@Override
public void onOpen(WebSocket webSocket, Response response) {
super.onOpen(webSocket, response);
try {
connecting = false;
models.invalidateAll();
LOG.debug("WS open: [{}]", response.body().string());
} catch (IOException e) {
LOG.error("Error while processing onOpen event", e);
}
}
@Override
public void onClosed(WebSocket webSocket, int code, String reason) {
super.onClosed(webSocket, code, reason);
connecting = false;
LOG.info("MVLive websocket closed: {} {}", code, reason);
MVLiveClient.this.ws = null;
if (!running) {
site.getHttpClient().shutdown();
}
}
@Override
public void onFailure(WebSocket webSocket, Throwable t, Response response) {
super.onFailure(webSocket, t, response);
connecting = false;
if (response != null) {
int code = response.code();
String message = response.message();
LOG.error("MVLive websocket failure: {} {}", code, message, t);
response.close();
} else {
LOG.error("MVLive websocket failure", t);
}
MVLiveClient.this.ws = null;
}
@Override
public void onMessage(WebSocket webSocket, String text) {
super.onMessage(webSocket, text);
//msgBuffer.append(text);
LOG.debug("Message: {}", text);
text = Optional.ofNullable(text).orElse("");
if (Objects.equal("o", text)) {
sendMessages(new Ping());
String roomId = "3d9b273a-8039-41f7-a5d8-bdd6972d697c";
String showId = "HARIBOO";
sendMessages(new JoinChat(roomId, showId, (m, r) -> {
System.out.println(m);
System.out.println(r);
}));
} else if (text.startsWith("a")) {
JSONArray jsonArray = new JSONArray(text.substring(1));
for (int i = 0; i < jsonArray.length(); i++) {
String respJson = jsonArray.getString(i);
JSONObject response = new JSONObject(respJson);
String address = response.optString("address");
if (!address.isBlank()) {
SendMessage sendMessage = futureResponses.get(address);
if (sendMessage != null) {
sendMessage.handleResponse(response);
}
}
}
}
}
Map<String, SendMessage> futureResponses = new HashMap<>();
private void sendMessages(Message... messages) {
JSONArray msgs = new JSONArray();
for (Message msg : messages) {
msgs.put(msg.toString());
if(msg instanceof SendMessage) {
SendMessage sendMessage = (SendMessage) msg;
futureResponses.put(sendMessage.getReplyAddress(), sendMessage);
}
}
ws.send(msgs.toString());
}
@Override
public void onMessage(WebSocket webSocket, ByteString bytes) {
super.onMessage(webSocket, bytes);
LOG.debug("Binary Message: {}", bytes.hex());
}
});
return websocket;
}
}

34
common/src/main/java/ctbrec/sites/manyvids/wsmsg/JoinChat.java Normal file
View File

@ -0,0 +1,34 @@
package ctbrec.sites.manyvids.wsmsg;
import java.util.UUID;
import java.util.function.BiConsumer;
import org.json.JSONObject;
public class JoinChat extends SendMessage {
private String roomId;
private String showId;
public JoinChat(String roomId, String showId, BiConsumer<Message, JSONObject> responseConsumer) {
super(responseConsumer);
this.roomId = roomId;
this.showId = showId;
address = "api/ChatService";
action = "join";
}
@Override
public String toString() {
JSONObject msg = build();
JSONObject body = new JSONObject();
body.put("connectionId", "");
body.put("telemetryId", UUID.randomUUID().toString());
body.put("roomId", roomId);
body.put("showId", showId);
body.put("joinPrivateShow", false);
body.put("deviceSourceType", "DESKTOP");
msg.put("body", body);
return msg.toString();
}
}

19
common/src/main/java/ctbrec/sites/manyvids/wsmsg/Message.java Normal file
View File

@ -0,0 +1,19 @@
package ctbrec.sites.manyvids.wsmsg;
import java.util.Optional;
import java.util.function.BiConsumer;
import org.json.JSONObject;
public abstract class Message extends JSONObject {
private BiConsumer<Message, JSONObject> responseConsumer;
public Message(BiConsumer<Message, JSONObject> responseConsumer) {
this.responseConsumer = responseConsumer;
}
public void handleResponse(JSONObject response) {
Optional.ofNullable(responseConsumer).ifPresent(consumer -> consumer.accept(this, response));
}
}

9
common/src/main/java/ctbrec/sites/manyvids/wsmsg/Ping.java Normal file
View File

@ -0,0 +1,9 @@
package ctbrec.sites.manyvids.wsmsg;
public class Ping extends Message {
public Ping() {
super(null);
put("type", "ping");
}
}

7
common/src/main/java/ctbrec/sites/manyvids/wsmsg/Response.java Normal file
View File

@ -0,0 +1,7 @@
package ctbrec.sites.manyvids.wsmsg;
import org.json.JSONObject;
public class Response extends JSONObject {
}

33
common/src/main/java/ctbrec/sites/manyvids/wsmsg/SendMessage.java Normal file
View File

@ -0,0 +1,33 @@
package ctbrec.sites.manyvids.wsmsg;
import java.util.UUID;
import java.util.function.BiConsumer;
import org.json.JSONObject;
public class SendMessage extends Message {
protected String address;
protected String action;
protected String replyAddress;
public SendMessage(BiConsumer<Message, JSONObject> responseConsumer) {
super(responseConsumer);
replyAddress = UUID.randomUUID().toString();
}
public String getReplyAddress() {
return replyAddress;
}
protected JSONObject build() {
JSONObject msg = new JSONObject();
msg.put("type", "send");
msg.put("address", address);
msg.put("replyAddress", replyAddress);
JSONObject headers = new JSONObject();
headers.put("action", action);
msg.put("headers", headers);
return msg;
}
}