From 096ed7a7341e3847fe7dc1fee1a413406abc003f Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Tue, 18 Aug 2020 19:58:48 +0200 Subject: [PATCH] Implement proper online check and add search --- .../ui/sites/manyvids/MVLiveTabProvider.java | 6 +- .../sites/manyvids/MVLiveUpdateService.java | 63 +------- .../java/ctbrec/sites/manyvids/MVLive.java | 152 +++++++++++++++++- .../ctbrec/sites/manyvids/MVLiveClient.java | 46 +++--- .../ctbrec/sites/manyvids/MVLiveModel.java | 31 +++- 5 files changed, 200 insertions(+), 98 deletions(-) diff --git a/client/src/main/java/ctbrec/ui/sites/manyvids/MVLiveTabProvider.java b/client/src/main/java/ctbrec/ui/sites/manyvids/MVLiveTabProvider.java index 51bbd0ae..cdfa829e 100644 --- a/client/src/main/java/ctbrec/ui/sites/manyvids/MVLiveTabProvider.java +++ b/client/src/main/java/ctbrec/ui/sites/manyvids/MVLiveTabProvider.java @@ -21,12 +21,12 @@ public class MVLiveTabProvider extends TabProvider { @Override public List getTabs(Scene scene) { List tabs = new ArrayList<>(); - tabs.add(createTab("Online", mvlive.getBaseUrl())); + tabs.add(createTab("Online")); return tabs; } - private Tab createTab(String title, String url) { - MVLiveUpdateService updateService = new MVLiveUpdateService(url, mvlive); + private Tab createTab(String title) { + MVLiveUpdateService updateService = new MVLiveUpdateService(mvlive); ThumbOverviewTab tab = new ThumbOverviewTab(title, updateService, mvlive); tab.setRecorder(mvlive.getRecorder()); return tab; diff --git a/client/src/main/java/ctbrec/ui/sites/manyvids/MVLiveUpdateService.java b/client/src/main/java/ctbrec/ui/sites/manyvids/MVLiveUpdateService.java index ede57711..02103906 100644 --- a/client/src/main/java/ctbrec/ui/sites/manyvids/MVLiveUpdateService.java +++ b/client/src/main/java/ctbrec/ui/sites/manyvids/MVLiveUpdateService.java @@ -1,33 +1,18 @@ package ctbrec.ui.sites.manyvids; import java.io.IOException; -import java.util.ArrayList; import java.util.List; -import org.jsoup.nodes.Element; -import org.jsoup.select.Elements; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import ctbrec.Config; import ctbrec.Model; -import ctbrec.io.HtmlParser; -import ctbrec.io.HttpException; import ctbrec.sites.manyvids.MVLive; -import ctbrec.sites.manyvids.MVLiveModel; import ctbrec.ui.tabs.PaginatedScheduledService; import javafx.concurrent.Task; -import okhttp3.Request; -import okhttp3.Response; public class MVLiveUpdateService extends PaginatedScheduledService { - private static final Logger LOG = LoggerFactory.getLogger(MVLiveUpdateService.class); - private String url; private MVLive mvlive; - public MVLiveUpdateService(String url, MVLive mvlive) { - this.url = url; + public MVLiveUpdateService(MVLive mvlive) { this.mvlive = mvlive; } @@ -36,51 +21,7 @@ public class MVLiveUpdateService extends PaginatedScheduledService { return new Task>() { @Override public List call() throws IOException { - LOG.debug("Fetching page {}", url); - Request request = new Request.Builder() - .url(url) - .header("User-Agent", Config.getInstance().getSettings().httpUserAgent) - .header("Referer", MVLive.BASE_URL) - .build(); - try (Response response = mvlive.getHttpClient().execute(request)) { - if (response.isSuccessful()) { - List models = new ArrayList<>(); - String content = response.body().string(); - Elements cards = HtmlParser.getTags(content, "div[class~=mv-live-model]"); - for (Element card : cards) { - try { - String cardHtml = card.html(); - Element link = HtmlParser.getTag(cardHtml, "a"); - link.setBaseUri(mvlive.getBaseUrl()); - String name = HtmlParser.getText(cardHtml, "h4 a"); - MVLiveModel model = (MVLiveModel) mvlive.createModel(name); - model.setUrl(link.absUrl("href")); - Element thumb = HtmlParser.getTag(cardHtml, "a img.b-lazy"); - thumb.setBaseUri(mvlive.getBaseUrl()); - model.setPreview(thumb.absUrl("data-src")); - // Element status = HtmlParser.getTag(cardHtml, "div[class~=model-status]"); - // String cssClass = status.attr("class"); - // if(cssClass.contains("live")) { - // model.setOnlineState(Model.State.ONLINE); - // } else if(cssClass.contains("private")) { - // model.setOnlineState(Model.State.PRIVATE); - // } else { - // model.setOnlineState(Model.State.UNKNOWN); - // } - models.add(model); - } catch(RuntimeException e) { - if(e.getMessage().contains("No element selected by")) { - // ignore - } else { - throw e; - } - } - } - return models; - } else { - throw new HttpException(response.code(), response.message()); - } - } + return mvlive.getModels(); } }; } diff --git a/common/src/main/java/ctbrec/sites/manyvids/MVLive.java b/common/src/main/java/ctbrec/sites/manyvids/MVLive.java index e3cd81ad..03a9c819 100644 --- a/common/src/main/java/ctbrec/sites/manyvids/MVLive.java +++ b/common/src/main/java/ctbrec/sites/manyvids/MVLive.java @@ -1,10 +1,28 @@ package ctbrec.sites.manyvids; -import java.io.IOException; +import static ctbrec.io.HttpConstants.*; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; + +import org.json.JSONArray; +import org.json.JSONObject; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; + +import ctbrec.Config; import ctbrec.Model; +import ctbrec.Model.State; +import ctbrec.io.HtmlParser; import ctbrec.io.HttpClient; +import ctbrec.io.HttpException; import ctbrec.sites.AbstractSite; +import okhttp3.FormBody; +import okhttp3.Request; +import okhttp3.RequestBody; +import okhttp3.Response; public class MVLive extends AbstractSite { @@ -14,6 +32,7 @@ public class MVLive extends AbstractSite { public static final String BASE_URL = "https://www.manyvids.com/MVLive/"; private MVLiveHttpClient httpClient; + private String mvtoken; @Override public String getName() { @@ -31,7 +50,7 @@ public class MVLive extends AbstractSite { } @Override - public Model createModel(String name) { + public MVLiveModel createModel(String name) { MVLiveModel model = new MVLiveModel(); model.setName(name); model.setDescription(""); @@ -91,7 +110,134 @@ public class MVLive extends AbstractSite { @Override public void init() throws IOException { - } + public List getModels() throws IOException { + Request request = new Request.Builder() + .url(getBaseUrl()) + .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) + .header(REFERER, MVLive.BASE_URL) + .build(); + try (Response response = getHttpClient().execute(request)) { + if (response.isSuccessful()) { + List models = new ArrayList<>(); + String content = response.body().string(); + Elements cards = HtmlParser.getTags(content, "div[class*=-model]"); + for (Element card : cards) { + try { + String cardHtml = card.html(); + Element link = HtmlParser.getTag(cardHtml, "a"); + link.setBaseUri(getBaseUrl()); + String name = HtmlParser.getText(cardHtml, "h4 a"); + MVLiveModel model = createModel(name); + model.setUrl(link.absUrl("href")); + Element thumb = HtmlParser.getTag(cardHtml, "a img.b-lazy"); + thumb.setBaseUri(getBaseUrl()); + model.setPreview(thumb.absUrl("data-src")); + + Element status = HtmlParser.getTag(cardHtml, "h4[class~=profile-pic-name]"); + String cssClass = status.attr("class"); + if(cssClass.contains("live")) { + model.setOnlineState(Model.State.ONLINE); + } else if(cssClass.contains("private")) { + model.setOnlineState(Model.State.PRIVATE); + } else { + model.setOnlineState(Model.State.UNKNOWN); + } + models.add(model); + } catch(RuntimeException e) { + if(e.getMessage().contains("No element selected by")) { + // ignore + } else { + throw e; + } + } + } + return models; + } else { + throw new HttpException(response.code(), response.message()); + } + } + } + + @Override + public boolean supportsSearch() { + return true; + } + + @Override + public boolean searchRequiresLogin() { + return false; + } + + String getMvToken() throws IOException { + if (mvtoken == null) { + Request request = new Request.Builder() + .url(getBaseUrl()) + .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) + .header(REFERER, MVLive.BASE_URL) + .build(); + try (Response response = getHttpClient().execute(request)) { + if (response.isSuccessful()) { + Element tag = HtmlParser.getTag(response.body().string(), "html"); + mvtoken = tag.attr("data-mvtoken"); + } else { + throw new HttpException(response.code(), response.message()); + } + } + } + return mvtoken; + } + + @Override + public List search(String q) throws IOException, InterruptedException { + List result = new ArrayList<>(); + RequestBody body = new FormBody.Builder() + .add("mvtoken", getMvToken()) + .add("type", "search") + .add("category", "stars") + .add("search", q) + .build(); + Request request = new Request.Builder() + .url("https://www.manyvids.com/includes/filterSearch.php") + .header(ACCEPT, MIMETYPE_APPLICATION_JSON) + .header(ACCEPT_LANGUAGE, Locale.ENGLISH.getLanguage()) + .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) + .header(ORIGIN, MVLive.BASE_URL) + .header(REFERER, MVLive.BASE_URL) + .header(X_REQUESTED_WITH, XML_HTTP_REQUEST) + .post(body) + .build(); + try (Response response = getHttpClient().execute(request)) { + if (response.isSuccessful()) { + String responseBody = response.body().string(); + JSONObject json = new JSONObject(responseBody); + if(json.has("stars")) { + JSONArray stars = json.getJSONArray("stars"); + for (int i = 0; i < stars.length(); i++) { + JSONObject star = stars.getJSONObject(i); + String name = star.getString("label"); + MVLiveModel model = createModel(name); + long id = star.getLong("id"); + String url = "https://www.manyvids.com/MVLive/" + model.getDisplayName() + '/' + id + '/'; + model.setUrl(url); + model.setPreview(star.getString("img")); + if (star.optString("is_live").equals("1")) { + if (star.optString("is_private").equals("1")) { + model.setOnlineState(State.PRIVATE); + } else { + model.setOnlineState(State.ONLINE); + } + } else { + model.setOnlineState(State.OFFLINE); + } + result.add(model); + } + } + } else { + throw new HttpException(response.code(), response.message()); + } + } + return result; + } } diff --git a/common/src/main/java/ctbrec/sites/manyvids/MVLiveClient.java b/common/src/main/java/ctbrec/sites/manyvids/MVLiveClient.java index 423f821b..4b0180a5 100644 --- a/common/src/main/java/ctbrec/sites/manyvids/MVLiveClient.java +++ b/common/src/main/java/ctbrec/sites/manyvids/MVLiveClient.java @@ -48,7 +48,7 @@ public class MVLiveClient { private String roomNumber; private String roomId; private ScheduledExecutorService scheduler; - + private Map futureResponses = new HashMap<>(); private MVLiveHttpClient httpClient; public MVLiveClient(MVLiveHttpClient httpClient) { @@ -68,13 +68,13 @@ public class MVLiveClient { String wsUrl = String.format("%s/api/%s/eventbus/%s/%s/websocket", WS_URL, roomNumber, randomNumber, randomString); LOG.info("Websocket is null. Starting a new connection to {}", wsUrl); - ws = createWebSocket(wsUrl, roomId, model.getName()); + ws = createWebSocket(wsUrl, roomId, model.getDisplayName()); } } private JSONObject getRoomLocation(MVLiveModel model) throws IOException { Request req = new Request.Builder() - .url(WS_ORIGIN + "/api/roomlocation/" + model.getName() + "?private=false") + .url(WS_ORIGIN + "/api/roomlocation/" + model.getDisplayName() + "?private=false") .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) .header(ACCEPT, MIMETYPE_APPLICATION_JSON) .header(COOKIE, getPhpSessionIdCookie()) @@ -170,7 +170,7 @@ public class MVLiveClient { LOG.debug("<-- {}", rr); masterPlaylist = rr.getJSONObject("body").optString("videoUrl"); LOG.debug("Got the URL: {}", masterPlaylist); - //stop(); + stop(); synchronized (streamUrlMonitor) { streamUrlMonitor.notifyAll(); } @@ -195,23 +195,6 @@ public class MVLiveClient { } } - Map futureResponses = new HashMap<>(); - - private void sendMessages(Message... messages) { - JSONArray msgs = new JSONArray(); - for (Message msg : messages) { - msgs.put(msg.toString()); - if (msg instanceof SendMessage) { - SendMessage sendMessage = (SendMessage) msg; - futureResponses.put(sendMessage.getReplyAddress(), sendMessage); - } else if (msg instanceof RegisterMessage) { - RegisterMessage registerMessage = (RegisterMessage) msg; - futureResponses.put(registerMessage.getAddress(), registerMessage); - } - } - ws.send(msgs.toString()); - } - @Override public void onMessage(WebSocket webSocket, ByteString bytes) { super.onMessage(webSocket, bytes); @@ -221,6 +204,21 @@ public class MVLiveClient { return websocket; } + void sendMessages(Message... messages) { + JSONArray msgs = new JSONArray(); + for (Message msg : messages) { + msgs.put(msg.toString()); + if (msg instanceof SendMessage) { + SendMessage sendMessage = (SendMessage) msg; + futureResponses.put(sendMessage.getReplyAddress(), sendMessage); + } else if (msg instanceof RegisterMessage) { + RegisterMessage registerMessage = (RegisterMessage) msg; + futureResponses.put(registerMessage.getAddress(), registerMessage); + } + } + ws.send(msgs.toString()); + } + public StreamLocation getStreamLocation(MVLiveModel model) throws IOException, InterruptedException { start(model); while (running) { @@ -229,9 +227,9 @@ public class MVLiveClient { break; } } - // if (ws != null) { - // stop(); - // } + if (ws != null) { + stop(); + } return new StreamLocation(roomId, roomNumber, masterPlaylist); } } diff --git a/common/src/main/java/ctbrec/sites/manyvids/MVLiveModel.java b/common/src/main/java/ctbrec/sites/manyvids/MVLiveModel.java index c03646dc..4812795b 100644 --- a/common/src/main/java/ctbrec/sites/manyvids/MVLiveModel.java +++ b/common/src/main/java/ctbrec/sites/manyvids/MVLiveModel.java @@ -1,5 +1,6 @@ package ctbrec.sites.manyvids; +import static ctbrec.Model.State.*; import static ctbrec.io.HttpConstants.*; import static java.nio.charset.StandardCharsets.*; @@ -28,6 +29,8 @@ import com.iheartradio.m3u8.data.PlaylistData; import ctbrec.AbstractModel; import ctbrec.Config; +import ctbrec.Model; +import ctbrec.StringUtil; import ctbrec.io.HttpException; import ctbrec.recorder.download.Download; import ctbrec.recorder.download.StreamSource; @@ -45,8 +48,19 @@ public class MVLiveModel extends AbstractModel { @Override public boolean isOnline(boolean ignoreCache) throws IOException, ExecutionException, InterruptedException { - JSONObject json = getRoomLocation(); - return json.optBoolean("success"); + if (ignoreCache) { + MVLive site = (MVLive) getSite(); + for (Model model : site.getModels()) { + if (model.getName().equalsIgnoreCase(getName()) || model.getDisplayName().equalsIgnoreCase(getName())) { + this.onlineState = model.getOnlineState(true); + setName(model.getName()); + setDisplayName(model.getDisplayName()); + setUrl(model.getUrl()); + break; + } + } + } + return this.onlineState == ONLINE; } @Override @@ -91,7 +105,6 @@ public class MVLiveModel extends AbstractModel { try (Response response = getHttpClient().execute(req)) { if (response.isSuccessful()) { String body = response.body().string(); - LOG.debug(body); InputStream inputStream = new ByteArrayInputStream(body.getBytes(UTF_8)); PlaylistParser parser = new PlaylistParser(inputStream, Format.EXT_M3U, Encoding.UTF_8, ParsingMode.LENIENT); Playlist playlist = parser.parse(); @@ -104,7 +117,7 @@ public class MVLiveModel extends AbstractModel { } public void updateCloudFlareCookies() throws IOException, InterruptedException { - String url = MVLive.WS_ORIGIN + "/api/" + getRoomNumber() + "/player-settings/" + getName(); + String url = MVLive.WS_ORIGIN + "/api/" + getRoomNumber() + "/player-settings/" + getDisplayName(); LOG.trace("Getting CF cookies: {}", url); Request req = new Request.Builder() .url(url) @@ -112,17 +125,19 @@ public class MVLiveModel extends AbstractModel { .build(); try (Response response = getHttpClient().execute(req)) { if (!response.isSuccessful()) { + LOG.debug(response.body().string()); throw new HttpException(response.code(), response.message()); } } } public String getRoomNumber() throws IOException, InterruptedException { - if (roomNumber == null) { + if (StringUtil.isBlank(roomNumber)) { JSONObject json = getRoomLocation(); if (json.optBoolean("success")) { roomNumber = json.getString("floorId"); } else { + LOG.debug("Room number response: {}", json.toString(2)); throw new RuntimeException(getName() + " is offline"); } } @@ -144,7 +159,8 @@ public class MVLiveModel extends AbstractModel { public JSONObject getRoomLocation() throws IOException { fetchGeneralCookies(); httpClient.fetchAuthenticationCookies(); - String url = MVLive.WS_ORIGIN + "/api/roomlocation/" + getName() + "?private=false"; + String url = MVLive.WS_ORIGIN + "/api/roomlocation/" + getDisplayName() + "?private=false"; + LOG.trace("Fetching room location from {}", url); Request req = new Request.Builder() .url(url) .header(ACCEPT, MIMETYPE_APPLICATION_JSON) @@ -155,6 +171,7 @@ public class MVLiveModel extends AbstractModel { try (Response response = getHttpClient().execute(req)) { if (response.isSuccessful()) { JSONObject json = new JSONObject(response.body().string()); + LOG.trace("Room location response: {}", json.toString(2)); return json; } else { throw new HttpException(response.code(), response.message()); @@ -180,7 +197,7 @@ public class MVLiveModel extends AbstractModel { @Override public int[] getStreamResolution(boolean failFast) throws ExecutionException { - return new int[2]; + return new int[] {1280, 720}; } @Override