From 6600b0da940c1340f44009d9fec168ea4f8d8e32 Mon Sep 17 00:00:00 2001 From: 0xboobface <0xboobface@gmail.com> Date: Fri, 10 Jul 2020 22:12:28 +0200 Subject: [PATCH] Get the authentication and the stream working --- .../java/ctbrec/ui/settings/SettingsTab.java | 6 +- .../java/ctbrec/sites/manyvids/MVLive.java | 12 +- .../ctbrec/sites/manyvids/MVLiveClient.java | 171 ++++++++++-------- .../ctbrec/sites/manyvids/MVLiveModel.java | 87 ++++++++- .../ctbrec/sites/manyvids/StreamLocation.java | 14 ++ .../manyvids/wsmsg/GetBroadcastHealth.java | 32 ++++ .../sites/manyvids/wsmsg/RegisterMessage.java | 28 +++ 7 files changed, 263 insertions(+), 87 deletions(-) create mode 100644 common/src/main/java/ctbrec/sites/manyvids/StreamLocation.java create mode 100644 common/src/main/java/ctbrec/sites/manyvids/wsmsg/GetBroadcastHealth.java create mode 100644 common/src/main/java/ctbrec/sites/manyvids/wsmsg/RegisterMessage.java diff --git a/client/src/main/java/ctbrec/ui/settings/SettingsTab.java b/client/src/main/java/ctbrec/ui/settings/SettingsTab.java index 7d3a3c52..0f8e7cbc 100644 --- a/client/src/main/java/ctbrec/ui/settings/SettingsTab.java +++ b/client/src/main/java/ctbrec/ui/settings/SettingsTab.java @@ -2,6 +2,7 @@ package ctbrec.ui.settings; import static ctbrec.Settings.DirectoryStructure.*; import static ctbrec.Settings.ProxyType.*; +import static java.util.Optional.*; import java.io.IOException; import java.util.ArrayList; @@ -19,6 +20,7 @@ import ctbrec.Settings.DirectoryStructure; import ctbrec.Settings.ProxyType; import ctbrec.recorder.Recorder; import ctbrec.sites.Site; +import ctbrec.ui.SiteUI; import ctbrec.ui.SiteUiFactory; import ctbrec.ui.controls.range.DiscreteRange; import ctbrec.ui.settings.api.Category; @@ -31,6 +33,7 @@ import ctbrec.ui.settings.api.SimpleDirectoryProperty; import ctbrec.ui.settings.api.SimpleFileProperty; import ctbrec.ui.settings.api.SimpleRangeProperty; import ctbrec.ui.settings.api.ValueConverter; +import ctbrec.ui.sites.ConfigUI; import ctbrec.ui.tabs.TabSelectionListener; import javafx.beans.binding.BooleanExpression; import javafx.beans.property.SimpleBooleanProperty; @@ -152,7 +155,8 @@ public class SettingsTab extends Tab implements TabSelectionListener { ignoreList = new IgnoreList(sites); List siteCategories = new ArrayList<>(); for (Site site : sites) { - siteCategories.add(Category.of(site.getName(), SiteUiFactory.getUi(site).getConfigUI().createConfigPanel())); + ofNullable(SiteUiFactory.getUi(site)).map(SiteUI::getConfigUI).map(ConfigUI::createConfigPanel) + .ifPresent(configPanel -> siteCategories.add(Category.of(site.getName(), configPanel))); } Preferences prefs = Preferences.of(new CtbrecPreferencesStorage(config), diff --git a/common/src/main/java/ctbrec/sites/manyvids/MVLive.java b/common/src/main/java/ctbrec/sites/manyvids/MVLive.java index 9b2a44fb..73b2ca4e 100644 --- a/common/src/main/java/ctbrec/sites/manyvids/MVLive.java +++ b/common/src/main/java/ctbrec/sites/manyvids/MVLive.java @@ -8,7 +8,9 @@ import ctbrec.sites.AbstractSite; public class MVLive extends AbstractSite { - public static final String WS_URL = "https://live.manyvids.com"; + public static final String WS_URL = "wss://live.manyvids.com"; + //public static final String WS_URL = "http://localhost:8080"; + public static final String WS_ORIGIN = "https://live.manyvids.com"; public static final String BASE_URL = "https://www.manyvids.com/MVLive/"; private MVLiveHttpClient httpClient; @@ -62,13 +64,7 @@ public class MVLive extends AbstractSite { @Override public void init() throws IOException { - try { - MVLiveClient.getInstance().setSite(this); - MVLiveClient.getInstance().start(); - } catch (IOException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } + MVLiveClient.getInstance().setSite(this); } @Override diff --git a/common/src/main/java/ctbrec/sites/manyvids/MVLiveClient.java b/common/src/main/java/ctbrec/sites/manyvids/MVLiveClient.java index 04ef34bb..0d136958 100644 --- a/common/src/main/java/ctbrec/sites/manyvids/MVLiveClient.java +++ b/common/src/main/java/ctbrec/sites/manyvids/MVLiveClient.java @@ -4,16 +4,13 @@ import static ctbrec.io.HttpConstants.*; import static ctbrec.sites.manyvids.MVLive.*; import java.io.IOException; -import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Random; import java.util.UUID; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; -import java.util.stream.Collectors; +import java.util.concurrent.TimeUnit; import org.json.JSONArray; import org.json.JSONObject; @@ -21,15 +18,13 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.common.base.Objects; -import com.google.common.cache.Cache; -import com.google.common.cache.CacheBuilder; -import com.squareup.moshi.Moshi; import ctbrec.Config; import ctbrec.io.HttpException; -import ctbrec.sites.manyvids.wsmsg.JoinChat; +import ctbrec.sites.manyvids.wsmsg.GetBroadcastHealth; import ctbrec.sites.manyvids.wsmsg.Message; import ctbrec.sites.manyvids.wsmsg.Ping; +import ctbrec.sites.manyvids.wsmsg.RegisterMessage; import ctbrec.sites.manyvids.wsmsg.SendMessage; import okhttp3.Cookie; import okhttp3.Request; @@ -45,16 +40,15 @@ public class MVLiveClient { private static MVLiveClient instance; private MVLive site; private WebSocket ws; - private Moshi moshi; private Random rng = new Random(); private volatile boolean running = false; - - private Cache models = CacheBuilder.newBuilder().maximumSize(4000).build(); - private Lock lock = new ReentrantLock(); private volatile boolean connecting = false; + private Object streamUrlMonitor = new Object(); + private String masterPlaylist = null; + private String roomNumber; + private String roomId; private MVLiveClient() { - moshi = new Moshi.Builder().build(); } public static synchronized MVLiveClient getInstance() { @@ -68,13 +62,42 @@ public class MVLiveClient { this.site = site; } - public void start() throws IOException { + public void start(MVLiveModel model) throws IOException { running = true; + if (ws == null && !connecting) { + fetchAuthenticationCookies(); + JSONObject response = getRoomLocation(model); + roomNumber = response.optString("floorId"); + roomId = response.optString("roomId"); + int randomNumber = 100 + rng.nextInt(800); + String randomString = UUID.randomUUID().toString().replace("-", "").substring(0, 8); + String wsUrl = String.format("%s/api/%s/eventbus/%s/%s/websocket", WS_URL, roomNumber, randomNumber, randomString); - // make a call to the website to get necessary cookies + LOG.info("Websocket is null. Starting a new connection to {}", wsUrl); + ws = createWebSocket(wsUrl, roomId, model.getName()); + } + } + + private JSONObject getRoomLocation(MVLiveModel model) throws IOException { Request req = new Request.Builder() - .url(site.getBaseUrl()) + .url(WS_ORIGIN + "/api/roomlocation/" + model.getName() + "?private=false") + .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) + .header(ACCEPT, MIMETYPE_APPLICATION_JSON) + .header(COOKIE, getPhpSessionIdCookie()) + .build(); + try (Response response = site.getHttpClient().execute(req)) { + if (response.isSuccessful()) { + return new JSONObject(response.body().string()); + } else { + throw new HttpException(response.code(), response.message()); + } + } + } + + private void fetchAuthenticationCookies() throws IOException { + Request req = new Request.Builder() + .url("https://www.manyvids.com/tak-live-redirect.php") .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) .build(); try (Response response = site.getHttpClient().execute(req)) { @@ -82,67 +105,33 @@ public class MVLiveClient { throw new HttpException(response.code(), response.message()); } } + } - int randomNumber = 100 + rng.nextInt(800); - String randomString = UUID.randomUUID().toString().replace("-", "").substring(0, 8); - String roomNumber = "176"; - String wsUrl = String.format("%s/api/%s/eventbus/%s/%s/websocket", WS_URL, roomNumber, randomNumber, randomString); - List cookies = site.getHttpClient().getCookiesByName("PHPSESSID"); //, "XSRF-TOKEN"); - String cookieHeaderValue = cookies.stream().map(Object::toString).collect(Collectors.joining("; ")); - - Thread watchDog = new Thread(() -> { - while (running) { - if (ws == null && !connecting) { - LOG.info("Websocket is null. Starting a new connection to {}", wsUrl); - Request websocketRequest = new Request.Builder() - .url(wsUrl) - .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) - .header(ORIGIN, WS_URL) - .header(COOKIE, cookieHeaderValue) - .build(); - ws = createWebSocket(websocketRequest); - } - - try { - Thread.sleep(10000); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - LOG.error("WatchDog couldn't sleep", e); - stop(); - running = false; - } - } - }); - watchDog.setDaemon(true); - watchDog.setName("MVLive WebSocket WatchDog"); - watchDog.setPriority(Thread.MIN_PRIORITY); - watchDog.start(); + private String getPhpSessionIdCookie() { + List cookies = site.getHttpClient().getCookiesByName("PHPSESSID"); + return cookies.stream().map(c -> c.name() + "=" + c.value()).findFirst().orElse(""); } public void stop() { running = false; ws.close(1000, "Good Bye"); // terminate normally (1000) + ws = null; } - public List getModels() { - lock.lock(); - try { - LOG.trace("Models: {}", models.size()); - return new ArrayList<>(this.models.asMap().values()); - } finally { - lock.unlock(); - } - } - - private WebSocket createWebSocket(Request req) { + private WebSocket createWebSocket(String wsUrl, String roomId, String modelName) { connecting = true; + Request req = new Request.Builder() + .url(wsUrl) + .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) + .header(ORIGIN, WS_ORIGIN) + .header(COOKIE, getPhpSessionIdCookie()) + .build(); WebSocket websocket = site.getHttpClient().newWebSocket(req, new WebSocketListener() { @Override public void onOpen(WebSocket webSocket, Response response) { super.onOpen(webSocket, response); try { connecting = false; - models.invalidateAll(); LOG.debug("WS open: [{}]", response.body().string()); } catch (IOException e) { LOG.error("Error while processing onOpen event", e); @@ -155,14 +144,19 @@ public class MVLiveClient { connecting = false; LOG.info("MVLive websocket closed: {} {}", code, reason); MVLiveClient.this.ws = null; - if (!running) { - site.getHttpClient().shutdown(); + running = false; + synchronized (streamUrlMonitor) { + streamUrlMonitor.notifyAll(); } } @Override public void onFailure(WebSocket webSocket, Throwable t, Response response) { super.onFailure(webSocket, t, response); + running = false; + synchronized (streamUrlMonitor) { + streamUrlMonitor.notifyAll(); + } connecting = false; if (response != null) { int code = response.code(); @@ -183,11 +177,20 @@ public class MVLiveClient { text = Optional.ofNullable(text).orElse(""); if (Objects.equal("o", text)) { sendMessages(new Ping()); - String roomId = "3d9b273a-8039-41f7-a5d8-bdd6972d697c"; - String showId = "HARIBOO"; - sendMessages(new JoinChat(roomId, showId, (m, r) -> { - System.out.println(m); - System.out.println(r); + sendMessages(new GetBroadcastHealth(roomId, modelName, (m, r) -> { + LOG.debug("--> {}", m); + LOG.debug("<-- {}", r); + String addr = r.getJSONObject("body").optString("subscribeAddress"); + sendMessages(new RegisterMessage(addr, (mr, rr) -> { + LOG.debug("--> {}", mr); + LOG.debug("<-- {}", rr); + masterPlaylist = rr.getJSONObject("body").optString("videoUrl"); + LOG.debug("Got the URL: {}", masterPlaylist); + stop(); + synchronized (streamUrlMonitor) { + streamUrlMonitor.notifyAll(); + } + })); })); } else if (text.startsWith("a")) { JSONArray jsonArray = new JSONArray(text.substring(1)); @@ -196,24 +199,30 @@ public class MVLiveClient { JSONObject response = new JSONObject(respJson); String address = response.optString("address"); if (!address.isBlank()) { - SendMessage sendMessage = futureResponses.get(address); - if (sendMessage != null) { - sendMessage.handleResponse(response); + Message message = futureResponses.get(address); + if (message != null) { + message.handleResponse(response); + if (!(message instanceof RegisterMessage)) { + futureResponses.remove(address); + } } } } } } - Map futureResponses = new HashMap<>(); + Map futureResponses = new HashMap<>(); private void sendMessages(Message... messages) { JSONArray msgs = new JSONArray(); for (Message msg : messages) { msgs.put(msg.toString()); - if(msg instanceof SendMessage) { + if (msg instanceof SendMessage) { SendMessage sendMessage = (SendMessage) msg; futureResponses.put(sendMessage.getReplyAddress(), sendMessage); + } else if (msg instanceof RegisterMessage) { + RegisterMessage registerMessage = (RegisterMessage) msg; + futureResponses.put(registerMessage.getAddress(), registerMessage); } } ws.send(msgs.toString()); @@ -228,5 +237,17 @@ public class MVLiveClient { return websocket; } - + public StreamLocation getStreamLocation(MVLiveModel model) throws IOException, InterruptedException { + start(model); + while (running) { + synchronized (streamUrlMonitor) { + streamUrlMonitor.wait(TimeUnit.SECONDS.toMillis(20000)); + running = false; + } + } + if (ws != null) { + stop(); + } + return new StreamLocation(roomId, roomNumber, masterPlaylist); + } } diff --git a/common/src/main/java/ctbrec/sites/manyvids/MVLiveModel.java b/common/src/main/java/ctbrec/sites/manyvids/MVLiveModel.java index de2f98d0..8e702fb8 100644 --- a/common/src/main/java/ctbrec/sites/manyvids/MVLiveModel.java +++ b/common/src/main/java/ctbrec/sites/manyvids/MVLiveModel.java @@ -1,18 +1,35 @@ package ctbrec.sites.manyvids; +import static ctbrec.io.HttpConstants.*; +import static java.nio.charset.StandardCharsets.*; + +import java.io.ByteArrayInputStream; import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.concurrent.ExecutionException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.iheartradio.m3u8.Encoding; +import com.iheartradio.m3u8.Format; import com.iheartradio.m3u8.ParseException; +import com.iheartradio.m3u8.ParsingMode; import com.iheartradio.m3u8.PlaylistException; +import com.iheartradio.m3u8.PlaylistParser; +import com.iheartradio.m3u8.data.MasterPlaylist; +import com.iheartradio.m3u8.data.Playlist; +import com.iheartradio.m3u8.data.PlaylistData; import ctbrec.AbstractModel; -import ctbrec.Model; +import ctbrec.Config; +import ctbrec.io.HttpException; import ctbrec.recorder.download.StreamSource; +import okhttp3.Request; +import okhttp3.Response; public class MVLiveModel extends AbstractModel { @@ -20,13 +37,77 @@ public class MVLiveModel extends AbstractModel { @Override public boolean isOnline(boolean ignoreCache) throws IOException, ExecutionException, InterruptedException { - return getOnlineState(true) == Model.State.ONLINE; + //return getOnlineState(true) == Model.State.ONLINE; + return true; } @Override public List getStreamSources() throws IOException, ExecutionException, ParseException, PlaylistException { LOG.debug("Loading {}", getUrl()); - return null; + MVLiveClient client = MVLiveClient.getInstance(); + try { + StreamLocation streamLocation = client.getStreamLocation(this); + getCloudFlareCookies(streamLocation.roomNumber); + MasterPlaylist masterPlaylist = getMasterPlaylist(streamLocation.masterPlaylist); + List sources = new ArrayList<>(); + for (PlaylistData playlist : masterPlaylist.getPlaylists()) { + if (playlist.hasStreamInfo()) { + StreamSource src = new StreamSource(); + src.bandwidth = playlist.getStreamInfo().getBandwidth(); + src.height = playlist.getStreamInfo().getResolution().height; + String masterUrl = streamLocation.masterPlaylist; + String baseUrl = masterUrl.substring(0, masterUrl.lastIndexOf('/') + 1); + String segmentUri = baseUrl + playlist.getUri(); + src.mediaPlaylistUrl = segmentUri; + if(src.mediaPlaylistUrl.contains("?")) { + src.mediaPlaylistUrl = src.mediaPlaylistUrl.substring(0, src.mediaPlaylistUrl.lastIndexOf('?')); + } + LOG.debug("Media playlist {}", src.mediaPlaylistUrl); + sources.add(src); + } + } + return sources; + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + return Collections.emptyList(); + } + + private MasterPlaylist getMasterPlaylist(String url) throws IOException, ParseException, PlaylistException { + LOG.trace("Loading master playlist {}", url); + Request req = new Request.Builder() + .url(url) + .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) + .build(); + try (Response response = getSite().getHttpClient().execute(req)) { + if (response.isSuccessful()) { + String body = response.body().string(); + LOG.debug(body); + InputStream inputStream = new ByteArrayInputStream(body.getBytes(UTF_8)); + PlaylistParser parser = new PlaylistParser(inputStream, Format.EXT_M3U, Encoding.UTF_8, ParsingMode.LENIENT); + Playlist playlist = parser.parse(); + MasterPlaylist master = playlist.getMasterPlaylist(); + return master; + } else { + throw new HttpException(response.code(), response.message()); + } + } + } + + private void getCloudFlareCookies(String roomNumber) throws IOException { + String url = MVLive.WS_ORIGIN + "/api/" + roomNumber + "/player-settings/" + getName(); + LOG.debug("Getting CF cookies: {}", url); + Request req = new Request.Builder() + .url(url) + .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) + .build(); + try (Response response = site.getHttpClient().execute(req)) { + if (!response.isSuccessful()) { + throw new HttpException(response.code(), response.message()); + } else { + LOG.debug("headers: {}", response.headers()); + } + } } @Override diff --git a/common/src/main/java/ctbrec/sites/manyvids/StreamLocation.java b/common/src/main/java/ctbrec/sites/manyvids/StreamLocation.java new file mode 100644 index 00000000..3d098421 --- /dev/null +++ b/common/src/main/java/ctbrec/sites/manyvids/StreamLocation.java @@ -0,0 +1,14 @@ +package ctbrec.sites.manyvids; + +public class StreamLocation { + + public String roomId; + public String roomNumber; + public String masterPlaylist; + + public StreamLocation(String roomId, String roomNumber, String masterPlaylist) { + this.roomId = roomId; + this.roomNumber = roomNumber; + this.masterPlaylist = masterPlaylist; + } +} diff --git a/common/src/main/java/ctbrec/sites/manyvids/wsmsg/GetBroadcastHealth.java b/common/src/main/java/ctbrec/sites/manyvids/wsmsg/GetBroadcastHealth.java new file mode 100644 index 00000000..871ab1a3 --- /dev/null +++ b/common/src/main/java/ctbrec/sites/manyvids/wsmsg/GetBroadcastHealth.java @@ -0,0 +1,32 @@ +package ctbrec.sites.manyvids.wsmsg; + +import java.util.UUID; +import java.util.function.BiConsumer; + +import org.json.JSONObject; + +public class GetBroadcastHealth extends SendMessage { + + private String roomId; + private String showId; + + public GetBroadcastHealth(String roomId, String showId, BiConsumer responseConsumer) { + super(responseConsumer); + this.roomId = roomId; + this.showId = showId; + address = "api/StreamService"; + action = "getBroadcastHealth"; + } + + @Override + public String toString() { + JSONObject msg = build(); + JSONObject body = new JSONObject(); + body.put("connectionId", ""); + body.put("telemetryId", UUID.randomUUID().toString()); + body.put("roomId", roomId); + body.put("showId", showId); + msg.put("body", body); + return msg.toString(); + } +} diff --git a/common/src/main/java/ctbrec/sites/manyvids/wsmsg/RegisterMessage.java b/common/src/main/java/ctbrec/sites/manyvids/wsmsg/RegisterMessage.java new file mode 100644 index 00000000..ecc8cc57 --- /dev/null +++ b/common/src/main/java/ctbrec/sites/manyvids/wsmsg/RegisterMessage.java @@ -0,0 +1,28 @@ +package ctbrec.sites.manyvids.wsmsg; + +import java.util.function.BiConsumer; + +import org.json.JSONObject; + +public class RegisterMessage extends Message { + + protected String address; + + public RegisterMessage(String address, BiConsumer responseConsumer) { + super(responseConsumer); + this.address = address; + } + + public String getAddress() { + return address; + } + + @Override + public String toString() { + JSONObject json = new JSONObject(); + json.put("type", "register"); + json.put("address", address); + json.put("headers", new JSONObject()); + return json.toString(); + } +}