Get the authentication and the stream working

client/src/main/java/ctbrec/ui/settings/SettingsTab.java

@@ -2,6 +2,7 @@ package ctbrec.ui.settings;
 
 import static ctbrec.Settings.DirectoryStructure.*;
 import static ctbrec.Settings.ProxyType.*;
+import static java.util.Optional.*;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -19,6 +20,7 @@ import ctbrec.Settings.DirectoryStructure;
 import ctbrec.Settings.ProxyType;
 import ctbrec.recorder.Recorder;
 import ctbrec.sites.Site;
+import ctbrec.ui.SiteUI;
 import ctbrec.ui.SiteUiFactory;
 import ctbrec.ui.controls.range.DiscreteRange;
 import ctbrec.ui.settings.api.Category;
@@ -31,6 +33,7 @@ import ctbrec.ui.settings.api.SimpleDirectoryProperty;
 import ctbrec.ui.settings.api.SimpleFileProperty;
 import ctbrec.ui.settings.api.SimpleRangeProperty;
 import ctbrec.ui.settings.api.ValueConverter;
+import ctbrec.ui.sites.ConfigUI;
 import ctbrec.ui.tabs.TabSelectionListener;
 import javafx.beans.binding.BooleanExpression;
 import javafx.beans.property.SimpleBooleanProperty;
@@ -152,7 +155,8 @@ public class SettingsTab extends Tab implements TabSelectionListener {
         ignoreList = new IgnoreList(sites);
         List<Category> siteCategories = new ArrayList<>();
         for (Site site : sites) {
-            siteCategories.add(Category.of(site.getName(), SiteUiFactory.getUi(site).getConfigUI().createConfigPanel()));
+            ofNullable(SiteUiFactory.getUi(site)).map(SiteUI::getConfigUI).map(ConfigUI::createConfigPanel)
+            .ifPresent(configPanel -> siteCategories.add(Category.of(site.getName(), configPanel)));
         }
 
         Preferences prefs = Preferences.of(new CtbrecPreferencesStorage(config),

common/src/main/java/ctbrec/sites/manyvids/MVLive.java

@@ -8,7 +8,9 @@ import ctbrec.sites.AbstractSite;
 
 public class MVLive extends AbstractSite {
 
-    public static final String WS_URL = "https://live.manyvids.com";
+    public static final String WS_URL = "wss://live.manyvids.com";
+    //public static final String WS_URL = "http://localhost:8080";
+    public static final String WS_ORIGIN = "https://live.manyvids.com";
     public static final String BASE_URL = "https://www.manyvids.com/MVLive/";
 
     private MVLiveHttpClient httpClient;
@@ -62,13 +64,7 @@ public class MVLive extends AbstractSite {
 
     @Override
     public void init() throws IOException {
-        try {
+        MVLiveClient.getInstance().setSite(this);
-            MVLiveClient.getInstance().setSite(this);
-            MVLiveClient.getInstance().start();
-        } catch (IOException e) {
-            // TODO Auto-generated catch block
-            e.printStackTrace();
-        }
     }
 
     @Override

common/src/main/java/ctbrec/sites/manyvids/MVLiveClient.java

@@ -4,16 +4,13 @@ import static ctbrec.io.HttpConstants.*;
 import static ctbrec.sites.manyvids.MVLive.*;
 
 import java.io.IOException;
-import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.Random;
 import java.util.UUID;
-import java.util.concurrent.locks.Lock;
+import java.util.concurrent.TimeUnit;
-import java.util.concurrent.locks.ReentrantLock;
-import java.util.stream.Collectors;
 
 import org.json.JSONArray;
 import org.json.JSONObject;
@@ -21,15 +18,13 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import com.google.common.base.Objects;
-import com.google.common.cache.Cache;
-import com.google.common.cache.CacheBuilder;
-import com.squareup.moshi.Moshi;
 
 import ctbrec.Config;
 import ctbrec.io.HttpException;
-import ctbrec.sites.manyvids.wsmsg.JoinChat;
+import ctbrec.sites.manyvids.wsmsg.GetBroadcastHealth;
 import ctbrec.sites.manyvids.wsmsg.Message;
 import ctbrec.sites.manyvids.wsmsg.Ping;
+import ctbrec.sites.manyvids.wsmsg.RegisterMessage;
 import ctbrec.sites.manyvids.wsmsg.SendMessage;
 import okhttp3.Cookie;
 import okhttp3.Request;
@@ -45,16 +40,15 @@ public class MVLiveClient {
     private static MVLiveClient instance;
     private MVLive site;
     private WebSocket ws;
-    private Moshi moshi;
     private Random rng = new Random();
     private volatile boolean running = false;
-
-    private Cache<Integer, MVLiveModel> models = CacheBuilder.newBuilder().maximumSize(4000).build();
-    private Lock lock = new ReentrantLock();
     private volatile boolean connecting = false;
+    private Object streamUrlMonitor = new Object();
+    private String masterPlaylist = null;
+    private String roomNumber;
+    private String roomId;
 
     private MVLiveClient() {
-        moshi = new Moshi.Builder().build();
     }
 
     public static synchronized MVLiveClient getInstance() {
@@ -68,13 +62,42 @@ public class MVLiveClient {
         this.site = site;
     }
 
-    public void start() throws IOException {
+    public void start(MVLiveModel model) throws IOException {
         running = true;
 
+        if (ws == null && !connecting) {
+            fetchAuthenticationCookies();
+            JSONObject response = getRoomLocation(model);
+            roomNumber = response.optString("floorId");
+            roomId = response.optString("roomId");
+            int randomNumber = 100 + rng.nextInt(800);
+            String randomString = UUID.randomUUID().toString().replace("-", "").substring(0, 8);
+            String wsUrl = String.format("%s/api/%s/eventbus/%s/%s/websocket", WS_URL, roomNumber, randomNumber, randomString);
 
-        // make a call to the website to get necessary cookies
+            LOG.info("Websocket is null. Starting a new connection to {}", wsUrl);
+            ws = createWebSocket(wsUrl, roomId, model.getName());
+        }
+    }
+
+    private JSONObject getRoomLocation(MVLiveModel model) throws IOException {
         Request req = new Request.Builder()
-                .url(site.getBaseUrl())
+                .url(WS_ORIGIN + "/api/roomlocation/" + model.getName() + "?private=false")
+                .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
+                .header(ACCEPT, MIMETYPE_APPLICATION_JSON)
+                .header(COOKIE, getPhpSessionIdCookie())
+                .build();
+        try (Response response = site.getHttpClient().execute(req)) {
+            if (response.isSuccessful()) {
+                return new JSONObject(response.body().string());
+            } else {
+                throw new HttpException(response.code(), response.message());
+            }
+        }
+    }
+
+    private void fetchAuthenticationCookies() throws IOException {
+        Request req = new Request.Builder()
+                .url("https://www.manyvids.com/tak-live-redirect.php")
                 .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
                 .build();
         try (Response response = site.getHttpClient().execute(req)) {
@@ -82,67 +105,33 @@ public class MVLiveClient {
                 throw new HttpException(response.code(), response.message());
             }
         }
+    }
 
-        int randomNumber = 100 + rng.nextInt(800);
+    private String getPhpSessionIdCookie() {
-        String randomString = UUID.randomUUID().toString().replace("-", "").substring(0, 8);
+        List<Cookie> cookies = site.getHttpClient().getCookiesByName("PHPSESSID");
-        String roomNumber = "176";
+        return cookies.stream().map(c -> c.name() + "=" + c.value()).findFirst().orElse("");
-        String wsUrl = String.format("%s/api/%s/eventbus/%s/%s/websocket", WS_URL, roomNumber, randomNumber, randomString);
-        List<Cookie> cookies = site.getHttpClient().getCookiesByName("PHPSESSID"); //, "XSRF-TOKEN");
-        String cookieHeaderValue = cookies.stream().map(Object::toString).collect(Collectors.joining("; "));
-
-        Thread watchDog = new Thread(() -> {
-            while (running) {
-                if (ws == null && !connecting) {
-                    LOG.info("Websocket is null. Starting a new connection to {}", wsUrl);
-                    Request websocketRequest = new Request.Builder()
-                            .url(wsUrl)
-                            .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
-                            .header(ORIGIN, WS_URL)
-                            .header(COOKIE, cookieHeaderValue)
-                            .build();
-                    ws = createWebSocket(websocketRequest);
-                }
-
-                try {
-                    Thread.sleep(10000);
-                } catch (InterruptedException e) {
-                    Thread.currentThread().interrupt();
-                    LOG.error("WatchDog couldn't sleep", e);
-                    stop();
-                    running = false;
-                }
-            }
-        });
-        watchDog.setDaemon(true);
-        watchDog.setName("MVLive WebSocket WatchDog");
-        watchDog.setPriority(Thread.MIN_PRIORITY);
-        watchDog.start();
     }
 
     public void stop() {
         running = false;
         ws.close(1000, "Good Bye"); // terminate normally (1000)
+        ws = null;
     }
 
-    public List<MVLiveModel> getModels() {
+    private WebSocket createWebSocket(String wsUrl, String roomId, String modelName) {
-        lock.lock();
-        try {
-            LOG.trace("Models: {}", models.size());
-            return new ArrayList<>(this.models.asMap().values());
-        } finally {
-            lock.unlock();
-        }
-    }
-
-    private WebSocket createWebSocket(Request req) {
         connecting = true;
+        Request req = new Request.Builder()
+                .url(wsUrl)
+                .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
+                .header(ORIGIN, WS_ORIGIN)
+                .header(COOKIE, getPhpSessionIdCookie())
+                .build();
         WebSocket websocket = site.getHttpClient().newWebSocket(req, new WebSocketListener() {
             @Override
             public void onOpen(WebSocket webSocket, Response response) {
                 super.onOpen(webSocket, response);
                 try {
                     connecting = false;
-                    models.invalidateAll();
                     LOG.debug("WS open: [{}]", response.body().string());
                 } catch (IOException e) {
                     LOG.error("Error while processing onOpen event", e);
@@ -155,14 +144,19 @@ public class MVLiveClient {
                 connecting = false;
                 LOG.info("MVLive websocket closed: {} {}", code, reason);
                 MVLiveClient.this.ws = null;
-                if (!running) {
+                running = false;
-                    site.getHttpClient().shutdown();
+                synchronized (streamUrlMonitor) {
+                    streamUrlMonitor.notifyAll();
                 }
             }
 
             @Override
             public void onFailure(WebSocket webSocket, Throwable t, Response response) {
                 super.onFailure(webSocket, t, response);
+                running = false;
+                synchronized (streamUrlMonitor) {
+                    streamUrlMonitor.notifyAll();
+                }
                 connecting = false;
                 if (response != null) {
                     int code = response.code();
@@ -183,11 +177,20 @@ public class MVLiveClient {
                 text = Optional.ofNullable(text).orElse("");
                 if (Objects.equal("o", text)) {
                     sendMessages(new Ping());
-                    String roomId = "3d9b273a-8039-41f7-a5d8-bdd6972d697c";
+                    sendMessages(new GetBroadcastHealth(roomId, modelName, (m, r) -> {
-                    String showId = "HARIBOO";
+                        LOG.debug("--> {}", m);
-                    sendMessages(new JoinChat(roomId, showId, (m, r) -> {
+                        LOG.debug("<-- {}", r);
-                        System.out.println(m);
+                        String addr = r.getJSONObject("body").optString("subscribeAddress");
-                        System.out.println(r);
+                        sendMessages(new RegisterMessage(addr, (mr, rr) -> {
+                            LOG.debug("--> {}", mr);
+                            LOG.debug("<-- {}", rr);
+                            masterPlaylist = rr.getJSONObject("body").optString("videoUrl");
+                            LOG.debug("Got the URL: {}", masterPlaylist);
+                            stop();
+                            synchronized (streamUrlMonitor) {
+                                streamUrlMonitor.notifyAll();
+                            }
+                        }));
                     }));
                 } else if (text.startsWith("a")) {
                     JSONArray jsonArray = new JSONArray(text.substring(1));
@@ -196,24 +199,30 @@ public class MVLiveClient {
                         JSONObject response = new JSONObject(respJson);
                         String address = response.optString("address");
                         if (!address.isBlank()) {
-                            SendMessage sendMessage = futureResponses.get(address);
+                            Message message = futureResponses.get(address);
-                            if (sendMessage != null) {
+                            if (message != null) {
-                                sendMessage.handleResponse(response);
+                                message.handleResponse(response);
+                                if (!(message instanceof RegisterMessage)) {
+                                    futureResponses.remove(address);
+                                }
                             }
                         }
                     }
                 }
             }
 
-            Map<String, SendMessage> futureResponses = new HashMap<>();
+            Map<String, Message> futureResponses = new HashMap<>();
 
             private void sendMessages(Message... messages) {
                 JSONArray msgs = new JSONArray();
                 for (Message msg : messages) {
                     msgs.put(msg.toString());
-                    if(msg instanceof SendMessage) {
+                    if (msg instanceof SendMessage) {
                         SendMessage sendMessage = (SendMessage) msg;
                         futureResponses.put(sendMessage.getReplyAddress(), sendMessage);
+                    } else if (msg instanceof RegisterMessage) {
+                        RegisterMessage registerMessage = (RegisterMessage) msg;
+                        futureResponses.put(registerMessage.getAddress(), registerMessage);
                     }
                 }
                 ws.send(msgs.toString());
@@ -228,5 +237,17 @@ public class MVLiveClient {
         return websocket;
     }
 
-
+    public StreamLocation getStreamLocation(MVLiveModel model) throws IOException, InterruptedException {
+        start(model);
+        while (running) {
+            synchronized (streamUrlMonitor) {
+                streamUrlMonitor.wait(TimeUnit.SECONDS.toMillis(20000));
+                running = false;
+            }
+        }
+        if (ws != null) {
+            stop();
+        }
+        return new StreamLocation(roomId, roomNumber, masterPlaylist);
+    }
 }

common/src/main/java/ctbrec/sites/manyvids/MVLiveModel.java

@@ -1,18 +1,35 @@
 package ctbrec.sites.manyvids;
 
+import static ctbrec.io.HttpConstants.*;
+import static java.nio.charset.StandardCharsets.*;
+
+import java.io.ByteArrayInputStream;
 import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 import java.util.concurrent.ExecutionException;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.iheartradio.m3u8.Encoding;
+import com.iheartradio.m3u8.Format;
 import com.iheartradio.m3u8.ParseException;
+import com.iheartradio.m3u8.ParsingMode;
 import com.iheartradio.m3u8.PlaylistException;
+import com.iheartradio.m3u8.PlaylistParser;
+import com.iheartradio.m3u8.data.MasterPlaylist;
+import com.iheartradio.m3u8.data.Playlist;
+import com.iheartradio.m3u8.data.PlaylistData;
 
 import ctbrec.AbstractModel;
-import ctbrec.Model;
+import ctbrec.Config;
+import ctbrec.io.HttpException;
 import ctbrec.recorder.download.StreamSource;
+import okhttp3.Request;
+import okhttp3.Response;
 
 public class MVLiveModel extends AbstractModel {
 
@@ -20,13 +37,77 @@ public class MVLiveModel extends AbstractModel {
 
     @Override
     public boolean isOnline(boolean ignoreCache) throws IOException, ExecutionException, InterruptedException {
-        return getOnlineState(true) == Model.State.ONLINE;
+        //return getOnlineState(true) == Model.State.ONLINE;
+        return true;
     }
 
     @Override
     public List<StreamSource> getStreamSources() throws IOException, ExecutionException, ParseException, PlaylistException {
         LOG.debug("Loading {}", getUrl());
-        return null;
+        MVLiveClient client = MVLiveClient.getInstance();
+        try {
+            StreamLocation streamLocation = client.getStreamLocation(this);
+            getCloudFlareCookies(streamLocation.roomNumber);
+            MasterPlaylist masterPlaylist = getMasterPlaylist(streamLocation.masterPlaylist);
+            List<StreamSource> sources = new ArrayList<>();
+            for (PlaylistData playlist : masterPlaylist.getPlaylists()) {
+                if (playlist.hasStreamInfo()) {
+                    StreamSource src = new StreamSource();
+                    src.bandwidth = playlist.getStreamInfo().getBandwidth();
+                    src.height = playlist.getStreamInfo().getResolution().height;
+                    String masterUrl = streamLocation.masterPlaylist;
+                    String baseUrl = masterUrl.substring(0, masterUrl.lastIndexOf('/') + 1);
+                    String segmentUri = baseUrl + playlist.getUri();
+                    src.mediaPlaylistUrl = segmentUri;
+                    if(src.mediaPlaylistUrl.contains("?")) {
+                        src.mediaPlaylistUrl = src.mediaPlaylistUrl.substring(0, src.mediaPlaylistUrl.lastIndexOf('?'));
+                    }
+                    LOG.debug("Media playlist {}", src.mediaPlaylistUrl);
+                    sources.add(src);
+                }
+            }
+            return sources;
+        } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
+        }
+        return Collections.emptyList();
+    }
+
+    private MasterPlaylist getMasterPlaylist(String url) throws IOException, ParseException, PlaylistException {
+        LOG.trace("Loading master playlist {}", url);
+        Request req = new Request.Builder()
+                .url(url)
+                .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
+                .build();
+        try (Response response = getSite().getHttpClient().execute(req)) {
+            if (response.isSuccessful()) {
+                String body = response.body().string();
+                LOG.debug(body);
+                InputStream inputStream = new ByteArrayInputStream(body.getBytes(UTF_8));
+                PlaylistParser parser = new PlaylistParser(inputStream, Format.EXT_M3U, Encoding.UTF_8, ParsingMode.LENIENT);
+                Playlist playlist = parser.parse();
+                MasterPlaylist master = playlist.getMasterPlaylist();
+                return master;
+            } else {
+                throw new HttpException(response.code(), response.message());
+            }
+        }
+    }
+
+    private void getCloudFlareCookies(String roomNumber) throws IOException {
+        String url = MVLive.WS_ORIGIN + "/api/" + roomNumber + "/player-settings/" + getName();
+        LOG.debug("Getting CF cookies: {}", url);
+        Request req = new Request.Builder()
+                .url(url)
+                .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
+                .build();
+        try (Response response = site.getHttpClient().execute(req)) {
+            if (!response.isSuccessful()) {
+                throw new HttpException(response.code(), response.message());
+            } else {
+                LOG.debug("headers: {}", response.headers());
+            }
+        }
     }
 
     @Override

common/src/main/java/ctbrec/sites/manyvids/StreamLocation.java

@@ -0,0 +1,14 @@
+package ctbrec.sites.manyvids;
+
+public class StreamLocation {
+
+    public String roomId;
+    public String roomNumber;
+    public String masterPlaylist;
+
+    public StreamLocation(String roomId, String roomNumber, String masterPlaylist) {
+        this.roomId = roomId;
+        this.roomNumber = roomNumber;
+        this.masterPlaylist = masterPlaylist;
+    }
+}

common/src/main/java/ctbrec/sites/manyvids/wsmsg/GetBroadcastHealth.java

@@ -0,0 +1,32 @@
+package ctbrec.sites.manyvids.wsmsg;
+
+import java.util.UUID;
+import java.util.function.BiConsumer;
+
+import org.json.JSONObject;
+
+public class GetBroadcastHealth extends SendMessage {
+
+    private String roomId;
+    private String showId;
+
+    public GetBroadcastHealth(String roomId, String showId, BiConsumer<Message, JSONObject> responseConsumer) {
+        super(responseConsumer);
+        this.roomId = roomId;
+        this.showId = showId;
+        address = "api/StreamService";
+        action = "getBroadcastHealth";
+    }
+
+    @Override
+    public String toString() {
+        JSONObject msg = build();
+        JSONObject body = new JSONObject();
+        body.put("connectionId", "");
+        body.put("telemetryId", UUID.randomUUID().toString());
+        body.put("roomId", roomId);
+        body.put("showId", showId);
+        msg.put("body", body);
+        return msg.toString();
+    }
+}

common/src/main/java/ctbrec/sites/manyvids/wsmsg/RegisterMessage.java

@@ -0,0 +1,28 @@
+package ctbrec.sites.manyvids.wsmsg;
+
+import java.util.function.BiConsumer;
+
+import org.json.JSONObject;
+
+public class RegisterMessage extends Message {
+
+    protected String address;
+
+    public RegisterMessage(String address, BiConsumer<Message, JSONObject> responseConsumer) {
+        super(responseConsumer);
+        this.address = address;
+    }
+
+    public String getAddress() {
+        return address;
+    }
+
+    @Override
+    public String toString() {
+        JSONObject json = new JSONObject();
+        json.put("type", "register");
+        json.put("address", address);
+        json.put("headers", new JSONObject());
+        return json.toString();
+    }
+}