Implement websocket MP4 recording

common/src/main/java/ctbrec/sites/secretfriends/SecretFriendsModel.java

@@ -6,26 +6,35 @@ import ctbrec.AbstractModel;
 import ctbrec.Config;
 import ctbrec.io.HtmlParser;
 import ctbrec.io.HttpException;
+import ctbrec.recorder.download.Download;
 import ctbrec.recorder.download.StreamSource;
+import okhttp3.HttpUrl;
 import okhttp3.Request;
 import okhttp3.Response;
+import org.json.JSONObject;
 import org.jsoup.nodes.Element;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import javax.xml.bind.JAXBException;
 import java.io.IOException;
-import java.util.Collections;
+import java.time.Instant;
-import java.util.List;
+import java.util.*;
-import java.util.Locale;
-import java.util.Objects;
 import java.util.concurrent.ExecutionException;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 import static ctbrec.Model.State.ONLINE;
 import static ctbrec.io.HttpConstants.*;
 
 public class SecretFriendsModel extends AbstractModel {
-
+    private static final Logger LOG = LoggerFactory.getLogger(SecretFriendsModel.class);
     private String status = null;
     private int[] resolution = new int[]{0, 0};
+    private static final Random RNG = new Random();
+
+    private static final String H5LIVE = "h5live";
+    private static final String SECURITY = "security";
 
     @Override
     public boolean isOnline(boolean ignoreCache) throws IOException, ExecutionException, InterruptedException {
@@ -58,8 +67,51 @@ public class SecretFriendsModel extends AbstractModel {
 
     @Override
     public List<StreamSource> getStreamSources() throws IOException, ExecutionException, ParseException, PlaylistException, JAXBException {
-        String name = getName();
+        String bioPage = loadBioPage();
-        String url = getSite().getBaseUrl() + "/api/front/models/username/" + name + "/cam?triggerRequest=loadCam";
+        String streamName = getStreamName(bioPage);
+        String streamId = getStreamId(bioPage);
+        JSONObject token = getToken(streamName);
+
+        String stream = streamName + "?host=www.secretfriends.com"
+                + "&startAt=" + Instant.now().getEpochSecond()
+                + "&userId=null&ip=0.0.0.0&cSessionId=guestKey"
+                + "&streamId=" + streamId
+                + "&groupId=null"
+                + "&userAgent=" + Config.getInstance().getSettings().httpUserAgent;
+
+        HttpUrl wsUrl = new HttpUrl.Builder()
+                .scheme("https")
+                .host("bintu-splay.nanocosmos.de")
+                .addPathSegments("h5live/authstream")
+                .addQueryParameter("url", "rtmp://bintu-splay.nanocosmos.de/splay")
+                .addQueryParameter("stream", stream)
+                .addQueryParameter("cid", String.valueOf(RNG.nextInt(899000) + 100000))
+                .addQueryParameter("pid", String.valueOf(RNG.nextLong() + 10_000_000_000L))
+                .addQueryParameter("token", token.getJSONObject(H5LIVE).getJSONObject(SECURITY).getString("token"))
+                .addQueryParameter("expires", token.getJSONObject(H5LIVE).getJSONObject(SECURITY).getString("expires"))
+                .addQueryParameter("options", token.getJSONObject(H5LIVE).getJSONObject(SECURITY).getString("options"))
+                .build();
+
+        StreamSource src = new StreamSource();
+        src.width = 1280;
+        src.height = 720;
+        src.mediaPlaylistUrl = wsUrl.toString();
+        return Collections.singletonList(src);
+    }
+
+    private String getStreamId(String bioPage) throws IOException {
+        Pattern p = Pattern.compile("app.configure\\((.*?)\\);");
+        Matcher m = p.matcher(bioPage);
+        if (m.find()) {
+            JSONObject appConfig = new JSONObject(m.group(1));
+            return appConfig.getJSONObject("page").getJSONObject("user").getString("id");
+        } else {
+            throw new IOException("app configuration not found in HTML");
+        }
+    }
+
+    private JSONObject getToken(String streamName) throws IOException {
+        String url = SecretFriends.BASE_URI + "/nano/generateToken?streamName=" + streamName;
         Request req = new Request.Builder()
                 .url(url)
                 .header(ACCEPT, MIMETYPE_APPLICATION_JSON)
@@ -70,13 +122,41 @@ public class SecretFriendsModel extends AbstractModel {
                 .build();
         try (Response response = site.getHttpClient().execute(req)) {
             if (response.isSuccessful()) {
-                return Collections.emptyList();
+                String body = Objects.requireNonNull(response.body(), "HTTP response body is null").string();
+                return new JSONObject(body);
             } else {
                 throw new HttpException(response.code(), response.message());
             }
         }
     }
 
+    private String loadBioPage() throws IOException {
+        String url = SecretFriends.BASE_URI + "/friends/" + getName();
+        Request req = new Request.Builder()
+                .url(url)
+                .header(ACCEPT, MIMETYPE_APPLICATION_JSON)
+                .header(ACCEPT_LANGUAGE, Locale.ENGLISH.getLanguage())
+                .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
+                .header(REFERER, getUrl())
+                .build();
+        try (Response response = site.getHttpClient().execute(req)) {
+            if (response.isSuccessful()) {
+                return Objects.requireNonNull(response.body(), "HTTP response body is null").string();
+            } else {
+                throw new HttpException(response.code(), response.message());
+            }
+        }
+    }
+
+    private String getStreamName(String bioPage) throws IOException {
+        Pattern p = Pattern.compile("'streamName'\\s*:\\s*\"(.*?)\",");
+        Matcher m = p.matcher(bioPage);
+        if (m.find()) {
+            return m.group(1);
+        } else {
+            throw new IOException("Stream name not found in HTML");
+        }
+    }
 
     @Override
     public void invalidateCacheEntries() {
@@ -114,4 +194,9 @@ public class SecretFriendsModel extends AbstractModel {
     public boolean unfollow() throws IOException {
         return false;
     }
+
+    @Override
+    public Download createDownload() {
+        return new SecretFriendsWebrtcDownload(getSite().getHttpClient());
+    }
 }

common/src/main/java/ctbrec/sites/secretfriends/SecretFriendsModelParser.java

@@ -39,6 +39,9 @@ public class SecretFriendsModelParser {
         Element bioLink = Objects.requireNonNull(div.selectFirst("a[href*=/friend]"), "a[href*=/friend] not found");
         bioLink.setBaseUri(SecretFriends.BASE_URI);
         String href = bioLink.attr("href");
+        if (href.contains("signup")) {
+            return href.substring(href.indexOf('=') + 1);
+        } else {
             String name = href.substring(href.lastIndexOf('/') + 1);
             if (name.indexOf('?') >= 0) {
                 name = name.substring(0, name.indexOf('?'));
@@ -48,6 +51,7 @@ public class SecretFriendsModelParser {
             }
             return name;
         }
+    }
 
     private static Model.State extractOnlineState(Element div) {
         Element modelTag = Objects.requireNonNull(div.selectFirst("div[class~=model-tag]"), "div.model-tag not found");

common/src/main/java/ctbrec/sites/secretfriends/SecretFriendsWebrtcDownload.java

@@ -0,0 +1,224 @@
+package ctbrec.sites.secretfriends;
+
+import ctbrec.Config;
+import ctbrec.Model;
+import ctbrec.Recording;
+import ctbrec.io.HttpClient;
+import ctbrec.recorder.download.AbstractDownload;
+import ctbrec.recorder.download.Download;
+import okhttp3.Request;
+import okhttp3.Response;
+import okhttp3.WebSocket;
+import okhttp3.WebSocketListener;
+import okio.ByteString;
+import org.json.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.EOFException;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.concurrent.ExecutorService;
+import java.util.regex.Pattern;
+
+import static ctbrec.io.HttpConstants.*;
+
+public class SecretFriendsWebrtcDownload extends AbstractDownload {
+
+    private static final Logger LOG = LoggerFactory.getLogger(SecretFriendsWebrtcDownload.class);
+    private static final int MAX_SECONDS_WITHOUT_TRANSFER = 20;
+
+    private final HttpClient httpClient;
+    private WebSocket ws;
+    private FileOutputStream fout;
+    private Instant timeOfLastTransfer = Instant.MAX;
+
+    private volatile boolean running;
+    private volatile boolean started;
+
+
+    private File targetFile;
+
+    public SecretFriendsWebrtcDownload(HttpClient httpClient) {
+        this.httpClient = httpClient;
+    }
+
+    @Override
+    public void init(Config config, Model model, Instant startTime, ExecutorService executorService) throws IOException {
+        this.config = config;
+        this.model = model;
+        this.startTime = startTime;
+        this.downloadExecutor = executorService;
+        splittingStrategy = initSplittingStrategy(config.getSettings());
+        targetFile = config.getFileForRecording(model, "mp4", startTime);
+        timeOfLastTransfer = Instant.now();
+    }
+
+    @Override
+    public void stop() {
+        running = false;
+        if (ws != null) {
+            ws.close(1000, "");
+            ws = null;
+        }
+    }
+
+    @Override
+    public void finalizeDownload() {
+        if (fout != null) {
+            try {
+                LOG.debug("Closing recording file {}", targetFile);
+                fout.close();
+            } catch (IOException e) {
+                LOG.error("Error while closing recording file {}", targetFile, e);
+            }
+        }
+    }
+
+    @Override
+    public boolean isRunning() {
+        return running;
+    }
+
+    @Override
+    public void postprocess(Recording recording) {
+        // nothing to do
+    }
+
+    @Override
+    public File getTarget() {
+        return targetFile;
+    }
+
+    @Override
+    public String getPath(Model model) {
+        String absolutePath = targetFile.getAbsolutePath();
+        String recordingsDir = Config.getInstance().getSettings().recordingsDir;
+        String relativePath = absolutePath.replaceFirst(Pattern.quote(recordingsDir), "");
+        return relativePath;
+    }
+
+    @Override
+    public boolean isSingleFile() {
+        return true;
+    }
+
+    @Override
+    public long getSizeInByte() {
+        return getTarget().length();
+    }
+
+    @Override
+    public Download call() throws Exception {
+        if (!started) {
+            started = true;
+            startDownload();
+        }
+
+        if (splittingStrategy.splitNecessary(this)) {
+            stop();
+            rescheduleTime = Instant.now();
+        } else {
+            rescheduleTime = Instant.now().plusSeconds(5);
+        }
+        if (!model.isOnline(true)) {
+            stop();
+        }
+        if (Duration.between(timeOfLastTransfer, Instant.now()).getSeconds() > MAX_SECONDS_WITHOUT_TRANSFER) {
+            LOG.info("No video data received for {} seconds. Stopping recording for model {}", MAX_SECONDS_WITHOUT_TRANSFER, model);
+            stop();
+        }
+        return this;
+    }
+
+    private void startDownload() throws IOException {
+        Request request;
+        try {
+            request = new Request.Builder()
+                    .url(model.getStreamSources().get(0).getMediaPlaylistUrl())
+                    .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
+                    .header(ACCEPT, "*/*")
+                    .header(ACCEPT_LANGUAGE, "pl")
+                    .header(ORIGIN, model.getSite().getBaseUrl())
+                    .build();
+        } catch (Exception e) {
+            throw new IOException(e);
+        }
+
+        running = true;
+        LOG.debug("Opening webrtc connection {}", request.url());
+        ws = httpClient.newWebSocket(request, new WebSocketListener() {
+            @Override
+            public void onOpen(WebSocket webSocket, Response response) {
+                super.onOpen(webSocket, response);
+                LOG.trace("onOpen {} {}", webSocket, response);
+                response.close();
+                try {
+                    LOG.debug("Recording video stream to {}", targetFile);
+                    Files.createDirectories(targetFile.getParentFile().toPath());
+                    fout = new FileOutputStream(targetFile);
+                } catch (Exception e) {
+                    LOG.error("Couldn't open file {} to save the video stream", targetFile, e);
+                    stop();
+                }
+            }
+
+            @Override
+            public void onMessage(WebSocket webSocket, ByteString bytes) {
+                super.onMessage(webSocket, bytes);
+                timeOfLastTransfer = Instant.now();
+                try {
+                    fout.write(bytes.toByteArray());
+                } catch (IOException e) {
+                    if (running) {
+                        LOG.error("Couldn't write video stream to file", e);
+                        stop();
+                    }
+                }
+            }
+
+            @Override
+            public void onMessage(WebSocket webSocket, String text) {
+                super.onMessage(webSocket, text);
+                LOG.trace("onMessageT {} {}", webSocket, text);
+                JSONObject msg = new JSONObject(text);
+                if (msg.optString("eventType").equals("onStreamInfo")) {
+                    JSONObject streamInfo = msg.getJSONObject("onStreamInfo");
+                    JSONObject videoInfo = streamInfo.getJSONObject("videoInfo");
+                    LOG.info("Stream resolution for {} is {}", model, videoInfo.getInt("height"));
+                }
+            }
+
+            @Override
+            public void onFailure(WebSocket webSocket, Throwable t, Response response) {
+                super.onFailure(webSocket, t, response);
+                stop();
+                if (t instanceof EOFException) {
+                    LOG.info("End of stream detected for model {}", model);
+                } else {
+                    LOG.error("Websocket failure for model {} {} {}", model, response, t);
+                }
+                if (response != null) {
+                    response.close();
+                }
+            }
+
+            @Override
+            public void onClosing(WebSocket webSocket, int code, String reason) {
+                super.onClosing(webSocket, code, reason);
+                LOG.trace("Websocket closing for model {} {} {}", model, code, reason);
+            }
+
+            @Override
+            public void onClosed(WebSocket webSocket, int code, String reason) {
+                super.onClosed(webSocket, code, reason);
+                LOG.debug("Websocket closed for model {} {} {}", model, code, reason);
+                stop();
+            }
+        });
+    }
+}