diff --git a/common/src/main/java/ctbrec/sites/secretfriends/SecretFriendsModel.java b/common/src/main/java/ctbrec/sites/secretfriends/SecretFriendsModel.java index 4d06f2f1..ae77206b 100644 --- a/common/src/main/java/ctbrec/sites/secretfriends/SecretFriendsModel.java +++ b/common/src/main/java/ctbrec/sites/secretfriends/SecretFriendsModel.java @@ -6,26 +6,35 @@ import ctbrec.AbstractModel; import ctbrec.Config; import ctbrec.io.HtmlParser; import ctbrec.io.HttpException; +import ctbrec.recorder.download.Download; import ctbrec.recorder.download.StreamSource; +import okhttp3.HttpUrl; import okhttp3.Request; import okhttp3.Response; +import org.json.JSONObject; import org.jsoup.nodes.Element; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import javax.xml.bind.JAXBException; import java.io.IOException; -import java.util.Collections; -import java.util.List; -import java.util.Locale; -import java.util.Objects; +import java.time.Instant; +import java.util.*; import java.util.concurrent.ExecutionException; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import static ctbrec.Model.State.ONLINE; import static ctbrec.io.HttpConstants.*; public class SecretFriendsModel extends AbstractModel { - + private static final Logger LOG = LoggerFactory.getLogger(SecretFriendsModel.class); private String status = null; private int[] resolution = new int[]{0, 0}; + private static final Random RNG = new Random(); + + private static final String H5LIVE = "h5live"; + private static final String SECURITY = "security"; @Override public boolean isOnline(boolean ignoreCache) throws IOException, ExecutionException, InterruptedException { @@ -58,8 +67,51 @@ public class SecretFriendsModel extends AbstractModel { @Override public List getStreamSources() throws IOException, ExecutionException, ParseException, PlaylistException, JAXBException { - String name = getName(); - String url = getSite().getBaseUrl() + "/api/front/models/username/" + name + "/cam?triggerRequest=loadCam"; + String bioPage = loadBioPage(); + String streamName = getStreamName(bioPage); + String streamId = getStreamId(bioPage); + JSONObject token = getToken(streamName); + + String stream = streamName + "?host=www.secretfriends.com" + + "&startAt=" + Instant.now().getEpochSecond() + + "&userId=null&ip=0.0.0.0&cSessionId=guestKey" + + "&streamId=" + streamId + + "&groupId=null" + + "&userAgent=" + Config.getInstance().getSettings().httpUserAgent; + + HttpUrl wsUrl = new HttpUrl.Builder() + .scheme("https") + .host("bintu-splay.nanocosmos.de") + .addPathSegments("h5live/authstream") + .addQueryParameter("url", "rtmp://bintu-splay.nanocosmos.de/splay") + .addQueryParameter("stream", stream) + .addQueryParameter("cid", String.valueOf(RNG.nextInt(899000) + 100000)) + .addQueryParameter("pid", String.valueOf(RNG.nextLong() + 10_000_000_000L)) + .addQueryParameter("token", token.getJSONObject(H5LIVE).getJSONObject(SECURITY).getString("token")) + .addQueryParameter("expires", token.getJSONObject(H5LIVE).getJSONObject(SECURITY).getString("expires")) + .addQueryParameter("options", token.getJSONObject(H5LIVE).getJSONObject(SECURITY).getString("options")) + .build(); + + StreamSource src = new StreamSource(); + src.width = 1280; + src.height = 720; + src.mediaPlaylistUrl = wsUrl.toString(); + return Collections.singletonList(src); + } + + private String getStreamId(String bioPage) throws IOException { + Pattern p = Pattern.compile("app.configure\\((.*?)\\);"); + Matcher m = p.matcher(bioPage); + if (m.find()) { + JSONObject appConfig = new JSONObject(m.group(1)); + return appConfig.getJSONObject("page").getJSONObject("user").getString("id"); + } else { + throw new IOException("app configuration not found in HTML"); + } + } + + private JSONObject getToken(String streamName) throws IOException { + String url = SecretFriends.BASE_URI + "/nano/generateToken?streamName=" + streamName; Request req = new Request.Builder() .url(url) .header(ACCEPT, MIMETYPE_APPLICATION_JSON) @@ -70,13 +122,41 @@ public class SecretFriendsModel extends AbstractModel { .build(); try (Response response = site.getHttpClient().execute(req)) { if (response.isSuccessful()) { - return Collections.emptyList(); + String body = Objects.requireNonNull(response.body(), "HTTP response body is null").string(); + return new JSONObject(body); } else { throw new HttpException(response.code(), response.message()); } } } + private String loadBioPage() throws IOException { + String url = SecretFriends.BASE_URI + "/friends/" + getName(); + Request req = new Request.Builder() + .url(url) + .header(ACCEPT, MIMETYPE_APPLICATION_JSON) + .header(ACCEPT_LANGUAGE, Locale.ENGLISH.getLanguage()) + .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) + .header(REFERER, getUrl()) + .build(); + try (Response response = site.getHttpClient().execute(req)) { + if (response.isSuccessful()) { + return Objects.requireNonNull(response.body(), "HTTP response body is null").string(); + } else { + throw new HttpException(response.code(), response.message()); + } + } + } + + private String getStreamName(String bioPage) throws IOException { + Pattern p = Pattern.compile("'streamName'\\s*:\\s*\"(.*?)\","); + Matcher m = p.matcher(bioPage); + if (m.find()) { + return m.group(1); + } else { + throw new IOException("Stream name not found in HTML"); + } + } @Override public void invalidateCacheEntries() { @@ -114,4 +194,9 @@ public class SecretFriendsModel extends AbstractModel { public boolean unfollow() throws IOException { return false; } + + @Override + public Download createDownload() { + return new SecretFriendsWebrtcDownload(getSite().getHttpClient()); + } } diff --git a/common/src/main/java/ctbrec/sites/secretfriends/SecretFriendsModelParser.java b/common/src/main/java/ctbrec/sites/secretfriends/SecretFriendsModelParser.java index 623accd9..fa4b017b 100644 --- a/common/src/main/java/ctbrec/sites/secretfriends/SecretFriendsModelParser.java +++ b/common/src/main/java/ctbrec/sites/secretfriends/SecretFriendsModelParser.java @@ -39,14 +39,18 @@ public class SecretFriendsModelParser { Element bioLink = Objects.requireNonNull(div.selectFirst("a[href*=/friend]"), "a[href*=/friend] not found"); bioLink.setBaseUri(SecretFriends.BASE_URI); String href = bioLink.attr("href"); - String name = href.substring(href.lastIndexOf('/') + 1); - if (name.indexOf('?') >= 0) { - name = name.substring(0, name.indexOf('?')); + if (href.contains("signup")) { + return href.substring(href.indexOf('=') + 1); + } else { + String name = href.substring(href.lastIndexOf('/') + 1); + if (name.indexOf('?') >= 0) { + name = name.substring(0, name.indexOf('?')); + } + if (name.indexOf('#') >= 0) { + name = name.substring(0, name.indexOf('#')); + } + return name; } - if (name.indexOf('#') >= 0) { - name = name.substring(0, name.indexOf('#')); - } - return name; } private static Model.State extractOnlineState(Element div) { diff --git a/common/src/main/java/ctbrec/sites/secretfriends/SecretFriendsWebrtcDownload.java b/common/src/main/java/ctbrec/sites/secretfriends/SecretFriendsWebrtcDownload.java new file mode 100644 index 00000000..949c6dc3 --- /dev/null +++ b/common/src/main/java/ctbrec/sites/secretfriends/SecretFriendsWebrtcDownload.java @@ -0,0 +1,224 @@ +package ctbrec.sites.secretfriends; + +import ctbrec.Config; +import ctbrec.Model; +import ctbrec.Recording; +import ctbrec.io.HttpClient; +import ctbrec.recorder.download.AbstractDownload; +import ctbrec.recorder.download.Download; +import okhttp3.Request; +import okhttp3.Response; +import okhttp3.WebSocket; +import okhttp3.WebSocketListener; +import okio.ByteString; +import org.json.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.EOFException; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.file.Files; +import java.time.Duration; +import java.time.Instant; +import java.util.concurrent.ExecutorService; +import java.util.regex.Pattern; + +import static ctbrec.io.HttpConstants.*; + +public class SecretFriendsWebrtcDownload extends AbstractDownload { + + private static final Logger LOG = LoggerFactory.getLogger(SecretFriendsWebrtcDownload.class); + private static final int MAX_SECONDS_WITHOUT_TRANSFER = 20; + + private final HttpClient httpClient; + private WebSocket ws; + private FileOutputStream fout; + private Instant timeOfLastTransfer = Instant.MAX; + + private volatile boolean running; + private volatile boolean started; + + + private File targetFile; + + public SecretFriendsWebrtcDownload(HttpClient httpClient) { + this.httpClient = httpClient; + } + + @Override + public void init(Config config, Model model, Instant startTime, ExecutorService executorService) throws IOException { + this.config = config; + this.model = model; + this.startTime = startTime; + this.downloadExecutor = executorService; + splittingStrategy = initSplittingStrategy(config.getSettings()); + targetFile = config.getFileForRecording(model, "mp4", startTime); + timeOfLastTransfer = Instant.now(); + } + + @Override + public void stop() { + running = false; + if (ws != null) { + ws.close(1000, ""); + ws = null; + } + } + + @Override + public void finalizeDownload() { + if (fout != null) { + try { + LOG.debug("Closing recording file {}", targetFile); + fout.close(); + } catch (IOException e) { + LOG.error("Error while closing recording file {}", targetFile, e); + } + } + } + + @Override + public boolean isRunning() { + return running; + } + + @Override + public void postprocess(Recording recording) { + // nothing to do + } + + @Override + public File getTarget() { + return targetFile; + } + + @Override + public String getPath(Model model) { + String absolutePath = targetFile.getAbsolutePath(); + String recordingsDir = Config.getInstance().getSettings().recordingsDir; + String relativePath = absolutePath.replaceFirst(Pattern.quote(recordingsDir), ""); + return relativePath; + } + + @Override + public boolean isSingleFile() { + return true; + } + + @Override + public long getSizeInByte() { + return getTarget().length(); + } + + @Override + public Download call() throws Exception { + if (!started) { + started = true; + startDownload(); + } + + if (splittingStrategy.splitNecessary(this)) { + stop(); + rescheduleTime = Instant.now(); + } else { + rescheduleTime = Instant.now().plusSeconds(5); + } + if (!model.isOnline(true)) { + stop(); + } + if (Duration.between(timeOfLastTransfer, Instant.now()).getSeconds() > MAX_SECONDS_WITHOUT_TRANSFER) { + LOG.info("No video data received for {} seconds. Stopping recording for model {}", MAX_SECONDS_WITHOUT_TRANSFER, model); + stop(); + } + return this; + } + + private void startDownload() throws IOException { + Request request; + try { + request = new Request.Builder() + .url(model.getStreamSources().get(0).getMediaPlaylistUrl()) + .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) + .header(ACCEPT, "*/*") + .header(ACCEPT_LANGUAGE, "pl") + .header(ORIGIN, model.getSite().getBaseUrl()) + .build(); + } catch (Exception e) { + throw new IOException(e); + } + + running = true; + LOG.debug("Opening webrtc connection {}", request.url()); + ws = httpClient.newWebSocket(request, new WebSocketListener() { + @Override + public void onOpen(WebSocket webSocket, Response response) { + super.onOpen(webSocket, response); + LOG.trace("onOpen {} {}", webSocket, response); + response.close(); + try { + LOG.debug("Recording video stream to {}", targetFile); + Files.createDirectories(targetFile.getParentFile().toPath()); + fout = new FileOutputStream(targetFile); + } catch (Exception e) { + LOG.error("Couldn't open file {} to save the video stream", targetFile, e); + stop(); + } + } + + @Override + public void onMessage(WebSocket webSocket, ByteString bytes) { + super.onMessage(webSocket, bytes); + timeOfLastTransfer = Instant.now(); + try { + fout.write(bytes.toByteArray()); + } catch (IOException e) { + if (running) { + LOG.error("Couldn't write video stream to file", e); + stop(); + } + } + } + + @Override + public void onMessage(WebSocket webSocket, String text) { + super.onMessage(webSocket, text); + LOG.trace("onMessageT {} {}", webSocket, text); + JSONObject msg = new JSONObject(text); + if (msg.optString("eventType").equals("onStreamInfo")) { + JSONObject streamInfo = msg.getJSONObject("onStreamInfo"); + JSONObject videoInfo = streamInfo.getJSONObject("videoInfo"); + LOG.info("Stream resolution for {} is {}", model, videoInfo.getInt("height")); + } + } + + @Override + public void onFailure(WebSocket webSocket, Throwable t, Response response) { + super.onFailure(webSocket, t, response); + stop(); + if (t instanceof EOFException) { + LOG.info("End of stream detected for model {}", model); + } else { + LOG.error("Websocket failure for model {} {} {}", model, response, t); + } + if (response != null) { + response.close(); + } + } + + @Override + public void onClosing(WebSocket webSocket, int code, String reason) { + super.onClosing(webSocket, code, reason); + LOG.trace("Websocket closing for model {} {} {}", model, code, reason); + } + + @Override + public void onClosed(WebSocket webSocket, int code, String reason) { + super.onClosed(webSocket, code, reason); + LOG.debug("Websocket closed for model {} {} {}", model, code, reason); + stop(); + } + }); + } +}