Implement websocket MP4 recording
This commit is contained in:
0xb00bface
parent
1d9a15ea6b
commit
de9c11baf5
|
|
@ -6,26 +6,35 @@ import ctbrec.AbstractModel;
|
|||
import ctbrec.Config;
|
||||
import ctbrec.io.HtmlParser;
|
||||
import ctbrec.io.HttpException;
|
||||
import ctbrec.recorder.download.Download;
|
||||
import ctbrec.recorder.download.StreamSource;
|
||||
import okhttp3.HttpUrl;
|
||||
import okhttp3.Request;
|
||||
import okhttp3.Response;
|
||||
import org.json.JSONObject;
|
||||
import org.jsoup.nodes.Element;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import javax.xml.bind.JAXBException;
|
||||
import java.io.IOException;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Objects;
|
||||
import java.time.Instant;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import static ctbrec.Model.State.ONLINE;
|
||||
import static ctbrec.io.HttpConstants.*;
|
||||
|
||||
public class SecretFriendsModel extends AbstractModel {
|
||||
|
||||
private static final Logger LOG = LoggerFactory.getLogger(SecretFriendsModel.class);
|
||||
private String status = null;
|
||||
private int[] resolution = new int[]{0, 0};
|
||||
private static final Random RNG = new Random();
|
||||
|
||||
private static final String H5LIVE = "h5live";
|
||||
private static final String SECURITY = "security";
|
||||
|
||||
@Override
|
||||
public boolean isOnline(boolean ignoreCache) throws IOException, ExecutionException, InterruptedException {
|
||||
|
|
@ -58,8 +67,51 @@ public class SecretFriendsModel extends AbstractModel {
|
|||
|
||||
@Override
|
||||
public List<StreamSource> getStreamSources() throws IOException, ExecutionException, ParseException, PlaylistException, JAXBException {
|
||||
String name = getName();
|
||||
String url = getSite().getBaseUrl() + "/api/front/models/username/" + name + "/cam?triggerRequest=loadCam";
|
||||
String bioPage = loadBioPage();
|
||||
String streamName = getStreamName(bioPage);
|
||||
String streamId = getStreamId(bioPage);
|
||||
JSONObject token = getToken(streamName);
|
||||
|
||||
String stream = streamName + "?host=www.secretfriends.com"
|
||||
+ "&startAt=" + Instant.now().getEpochSecond()
|
||||
+ "&userId=null&ip=0.0.0.0&cSessionId=guestKey"
|
||||
+ "&streamId=" + streamId
|
||||
+ "&groupId=null"
|
||||
+ "&userAgent=" + Config.getInstance().getSettings().httpUserAgent;
|
||||
|
||||
HttpUrl wsUrl = new HttpUrl.Builder()
|
||||
.scheme("https")
|
||||
.host("bintu-splay.nanocosmos.de")
|
||||
.addPathSegments("h5live/authstream")
|
||||
.addQueryParameter("url", "rtmp://bintu-splay.nanocosmos.de/splay")
|
||||
.addQueryParameter("stream", stream)
|
||||
.addQueryParameter("cid", String.valueOf(RNG.nextInt(899000) + 100000))
|
||||
.addQueryParameter("pid", String.valueOf(RNG.nextLong() + 10_000_000_000L))
|
||||
.addQueryParameter("token", token.getJSONObject(H5LIVE).getJSONObject(SECURITY).getString("token"))
|
||||
.addQueryParameter("expires", token.getJSONObject(H5LIVE).getJSONObject(SECURITY).getString("expires"))
|
||||
.addQueryParameter("options", token.getJSONObject(H5LIVE).getJSONObject(SECURITY).getString("options"))
|
||||
.build();
|
||||
|
||||
StreamSource src = new StreamSource();
|
||||
src.width = 1280;
|
||||
src.height = 720;
|
||||
src.mediaPlaylistUrl = wsUrl.toString();
|
||||
return Collections.singletonList(src);
|
||||
}
|
||||
|
||||
private String getStreamId(String bioPage) throws IOException {
|
||||
Pattern p = Pattern.compile("app.configure\\((.*?)\\);");
|
||||
Matcher m = p.matcher(bioPage);
|
||||
if (m.find()) {
|
||||
JSONObject appConfig = new JSONObject(m.group(1));
|
||||
return appConfig.getJSONObject("page").getJSONObject("user").getString("id");
|
||||
} else {
|
||||
throw new IOException("app configuration not found in HTML");
|
||||
}
|
||||
}
|
||||
|
||||
private JSONObject getToken(String streamName) throws IOException {
|
||||
String url = SecretFriends.BASE_URI + "/nano/generateToken?streamName=" + streamName;
|
||||
Request req = new Request.Builder()
|
||||
.url(url)
|
||||
.header(ACCEPT, MIMETYPE_APPLICATION_JSON)
|
||||
|
|
@ -70,13 +122,41 @@ public class SecretFriendsModel extends AbstractModel {
|
|||
.build();
|
||||
try (Response response = site.getHttpClient().execute(req)) {
|
||||
if (response.isSuccessful()) {
|
||||
return Collections.emptyList();
|
||||
String body = Objects.requireNonNull(response.body(), "HTTP response body is null").string();
|
||||
return new JSONObject(body);
|
||||
} else {
|
||||
throw new HttpException(response.code(), response.message());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private String loadBioPage() throws IOException {
|
||||
String url = SecretFriends.BASE_URI + "/friends/" + getName();
|
||||
Request req = new Request.Builder()
|
||||
.url(url)
|
||||
.header(ACCEPT, MIMETYPE_APPLICATION_JSON)
|
||||
.header(ACCEPT_LANGUAGE, Locale.ENGLISH.getLanguage())
|
||||
.header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
|
||||
.header(REFERER, getUrl())
|
||||
.build();
|
||||
try (Response response = site.getHttpClient().execute(req)) {
|
||||
if (response.isSuccessful()) {
|
||||
return Objects.requireNonNull(response.body(), "HTTP response body is null").string();
|
||||
} else {
|
||||
throw new HttpException(response.code(), response.message());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private String getStreamName(String bioPage) throws IOException {
|
||||
Pattern p = Pattern.compile("'streamName'\\s*:\\s*\"(.*?)\",");
|
||||
Matcher m = p.matcher(bioPage);
|
||||
if (m.find()) {
|
||||
return m.group(1);
|
||||
} else {
|
||||
throw new IOException("Stream name not found in HTML");
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void invalidateCacheEntries() {
|
||||
|
|
@ -114,4 +194,9 @@ public class SecretFriendsModel extends AbstractModel {
|
|||
public boolean unfollow() throws IOException {
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Download createDownload() {
|
||||
return new SecretFriendsWebrtcDownload(getSite().getHttpClient());
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -39,14 +39,18 @@ public class SecretFriendsModelParser {
|
|||
Element bioLink = Objects.requireNonNull(div.selectFirst("a[href*=/friend]"), "a[href*=/friend] not found");
|
||||
bioLink.setBaseUri(SecretFriends.BASE_URI);
|
||||
String href = bioLink.attr("href");
|
||||
String name = href.substring(href.lastIndexOf('/') + 1);
|
||||
if (name.indexOf('?') >= 0) {
|
||||
name = name.substring(0, name.indexOf('?'));
|
||||
if (href.contains("signup")) {
|
||||
return href.substring(href.indexOf('=') + 1);
|
||||
} else {
|
||||
String name = href.substring(href.lastIndexOf('/') + 1);
|
||||
if (name.indexOf('?') >= 0) {
|
||||
name = name.substring(0, name.indexOf('?'));
|
||||
}
|
||||
if (name.indexOf('#') >= 0) {
|
||||
name = name.substring(0, name.indexOf('#'));
|
||||
}
|
||||
return name;
|
||||
}
|
||||
if (name.indexOf('#') >= 0) {
|
||||
name = name.substring(0, name.indexOf('#'));
|
||||
}
|
||||
return name;
|
||||
}
|
||||
|
||||
private static Model.State extractOnlineState(Element div) {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,224 @@
|
|||
package ctbrec.sites.secretfriends;
|
||||
|
||||
import ctbrec.Config;
|
||||
import ctbrec.Model;
|
||||
import ctbrec.Recording;
|
||||
import ctbrec.io.HttpClient;
|
||||
import ctbrec.recorder.download.AbstractDownload;
|
||||
import ctbrec.recorder.download.Download;
|
||||
import okhttp3.Request;
|
||||
import okhttp3.Response;
|
||||
import okhttp3.WebSocket;
|
||||
import okhttp3.WebSocketListener;
|
||||
import okio.ByteString;
|
||||
import org.json.JSONObject;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.io.EOFException;
|
||||
import java.io.File;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import static ctbrec.io.HttpConstants.*;
|
||||
|
||||
public class SecretFriendsWebrtcDownload extends AbstractDownload {
|
||||
|
||||
private static final Logger LOG = LoggerFactory.getLogger(SecretFriendsWebrtcDownload.class);
|
||||
private static final int MAX_SECONDS_WITHOUT_TRANSFER = 20;
|
||||
|
||||
private final HttpClient httpClient;
|
||||
private WebSocket ws;
|
||||
private FileOutputStream fout;
|
||||
private Instant timeOfLastTransfer = Instant.MAX;
|
||||
|
||||
private volatile boolean running;
|
||||
private volatile boolean started;
|
||||
|
||||
|
||||
private File targetFile;
|
||||
|
||||
public SecretFriendsWebrtcDownload(HttpClient httpClient) {
|
||||
this.httpClient = httpClient;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void init(Config config, Model model, Instant startTime, ExecutorService executorService) throws IOException {
|
||||
this.config = config;
|
||||
this.model = model;
|
||||
this.startTime = startTime;
|
||||
this.downloadExecutor = executorService;
|
||||
splittingStrategy = initSplittingStrategy(config.getSettings());
|
||||
targetFile = config.getFileForRecording(model, "mp4", startTime);
|
||||
timeOfLastTransfer = Instant.now();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void stop() {
|
||||
running = false;
|
||||
if (ws != null) {
|
||||
ws.close(1000, "");
|
||||
ws = null;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void finalizeDownload() {
|
||||
if (fout != null) {
|
||||
try {
|
||||
LOG.debug("Closing recording file {}", targetFile);
|
||||
fout.close();
|
||||
} catch (IOException e) {
|
||||
LOG.error("Error while closing recording file {}", targetFile, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isRunning() {
|
||||
return running;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void postprocess(Recording recording) {
|
||||
// nothing to do
|
||||
}
|
||||
|
||||
@Override
|
||||
public File getTarget() {
|
||||
return targetFile;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getPath(Model model) {
|
||||
String absolutePath = targetFile.getAbsolutePath();
|
||||
String recordingsDir = Config.getInstance().getSettings().recordingsDir;
|
||||
String relativePath = absolutePath.replaceFirst(Pattern.quote(recordingsDir), "");
|
||||
return relativePath;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isSingleFile() {
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getSizeInByte() {
|
||||
return getTarget().length();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Download call() throws Exception {
|
||||
if (!started) {
|
||||
started = true;
|
||||
startDownload();
|
||||
}
|
||||
|
||||
if (splittingStrategy.splitNecessary(this)) {
|
||||
stop();
|
||||
rescheduleTime = Instant.now();
|
||||
} else {
|
||||
rescheduleTime = Instant.now().plusSeconds(5);
|
||||
}
|
||||
if (!model.isOnline(true)) {
|
||||
stop();
|
||||
}
|
||||
if (Duration.between(timeOfLastTransfer, Instant.now()).getSeconds() > MAX_SECONDS_WITHOUT_TRANSFER) {
|
||||
LOG.info("No video data received for {} seconds. Stopping recording for model {}", MAX_SECONDS_WITHOUT_TRANSFER, model);
|
||||
stop();
|
||||
}
|
||||
return this;
|
||||
}
|
||||
|
||||
private void startDownload() throws IOException {
|
||||
Request request;
|
||||
try {
|
||||
request = new Request.Builder()
|
||||
.url(model.getStreamSources().get(0).getMediaPlaylistUrl())
|
||||
.header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
|
||||
.header(ACCEPT, "*/*")
|
||||
.header(ACCEPT_LANGUAGE, "pl")
|
||||
.header(ORIGIN, model.getSite().getBaseUrl())
|
||||
.build();
|
||||
} catch (Exception e) {
|
||||
throw new IOException(e);
|
||||
}
|
||||
|
||||
running = true;
|
||||
LOG.debug("Opening webrtc connection {}", request.url());
|
||||
ws = httpClient.newWebSocket(request, new WebSocketListener() {
|
||||
@Override
|
||||
public void onOpen(WebSocket webSocket, Response response) {
|
||||
super.onOpen(webSocket, response);
|
||||
LOG.trace("onOpen {} {}", webSocket, response);
|
||||
response.close();
|
||||
try {
|
||||
LOG.debug("Recording video stream to {}", targetFile);
|
||||
Files.createDirectories(targetFile.getParentFile().toPath());
|
||||
fout = new FileOutputStream(targetFile);
|
||||
} catch (Exception e) {
|
||||
LOG.error("Couldn't open file {} to save the video stream", targetFile, e);
|
||||
stop();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onMessage(WebSocket webSocket, ByteString bytes) {
|
||||
super.onMessage(webSocket, bytes);
|
||||
timeOfLastTransfer = Instant.now();
|
||||
try {
|
||||
fout.write(bytes.toByteArray());
|
||||
} catch (IOException e) {
|
||||
if (running) {
|
||||
LOG.error("Couldn't write video stream to file", e);
|
||||
stop();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onMessage(WebSocket webSocket, String text) {
|
||||
super.onMessage(webSocket, text);
|
||||
LOG.trace("onMessageT {} {}", webSocket, text);
|
||||
JSONObject msg = new JSONObject(text);
|
||||
if (msg.optString("eventType").equals("onStreamInfo")) {
|
||||
JSONObject streamInfo = msg.getJSONObject("onStreamInfo");
|
||||
JSONObject videoInfo = streamInfo.getJSONObject("videoInfo");
|
||||
LOG.info("Stream resolution for {} is {}", model, videoInfo.getInt("height"));
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onFailure(WebSocket webSocket, Throwable t, Response response) {
|
||||
super.onFailure(webSocket, t, response);
|
||||
stop();
|
||||
if (t instanceof EOFException) {
|
||||
LOG.info("End of stream detected for model {}", model);
|
||||
} else {
|
||||
LOG.error("Websocket failure for model {} {} {}", model, response, t);
|
||||
}
|
||||
if (response != null) {
|
||||
response.close();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onClosing(WebSocket webSocket, int code, String reason) {
|
||||
super.onClosing(webSocket, code, reason);
|
||||
LOG.trace("Websocket closing for model {} {} {}", model, code, reason);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onClosed(WebSocket webSocket, int code, String reason) {
|
||||
super.onClosed(webSocket, code, reason);
|
||||
LOG.debug("Websocket closed for model {} {} {}", model, code, reason);
|
||||
stop();
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue