From ae6fa229e7b3ac79d2560c5a5cd4b6cee40463a9 Mon Sep 17 00:00:00 2001 From: 0xboobface <0xboobface@gmail.com> Date: Mon, 13 Apr 2020 18:42:57 +0200 Subject: [PATCH] Add user-agent agent header to every http request in MFC classes --- .../sites/myfreecams/MyFreeCamsTableTab.java | 2 +- .../recorder/download/StreamSource.java | 36 ++++++++ .../download/hls/MergedFfmpegHlsDownload.java | 6 +- .../sites/mfc/HlsStreamSourceProvider.java | 19 ++++- .../java/ctbrec/sites/mfc/MyFreeCams.java | 11 ++- .../ctbrec/sites/mfc/MyFreeCamsClient.java | 83 ++++++++++--------- .../sites/mfc/MyFreeCamsHttpClient.java | 13 ++- .../ctbrec/sites/mfc/MyFreeCamsModel.java | 33 +++++--- .../java/ctbrec/sites/mfc/ServerConfig.java | 18 ++-- .../src/main/java/ctbrec/sites/mfc/Share.java | 8 +- 10 files changed, 158 insertions(+), 71 deletions(-) diff --git a/client/src/main/java/ctbrec/ui/sites/myfreecams/MyFreeCamsTableTab.java b/client/src/main/java/ctbrec/ui/sites/myfreecams/MyFreeCamsTableTab.java index 5b3a50b1..db3d7e94 100644 --- a/client/src/main/java/ctbrec/ui/sites/myfreecams/MyFreeCamsTableTab.java +++ b/client/src/main/java/ctbrec/ui/sites/myfreecams/MyFreeCamsTableTab.java @@ -357,7 +357,7 @@ public class MyFreeCamsTableTab extends Tab implements TabSelectionListener { try { List sources = m.getStreamSources(); for (StreamSource src : sources) { - LOG.info("{} {} {}", m.getUrl(), src.bandwidth, src.height); + LOG.info("m:{} s:{} bandwidth:{} height:{}", m.getName(), src.mediaPlaylistUrl, src.bandwidth, src.height); } LOG.info("==============="); } catch (IOException | ExecutionException | ParseException | PlaylistException | JAXBException e1) { diff --git a/common/src/main/java/ctbrec/recorder/download/StreamSource.java b/common/src/main/java/ctbrec/recorder/download/StreamSource.java index 6044716b..76ca84e6 100644 --- a/common/src/main/java/ctbrec/recorder/download/StreamSource.java +++ b/common/src/main/java/ctbrec/recorder/download/StreamSource.java @@ -64,4 +64,40 @@ public class StreamSource implements Comparable { return bandwidth - o.bandwidth; } } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + bandwidth; + result = prime * result + height; + result = prime * result + ((mediaPlaylistUrl == null) ? 0 : mediaPlaylistUrl.hashCode()); + result = prime * result + width; + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + StreamSource other = (StreamSource) obj; + if (bandwidth != other.bandwidth) + return false; + if (height != other.height) + return false; + if (mediaPlaylistUrl == null) { + if (other.mediaPlaylistUrl != null) + return false; + } else if (!mediaPlaylistUrl.equals(other.mediaPlaylistUrl)) + return false; + if (width != other.width) + return false; + return true; + } + + } diff --git a/common/src/main/java/ctbrec/recorder/download/hls/MergedFfmpegHlsDownload.java b/common/src/main/java/ctbrec/recorder/download/hls/MergedFfmpegHlsDownload.java index 4851c844..cc151b01 100644 --- a/common/src/main/java/ctbrec/recorder/download/hls/MergedFfmpegHlsDownload.java +++ b/common/src/main/java/ctbrec/recorder/download/hls/MergedFfmpegHlsDownload.java @@ -288,7 +288,11 @@ public class MergedFfmpegHlsDownload extends AbstractHlsDownload { private void writeSegment(byte[] segmentData) throws IOException { if (running) { - ffmpegStdIn.write(segmentData); + if (ffmpegStdIn != null) { + ffmpegStdIn.write(segmentData); + } else { + LOG.error("FFmpeg stdin stream is null - skipping writing of segment"); + } } } diff --git a/common/src/main/java/ctbrec/sites/mfc/HlsStreamSourceProvider.java b/common/src/main/java/ctbrec/sites/mfc/HlsStreamSourceProvider.java index 4cba4bc1..a30c5644 100644 --- a/common/src/main/java/ctbrec/sites/mfc/HlsStreamSourceProvider.java +++ b/common/src/main/java/ctbrec/sites/mfc/HlsStreamSourceProvider.java @@ -1,9 +1,12 @@ package ctbrec.sites.mfc; +import static ctbrec.io.HttpConstants.*; + import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; import java.util.List; +import java.util.Locale; import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; @@ -73,16 +76,24 @@ public class HlsStreamSourceProvider implements StreamSourceProvider { throw new IllegalStateException("Stream url unknown"); } LOG.trace("Loading master playlist {}", streamUrl); - Request req = new Request.Builder().url(streamUrl).build(); - try(Response response = httpClient.execute(req)) { - if(response.isSuccessful()) { + Request req = new Request.Builder() + .url(streamUrl) + .header(ACCEPT, "*/*") + .header(ACCEPT_LANGUAGE, Locale.ENGLISH.getLanguage()) + .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) + .header(CONNECTION, KEEP_ALIVE) + .header(ORIGIN, MyFreeCams.baseUrl) + .header(REFERER, MyFreeCams.baseUrl + '/') + .build(); + try (Response response = httpClient.execute(req)) { + if (response.isSuccessful()) { InputStream inputStream = response.body().byteStream(); PlaylistParser parser = new PlaylistParser(inputStream, Format.EXT_M3U, Encoding.UTF_8, ParsingMode.LENIENT); Playlist playlist = parser.parse(); MasterPlaylist master = playlist.getMasterPlaylist(); return master; } else { - throw new HttpException(response.code(), response.message()); + throw new HttpException(response.code(), response.message() + " - Forbidden: " + streamUrl); } } } diff --git a/common/src/main/java/ctbrec/sites/mfc/MyFreeCams.java b/common/src/main/java/ctbrec/sites/mfc/MyFreeCams.java index ae834d63..41ada179 100644 --- a/common/src/main/java/ctbrec/sites/mfc/MyFreeCams.java +++ b/common/src/main/java/ctbrec/sites/mfc/MyFreeCams.java @@ -1,7 +1,10 @@ package ctbrec.sites.mfc; +import static ctbrec.io.HttpConstants.*; + import java.io.IOException; import java.util.List; +import java.util.Locale; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -60,7 +63,13 @@ public class MyFreeCams extends AbstractSite { @Override public Double getTokenBalance() throws IOException { - Request req = new Request.Builder().url(baseUrl + "/php/account.php?request=status").build(); + Request req = new Request.Builder() + .url(baseUrl + "/php/account.php?request=status") + .header(ACCEPT, "*/*") + .header(ACCEPT_LANGUAGE, Locale.ENGLISH.getLanguage()) + .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) + .header(CONNECTION, KEEP_ALIVE) + .build(); try(Response response = getHttpClient().execute(req)) { if(response.isSuccessful()) { String content = response.body().string(); diff --git a/common/src/main/java/ctbrec/sites/mfc/MyFreeCamsClient.java b/common/src/main/java/ctbrec/sites/mfc/MyFreeCamsClient.java index c877ccc1..9afe7175 100644 --- a/common/src/main/java/ctbrec/sites/mfc/MyFreeCamsClient.java +++ b/common/src/main/java/ctbrec/sites/mfc/MyFreeCamsClient.java @@ -16,6 +16,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.LinkedList; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Map.Entry; import java.util.Objects; @@ -62,6 +63,7 @@ public class MyFreeCamsClient { private Cache models = CacheBuilder.newBuilder().maximumSize(4000).build(); private Lock lock = new ReentrantLock(); private ServerConfig serverConfig; + @SuppressWarnings("unused") private String tkx; private Integer cxid; private int[] ctx; @@ -100,7 +102,7 @@ public class MyFreeCamsClient { } } - String server = websocketServers.get(new Random().nextInt(websocketServers.size()-1)); + String server = websocketServers.get(new Random().nextInt(websocketServers.size() - 1)); String wsUrl = "ws://" + server + ".myfreecams.com:8080/fcsl"; LOG.debug("Connecting to random websocket server {}", wsUrl); @@ -110,9 +112,9 @@ public class MyFreeCamsClient { LOG.info("Websocket is null. Starting a new connection"); Request req = new Request.Builder() .url(wsUrl) - .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgentMobile) - .header(REFERER, "http://m.myfreecams.com") - .header(ORIGIN, "http://m.myfreecams.com").build(); + .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) + .header(ORIGIN, MyFreeCams.baseUrl) + .build(); ws = createWebSocket(req); } @@ -160,9 +162,6 @@ public class MyFreeCamsClient { LOG.trace("open: [{}]", response.body().string()); webSocket.send("hello fcserver\n"); webSocket.send("fcsws_20180422\n"); - // TODO find out, what the values in the json message mean, at the moment we hust send 0s, which seems to work, too - // webSocket.send("1 0 0 81 0 - // %7B%22err%22%3A0%2C%22start%22%3A1540159843072%2C%22stop%22%3A1540159844121%2C%22a%22%3A6392%2C%22time%22%3A1540159844%2C%22key%22%3A%228da80f985c9db390809713dac71df297%22%2C%22cid%22%3A%22c504d684%22%2C%22pid%22%3A1%2C%22site%22%3A%22www%22%7D\n"); webSocket.send( "1 0 0 81 0 %7B%22err%22%3A0%2C%22start%22%3A0%2C%22stop%22%3A0%2C%22a%22%3A0%2C%22time%22%3A0%2C%22key%22%3A%22%22%2C%22cid%22%3A%22%22%2C%22pid%22%3A1%2C%22site%22%3A%22www%22%7D\n"); heartBeat = System.currentTimeMillis(); @@ -331,7 +330,13 @@ public class MyFreeCamsClient { long type = json.getInt("type"); String base = mfc.getBaseUrl() + "/php/FcwExtResp.php"; String url = base + "?respkey=" + respkey + "&opts=" + opts + "&serv=" + serv + "&type=" + type; - Request req = new Request.Builder().url(url).build(); + Request req = new Request.Builder() + .url(url) + .header(ACCEPT, "*/*") + .header(ACCEPT_LANGUAGE, Locale.ENGLISH.getLanguage()) + .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) + .header(CONNECTION, KEEP_ALIVE) + .build(); LOG.trace("Requesting EXTDATA {}", url); try (Response resp = mfc.getHttpClient().execute(req)) { if (resp.isSuccessful()) { @@ -520,30 +525,38 @@ public class MyFreeCamsClient { } } - protected void authorizeForStream(SessionState state) { + protected String getWebrtcAuthCommand(SessionState state) { JSONObject streamInfo = new JSONObject(); - streamInfo.put("applicationName", "NxServer"); int userChannel = 100000000 + state.getUid(); - String phase = state.getU().getPhase() != null ? state.getU().getPhase() : "z"; + String phase = Optional.ofNullable(state.getU()).map(User::getPhase).orElse("z"); String phasePrefix = phase.equals("z") ? "" : '_' + phase; String streamName = "mfc" + phasePrefix + '_' + userChannel + ".f4v"; + streamInfo.put("applicationName", "NxServer"); streamInfo.put("streamName", streamName); streamInfo.put("sessionId", "[empty]"); JSONObject userData = new JSONObject(); userData.put("sessionId", sessionId); - userData.put("password", tkx); + userData.put("password", ""); // tkx or "" ?!? userData.put("roomId", userChannel); userData.put("modelId", state.getUid()); - JSONArray array = new JSONArray(); - Arrays.stream(ctx).forEach(array::put); - userData.put("vidctx", Base64.getEncoder().encodeToString(array.toString().getBytes(StandardCharsets.UTF_8))); - userData.put("cxid", cxid); + if (isBroadcasterOnOBS(state)) { + JSONArray array = new JSONArray(); + Arrays.stream(ctx).forEach(array::put); + userData.put("vidctx", Base64.getEncoder().encodeToString(array.toString().getBytes(StandardCharsets.UTF_8))); + userData.put("cxid", cxid); + } userData.put("mode", "DOWNLOAD"); JSONObject authCommand = new JSONObject(); - authCommand.put("userData", userData); - authCommand.put("streamInfo", streamInfo); authCommand.put("command", "auth"); + authCommand.put("streamInfo", streamInfo); + authCommand.put("userData", userData); LOG.info("auth command {}", authCommand.toString(2)); + return streamInfo.toString(); + } + + private boolean isBroadcasterOnOBS(SessionState state) { + String phase = Optional.ofNullable(state.getU()).map(User::getPhase).orElse("z"); + return !phase.equals("z"); } private void startKeepAlive(WebSocket ws) { @@ -598,28 +611,8 @@ public class MyFreeCamsClient { String phase = state.getU().getPhase() != null ? state.getU().getPhase() : "z"; String phasePrefix = phase.equals("z") ? "" : '_' + phase; String server = "video" + getCamServ(state).replaceAll("^\\D+", ""); - boolean useHls = serverConfig.isOnObsServer(state); - String streamUrl; - boolean dontUseDash = !Config.getInstance().getSettings().mfcUseDash; - if (serverConfig.isOnWzObsVideoServer(state) || !serverConfig.isOnObsServer(state)) { - // wowza server - if (dontUseDash || useHls) { - String nonce = Double.toString(Math.random()); - streamUrl = HTTPS + server + ".myfreecams.com/NxServer/ngrp:mfc" + phasePrefix + '_' + userChannel + ".f4v_mobile/playlist.m3u8?nc=" + nonce; - } else { - streamUrl = HTTPS + server + ".myfreecams.com/NxServer/ngrp:mfc" + phasePrefix + '_' + userChannel + ".f4v_desktop/manifest.mpd"; - } - } else { - // nginx server - if (dontUseDash || useHls) { - String nonce = Double.toString(Math.random()); - streamUrl = HTTPS + server + ".myfreecams.com:8444/x-hls/" + cxid + '/' + userChannel + '/' + ctxenc + "/mfc" + phasePrefix + '_' + userChannel - + ".m3u8?nc=" + nonce; - } else { - streamUrl = HTTPS + server + ".myfreecams.com:8444/x-dsh/" + cxid + '/' + userChannel + '/' + ctxenc + "/mfc" + phasePrefix + '_' + userChannel - + ".mpd"; - } - } + String nonce = Double.toString(Math.random()); + String streamUrl = HTTPS + server + ".myfreecams.com/NxServer/ngrp:mfc" + phasePrefix + '_' + userChannel + ".f4v_mobile/playlist.m3u8?nc=" + nonce; return streamUrl; } @@ -633,12 +626,20 @@ public class MyFreeCamsClient { } else if (serverConfig.isOnHtml5VideoServer(state)) { camservString = serverConfig.h5Servers.get(camserv.toString()); } else if (camserv > 500) { - camserv -= 500; + if (camserv >= 3000) { + camserv -= 1000; + } else { + camserv -= 500; + } camservString = camserv.toString(); } return camservString; } + private boolean isBroadcasterOnWebRTC(SessionState state) { + return (Optional.ofNullable(state).map(SessionState::getM).map(Model::getFlags).orElse(0) & 524288) == 524288; + } + public MyFreeCamsModel getModel(int uid) { return models.getIfPresent(uid); } diff --git a/common/src/main/java/ctbrec/sites/mfc/MyFreeCamsHttpClient.java b/common/src/main/java/ctbrec/sites/mfc/MyFreeCamsHttpClient.java index ce0b7fa3..faa9092a 100644 --- a/common/src/main/java/ctbrec/sites/mfc/MyFreeCamsHttpClient.java +++ b/common/src/main/java/ctbrec/sites/mfc/MyFreeCamsHttpClient.java @@ -4,6 +4,7 @@ import static ctbrec.io.HttpConstants.*; import java.io.IOException; import java.util.List; +import java.util.Locale; import java.util.NoSuchElementException; import java.util.Objects; @@ -56,6 +57,10 @@ public class MyFreeCamsHttpClient extends HttpClient { .build(); Request req = new Request.Builder() .url(MyFreeCams.baseUrl + "/php/login.php") + .header(ACCEPT, "*/*") + .header(ACCEPT_LANGUAGE, Locale.ENGLISH.getLanguage()) + .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) + .header(CONNECTION, KEEP_ALIVE) .header(REFERER, MyFreeCams.baseUrl) .header("Content-Type", "application/x-www-form-urlencoded") .post(body) @@ -77,7 +82,13 @@ public class MyFreeCamsHttpClient extends HttpClient { } private boolean checkLogin() throws IOException { - Request req = new Request.Builder().url(MyFreeCams.baseUrl + "/php/account.php?request=status").build(); + Request req = new Request.Builder() + .url(MyFreeCams.baseUrl + "/php/account.php?request=status") + .header(ACCEPT, "*/*") + .header(ACCEPT_LANGUAGE, Locale.ENGLISH.getLanguage()) + .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) + .header(CONNECTION, KEEP_ALIVE) + .build(); try(Response response = execute(req)) { if(response.isSuccessful()) { String content = response.body().string(); diff --git a/common/src/main/java/ctbrec/sites/mfc/MyFreeCamsModel.java b/common/src/main/java/ctbrec/sites/mfc/MyFreeCamsModel.java index 05014a10..fbd304c1 100644 --- a/common/src/main/java/ctbrec/sites/mfc/MyFreeCamsModel.java +++ b/common/src/main/java/ctbrec/sites/mfc/MyFreeCamsModel.java @@ -8,6 +8,7 @@ import java.net.URLDecoder; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Locale; import java.util.Objects; import java.util.Optional; import java.util.concurrent.ExecutionException; @@ -29,7 +30,6 @@ import ctbrec.io.HtmlParser; import ctbrec.io.HttpException; import ctbrec.recorder.download.Download; import ctbrec.recorder.download.StreamSource; -import ctbrec.recorder.download.dash.DashDownload; import okhttp3.FormBody; import okhttp3.Request; import okhttp3.RequestBody; @@ -98,11 +98,7 @@ public class MyFreeCamsModel extends AbstractModel { } private StreamSourceProvider getStreamSourceProvider() { - if(isHlsStream()) { - return new HlsStreamSourceProvider(getSite().getHttpClient()); - } else { - return new DashStreamSourceProvider(Config.getInstance(), site); - } + return new HlsStreamSourceProvider(getSite().getHttpClient()); } private boolean isHlsStream() { @@ -126,7 +122,13 @@ public class MyFreeCamsModel extends AbstractModel { public void receiveTip(Double tokens) throws IOException { String tipUrl = MyFreeCams.baseUrl + "/php/tip.php"; String initUrl = tipUrl + "?request=tip&username="+getName()+"&broadcaster_id="+getUid(); - Request req = new Request.Builder().url(initUrl).build(); + Request req = new Request.Builder() + .url(initUrl) + .header(ACCEPT, "*/*") + .header(ACCEPT_LANGUAGE, Locale.ENGLISH.getLanguage()) + .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) + .header(CONNECTION, KEEP_ALIVE) + .build(); try(Response resp = site.getHttpClient().execute(req)) { if(resp.isSuccessful()) { String page = resp.body().string(); @@ -150,7 +152,11 @@ public class MyFreeCamsModel extends AbstractModel { req = new Request.Builder() .url(tipUrl) .post(body) - .addHeader(REFERER, initUrl) + .header(ACCEPT, "*/*") + .header(ACCEPT_LANGUAGE, Locale.ENGLISH.getLanguage()) + .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) + .header(CONNECTION, KEEP_ALIVE) + .header(REFERER, initUrl) .build(); try(Response response = site.getHttpClient().execute(req)) { if(!response.isSuccessful()) { @@ -329,10 +335,11 @@ public class MyFreeCamsModel extends AbstractModel { if(streamUrl == null) { updateStreamUrl(); } - if(isHlsStream()) { - return super.createDownload(); - } else { - return new DashDownload(getSite().getHttpClient(), streamUrl); - } + return super.createDownload(); + // if(isHlsStream()) { + // return super.createDownload(); + // } else { + // return new MyFreeCamsWebrtcDownload(uid, streamUrl, ((MyFreeCams)site).getClient()); + // } } } diff --git a/common/src/main/java/ctbrec/sites/mfc/ServerConfig.java b/common/src/main/java/ctbrec/sites/mfc/ServerConfig.java index 8c74fabf..96c7fd65 100644 --- a/common/src/main/java/ctbrec/sites/mfc/ServerConfig.java +++ b/common/src/main/java/ctbrec/sites/mfc/ServerConfig.java @@ -1,5 +1,7 @@ package ctbrec.sites.mfc; +import static ctbrec.io.HttpConstants.*; + import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; @@ -12,12 +14,13 @@ import org.json.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import ctbrec.Config; import okhttp3.Request; import okhttp3.Response; public class ServerConfig { - private static final transient Logger LOG = LoggerFactory.getLogger(ServerConfig.class); + private static final Logger LOG = LoggerFactory.getLogger(ServerConfig.class); List ajaxServers; List videoServers; @@ -30,7 +33,15 @@ public class ServerConfig { public ServerConfig(MyFreeCams mfc) throws IOException { String url = mfc.getBaseUrl() + "/_js/serverconfig.js"; LOG.debug("Loading server config from {}", url); - Request req = new Request.Builder().url(url).build(); + Request req = new Request.Builder() + .url(url) + .header(ACCEPT, "*/*") + .header(ACCEPT_LANGUAGE, "en-US,en;q=0.5") + .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) + .header(ORIGIN, mfc.getBaseUrl()) + .header(REFERER, mfc.getBaseUrl()) + .header(CONNECTION, KEEP_ALIVE) + .build(); Response resp = mfc.getHttpClient().execute(req); String json = resp.body().string(); @@ -42,9 +53,6 @@ public class ServerConfig { wsServers = parseMap(serverConfig, "websocket_servers"); wzobsServers = parseMap(serverConfig, "wzobs_servers"); ngVideoServers = parseMap(serverConfig, "ngvideo_servers"); - // System.out.println("wz " + wzobsServers); - // System.out.println("ng " + ngVideoServers); - // System.out.println("h5 " + h5Servers); } private static Map parseMap(JSONObject serverConfig, String name) { diff --git a/common/src/main/java/ctbrec/sites/mfc/Share.java b/common/src/main/java/ctbrec/sites/mfc/Share.java index ab6d5808..7c95ef1b 100644 --- a/common/src/main/java/ctbrec/sites/mfc/Share.java +++ b/common/src/main/java/ctbrec/sites/mfc/Share.java @@ -5,7 +5,7 @@ import java.util.Map; public class Share { - private Integer albums; + private String albums; private Integer follows; private Integer tmAlbum; private Integer things; @@ -14,13 +14,13 @@ public class Share { private Integer stores; private Integer goals; private Integer polls; - private Map additionalProperties = new HashMap(); + private Map additionalProperties = new HashMap<>(); - public Integer getAlbums() { + public String getAlbums() { return albums; } - public void setAlbums(Integer albums) { + public void setAlbums(String albums) { this.albums = albums; }