Add user-agent agent header to every http request in MFC classes

client/src/main/java/ctbrec/ui/sites/myfreecams/MyFreeCamsTableTab.java

@@ -357,7 +357,7 @@ public class MyFreeCamsTableTab extends Tab implements TabSelectionListener {
                     try {
                         List<StreamSource> sources = m.getStreamSources();
                         for (StreamSource src : sources) {
-                            LOG.info("{} {} {}", m.getUrl(), src.bandwidth, src.height);
+                            LOG.info("m:{} s:{} bandwidth:{} height:{}", m.getName(), src.mediaPlaylistUrl, src.bandwidth, src.height);
                         }
                         LOG.info("===============");
                     } catch (IOException | ExecutionException | ParseException | PlaylistException | JAXBException e1) {

common/src/main/java/ctbrec/recorder/download/StreamSource.java

@@ -64,4 +64,40 @@ public class StreamSource implements Comparable<StreamSource> {
             return bandwidth - o.bandwidth;
         }
     }
+
+    @Override
+    public int hashCode() {
+        final int prime = 31;
+        int result = 1;
+        result = prime * result + bandwidth;
+        result = prime * result + height;
+        result = prime * result + ((mediaPlaylistUrl == null) ? 0 : mediaPlaylistUrl.hashCode());
+        result = prime * result + width;
+        return result;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj)
+            return true;
+        if (obj == null)
+            return false;
+        if (getClass() != obj.getClass())
+            return false;
+        StreamSource other = (StreamSource) obj;
+        if (bandwidth != other.bandwidth)
+            return false;
+        if (height != other.height)
+            return false;
+        if (mediaPlaylistUrl == null) {
+            if (other.mediaPlaylistUrl != null)
+                return false;
+        } else if (!mediaPlaylistUrl.equals(other.mediaPlaylistUrl))
+            return false;
+        if (width != other.width)
+            return false;
+        return true;
+    }
+
+
 }

common/src/main/java/ctbrec/recorder/download/hls/MergedFfmpegHlsDownload.java

@@ -288,7 +288,11 @@ public class MergedFfmpegHlsDownload extends AbstractHlsDownload {
 
     private void writeSegment(byte[] segmentData) throws IOException {
         if (running) {
-            ffmpegStdIn.write(segmentData);
+            if (ffmpegStdIn != null) {
+                ffmpegStdIn.write(segmentData);
+            } else {
+                LOG.error("FFmpeg stdin stream is null - skipping writing of segment");
+            }
         }
     }
 

common/src/main/java/ctbrec/sites/mfc/HlsStreamSourceProvider.java

@@ -1,9 +1,12 @@
 package ctbrec.sites.mfc;
 
+import static ctbrec.io.HttpConstants.*;
+
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Locale;
 import java.util.concurrent.ExecutionException;
 import java.util.stream.Collectors;
 
@@ -73,16 +76,24 @@ public class HlsStreamSourceProvider implements StreamSourceProvider {
             throw new IllegalStateException("Stream url unknown");
         }
         LOG.trace("Loading master playlist {}", streamUrl);
-        Request req = new Request.Builder().url(streamUrl).build();
+        Request req = new Request.Builder()
-        try(Response response = httpClient.execute(req)) {
+                .url(streamUrl)
-            if(response.isSuccessful()) {
+                .header(ACCEPT, "*/*")
+                .header(ACCEPT_LANGUAGE, Locale.ENGLISH.getLanguage())
+                .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
+                .header(CONNECTION, KEEP_ALIVE)
+                .header(ORIGIN, MyFreeCams.baseUrl)
+                .header(REFERER, MyFreeCams.baseUrl + '/')
+                .build();
+        try (Response response = httpClient.execute(req)) {
+            if (response.isSuccessful()) {
                 InputStream inputStream = response.body().byteStream();
                 PlaylistParser parser = new PlaylistParser(inputStream, Format.EXT_M3U, Encoding.UTF_8, ParsingMode.LENIENT);
                 Playlist playlist = parser.parse();
                 MasterPlaylist master = playlist.getMasterPlaylist();
                 return master;
             } else {
-                throw new HttpException(response.code(), response.message());
+                throw new HttpException(response.code(), response.message() + " - Forbidden: " + streamUrl);
             }
         }
     }

common/src/main/java/ctbrec/sites/mfc/MyFreeCams.java

@@ -1,7 +1,10 @@
 package ctbrec.sites.mfc;
 
+import static ctbrec.io.HttpConstants.*;
+
 import java.io.IOException;
 import java.util.List;
+import java.util.Locale;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -60,7 +63,13 @@ public class MyFreeCams extends AbstractSite {
 
     @Override
     public Double getTokenBalance() throws IOException {
-        Request req = new Request.Builder().url(baseUrl + "/php/account.php?request=status").build();
+        Request req = new Request.Builder()
+                .url(baseUrl + "/php/account.php?request=status")
+                .header(ACCEPT, "*/*")
+                .header(ACCEPT_LANGUAGE, Locale.ENGLISH.getLanguage())
+                .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
+                .header(CONNECTION, KEEP_ALIVE)
+                .build();
         try(Response response = getHttpClient().execute(req)) {
             if(response.isSuccessful()) {
                 String content = response.body().string();

common/src/main/java/ctbrec/sites/mfc/MyFreeCamsClient.java

@@ -16,6 +16,7 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Objects;
@@ -62,6 +63,7 @@ public class MyFreeCamsClient {
     private Cache<Integer, MyFreeCamsModel> models = CacheBuilder.newBuilder().maximumSize(4000).build();
     private Lock lock = new ReentrantLock();
     private ServerConfig serverConfig;
+    @SuppressWarnings("unused")
     private String tkx;
     private Integer cxid;
     private int[] ctx;
@@ -100,7 +102,7 @@ public class MyFreeCamsClient {
             }
         }
 
-        String server = websocketServers.get(new Random().nextInt(websocketServers.size()-1));
+        String server = websocketServers.get(new Random().nextInt(websocketServers.size() - 1));
         String wsUrl = "ws://" + server + ".myfreecams.com:8080/fcsl";
         LOG.debug("Connecting to random websocket server {}", wsUrl);
 
@@ -110,9 +112,9 @@ public class MyFreeCamsClient {
                     LOG.info("Websocket is null. Starting a new connection");
                     Request req = new Request.Builder()
                             .url(wsUrl)
-                            .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgentMobile)
+                            .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
-                            .header(REFERER, "http://m.myfreecams.com")
+                            .header(ORIGIN, MyFreeCams.baseUrl)
-                            .header(ORIGIN, "http://m.myfreecams.com").build();
+                            .build();
                     ws = createWebSocket(req);
                 }
 
@@ -160,9 +162,6 @@ public class MyFreeCamsClient {
                     LOG.trace("open: [{}]", response.body().string());
                     webSocket.send("hello fcserver\n");
                     webSocket.send("fcsws_20180422\n");
-                    // TODO find out, what the values in the json message mean, at the moment we hust send 0s, which seems to work, too
-                    // webSocket.send("1 0 0 81 0
-                    // %7B%22err%22%3A0%2C%22start%22%3A1540159843072%2C%22stop%22%3A1540159844121%2C%22a%22%3A6392%2C%22time%22%3A1540159844%2C%22key%22%3A%228da80f985c9db390809713dac71df297%22%2C%22cid%22%3A%22c504d684%22%2C%22pid%22%3A1%2C%22site%22%3A%22www%22%7D\n");
                     webSocket.send(
                             "1 0 0 81 0 %7B%22err%22%3A0%2C%22start%22%3A0%2C%22stop%22%3A0%2C%22a%22%3A0%2C%22time%22%3A0%2C%22key%22%3A%22%22%2C%22cid%22%3A%22%22%2C%22pid%22%3A1%2C%22site%22%3A%22www%22%7D\n");
                     heartBeat = System.currentTimeMillis();
@@ -331,7 +330,13 @@ public class MyFreeCamsClient {
                     long type = json.getInt("type");
                     String base = mfc.getBaseUrl() + "/php/FcwExtResp.php";
                     String url = base + "?respkey=" + respkey + "&opts=" + opts + "&serv=" + serv + "&type=" + type;
-                    Request req = new Request.Builder().url(url).build();
+                    Request req = new Request.Builder()
+                            .url(url)
+                            .header(ACCEPT, "*/*")
+                            .header(ACCEPT_LANGUAGE, Locale.ENGLISH.getLanguage())
+                            .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
+                            .header(CONNECTION, KEEP_ALIVE)
+                            .build();
                     LOG.trace("Requesting EXTDATA {}", url);
                     try (Response resp = mfc.getHttpClient().execute(req)) {
                         if (resp.isSuccessful()) {
@@ -520,30 +525,38 @@ public class MyFreeCamsClient {
         }
     }
 
-    protected void authorizeForStream(SessionState state) {
+    protected String getWebrtcAuthCommand(SessionState state) {
         JSONObject streamInfo = new JSONObject();
-        streamInfo.put("applicationName", "NxServer");
         int userChannel = 100000000 + state.getUid();
-        String phase = state.getU().getPhase() != null ? state.getU().getPhase() : "z";
+        String phase = Optional.ofNullable(state.getU()).map(User::getPhase).orElse("z");
         String phasePrefix = phase.equals("z") ? "" : '_' + phase;
         String streamName = "mfc" + phasePrefix + '_' + userChannel + ".f4v";
+        streamInfo.put("applicationName", "NxServer");
         streamInfo.put("streamName", streamName);
         streamInfo.put("sessionId", "[empty]");
         JSONObject userData = new JSONObject();
         userData.put("sessionId", sessionId);
-        userData.put("password", tkx);
+        userData.put("password", ""); // tkx or "" ?!?
         userData.put("roomId", userChannel);
         userData.put("modelId", state.getUid());
-        JSONArray array = new JSONArray();
+        if (isBroadcasterOnOBS(state)) {
-        Arrays.stream(ctx).forEach(array::put);
+            JSONArray array = new JSONArray();
-        userData.put("vidctx", Base64.getEncoder().encodeToString(array.toString().getBytes(StandardCharsets.UTF_8)));
+            Arrays.stream(ctx).forEach(array::put);
-        userData.put("cxid", cxid);
+            userData.put("vidctx", Base64.getEncoder().encodeToString(array.toString().getBytes(StandardCharsets.UTF_8)));
+            userData.put("cxid", cxid);
+        }
         userData.put("mode", "DOWNLOAD");
         JSONObject authCommand = new JSONObject();
-        authCommand.put("userData", userData);
-        authCommand.put("streamInfo", streamInfo);
         authCommand.put("command", "auth");
+        authCommand.put("streamInfo", streamInfo);
+        authCommand.put("userData", userData);
         LOG.info("auth command {}", authCommand.toString(2));
+        return streamInfo.toString();
+    }
+
+    private boolean isBroadcasterOnOBS(SessionState state) {
+        String phase = Optional.ofNullable(state.getU()).map(User::getPhase).orElse("z");
+        return !phase.equals("z");
     }
 
     private void startKeepAlive(WebSocket ws) {
@@ -598,28 +611,8 @@ public class MyFreeCamsClient {
         String phase = state.getU().getPhase() != null ? state.getU().getPhase() : "z";
         String phasePrefix = phase.equals("z") ? "" : '_' + phase;
         String server = "video" + getCamServ(state).replaceAll("^\\D+", "");
-        boolean useHls = serverConfig.isOnObsServer(state);
+        String nonce = Double.toString(Math.random());
-        String streamUrl;
+        String streamUrl = HTTPS + server + ".myfreecams.com/NxServer/ngrp:mfc" + phasePrefix + '_' + userChannel + ".f4v_mobile/playlist.m3u8?nc=" + nonce;
-        boolean dontUseDash = !Config.getInstance().getSettings().mfcUseDash;
-        if (serverConfig.isOnWzObsVideoServer(state) || !serverConfig.isOnObsServer(state)) {
-            // wowza server
-            if (dontUseDash || useHls) {
-                String nonce = Double.toString(Math.random());
-                streamUrl = HTTPS + server + ".myfreecams.com/NxServer/ngrp:mfc" + phasePrefix + '_' + userChannel + ".f4v_mobile/playlist.m3u8?nc=" + nonce;
-            } else {
-                streamUrl = HTTPS + server + ".myfreecams.com/NxServer/ngrp:mfc" + phasePrefix + '_' + userChannel + ".f4v_desktop/manifest.mpd";
-            }
-        } else {
-            // nginx server
-            if (dontUseDash || useHls) {
-                String nonce = Double.toString(Math.random());
-                streamUrl = HTTPS + server + ".myfreecams.com:8444/x-hls/" + cxid + '/' + userChannel + '/' + ctxenc + "/mfc" + phasePrefix + '_' + userChannel
-                        + ".m3u8?nc=" + nonce;
-            } else {
-                streamUrl = HTTPS + server + ".myfreecams.com:8444/x-dsh/" + cxid + '/' + userChannel + '/' + ctxenc + "/mfc" + phasePrefix + '_' + userChannel
-                        + ".mpd";
-            }
-        }
         return streamUrl;
     }
 
@@ -633,12 +626,20 @@ public class MyFreeCamsClient {
         } else if (serverConfig.isOnHtml5VideoServer(state)) {
             camservString = serverConfig.h5Servers.get(camserv.toString());
         } else if (camserv > 500) {
-            camserv -= 500;
+            if (camserv >= 3000) {
+                camserv -= 1000;
+            } else {
+                camserv -= 500;
+            }
             camservString = camserv.toString();
         }
         return camservString;
     }
 
+    private boolean isBroadcasterOnWebRTC(SessionState state) {
+        return (Optional.ofNullable(state).map(SessionState::getM).map(Model::getFlags).orElse(0) & 524288) == 524288;
+    }
+
     public MyFreeCamsModel getModel(int uid) {
         return models.getIfPresent(uid);
     }

common/src/main/java/ctbrec/sites/mfc/MyFreeCamsHttpClient.java

@@ -4,6 +4,7 @@ import static ctbrec.io.HttpConstants.*;
 
 import java.io.IOException;
 import java.util.List;
+import java.util.Locale;
 import java.util.NoSuchElementException;
 import java.util.Objects;
 
@@ -56,6 +57,10 @@ public class MyFreeCamsHttpClient extends HttpClient {
                 .build();
         Request req = new Request.Builder()
                 .url(MyFreeCams.baseUrl + "/php/login.php")
+                .header(ACCEPT, "*/*")
+                .header(ACCEPT_LANGUAGE, Locale.ENGLISH.getLanguage())
+                .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
+                .header(CONNECTION, KEEP_ALIVE)
                 .header(REFERER, MyFreeCams.baseUrl)
                 .header("Content-Type", "application/x-www-form-urlencoded")
                 .post(body)
@@ -77,7 +82,13 @@ public class MyFreeCamsHttpClient extends HttpClient {
     }
 
     private boolean checkLogin() throws IOException {
-        Request req = new Request.Builder().url(MyFreeCams.baseUrl + "/php/account.php?request=status").build();
+        Request req = new Request.Builder()
+                .url(MyFreeCams.baseUrl + "/php/account.php?request=status")
+                .header(ACCEPT, "*/*")
+                .header(ACCEPT_LANGUAGE, Locale.ENGLISH.getLanguage())
+                .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
+                .header(CONNECTION, KEEP_ALIVE)
+                .build();
         try(Response response = execute(req)) {
             if(response.isSuccessful()) {
                 String content = response.body().string();

common/src/main/java/ctbrec/sites/mfc/MyFreeCamsModel.java

@@ -8,6 +8,7 @@ import java.net.URLDecoder;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
+import java.util.Locale;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.concurrent.ExecutionException;
@@ -29,7 +30,6 @@ import ctbrec.io.HtmlParser;
 import ctbrec.io.HttpException;
 import ctbrec.recorder.download.Download;
 import ctbrec.recorder.download.StreamSource;
-import ctbrec.recorder.download.dash.DashDownload;
 import okhttp3.FormBody;
 import okhttp3.Request;
 import okhttp3.RequestBody;
@@ -98,11 +98,7 @@ public class MyFreeCamsModel extends AbstractModel {
     }
 
     private StreamSourceProvider getStreamSourceProvider() {
-        if(isHlsStream()) {
+        return new HlsStreamSourceProvider(getSite().getHttpClient());
-            return new HlsStreamSourceProvider(getSite().getHttpClient());
-        } else {
-            return new DashStreamSourceProvider(Config.getInstance(), site);
-        }
     }
 
     private boolean isHlsStream() {
@@ -126,7 +122,13 @@ public class MyFreeCamsModel extends AbstractModel {
     public void receiveTip(Double tokens) throws IOException {
         String tipUrl = MyFreeCams.baseUrl + "/php/tip.php";
         String initUrl = tipUrl + "?request=tip&username="+getName()+"&broadcaster_id="+getUid();
-        Request req = new Request.Builder().url(initUrl).build();
+        Request req = new Request.Builder()
+                .url(initUrl)
+                .header(ACCEPT, "*/*")
+                .header(ACCEPT_LANGUAGE, Locale.ENGLISH.getLanguage())
+                .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
+                .header(CONNECTION, KEEP_ALIVE)
+                .build();
         try(Response resp = site.getHttpClient().execute(req)) {
             if(resp.isSuccessful()) {
                 String page = resp.body().string();
@@ -150,7 +152,11 @@ public class MyFreeCamsModel extends AbstractModel {
                     req = new Request.Builder()
                             .url(tipUrl)
                             .post(body)
-                            .addHeader(REFERER, initUrl)
+                            .header(ACCEPT, "*/*")
+                            .header(ACCEPT_LANGUAGE, Locale.ENGLISH.getLanguage())
+                            .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
+                            .header(CONNECTION, KEEP_ALIVE)
+                            .header(REFERER, initUrl)
                             .build();
                     try(Response response = site.getHttpClient().execute(req)) {
                         if(!response.isSuccessful()) {
@@ -329,10 +335,11 @@ public class MyFreeCamsModel extends AbstractModel {
         if(streamUrl == null) {
             updateStreamUrl();
         }
-        if(isHlsStream()) {
+        return super.createDownload();
-            return super.createDownload();
+        //        if(isHlsStream()) {
-        } else {
+        //            return super.createDownload();
-            return new DashDownload(getSite().getHttpClient(), streamUrl);
+        //        } else {
-        }
+        //            return new MyFreeCamsWebrtcDownload(uid, streamUrl, ((MyFreeCams)site).getClient());
+        //        }
     }
 }

common/src/main/java/ctbrec/sites/mfc/ServerConfig.java

@@ -1,5 +1,7 @@
 package ctbrec.sites.mfc;
 
+import static ctbrec.io.HttpConstants.*;
+
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -12,12 +14,13 @@ import org.json.JSONObject;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import ctbrec.Config;
 import okhttp3.Request;
 import okhttp3.Response;
 
 public class ServerConfig {
 
-    private static final transient Logger LOG = LoggerFactory.getLogger(ServerConfig.class);
+    private static final Logger LOG = LoggerFactory.getLogger(ServerConfig.class);
 
     List<String> ajaxServers;
     List<String> videoServers;
@@ -30,7 +33,15 @@ public class ServerConfig {
     public ServerConfig(MyFreeCams mfc) throws IOException {
         String url = mfc.getBaseUrl() + "/_js/serverconfig.js";
         LOG.debug("Loading server config from {}", url);
-        Request req = new Request.Builder().url(url).build();
+        Request req = new Request.Builder()
+                .url(url)
+                .header(ACCEPT, "*/*")
+                .header(ACCEPT_LANGUAGE, "en-US,en;q=0.5")
+                .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
+                .header(ORIGIN, mfc.getBaseUrl())
+                .header(REFERER, mfc.getBaseUrl())
+                .header(CONNECTION, KEEP_ALIVE)
+                .build();
         Response resp = mfc.getHttpClient().execute(req);
         String json = resp.body().string();
 
@@ -42,9 +53,6 @@ public class ServerConfig {
         wsServers = parseMap(serverConfig, "websocket_servers");
         wzobsServers = parseMap(serverConfig, "wzobs_servers");
         ngVideoServers = parseMap(serverConfig, "ngvideo_servers");
-        //        System.out.println("wz " + wzobsServers);
-        //        System.out.println("ng " + ngVideoServers);
-        //        System.out.println("h5 " + h5Servers);
     }
 
     private static Map<String, String> parseMap(JSONObject serverConfig, String name) {

common/src/main/java/ctbrec/sites/mfc/Share.java

@@ -5,7 +5,7 @@ import java.util.Map;
 
 public class Share {
 
-    private Integer albums;
+    private String albums;
     private Integer follows;
     private Integer tmAlbum;
     private Integer things;
@@ -14,13 +14,13 @@ public class Share {
     private Integer stores;
     private Integer goals;
     private Integer polls;
-    private Map<String, Object> additionalProperties = new HashMap<String, Object>();
+    private Map<String, Object> additionalProperties = new HashMap<>();
 
-    public Integer getAlbums() {
+    public String getAlbums() {
         return albums;
     }
 
-    public void setAlbums(Integer albums) {
+    public void setAlbums(String albums) {
         this.albums = albums;
     }