davebluke
/
ctbrec
forked from j62/ctbrec
1
0
Fork 0
Code Pull Requests Activity

Add user-agent agent header to every http request in MFC classes

This commit is contained in:
0xboobface 2020-04-13 18:42:57 +02:00
parent 979c8b0a91
commit ae6fa229e7
10 changed files with 158 additions and 71 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
client/src/main/java/ctbrec/ui/sites/myfreecams/MyFreeCamsTableTab.java
View File

@ -357,7 +357,7 @@ public class MyFreeCamsTableTab extends Tab implements TabSelectionListener {
try {
List<StreamSource> sources = m.getStreamSources();
for (StreamSource src : sources) {
LOG.info("{} {} {}", m.getUrl(), src.bandwidth, src.height);
LOG.info("m:{} s:{} bandwidth:{} height:{}", m.getName(), src.mediaPlaylistUrl, src.bandwidth, src.height);
}
LOG.info("===============");
} catch (IOException | ExecutionException | ParseException | PlaylistException | JAXBException e1) {

36
common/src/main/java/ctbrec/recorder/download/StreamSource.java
View File

@ -64,4 +64,40 @@ public class StreamSource implements Comparable<StreamSource> {
return bandwidth - o.bandwidth;
}
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + bandwidth;
result = prime * result + height;
result = prime * result + ((mediaPlaylistUrl == null) ? 0 : mediaPlaylistUrl.hashCode());
result = prime * result + width;
return result;
}
@Override
public boolean equals(Object obj) {
if (this == obj)
return true;
if (obj == null)
return false;
if (getClass() != obj.getClass())
return false;
StreamSource other = (StreamSource) obj;
if (bandwidth != other.bandwidth)
return false;
if (height != other.height)
return false;
if (mediaPlaylistUrl == null) {
if (other.mediaPlaylistUrl != null)
return false;
} else if (!mediaPlaylistUrl.equals(other.mediaPlaylistUrl))
return false;
if (width != other.width)
return false;
return true;
}
}

4
common/src/main/java/ctbrec/recorder/download/hls/MergedFfmpegHlsDownload.java
View File

@ -288,7 +288,11 @@ public class MergedFfmpegHlsDownload extends AbstractHlsDownload {
private void writeSegment(byte[] segmentData) throws IOException {
if (running) {
if (ffmpegStdIn != null) {
ffmpegStdIn.write(segmentData);
} else {
LOG.error("FFmpeg stdin stream is null - skipping writing of segment");
}
}
}

15
common/src/main/java/ctbrec/sites/mfc/HlsStreamSourceProvider.java
View File

@ -1,9 +1,12 @@
package ctbrec.sites.mfc;
import static ctbrec.io.HttpConstants.*;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
import java.util.concurrent.ExecutionException;
import java.util.stream.Collectors;
@ -73,7 +76,15 @@ public class HlsStreamSourceProvider implements StreamSourceProvider {
throw new IllegalStateException("Stream url unknown");
}
LOG.trace("Loading master playlist {}", streamUrl);
Request req = new Request.Builder().url(streamUrl).build();
Request req = new Request.Builder()
.url(streamUrl)
.header(ACCEPT, "*/*")
.header(ACCEPT_LANGUAGE, Locale.ENGLISH.getLanguage())
.header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
.header(CONNECTION, KEEP_ALIVE)
.header(ORIGIN, MyFreeCams.baseUrl)
.header(REFERER, MyFreeCams.baseUrl + '/')
.build();
try (Response response = httpClient.execute(req)) {
if (response.isSuccessful()) {
InputStream inputStream = response.body().byteStream();
@ -82,7 +93,7 @@ public class HlsStreamSourceProvider implements StreamSourceProvider {
MasterPlaylist master = playlist.getMasterPlaylist();
return master;
} else {
throw new HttpException(response.code(), response.message());
throw new HttpException(response.code(), response.message() + " - Forbidden: " + streamUrl);
}
}
}

11
common/src/main/java/ctbrec/sites/mfc/MyFreeCams.java
View File

@ -1,7 +1,10 @@
package ctbrec.sites.mfc;
import static ctbrec.io.HttpConstants.*;
import java.io.IOException;
import java.util.List;
import java.util.Locale;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@ -60,7 +63,13 @@ public class MyFreeCams extends AbstractSite {
@Override
public Double getTokenBalance() throws IOException {
Request req = new Request.Builder().url(baseUrl + "/php/account.php?request=status").build();
Request req = new Request.Builder()
.url(baseUrl + "/php/account.php?request=status")
.header(ACCEPT, "*/*")
.header(ACCEPT_LANGUAGE, Locale.ENGLISH.getLanguage())
.header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
.header(CONNECTION, KEEP_ALIVE)
.build();
try(Response response = getHttpClient().execute(req)) {
if(response.isSuccessful()) {
String content = response.body().string();

69
common/src/main/java/ctbrec/sites/mfc/MyFreeCamsClient.java
View File

@ -16,6 +16,7 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Objects;
@ -62,6 +63,7 @@ public class MyFreeCamsClient {
private Cache<Integer, MyFreeCamsModel> models = CacheBuilder.newBuilder().maximumSize(4000).build();
private Lock lock = new ReentrantLock();
private ServerConfig serverConfig;
@SuppressWarnings("unused")
private String tkx;
private Integer cxid;
private int[] ctx;
@ -110,9 +112,9 @@ public class MyFreeCamsClient {
LOG.info("Websocket is null. Starting a new connection");
Request req = new Request.Builder()
.url(wsUrl)
.header(USER_AGENT, Config.getInstance().getSettings().httpUserAgentMobile)
.header(REFERER, "http://m.myfreecams.com")
.header(ORIGIN, "http://m.myfreecams.com").build();
.header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
.header(ORIGIN, MyFreeCams.baseUrl)
.build();
ws = createWebSocket(req);
}
@ -160,9 +162,6 @@ public class MyFreeCamsClient {
LOG.trace("open: [{}]", response.body().string());
webSocket.send("hello fcserver\n");
webSocket.send("fcsws_20180422\n");
// TODO find out, what the values in the json message mean, at the moment we hust send 0s, which seems to work, too
// webSocket.send("1 0 0 81 0
// %7B%22err%22%3A0%2C%22start%22%3A1540159843072%2C%22stop%22%3A1540159844121%2C%22a%22%3A6392%2C%22time%22%3A1540159844%2C%22key%22%3A%228da80f985c9db390809713dac71df297%22%2C%22cid%22%3A%22c504d684%22%2C%22pid%22%3A1%2C%22site%22%3A%22www%22%7D\n");
webSocket.send(
"1 0 0 81 0 %7B%22err%22%3A0%2C%22start%22%3A0%2C%22stop%22%3A0%2C%22a%22%3A0%2C%22time%22%3A0%2C%22key%22%3A%22%22%2C%22cid%22%3A%22%22%2C%22pid%22%3A1%2C%22site%22%3A%22www%22%7D\n");
heartBeat = System.currentTimeMillis();
@ -331,7 +330,13 @@ public class MyFreeCamsClient {
long type = json.getInt("type");
String base = mfc.getBaseUrl() + "/php/FcwExtResp.php";
String url = base + "?respkey=" + respkey + "&opts=" + opts + "&serv=" + serv + "&type=" + type;
Request req = new Request.Builder().url(url).build();
Request req = new Request.Builder()
.url(url)
.header(ACCEPT, "*/*")
.header(ACCEPT_LANGUAGE, Locale.ENGLISH.getLanguage())
.header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
.header(CONNECTION, KEEP_ALIVE)
.build();
LOG.trace("Requesting EXTDATA {}", url);
try (Response resp = mfc.getHttpClient().execute(req)) {
if (resp.isSuccessful()) {
@ -520,30 +525,38 @@ public class MyFreeCamsClient {
}
}
protected void authorizeForStream(SessionState state) {
protected String getWebrtcAuthCommand(SessionState state) {
JSONObject streamInfo = new JSONObject();
streamInfo.put("applicationName", "NxServer");
int userChannel = 100000000 + state.getUid();
String phase = state.getU().getPhase() != null ? state.getU().getPhase() : "z";
String phase = Optional.ofNullable(state.getU()).map(User::getPhase).orElse("z");
String phasePrefix = phase.equals("z") ? "" : '_' + phase;
String streamName = "mfc" + phasePrefix + '_' + userChannel + ".f4v";
streamInfo.put("applicationName", "NxServer");
streamInfo.put("streamName", streamName);
streamInfo.put("sessionId", "[empty]");
JSONObject userData = new JSONObject();
userData.put("sessionId", sessionId);
userData.put("password", tkx);
userData.put("password", ""); // tkx or "" ?!?
userData.put("roomId", userChannel);
userData.put("modelId", state.getUid());
if (isBroadcasterOnOBS(state)) {
JSONArray array = new JSONArray();
Arrays.stream(ctx).forEach(array::put);
userData.put("vidctx", Base64.getEncoder().encodeToString(array.toString().getBytes(StandardCharsets.UTF_8)));
userData.put("cxid", cxid);
}
userData.put("mode", "DOWNLOAD");
JSONObject authCommand = new JSONObject();
authCommand.put("userData", userData);
authCommand.put("streamInfo", streamInfo);
authCommand.put("command", "auth");
authCommand.put("streamInfo", streamInfo);
authCommand.put("userData", userData);
LOG.info("auth command {}", authCommand.toString(2));
return streamInfo.toString();
}
private boolean isBroadcasterOnOBS(SessionState state) {
String phase = Optional.ofNullable(state.getU()).map(User::getPhase).orElse("z");
return !phase.equals("z");
}
private void startKeepAlive(WebSocket ws) {
@ -598,28 +611,8 @@ public class MyFreeCamsClient {
String phase = state.getU().getPhase() != null ? state.getU().getPhase() : "z";
String phasePrefix = phase.equals("z") ? "" : '_' + phase;
String server = "video" + getCamServ(state).replaceAll("^\\D+", "");
boolean useHls = serverConfig.isOnObsServer(state);
String streamUrl;
boolean dontUseDash = !Config.getInstance().getSettings().mfcUseDash;
if (serverConfig.isOnWzObsVideoServer(state) || !serverConfig.isOnObsServer(state)) {
// wowza server
if (dontUseDash || useHls) {
String nonce = Double.toString(Math.random());
streamUrl = HTTPS + server + ".myfreecams.com/NxServer/ngrp:mfc" + phasePrefix + '_' + userChannel + ".f4v_mobile/playlist.m3u8?nc=" + nonce;
} else {
streamUrl = HTTPS + server + ".myfreecams.com/NxServer/ngrp:mfc" + phasePrefix + '_' + userChannel + ".f4v_desktop/manifest.mpd";
}
} else {
// nginx server
if (dontUseDash || useHls) {
String nonce = Double.toString(Math.random());
streamUrl = HTTPS + server + ".myfreecams.com:8444/x-hls/" + cxid + '/' + userChannel + '/' + ctxenc + "/mfc" + phasePrefix + '_' + userChannel
+ ".m3u8?nc=" + nonce;
} else {
streamUrl = HTTPS + server + ".myfreecams.com:8444/x-dsh/" + cxid + '/' + userChannel + '/' + ctxenc + "/mfc" + phasePrefix + '_' + userChannel
+ ".mpd";
}
}
String streamUrl = HTTPS + server + ".myfreecams.com/NxServer/ngrp:mfc" + phasePrefix + '_' + userChannel + ".f4v_mobile/playlist.m3u8?nc=" + nonce;
return streamUrl;
}
@ -633,12 +626,20 @@ public class MyFreeCamsClient {
} else if (serverConfig.isOnHtml5VideoServer(state)) {
camservString = serverConfig.h5Servers.get(camserv.toString());
} else if (camserv > 500) {
if (camserv >= 3000) {
camserv -= 1000;
} else {
camserv -= 500;
}
camservString = camserv.toString();
}
return camservString;
}
private boolean isBroadcasterOnWebRTC(SessionState state) {
return (Optional.ofNullable(state).map(SessionState::getM).map(Model::getFlags).orElse(0) & 524288) == 524288;
}
public MyFreeCamsModel getModel(int uid) {
return models.getIfPresent(uid);
}

13
common/src/main/java/ctbrec/sites/mfc/MyFreeCamsHttpClient.java
View File

@ -4,6 +4,7 @@ import static ctbrec.io.HttpConstants.*;
import java.io.IOException;
import java.util.List;
import java.util.Locale;
import java.util.NoSuchElementException;
import java.util.Objects;
@ -56,6 +57,10 @@ public class MyFreeCamsHttpClient extends HttpClient {
.build();
Request req = new Request.Builder()
.url(MyFreeCams.baseUrl + "/php/login.php")
.header(ACCEPT, "*/*")
.header(ACCEPT_LANGUAGE, Locale.ENGLISH.getLanguage())
.header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
.header(CONNECTION, KEEP_ALIVE)
.header(REFERER, MyFreeCams.baseUrl)
.header("Content-Type", "application/x-www-form-urlencoded")
.post(body)
@ -77,7 +82,13 @@ public class MyFreeCamsHttpClient extends HttpClient {
}
private boolean checkLogin() throws IOException {
Request req = new Request.Builder().url(MyFreeCams.baseUrl + "/php/account.php?request=status").build();
Request req = new Request.Builder()
.url(MyFreeCams.baseUrl + "/php/account.php?request=status")
.header(ACCEPT, "*/*")
.header(ACCEPT_LANGUAGE, Locale.ENGLISH.getLanguage())
.header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
.header(CONNECTION, KEEP_ALIVE)
.build();
try(Response response = execute(req)) {
if(response.isSuccessful()) {
String content = response.body().string();

29
common/src/main/java/ctbrec/sites/mfc/MyFreeCamsModel.java
View File

@ -8,6 +8,7 @@ import java.net.URLDecoder;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
import java.util.Objects;
import java.util.Optional;
import java.util.concurrent.ExecutionException;
@ -29,7 +30,6 @@ import ctbrec.io.HtmlParser;
import ctbrec.io.HttpException;
import ctbrec.recorder.download.Download;
import ctbrec.recorder.download.StreamSource;
import ctbrec.recorder.download.dash.DashDownload;
import okhttp3.FormBody;
import okhttp3.Request;
import okhttp3.RequestBody;
@ -98,11 +98,7 @@ public class MyFreeCamsModel extends AbstractModel {
}
private StreamSourceProvider getStreamSourceProvider() {
if(isHlsStream()) {
return new HlsStreamSourceProvider(getSite().getHttpClient());
} else {
return new DashStreamSourceProvider(Config.getInstance(), site);
}
}
private boolean isHlsStream() {
@ -126,7 +122,13 @@ public class MyFreeCamsModel extends AbstractModel {
public void receiveTip(Double tokens) throws IOException {
String tipUrl = MyFreeCams.baseUrl + "/php/tip.php";
String initUrl = tipUrl + "?request=tip&username="+getName()+"&broadcaster_id="+getUid();
Request req = new Request.Builder().url(initUrl).build();
Request req = new Request.Builder()
.url(initUrl)
.header(ACCEPT, "*/*")
.header(ACCEPT_LANGUAGE, Locale.ENGLISH.getLanguage())
.header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
.header(CONNECTION, KEEP_ALIVE)
.build();
try(Response resp = site.getHttpClient().execute(req)) {
if(resp.isSuccessful()) {
String page = resp.body().string();
@ -150,7 +152,11 @@ public class MyFreeCamsModel extends AbstractModel {
req = new Request.Builder()
.url(tipUrl)
.post(body)
.addHeader(REFERER, initUrl)
.header(ACCEPT, "*/*")
.header(ACCEPT_LANGUAGE, Locale.ENGLISH.getLanguage())
.header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
.header(CONNECTION, KEEP_ALIVE)
.header(REFERER, initUrl)
.build();
try(Response response = site.getHttpClient().execute(req)) {
if(!response.isSuccessful()) {
@ -329,10 +335,11 @@ public class MyFreeCamsModel extends AbstractModel {
if(streamUrl == null) {
updateStreamUrl();
}
if(isHlsStream()) {
return super.createDownload();
} else {
return new DashDownload(getSite().getHttpClient(), streamUrl);
}
// if(isHlsStream()) {
// return super.createDownload();
// } else {
// return new MyFreeCamsWebrtcDownload(uid, streamUrl, ((MyFreeCams)site).getClient());
// }
}
}

18
common/src/main/java/ctbrec/sites/mfc/ServerConfig.java
View File

@ -1,5 +1,7 @@
package ctbrec.sites.mfc;
import static ctbrec.io.HttpConstants.*;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
@ -12,12 +14,13 @@ import org.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import ctbrec.Config;
import okhttp3.Request;
import okhttp3.Response;
public class ServerConfig {
private static final transient Logger LOG = LoggerFactory.getLogger(ServerConfig.class);
private static final Logger LOG = LoggerFactory.getLogger(ServerConfig.class);
List<String> ajaxServers;
List<String> videoServers;
@ -30,7 +33,15 @@ public class ServerConfig {
public ServerConfig(MyFreeCams mfc) throws IOException {
String url = mfc.getBaseUrl() + "/_js/serverconfig.js";
LOG.debug("Loading server config from {}", url);
Request req = new Request.Builder().url(url).build();
Request req = new Request.Builder()
.url(url)
.header(ACCEPT, "*/*")
.header(ACCEPT_LANGUAGE, "en-US,en;q=0.5")
.header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
.header(ORIGIN, mfc.getBaseUrl())
.header(REFERER, mfc.getBaseUrl())
.header(CONNECTION, KEEP_ALIVE)
.build();
Response resp = mfc.getHttpClient().execute(req);
String json = resp.body().string();
@ -42,9 +53,6 @@ public class ServerConfig {
wsServers = parseMap(serverConfig, "websocket_servers");
wzobsServers = parseMap(serverConfig, "wzobs_servers");
ngVideoServers = parseMap(serverConfig, "ngvideo_servers");
// System.out.println("wz " + wzobsServers);
// System.out.println("ng " + ngVideoServers);
// System.out.println("h5 " + h5Servers);
}
private static Map<String, String> parseMap(JSONObject serverConfig, String name) {

8
common/src/main/java/ctbrec/sites/mfc/Share.java
View File

@ -5,7 +5,7 @@ import java.util.Map;
public class Share {
private Integer albums;
private String albums;
private Integer follows;
private Integer tmAlbum;
private Integer things;
@ -14,13 +14,13 @@ public class Share {
private Integer stores;
private Integer goals;
private Integer polls;
private Map<String, Object> additionalProperties = new HashMap<String, Object>();
private Map<String, Object> additionalProperties = new HashMap<>();
public Integer getAlbums() {
public String getAlbums() {
return albums;
}
public void setAlbums(Integer albums) {
public void setAlbums(String albums) {
this.albums = albums;
}