j62
/
ctbrec
2
2
Fork 2
Code Issues 3 Pull Requests Packages Projects Releases 3 Wiki Activity

Fix MVLive recordings

This commit is contained in:
0xb00bface 2023-04-09 12:41:40 +02:00
parent e436beaeeb
commit c8d9e53cc2
2 changed files with 66 additions and 200 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

174
common/src/main/java/ctbrec/sites/manyvids/MVLive.java
View File

@ -6,17 +6,17 @@ import ctbrec.io.HtmlParser;
import ctbrec.io.HttpClient;
import ctbrec.io.HttpException;
import ctbrec.sites.AbstractSite;
import okhttp3.*;
import okhttp3.FormBody;
import okhttp3.Request;
import okhttp3.RequestBody;
import okhttp3.Response;
import org.json.JSONArray;
import org.json.JSONObject;
import org.jsoup.nodes.Element;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.net.URLDecoder;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
import java.util.Locale;
import java.util.regex.Matcher;
@ -27,15 +27,10 @@ import static java.nio.charset.StandardCharsets.UTF_8;
public class MVLive extends AbstractSite {
private static final Logger LOG = LoggerFactory.getLogger(MVLive.class);
private static final String STARS = "stars";
public static final String WS_ORIGIN = "https://live.manyvids.com";
public static final String BASE_URL = "https://www.manyvids.com";
public static final String LIVE_URL = BASE_URL + "/mv-live/";
private final Pattern configPattern = Pattern.compile("<script type=\"combinativ-config\">(.*?)</script>");
private final Pattern graphQlUrlPattern = Pattern.compile("api:\\s*\"(https://.+?.appsync-api..+?.amazonaws.com/graphql)\"");
private String graphqlBaseUri;
public static final String BASE_URL = "https://www.manyvids.com/";
private MVLiveHttpClient httpClient;
private String mvtoken;
@ -47,7 +42,7 @@ public class MVLive extends AbstractSite {
@Override
public String getBaseUrl() {
return LIVE_URL;
return BASE_URL;
}
@Override
@ -120,157 +115,15 @@ public class MVLive extends AbstractSite {
// nothing to do
}
private String getGraphQlUrl() {
if (graphqlBaseUri == null) {
try {
String spaBundleUrl = getSpaBundleUrl();
loadGraphqlApiConfig(spaBundleUrl);
LOG.debug("Using graphql API at {}", graphqlBaseUri);
} catch (IOException e) {
LOG.error("Error while initializing {}", getName(), e);
}
}
return graphqlBaseUri;
}
private void loadGraphqlApiConfig(String spaBundleUrl) throws IOException {
Request request = new Request.Builder()
.url(spaBundleUrl)
.header(USER_AGENT, getConfig().getSettings().httpUserAgent)
.header(ORIGIN, MVLive.BASE_URL)
.header(REFERER, MVLive.BASE_URL)
.build();
LOG.debug("Loading graphql config {}", spaBundleUrl);
try (Response response = getHttpClient().execute(request)) {
if (response.isSuccessful()) {
String content = response.body().string();
Matcher m = graphQlUrlPattern.matcher(content);
if (m.find()) {
graphqlBaseUri = m.group(1);
} else {
throw new IllegalStateException("GraphQL URL not found");
}
} else {
throw new HttpException(response.code(), response.message());
}
}
}
private String getSpaBundleUrl() throws IOException {
Request request = new Request.Builder()
.url(getBaseUrl())
.header(USER_AGENT, getConfig().getSettings().httpUserAgent)
.header(REFERER, MVLive.BASE_URL)
.build();
try (Response response = getHttpClient().execute(request)) {
if (response.isSuccessful()) {
String content = response.body().string();
Matcher m = configPattern.matcher(content);
if (m.find()) {
JSONObject apps = new JSONObject(m.group(1)).getJSONObject("apps");
JSONObject mvlive = apps.getJSONObject("@manyvids/live");
String spaBundle = mvlive.getString("spaBundle");
return spaBundle;
} else {
throw new IllegalStateException("config not found");
}
} else {
throw new HttpException(response.code(), response.message());
}
}
}
public List<Model> getModels() throws IOException {
String body = new JSONObject()
.put("query", """
query LanderLiveSessions($nextToken: String, $limit: Int, $country: String, $subdivision: String, $audience: String) {
liveSessions(
nextToken: $nextToken
limit: $limit
country: $country
subdivision: $subdivision
audience: $audience
) {
presenters {
id
age
avatarSrc
city
country
name
previewImgSrc
starOfTheDay
status
sessionType
streamingStartDate
towerImgSrc
profileHandle
}
nextToken
}
}
""")
.put("variables", new JSONObject()
.put("audience", "public")
.put("country", Locale.getDefault().getDisplayCountry(Locale.ENGLISH))
.put("limit", 100)
.put("nextToken", JSONObject.NULL)
.put("subdivision", "")
).toString(2);
RequestBody requestBody = RequestBody.Companion.create(body, MediaType.parse("application/json"));
Request request = newRequestBuilder()
.post(requestBody)
.build();
try (Response response = getHttpClient().execute(request)) {
String content = response.body().string();
if (response.isSuccessful()) {
JSONObject responseBody = new JSONObject(content);
JSONArray presenters = responseBody.getJSONObject("data").getJSONObject("liveSessions").getJSONArray("presenters");
List<Model> models = new LinkedList<>();
for (int i = 0; i < presenters.length(); i++) {
JSONObject presenter = presenters.getJSONObject(i);
try {
MVLiveModel model = createModel(presenter.getString("name"));
model.setId(presenter.getString("id"));
model.setDisplayName(presenter.optString("name", model.getName()));
model.setPreview(presenter.optString("towerImgSrc"));
model.setOnlineState(mapState(presenter.getString("sessionType")));
models.add(model);
} catch (Exception e) {
LOG.error("Couldn't parse model: {}", presenter.toString(2), e);
}
}
return models;
} else {
LOG.debug("Response: {}", content);
throw new HttpException(response.code(), response.message());
}
}
}
private State mapState(String status) {
return switch (status) {
case "PUBLIC" -> State.ONLINE;
case "PRIVATE" -> State.PRIVATE;
default -> State.OFFLINE;
};
}
@Override
public boolean supportsSearch() {
return true;
}
@Override
public boolean searchRequiresLogin() {
return false;
}
String getMvToken() throws IOException {
if (mvtoken == null) {
Request request = new Request.Builder()
.url("https://www.manyvids.com/")
.url(BASE_URL)
.header(USER_AGENT, getConfig().getSettings().httpUserAgent)
.build();
try (Response response = getHttpClient().execute(request)) {
@ -356,17 +209,4 @@ public class MVLive extends AbstractSite {
return super.createModelFromUrl(url);
}
private Request.Builder newRequestBuilder() {
return new Request.Builder()
.url(getGraphQlUrl())
.header(ACCEPT, "*/*")
.header(ACCEPT_LANGUAGE, Locale.ENGLISH.getLanguage())
.header(USER_AGENT, getConfig().getSettings().httpUserAgent)
.header(ORIGIN, MVLive.BASE_URL)
.header(REFERER, MVLive.BASE_URL)
.header(AUTHORIZATION, "GUEST");
}
}

92
common/src/main/java/ctbrec/sites/manyvids/MVLiveModel.java
View File

@ -11,33 +11,28 @@ import ctbrec.io.HttpException;
import ctbrec.recorder.download.Download;
import ctbrec.recorder.download.StreamSource;
import ctbrec.sites.ModelOfflineException;
import lombok.extern.slf4j.Slf4j;
import okhttp3.Request;
import okhttp3.Response;
import org.json.JSONException;
import org.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.URLEncoder;
import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
import java.util.*;
import java.util.concurrent.ExecutionException;
import static ctbrec.Model.State.OFFLINE;
import static ctbrec.Model.State.ONLINE;
import static ctbrec.Model.State.*;
import static ctbrec.io.HttpConstants.*;
import static java.nio.charset.StandardCharsets.UTF_8;
@Slf4j
public class MVLiveModel extends AbstractModel {
private static final transient Logger LOG = LoggerFactory.getLogger(MVLiveModel.class);
private transient MVLiveHttpClient httpClient;
private transient MVLiveClient client;
private transient JSONObject roomLocation;
@ -48,31 +43,31 @@ public class MVLiveModel extends AbstractModel {
@Override
public boolean isOnline(boolean ignoreCache) throws IOException, ExecutionException, InterruptedException {
if (ignoreCache) {
boolean modelFound = false;
MVLive site = (MVLive) getSite();
for (Model model : site.getModels()) {
if (model.getName().equalsIgnoreCase(getName()) || model.getDisplayName().equalsIgnoreCase(getName())) {
this.onlineState = model.getOnlineState(true);
setName(model.getName());
setDisplayName(model.getDisplayName());
setUrl(model.getUrl());
modelFound = true;
break;
String url = "https://creator-api.vidchat.manyvids.com/creator?urlHandle=" + URLEncoder.encode(getName(), UTF_8);
Request req = new Request.Builder()
.url(url)
.header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
.build();
try (Response response = getHttpClient().execute(req)) {
if (response.isSuccessful()) {
String body = response.body().string();
JSONObject creator = new JSONObject(body);
updateStateFromJson(creator);
} else {
log.debug("{} URL: {}\n\tResponse: {}", response.code(), url, response.body().string());
throw new HttpException(response.code(), response.message());
}
}
if (!modelFound) {
this.onlineState = OFFLINE;
}
}
return this.onlineState == ONLINE;
}
@Override
public List<StreamSource> getStreamSources() throws IOException, ExecutionException, ParseException, PlaylistException {
LOG.debug("Loading {}", getUrl());
log.debug("Loading {}", getUrl());
try {
StreamLocation streamLocation = getClient().getStreamLocation(this);
LOG.debug("Got the stream location from WS {}", streamLocation.masterPlaylist);
log.debug("Got the stream location from WS {}", streamLocation.masterPlaylist);
roomNumber = streamLocation.roomNumber;
updateCloudFlareCookies();
MasterPlaylist masterPlaylist = getMasterPlaylist(streamLocation.masterPlaylist);
@ -89,7 +84,7 @@ public class MVLiveModel extends AbstractModel {
if (src.mediaPlaylistUrl.contains("?")) {
src.mediaPlaylistUrl = src.mediaPlaylistUrl.substring(0, src.mediaPlaylistUrl.lastIndexOf('?'));
}
LOG.debug("Media playlist {}", src.mediaPlaylistUrl);
log.debug("Media playlist {}", src.mediaPlaylistUrl);
sources.add(src);
}
}
@ -101,7 +96,7 @@ public class MVLiveModel extends AbstractModel {
}
private MasterPlaylist getMasterPlaylist(String url) throws IOException, ParseException, PlaylistException {
LOG.trace("Loading master playlist {}", url);
log.trace("Loading master playlist {}", url);
Request req = new Request.Builder()
.url(url)
.header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
@ -115,7 +110,7 @@ public class MVLiveModel extends AbstractModel {
MasterPlaylist master = playlist.getMasterPlaylist();
return master;
} else {
LOG.debug("{} URL: {}\n\tResponse: {}", response.code(), url, response.body().string());
log.debug("{} URL: {}\n\tResponse: {}", response.code(), url, response.body().string());
throw new HttpException(response.code(), response.message());
}
}
@ -123,14 +118,14 @@ public class MVLiveModel extends AbstractModel {
public void updateCloudFlareCookies() throws IOException {
String url = getApiUrl() + '/' + getRoomNumber() + "/player-settings/" + getDisplayName();
LOG.trace("Getting CF cookies: {}", url);
log.trace("Getting CF cookies: {}", url);
Request req = new Request.Builder()
.url(url)
.header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
.build();
try (Response response = getHttpClient().execute(req)) {
if (!response.isSuccessful()) {
LOG.debug("Loading CF cookies not successful: {}", response.body().string());
log.debug("Loading CF cookies not successful: {}", response.body().string());
throw new HttpException(response.code(), response.message());
}
}
@ -146,7 +141,7 @@ public class MVLiveModel extends AbstractModel {
if (json.optBoolean("success")) {
roomNumber = json.getString("floorId");
} else {
LOG.debug("Room number response: {}", json.toString(2));
log.debug("Room number response: {}", json.toString(2));
throw new ModelOfflineException(this);
}
}
@ -170,7 +165,7 @@ public class MVLiveModel extends AbstractModel {
fetchGeneralCookies();
httpClient.fetchAuthenticationCookies();
String url = "https://roompool.live.manyvids.com/roompool/" + getDisplayName() + "?private=false";
LOG.debug("Fetching room location from {}", url);
log.debug("Fetching room location from {}", url);
Request req = new Request.Builder()
.url(url)
.header(ACCEPT, MIMETYPE_APPLICATION_JSON)
@ -182,7 +177,7 @@ public class MVLiveModel extends AbstractModel {
if (response.isSuccessful()) {
String body = response.body().string();
roomLocation = new JSONObject(body);
LOG.trace("Room location response: {}", roomLocation);
log.debug("Room location response: {}", roomLocation);
lastRoomLocationUpdate = Instant.now();
return roomLocation;
} else {
@ -251,6 +246,7 @@ public class MVLiveModel extends AbstractModel {
@Override
public void readSiteSpecificData(JsonReader reader) throws IOException {
if (reader.hasNext()) {
//noinspection ResultOfMethodCallIgnored
reader.nextName();
id = reader.nextString();
}
@ -263,4 +259,34 @@ public class MVLiveModel extends AbstractModel {
public String getId() {
return id;
}
public void updateStateFromJson(JSONObject creator) {
setId(creator.getString("id"));
setDisplayName(creator.optString("display_name", null));
setUrl(creator.getString("session_url"));
setOnlineState(mapState(creator.optString("live_status"), creator.optString("session_type")));
setPreview(creator.optString("avatar", null));
}
protected Model.State mapState(String liveStatus, String sessionType) {
if (Objects.equals("ONLINE", liveStatus)) {
switch (sessionType) {
case "PUBLIC" -> {
return ONLINE;
}
case "PRIVATE" -> {
return PRIVATE;
}
case "OFFLINE" -> {
return OFFLINE;
}
default -> {
log.debug("Unknown state {}", sessionType);
return OFFLINE;
}
}
} else {
return OFFLINE;
}
}
}