j62
/
ctbrec
2
2
Fork 2
Code Issues 3 Pull Requests Packages Projects Releases 3 Wiki Activity

Fix MVLive tab

This commit is contained in:
0xb00bface 2022-02-19 20:09:45 +01:00
parent 26bf338807
commit bba353f309
4 changed files with 242 additions and 165 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
client/src/main/java/ctbrec/ui/sites/manyvids/MVLiveUpdateService.java
View File

@ -1,16 +1,16 @@
package ctbrec.ui.sites.manyvids;
import java.io.IOException;
import java.util.List;
import ctbrec.Model;
import ctbrec.sites.manyvids.MVLive;
import ctbrec.ui.tabs.PaginatedScheduledService;
import javafx.concurrent.Task;
import java.io.IOException;
import java.util.List;
public class MVLiveUpdateService extends PaginatedScheduledService {
private MVLive mvlive;
private final MVLive mvlive;
public MVLiveUpdateService(MVLive mvlive) {
this.mvlive = mvlive;

111
client/src/main/resources/logback.xml
View File

@ -1,61 +1,60 @@
<configuration scan="true" scanPeriod="10 seconds">
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<!-- encoders are assigned the type ch.qos.logback.classic.encoder.PatternLayoutEncoder
by default -->
<encoder>
<pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n
</pattern>
</encoder>
</appender>
<appender name="GUI" class="ctbrec.ui.tabs.logging.CtbrecAppender">
<encoder>
<pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern>
</encoder>
</appender>
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<!-- encoders are assigned the type ch.qos.logback.classic.encoder.PatternLayoutEncoder
by default -->
<encoder>
<pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n
</pattern>
</encoder>
</appender>
<appender name="FILE"
class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>ctbrec.log</file>
<append>true</append>
<filter class="ch.qos.logback.classic.filter.ThresholdFilter">
<level>DEBUG</level>
</filter>
<encoder>
<pattern>%date %level [%thread] %logger{10} [%file:%line] %msg%n</pattern>
</encoder>
<rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
<fileNamePattern>ctbrec.%i.log</fileNamePattern>
<minIndex>1</minIndex>
<maxIndex>3</maxIndex>
</rollingPolicy>
<triggeringPolicy class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
<maxFileSize>5MB</maxFileSize>
</triggeringPolicy>
</appender>
<appender name="GUI" class="ctbrec.ui.tabs.logging.CtbrecAppender">
<encoder>
<pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern>
</encoder>
</appender>
<appender name="FILE"
class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>ctbrec.log</file>
<append>true</append>
<filter class="ch.qos.logback.classic.filter.ThresholdFilter">
<level>DEBUG</level>
</filter>
<encoder>
<pattern>%date %level [%thread] %logger{10} [%file:%line] %msg%n</pattern>
</encoder>
<rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
<fileNamePattern>ctbrec.%i.log</fileNamePattern>
<minIndex>1</minIndex>
<maxIndex>3</maxIndex>
</rollingPolicy>
<triggeringPolicy class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
<maxFileSize>5MB</maxFileSize>
</triggeringPolicy>
</appender>
<root level="DEBUG">
<appender-ref ref="STDOUT"/>
<appender-ref ref="FILE"/>
<appender-ref ref="GUI"/>
</root>
<logger name="ctbrec.LoggingInterceptor" level="INFO"/>
<logger name="ctbrec.io.CookieJarImpl" level="INFO"/>
<logger name="ctbrec.recorder.FFmpeg" level="DEBUG"/>
<logger name="ctbrec.recorder.OnlineMonitor" level="INFO"/>
<logger name="ctbrec.recorder.RecordingFileMonitor" level="TRACE"/>
<logger name="ctbrec.recorder.download.dash.DashDownload" level="DEBUG"/>
<logger name="ctbrec.recorder.server.HlsServlet" level="INFO"/>
<logger name="ctbrec.recorder.server.RecorderServlet" level="INFO"/>
<logger name="ctbrec.recorder.ThreadPoolScaler" level="DEBUG"/>
<!-- <logger name="ctbrec.sites.cam4.Cam4Model" level="DEBUG"/> -->
<!-- <logger name="ctbrec.sites.showup.Showup" level="TRACE"/> -->
<logger name="ctbrec.ui.ExternalBrowser" level="DEBUG"/>
<logger name="ctbrec.ui.ThumbOverviewTab" level="DEBUG"/>
<logger name="org.eclipse.jetty" level="INFO"/>
<root level="DEBUG">
<appender-ref ref="STDOUT" />
<appender-ref ref="FILE" />
<appender-ref ref="GUI" />
</root>
<logger name="ctbrec.LoggingInterceptor" level="INFO"/>
<logger name="ctbrec.io.CookieJarImpl" level="INFO"/>
<logger name="ctbrec.recorder.FFmpeg" level="DEBUG"/>
<logger name="ctbrec.recorder.OnlineMonitor" level="INFO"/>
<logger name="ctbrec.recorder.RecordingFileMonitor" level="TRACE"/>
<logger name="ctbrec.recorder.download.dash.DashDownload" level="DEBUG"/>
<logger name="ctbrec.recorder.server.HlsServlet" level="INFO"/>
<logger name="ctbrec.recorder.server.RecorderServlet" level="INFO"/>
<logger name="ctbrec.recorder.ThreadPoolScaler" level="DEBUG"/>
<!-- <logger name="ctbrec.sites.cam4.Cam4Model" level="DEBUG"/> -->
<!-- <logger name="ctbrec.sites.showup.Showup" level="TRACE"/> -->
<logger name="ctbrec.ui.ExternalBrowser" level="DEBUG"/>
<logger name="ctbrec.ui.ThumbOverviewTab" level="DEBUG"/>
<logger name="org.eclipse.jetty" level="INFO" />
</configuration>

216
common/src/main/java/ctbrec/sites/manyvids/MVLive.java
View File

@ -1,40 +1,42 @@
package ctbrec.sites.manyvids;
import static ctbrec.io.HttpConstants.*;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URLDecoder;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.json.JSONArray;
import org.json.JSONObject;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import ctbrec.Model;
import ctbrec.Model.State;
import ctbrec.io.HtmlParser;
import ctbrec.io.HttpClient;
import ctbrec.io.HttpException;
import ctbrec.sites.AbstractSite;
import okhttp3.FormBody;
import okhttp3.Request;
import okhttp3.RequestBody;
import okhttp3.Response;
import okhttp3.*;
import org.json.JSONArray;
import org.json.JSONObject;
import org.jsoup.nodes.Element;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.net.URLDecoder;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
import java.util.Locale;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import static ctbrec.io.HttpConstants.*;
import static java.nio.charset.StandardCharsets.UTF_8;
public class MVLive extends AbstractSite {
private static final Logger LOG = LoggerFactory.getLogger(MVLive.class);
public static final String WS_ORIGIN = "https://live.manyvids.com";
public static final String BASE_URL = "https://www.manyvids.com/MVLive/";
public static final String BASE_URL = "https://www.manyvids.com";
public static final String LIVE_URL = BASE_URL + "/mv-live/";
private final Pattern configPattern = Pattern.compile("<script type=\"combinativ-config\">(.*?)</script>");
private final Pattern graphQlUrlPattern = Pattern.compile("url:\\s*\"(https://.+?.appsync-api..+?.amazonaws.com/graphql)\",");
private final Pattern graphQlApiKeyPattern = Pattern.compile("use:\\s*\\[ce\\(\"(.*?)\"\\)].concat\\(ue\\(Object\\(se\\[\"a\"]\\)\\(\\)\\)\\)");
private String graphqlBaseUri;
private String apiKey;
private MVLiveHttpClient httpClient;
private String mvtoken;
@ -46,7 +48,7 @@ public class MVLive extends AbstractSite {
@Override
public String getBaseUrl() {
return BASE_URL;
return LIVE_URL;
}
@Override
@ -60,6 +62,7 @@ public class MVLive extends AbstractSite {
model.setName(name);
model.setDescription("");
model.setSite(this);
model.setUrl(WS_ORIGIN + '/' + name);
return model;
}
@ -80,7 +83,7 @@ public class MVLive extends AbstractSite {
@Override
public HttpClient getHttpClient() {
if(httpClient == null) {
if (httpClient == null) {
httpClient = new MVLiveHttpClient(getConfig());
}
return httpClient;
@ -115,10 +118,51 @@ public class MVLive extends AbstractSite {
@Override
public void init() {
// nothing special to do for manyvids
// nothing to do
}
public List<Model> getModels() throws IOException {
private String getGraphQlUrl() {
if (graphqlBaseUri == null) {
try {
String spaBundleUrl = getSpaBundleUrl();
loadGraphqlApiConfig(spaBundleUrl);
LOG.debug("Using graphql API at {}", graphqlBaseUri);
} catch (IOException e) {
LOG.error("Error while initializing {}", getName(), e);
}
}
return graphqlBaseUri;
}
private void loadGraphqlApiConfig(String spaBundleUrl) throws IOException {
Request request = new Request.Builder()
.url(spaBundleUrl)
.header(USER_AGENT, getConfig().getSettings().httpUserAgent)
.header(ORIGIN, MVLive.BASE_URL)
.header(REFERER, MVLive.BASE_URL)
.build();
try (Response response = getHttpClient().execute(request)) {
if (response.isSuccessful()) {
String content = response.body().string();
Matcher m = graphQlUrlPattern.matcher(content);
if (m.find()) {
graphqlBaseUri = m.group(1);
m = graphQlApiKeyPattern.matcher(content);
if (m.find()) {
apiKey = m.group(1);
} else {
throw new IllegalStateException("GraphQL API key not found");
}
} else {
throw new IllegalStateException("GraphQL URL not found");
}
} else {
throw new HttpException(response.code(), response.message());
}
}
}
private String getSpaBundleUrl() throws IOException {
Request request = new Request.Builder()
.url(getBaseUrl())
.header(USER_AGENT, getConfig().getSettings().httpUserAgent)
@ -127,48 +171,76 @@ public class MVLive extends AbstractSite {
try (Response response = getHttpClient().execute(request)) {
if (response.isSuccessful()) {
String content = response.body().string();
Elements cards = HtmlParser.getTags(content, "div[class*=-model]");
return parseModelCards(cards);
Matcher m = configPattern.matcher(content);
if (m.find()) {
JSONObject apps = new JSONObject(m.group(1)).getJSONObject("apps");
JSONObject mvlive = apps.getJSONObject("@manyvids/live");
String spaBundle = mvlive.getString("spaBundle");
return spaBundle;
} else {
throw new IllegalStateException("config not found");
}
} else {
throw new HttpException(response.code(), response.message());
}
}
}
private List<Model> parseModelCards(Elements cards) {
List<Model> models = new ArrayList<>();
for (Element card : cards) {
try {
String cardHtml = card.html();
Element link = HtmlParser.getTag(cardHtml, "a");
link.setBaseUri(getBaseUrl());
String name = HtmlParser.getText(cardHtml, "h4 a");
MVLiveModel model = createModel(name);
model.setUrl(link.absUrl("href"));
Element thumb = HtmlParser.getTag(cardHtml, "a img.b-lazy");
thumb.setBaseUri(getBaseUrl());
model.setPreview(thumb.absUrl("data-src"));
public List<Model> getModels() throws IOException {
String body = new JSONObject()
.put("query", "query LanderLiveSessions($nextToken: String, $limit: Int, $country: String, $subdivision: String, $audience: String) { liveSessions( nextToken: $nextToken limit: $limit country: $country subdivision: $subdivision audience: $audience ) { presenters { id age avatarSrc city country name previewImgSrc starOfTheDay status sessionType streamingStartDate towerImgSrc profileHandle } nextToken }}")
.put("variables", new JSONObject()
.put("audience", "public")
.put("country", Locale.getDefault().getDisplayCountry(Locale.ENGLISH))
.put("limit", 100)
.put("nextToken", JSONObject.NULL)
.put("subdivision", "")
).toString(2);
RequestBody requestBody = RequestBody.Companion.create(body, MediaType.parse("application/json"));
Element status = HtmlParser.getTag(cardHtml, "h4[class~=profile-pic-name]");
String cssClass = status.attr("class");
if(cssClass.contains("live")) {
model.setOnlineState(Model.State.ONLINE);
} else if(cssClass.contains("private")) {
model.setOnlineState(Model.State.PRIVATE);
} else {
LOG.debug("Unknown online state {}", cssClass);
model.setOnlineState(Model.State.UNKNOWN);
}
models.add(model);
} catch(RuntimeException e) {
if(e.getMessage().contains("No element selected by")) {
// ignore
} else {
throw e;
Request request = new Request.Builder()
.url(getGraphQlUrl())
.header(ACCEPT, "*/*")
.header(ACCEPT_LANGUAGE, Locale.ENGLISH.getLanguage())
.header(USER_AGENT, getConfig().getSettings().httpUserAgent)
.header(ORIGIN, MVLive.BASE_URL)
.header(REFERER, MVLive.BASE_URL)
.header("x-api-key", apiKey)
.post(requestBody)
.build();
try (Response response = getHttpClient().execute(request)) {
String content = response.body().string();
if (response.isSuccessful()) {
JSONObject responseBody = new JSONObject(content);
JSONArray presenters = responseBody.getJSONObject("data").getJSONObject("liveSessions").getJSONArray("presenters");
List<Model> models = new LinkedList<>();
for (int i = 0; i < presenters.length(); i++) {
JSONObject presenter = presenters.getJSONObject(i);
try {
MVLiveModel model = createModel(presenter.getString("name"));
model.setId(presenter.getString("id"));
model.setDisplayName(presenter.optString("name", model.getName()));
model.setPreview(presenter.optString("towerImgSrc"));
model.setOnlineState(mapState(presenter.getString("sessionType")));
models.add(model);
} catch (Exception e) {
LOG.error("Couldn't parse model: {}", presenter.toString(2), e);
}
}
return models;
} else {
LOG.debug("Response: {}", content);
throw new HttpException(response.code(), response.message());
}
}
return models;
}
private State mapState(String status) {
return switch (status) {
case "PUBLIC" -> State.ONLINE;
case "PRIVATE" -> State.PRIVATE;
default -> State.OFFLINE;
};
}
@Override
@ -231,14 +303,14 @@ public class MVLive extends AbstractSite {
private void parseSearchResult(List<Model> result, String responseBody) {
JSONObject json = new JSONObject(responseBody);
if(json.has("stars")) {
if (json.has("stars")) {
JSONArray stars = json.getJSONArray("stars");
for (int i = 0; i < stars.length(); i++) {
JSONObject star = stars.getJSONObject(i);
String name = star.getString("label");
MVLiveModel model = createModel(name);
long id = star.getLong("id");
String url = "https://www.manyvids.com/MVLive/" + model.getDisplayName() + '/' + id + '/';
String url = BASE_URL + model.getDisplayName() + '/' + id + '/';
model.setUrl(url);
model.setPreview(star.getString("img"));
if (star.optString("is_live").equals("1")) {
@ -257,19 +329,15 @@ public class MVLive extends AbstractSite {
@Override
public Model createModelFromUrl(String url) {
try {
Matcher m = Pattern.compile("https://live.manyvids.com/(?:stream/)?(.*?)(?:/.*?)?").matcher(url.trim());
if (m.matches()) {
String modelName = URLDecoder.decode(m.group(1), "utf-8");
return createModel(modelName);
}
m = Pattern.compile("https://www.manyvids.com/MVLive/(.*?)/\\d+/?").matcher(url.trim());
if (m.matches()) {
String modelName = URLDecoder.decode(m.group(1), "utf-8");
return createModel(modelName);
}
} catch (UnsupportedEncodingException e) {
LOG.error("Couldn't decode model name from URL", e);
Matcher m = Pattern.compile("https://live.manyvids.com/(?:stream/)?(.*?)(?:/.*?)?").matcher(url.trim());
if (m.matches()) {
String modelName = URLDecoder.decode(m.group(1), UTF_8);
return createModel(modelName);
}
m = Pattern.compile("https://www.manyvids.com/MVLive/(.*?)/\\d+/?").matcher(url.trim());
if (m.matches()) {
String modelName = URLDecoder.decode(m.group(1), UTF_8);
return createModel(modelName);
}
return super.createModelFromUrl(url);

72
common/src/main/java/ctbrec/sites/manyvids/MVLiveModel.java
View File

@ -1,8 +1,22 @@
package ctbrec.sites.manyvids;
import static ctbrec.Model.State.*;
import static ctbrec.io.HttpConstants.*;
import static java.nio.charset.StandardCharsets.*;
import com.iheartradio.m3u8.*;
import com.iheartradio.m3u8.data.MasterPlaylist;
import com.iheartradio.m3u8.data.Playlist;
import com.iheartradio.m3u8.data.PlaylistData;
import com.squareup.moshi.JsonReader;
import com.squareup.moshi.JsonWriter;
import ctbrec.*;
import ctbrec.io.HttpException;
import ctbrec.recorder.download.Download;
import ctbrec.recorder.download.StreamSource;
import ctbrec.sites.ModelOfflineException;
import okhttp3.Request;
import okhttp3.Response;
import org.json.JSONException;
import org.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.ByteArrayInputStream;
import java.io.IOException;
@ -15,32 +29,10 @@ import java.util.List;
import java.util.Locale;
import java.util.concurrent.ExecutionException;
import org.json.JSONException;
import org.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.iheartradio.m3u8.Encoding;
import com.iheartradio.m3u8.Format;
import com.iheartradio.m3u8.ParseException;
import com.iheartradio.m3u8.ParsingMode;
import com.iheartradio.m3u8.PlaylistException;
import com.iheartradio.m3u8.PlaylistParser;
import com.iheartradio.m3u8.data.MasterPlaylist;
import com.iheartradio.m3u8.data.Playlist;
import com.iheartradio.m3u8.data.PlaylistData;
import ctbrec.AbstractModel;
import ctbrec.Config;
import ctbrec.Model;
import ctbrec.NotImplementedExcetion;
import ctbrec.StringUtil;
import ctbrec.io.HttpException;
import ctbrec.recorder.download.Download;
import ctbrec.recorder.download.StreamSource;
import ctbrec.sites.ModelOfflineException;
import okhttp3.Request;
import okhttp3.Response;
import static ctbrec.Model.State.OFFLINE;
import static ctbrec.Model.State.ONLINE;
import static ctbrec.io.HttpConstants.*;
import static java.nio.charset.StandardCharsets.UTF_8;
public class MVLiveModel extends AbstractModel {
@ -51,7 +43,7 @@ public class MVLiveModel extends AbstractModel {
private transient JSONObject roomLocation;
private transient Instant lastRoomLocationUpdate = Instant.EPOCH;
private String roomNumber;
private String id;
@Override
public boolean isOnline(boolean ignoreCache) throws IOException, ExecutionException, InterruptedException {
@ -221,7 +213,7 @@ public class MVLiveModel extends AbstractModel {
@Override
public int[] getStreamResolution(boolean failFast) throws ExecutionException {
return new int[] {1280, 720};
return new int[]{1280, 720};
}
@Override
@ -250,4 +242,22 @@ public class MVLiveModel extends AbstractModel {
}
return httpClient;
}
@Override
public void writeSiteSpecificData(JsonWriter writer) throws IOException {
writer.name("id").value(id);
}
@Override
public void readSiteSpecificData(JsonReader reader) throws IOException {
id = reader.nextString();
}
public void setId(String id) {
this.id = id;
}
public String getId() {
return id;
}
}