diff --git a/client/src/main/java/ctbrec/ui/sites/manyvids/MVLiveUpdateService.java b/client/src/main/java/ctbrec/ui/sites/manyvids/MVLiveUpdateService.java index 02103906..b31299b0 100644 --- a/client/src/main/java/ctbrec/ui/sites/manyvids/MVLiveUpdateService.java +++ b/client/src/main/java/ctbrec/ui/sites/manyvids/MVLiveUpdateService.java @@ -1,16 +1,16 @@ package ctbrec.ui.sites.manyvids; -import java.io.IOException; -import java.util.List; - import ctbrec.Model; import ctbrec.sites.manyvids.MVLive; import ctbrec.ui.tabs.PaginatedScheduledService; import javafx.concurrent.Task; +import java.io.IOException; +import java.util.List; + public class MVLiveUpdateService extends PaginatedScheduledService { - private MVLive mvlive; + private final MVLive mvlive; public MVLiveUpdateService(MVLive mvlive) { this.mvlive = mvlive; diff --git a/client/src/main/resources/logback.xml b/client/src/main/resources/logback.xml index a9e9fb01..62404622 100644 --- a/client/src/main/resources/logback.xml +++ b/client/src/main/resources/logback.xml @@ -1,61 +1,60 @@ - - - - %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n - - - - - - - %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n - - + + + + %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n + + + - - ctbrec.log - true - - DEBUG - - - %date %level [%thread] %logger{10} [%file:%line] %msg%n - - - ctbrec.%i.log - 1 - 3 - - - 5MB - - + + + %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n + + + + + ctbrec.log + true + + DEBUG + + + %date %level [%thread] %logger{10} [%file:%line] %msg%n + + + ctbrec.%i.log + 1 + 3 + + + 5MB + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - diff --git a/common/src/main/java/ctbrec/sites/manyvids/MVLive.java b/common/src/main/java/ctbrec/sites/manyvids/MVLive.java index 0c6804a6..182f3723 100644 --- a/common/src/main/java/ctbrec/sites/manyvids/MVLive.java +++ b/common/src/main/java/ctbrec/sites/manyvids/MVLive.java @@ -1,40 +1,42 @@ package ctbrec.sites.manyvids; -import static ctbrec.io.HttpConstants.*; - -import java.io.IOException; -import java.io.UnsupportedEncodingException; -import java.net.URLDecoder; -import java.util.ArrayList; -import java.util.List; -import java.util.Locale; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import org.json.JSONArray; -import org.json.JSONObject; -import org.jsoup.nodes.Element; -import org.jsoup.select.Elements; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import ctbrec.Model; import ctbrec.Model.State; import ctbrec.io.HtmlParser; import ctbrec.io.HttpClient; import ctbrec.io.HttpException; import ctbrec.sites.AbstractSite; -import okhttp3.FormBody; -import okhttp3.Request; -import okhttp3.RequestBody; -import okhttp3.Response; +import okhttp3.*; +import org.json.JSONArray; +import org.json.JSONObject; +import org.jsoup.nodes.Element; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.net.URLDecoder; +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; +import java.util.Locale; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static ctbrec.io.HttpConstants.*; +import static java.nio.charset.StandardCharsets.UTF_8; public class MVLive extends AbstractSite { private static final Logger LOG = LoggerFactory.getLogger(MVLive.class); public static final String WS_ORIGIN = "https://live.manyvids.com"; - public static final String BASE_URL = "https://www.manyvids.com/MVLive/"; + public static final String BASE_URL = "https://www.manyvids.com"; + public static final String LIVE_URL = BASE_URL + "/mv-live/"; + private final Pattern configPattern = Pattern.compile(""); + private final Pattern graphQlUrlPattern = Pattern.compile("url:\\s*\"(https://.+?.appsync-api..+?.amazonaws.com/graphql)\","); + private final Pattern graphQlApiKeyPattern = Pattern.compile("use:\\s*\\[ce\\(\"(.*?)\"\\)].concat\\(ue\\(Object\\(se\\[\"a\"]\\)\\(\\)\\)\\)"); + private String graphqlBaseUri; + private String apiKey; private MVLiveHttpClient httpClient; private String mvtoken; @@ -46,7 +48,7 @@ public class MVLive extends AbstractSite { @Override public String getBaseUrl() { - return BASE_URL; + return LIVE_URL; } @Override @@ -60,6 +62,7 @@ public class MVLive extends AbstractSite { model.setName(name); model.setDescription(""); model.setSite(this); + model.setUrl(WS_ORIGIN + '/' + name); return model; } @@ -80,7 +83,7 @@ public class MVLive extends AbstractSite { @Override public HttpClient getHttpClient() { - if(httpClient == null) { + if (httpClient == null) { httpClient = new MVLiveHttpClient(getConfig()); } return httpClient; @@ -115,10 +118,51 @@ public class MVLive extends AbstractSite { @Override public void init() { - // nothing special to do for manyvids + // nothing to do } - public List getModels() throws IOException { + private String getGraphQlUrl() { + if (graphqlBaseUri == null) { + try { + String spaBundleUrl = getSpaBundleUrl(); + loadGraphqlApiConfig(spaBundleUrl); + LOG.debug("Using graphql API at {}", graphqlBaseUri); + } catch (IOException e) { + LOG.error("Error while initializing {}", getName(), e); + } + } + return graphqlBaseUri; + } + + private void loadGraphqlApiConfig(String spaBundleUrl) throws IOException { + Request request = new Request.Builder() + .url(spaBundleUrl) + .header(USER_AGENT, getConfig().getSettings().httpUserAgent) + .header(ORIGIN, MVLive.BASE_URL) + .header(REFERER, MVLive.BASE_URL) + .build(); + try (Response response = getHttpClient().execute(request)) { + if (response.isSuccessful()) { + String content = response.body().string(); + Matcher m = graphQlUrlPattern.matcher(content); + if (m.find()) { + graphqlBaseUri = m.group(1); + m = graphQlApiKeyPattern.matcher(content); + if (m.find()) { + apiKey = m.group(1); + } else { + throw new IllegalStateException("GraphQL API key not found"); + } + } else { + throw new IllegalStateException("GraphQL URL not found"); + } + } else { + throw new HttpException(response.code(), response.message()); + } + } + } + + private String getSpaBundleUrl() throws IOException { Request request = new Request.Builder() .url(getBaseUrl()) .header(USER_AGENT, getConfig().getSettings().httpUserAgent) @@ -127,48 +171,76 @@ public class MVLive extends AbstractSite { try (Response response = getHttpClient().execute(request)) { if (response.isSuccessful()) { String content = response.body().string(); - Elements cards = HtmlParser.getTags(content, "div[class*=-model]"); - return parseModelCards(cards); + Matcher m = configPattern.matcher(content); + if (m.find()) { + JSONObject apps = new JSONObject(m.group(1)).getJSONObject("apps"); + JSONObject mvlive = apps.getJSONObject("@manyvids/live"); + String spaBundle = mvlive.getString("spaBundle"); + return spaBundle; + } else { + throw new IllegalStateException("config not found"); + } } else { throw new HttpException(response.code(), response.message()); } } } - private List parseModelCards(Elements cards) { - List models = new ArrayList<>(); - for (Element card : cards) { - try { - String cardHtml = card.html(); - Element link = HtmlParser.getTag(cardHtml, "a"); - link.setBaseUri(getBaseUrl()); - String name = HtmlParser.getText(cardHtml, "h4 a"); - MVLiveModel model = createModel(name); - model.setUrl(link.absUrl("href")); - Element thumb = HtmlParser.getTag(cardHtml, "a img.b-lazy"); - thumb.setBaseUri(getBaseUrl()); - model.setPreview(thumb.absUrl("data-src")); + public List getModels() throws IOException { + String body = new JSONObject() + .put("query", "query LanderLiveSessions($nextToken: String, $limit: Int, $country: String, $subdivision: String, $audience: String) { liveSessions( nextToken: $nextToken limit: $limit country: $country subdivision: $subdivision audience: $audience ) { presenters { id age avatarSrc city country name previewImgSrc starOfTheDay status sessionType streamingStartDate towerImgSrc profileHandle } nextToken }}") + .put("variables", new JSONObject() + .put("audience", "public") + .put("country", Locale.getDefault().getDisplayCountry(Locale.ENGLISH)) + .put("limit", 100) + .put("nextToken", JSONObject.NULL) + .put("subdivision", "") + ).toString(2); + RequestBody requestBody = RequestBody.Companion.create(body, MediaType.parse("application/json")); - Element status = HtmlParser.getTag(cardHtml, "h4[class~=profile-pic-name]"); - String cssClass = status.attr("class"); - if(cssClass.contains("live")) { - model.setOnlineState(Model.State.ONLINE); - } else if(cssClass.contains("private")) { - model.setOnlineState(Model.State.PRIVATE); - } else { - LOG.debug("Unknown online state {}", cssClass); - model.setOnlineState(Model.State.UNKNOWN); - } - models.add(model); - } catch(RuntimeException e) { - if(e.getMessage().contains("No element selected by")) { - // ignore - } else { - throw e; + Request request = new Request.Builder() + .url(getGraphQlUrl()) + .header(ACCEPT, "*/*") + .header(ACCEPT_LANGUAGE, Locale.ENGLISH.getLanguage()) + .header(USER_AGENT, getConfig().getSettings().httpUserAgent) + .header(ORIGIN, MVLive.BASE_URL) + .header(REFERER, MVLive.BASE_URL) + .header("x-api-key", apiKey) + .post(requestBody) + .build(); + try (Response response = getHttpClient().execute(request)) { + String content = response.body().string(); + if (response.isSuccessful()) { + JSONObject responseBody = new JSONObject(content); + JSONArray presenters = responseBody.getJSONObject("data").getJSONObject("liveSessions").getJSONArray("presenters"); + List models = new LinkedList<>(); + for (int i = 0; i < presenters.length(); i++) { + JSONObject presenter = presenters.getJSONObject(i); + try { + MVLiveModel model = createModel(presenter.getString("name")); + model.setId(presenter.getString("id")); + model.setDisplayName(presenter.optString("name", model.getName())); + model.setPreview(presenter.optString("towerImgSrc")); + model.setOnlineState(mapState(presenter.getString("sessionType"))); + models.add(model); + } catch (Exception e) { + LOG.error("Couldn't parse model: {}", presenter.toString(2), e); + } } + return models; + } else { + LOG.debug("Response: {}", content); + throw new HttpException(response.code(), response.message()); } } - return models; + } + + private State mapState(String status) { + return switch (status) { + case "PUBLIC" -> State.ONLINE; + case "PRIVATE" -> State.PRIVATE; + default -> State.OFFLINE; + }; } @Override @@ -231,14 +303,14 @@ public class MVLive extends AbstractSite { private void parseSearchResult(List result, String responseBody) { JSONObject json = new JSONObject(responseBody); - if(json.has("stars")) { + if (json.has("stars")) { JSONArray stars = json.getJSONArray("stars"); for (int i = 0; i < stars.length(); i++) { JSONObject star = stars.getJSONObject(i); String name = star.getString("label"); MVLiveModel model = createModel(name); long id = star.getLong("id"); - String url = "https://www.manyvids.com/MVLive/" + model.getDisplayName() + '/' + id + '/'; + String url = BASE_URL + model.getDisplayName() + '/' + id + '/'; model.setUrl(url); model.setPreview(star.getString("img")); if (star.optString("is_live").equals("1")) { @@ -257,19 +329,15 @@ public class MVLive extends AbstractSite { @Override public Model createModelFromUrl(String url) { - try { - Matcher m = Pattern.compile("https://live.manyvids.com/(?:stream/)?(.*?)(?:/.*?)?").matcher(url.trim()); - if (m.matches()) { - String modelName = URLDecoder.decode(m.group(1), "utf-8"); - return createModel(modelName); - } - m = Pattern.compile("https://www.manyvids.com/MVLive/(.*?)/\\d+/?").matcher(url.trim()); - if (m.matches()) { - String modelName = URLDecoder.decode(m.group(1), "utf-8"); - return createModel(modelName); - } - } catch (UnsupportedEncodingException e) { - LOG.error("Couldn't decode model name from URL", e); + Matcher m = Pattern.compile("https://live.manyvids.com/(?:stream/)?(.*?)(?:/.*?)?").matcher(url.trim()); + if (m.matches()) { + String modelName = URLDecoder.decode(m.group(1), UTF_8); + return createModel(modelName); + } + m = Pattern.compile("https://www.manyvids.com/MVLive/(.*?)/\\d+/?").matcher(url.trim()); + if (m.matches()) { + String modelName = URLDecoder.decode(m.group(1), UTF_8); + return createModel(modelName); } return super.createModelFromUrl(url); diff --git a/common/src/main/java/ctbrec/sites/manyvids/MVLiveModel.java b/common/src/main/java/ctbrec/sites/manyvids/MVLiveModel.java index e40be69b..f24281ed 100644 --- a/common/src/main/java/ctbrec/sites/manyvids/MVLiveModel.java +++ b/common/src/main/java/ctbrec/sites/manyvids/MVLiveModel.java @@ -1,8 +1,22 @@ package ctbrec.sites.manyvids; -import static ctbrec.Model.State.*; -import static ctbrec.io.HttpConstants.*; -import static java.nio.charset.StandardCharsets.*; +import com.iheartradio.m3u8.*; +import com.iheartradio.m3u8.data.MasterPlaylist; +import com.iheartradio.m3u8.data.Playlist; +import com.iheartradio.m3u8.data.PlaylistData; +import com.squareup.moshi.JsonReader; +import com.squareup.moshi.JsonWriter; +import ctbrec.*; +import ctbrec.io.HttpException; +import ctbrec.recorder.download.Download; +import ctbrec.recorder.download.StreamSource; +import ctbrec.sites.ModelOfflineException; +import okhttp3.Request; +import okhttp3.Response; +import org.json.JSONException; +import org.json.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.ByteArrayInputStream; import java.io.IOException; @@ -15,32 +29,10 @@ import java.util.List; import java.util.Locale; import java.util.concurrent.ExecutionException; -import org.json.JSONException; -import org.json.JSONObject; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.iheartradio.m3u8.Encoding; -import com.iheartradio.m3u8.Format; -import com.iheartradio.m3u8.ParseException; -import com.iheartradio.m3u8.ParsingMode; -import com.iheartradio.m3u8.PlaylistException; -import com.iheartradio.m3u8.PlaylistParser; -import com.iheartradio.m3u8.data.MasterPlaylist; -import com.iheartradio.m3u8.data.Playlist; -import com.iheartradio.m3u8.data.PlaylistData; - -import ctbrec.AbstractModel; -import ctbrec.Config; -import ctbrec.Model; -import ctbrec.NotImplementedExcetion; -import ctbrec.StringUtil; -import ctbrec.io.HttpException; -import ctbrec.recorder.download.Download; -import ctbrec.recorder.download.StreamSource; -import ctbrec.sites.ModelOfflineException; -import okhttp3.Request; -import okhttp3.Response; +import static ctbrec.Model.State.OFFLINE; +import static ctbrec.Model.State.ONLINE; +import static ctbrec.io.HttpConstants.*; +import static java.nio.charset.StandardCharsets.UTF_8; public class MVLiveModel extends AbstractModel { @@ -51,7 +43,7 @@ public class MVLiveModel extends AbstractModel { private transient JSONObject roomLocation; private transient Instant lastRoomLocationUpdate = Instant.EPOCH; private String roomNumber; - + private String id; @Override public boolean isOnline(boolean ignoreCache) throws IOException, ExecutionException, InterruptedException { @@ -221,7 +213,7 @@ public class MVLiveModel extends AbstractModel { @Override public int[] getStreamResolution(boolean failFast) throws ExecutionException { - return new int[] {1280, 720}; + return new int[]{1280, 720}; } @Override @@ -250,4 +242,22 @@ public class MVLiveModel extends AbstractModel { } return httpClient; } + + @Override + public void writeSiteSpecificData(JsonWriter writer) throws IOException { + writer.name("id").value(id); + } + + @Override + public void readSiteSpecificData(JsonReader reader) throws IOException { + id = reader.nextString(); + } + + public void setId(String id) { + this.id = id; + } + + public String getId() { + return id; + } }