davebluke
/
ctbrec
forked from j62/ctbrec
1
0
Fork 0
Code Pull Requests Activity

Implement proper online check and add search

This commit is contained in:
0xb00bface 2020-08-18 19:58:48 +02:00
parent 00f6a0fa0a
commit 096ed7a734
5 changed files with 200 additions and 98 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
client/src/main/java/ctbrec/ui/sites/manyvids/MVLiveTabProvider.java
View File

@ -21,12 +21,12 @@ public class MVLiveTabProvider extends TabProvider {
@Override
public List<Tab> getTabs(Scene scene) {
List<Tab> tabs = new ArrayList<>();
tabs.add(createTab("Online", mvlive.getBaseUrl()));
tabs.add(createTab("Online"));
return tabs;
}
private Tab createTab(String title, String url) {
MVLiveUpdateService updateService = new MVLiveUpdateService(url, mvlive);
private Tab createTab(String title) {
MVLiveUpdateService updateService = new MVLiveUpdateService(mvlive);
ThumbOverviewTab tab = new ThumbOverviewTab(title, updateService, mvlive);
tab.setRecorder(mvlive.getRecorder());
return tab;

63
client/src/main/java/ctbrec/ui/sites/manyvids/MVLiveUpdateService.java
View File

@ -1,33 +1,18 @@
package ctbrec.ui.sites.manyvids;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import ctbrec.Config;
import ctbrec.Model;
import ctbrec.io.HtmlParser;
import ctbrec.io.HttpException;
import ctbrec.sites.manyvids.MVLive;
import ctbrec.sites.manyvids.MVLiveModel;
import ctbrec.ui.tabs.PaginatedScheduledService;
import javafx.concurrent.Task;
import okhttp3.Request;
import okhttp3.Response;
public class MVLiveUpdateService extends PaginatedScheduledService {
private static final Logger LOG = LoggerFactory.getLogger(MVLiveUpdateService.class);
private String url;
private MVLive mvlive;
public MVLiveUpdateService(String url, MVLive mvlive) {
this.url = url;
public MVLiveUpdateService(MVLive mvlive) {
this.mvlive = mvlive;
}
@ -36,51 +21,7 @@ public class MVLiveUpdateService extends PaginatedScheduledService {
return new Task<List<Model>>() {
@Override
public List<Model> call() throws IOException {
LOG.debug("Fetching page {}", url);
Request request = new Request.Builder()
.url(url)
.header("User-Agent", Config.getInstance().getSettings().httpUserAgent)
.header("Referer", MVLive.BASE_URL)
.build();
try (Response response = mvlive.getHttpClient().execute(request)) {
if (response.isSuccessful()) {
List<Model> models = new ArrayList<>();
String content = response.body().string();
Elements cards = HtmlParser.getTags(content, "div[class~=mv-live-model]");
for (Element card : cards) {
try {
String cardHtml = card.html();
Element link = HtmlParser.getTag(cardHtml, "a");
link.setBaseUri(mvlive.getBaseUrl());
String name = HtmlParser.getText(cardHtml, "h4 a");
MVLiveModel model = (MVLiveModel) mvlive.createModel(name);
model.setUrl(link.absUrl("href"));
Element thumb = HtmlParser.getTag(cardHtml, "a img.b-lazy");
thumb.setBaseUri(mvlive.getBaseUrl());
model.setPreview(thumb.absUrl("data-src"));
// Element status = HtmlParser.getTag(cardHtml, "div[class~=model-status]");
// String cssClass = status.attr("class");
// if(cssClass.contains("live")) {
// model.setOnlineState(Model.State.ONLINE);
// } else if(cssClass.contains("private")) {
// model.setOnlineState(Model.State.PRIVATE);
// } else {
// model.setOnlineState(Model.State.UNKNOWN);
// }
models.add(model);
} catch(RuntimeException e) {
if(e.getMessage().contains("No element selected by")) {
// ignore
} else {
throw e;
}
}
}
return models;
} else {
throw new HttpException(response.code(), response.message());
}
}
return mvlive.getModels();
}
};
}

152
common/src/main/java/ctbrec/sites/manyvids/MVLive.java
View File

@ -1,10 +1,28 @@
package ctbrec.sites.manyvids;
import java.io.IOException;
import static ctbrec.io.HttpConstants.*;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
import org.json.JSONArray;
import org.json.JSONObject;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import ctbrec.Config;
import ctbrec.Model;
import ctbrec.Model.State;
import ctbrec.io.HtmlParser;
import ctbrec.io.HttpClient;
import ctbrec.io.HttpException;
import ctbrec.sites.AbstractSite;
import okhttp3.FormBody;
import okhttp3.Request;
import okhttp3.RequestBody;
import okhttp3.Response;
public class MVLive extends AbstractSite {
@ -14,6 +32,7 @@ public class MVLive extends AbstractSite {
public static final String BASE_URL = "https://www.manyvids.com/MVLive/";
private MVLiveHttpClient httpClient;
private String mvtoken;
@Override
public String getName() {
@ -31,7 +50,7 @@ public class MVLive extends AbstractSite {
}
@Override
public Model createModel(String name) {
public MVLiveModel createModel(String name) {
MVLiveModel model = new MVLiveModel();
model.setName(name);
model.setDescription("");
@ -91,7 +110,134 @@ public class MVLive extends AbstractSite {
@Override
public void init() throws IOException {
}
public List<Model> getModels() throws IOException {
Request request = new Request.Builder()
.url(getBaseUrl())
.header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
.header(REFERER, MVLive.BASE_URL)
.build();
try (Response response = getHttpClient().execute(request)) {
if (response.isSuccessful()) {
List<Model> models = new ArrayList<>();
String content = response.body().string();
Elements cards = HtmlParser.getTags(content, "div[class*=-model]");
for (Element card : cards) {
try {
String cardHtml = card.html();
Element link = HtmlParser.getTag(cardHtml, "a");
link.setBaseUri(getBaseUrl());
String name = HtmlParser.getText(cardHtml, "h4 a");
MVLiveModel model = createModel(name);
model.setUrl(link.absUrl("href"));
Element thumb = HtmlParser.getTag(cardHtml, "a img.b-lazy");
thumb.setBaseUri(getBaseUrl());
model.setPreview(thumb.absUrl("data-src"));
Element status = HtmlParser.getTag(cardHtml, "h4[class~=profile-pic-name]");
String cssClass = status.attr("class");
if(cssClass.contains("live")) {
model.setOnlineState(Model.State.ONLINE);
} else if(cssClass.contains("private")) {
model.setOnlineState(Model.State.PRIVATE);
} else {
model.setOnlineState(Model.State.UNKNOWN);
}
models.add(model);
} catch(RuntimeException e) {
if(e.getMessage().contains("No element selected by")) {
// ignore
} else {
throw e;
}
}
}
return models;
} else {
throw new HttpException(response.code(), response.message());
}
}
}
@Override
public boolean supportsSearch() {
return true;
}
@Override
public boolean searchRequiresLogin() {
return false;
}
String getMvToken() throws IOException {
if (mvtoken == null) {
Request request = new Request.Builder()
.url(getBaseUrl())
.header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
.header(REFERER, MVLive.BASE_URL)
.build();
try (Response response = getHttpClient().execute(request)) {
if (response.isSuccessful()) {
Element tag = HtmlParser.getTag(response.body().string(), "html");
mvtoken = tag.attr("data-mvtoken");
} else {
throw new HttpException(response.code(), response.message());
}
}
}
return mvtoken;
}
@Override
public List<Model> search(String q) throws IOException, InterruptedException {
List<Model> result = new ArrayList<>();
RequestBody body = new FormBody.Builder()
.add("mvtoken", getMvToken())
.add("type", "search")
.add("category", "stars")
.add("search", q)
.build();
Request request = new Request.Builder()
.url("https://www.manyvids.com/includes/filterSearch.php")
.header(ACCEPT, MIMETYPE_APPLICATION_JSON)
.header(ACCEPT_LANGUAGE, Locale.ENGLISH.getLanguage())
.header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
.header(ORIGIN, MVLive.BASE_URL)
.header(REFERER, MVLive.BASE_URL)
.header(X_REQUESTED_WITH, XML_HTTP_REQUEST)
.post(body)
.build();
try (Response response = getHttpClient().execute(request)) {
if (response.isSuccessful()) {
String responseBody = response.body().string();
JSONObject json = new JSONObject(responseBody);
if(json.has("stars")) {
JSONArray stars = json.getJSONArray("stars");
for (int i = 0; i < stars.length(); i++) {
JSONObject star = stars.getJSONObject(i);
String name = star.getString("label");
MVLiveModel model = createModel(name);
long id = star.getLong("id");
String url = "https://www.manyvids.com/MVLive/" + model.getDisplayName() + '/' + id + '/';
model.setUrl(url);
model.setPreview(star.getString("img"));
if (star.optString("is_live").equals("1")) {
if (star.optString("is_private").equals("1")) {
model.setOnlineState(State.PRIVATE);
} else {
model.setOnlineState(State.ONLINE);
}
} else {
model.setOnlineState(State.OFFLINE);
}
result.add(model);
}
}
} else {
throw new HttpException(response.code(), response.message());
}
}
return result;
}
}

34
common/src/main/java/ctbrec/sites/manyvids/MVLiveClient.java
View File

@ -48,7 +48,7 @@ public class MVLiveClient {
private String roomNumber;
private String roomId;
private ScheduledExecutorService scheduler;
private Map<String, Message> futureResponses = new HashMap<>();
private MVLiveHttpClient httpClient;
public MVLiveClient(MVLiveHttpClient httpClient) {
@ -68,13 +68,13 @@ public class MVLiveClient {
String wsUrl = String.format("%s/api/%s/eventbus/%s/%s/websocket", WS_URL, roomNumber, randomNumber, randomString);
LOG.info("Websocket is null. Starting a new connection to {}", wsUrl);
ws = createWebSocket(wsUrl, roomId, model.getName());
ws = createWebSocket(wsUrl, roomId, model.getDisplayName());
}
}
private JSONObject getRoomLocation(MVLiveModel model) throws IOException {
Request req = new Request.Builder()
.url(WS_ORIGIN + "/api/roomlocation/" + model.getName() + "?private=false")
.url(WS_ORIGIN + "/api/roomlocation/" + model.getDisplayName() + "?private=false")
.header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
.header(ACCEPT, MIMETYPE_APPLICATION_JSON)
.header(COOKIE, getPhpSessionIdCookie())
@ -170,7 +170,7 @@ public class MVLiveClient {
LOG.debug("<-- {}", rr);
masterPlaylist = rr.getJSONObject("body").optString("videoUrl");
LOG.debug("Got the URL: {}", masterPlaylist);
//stop();
stop();
synchronized (streamUrlMonitor) {
streamUrlMonitor.notifyAll();
}
@ -195,9 +195,16 @@ public class MVLiveClient {
}
}
Map<String, Message> futureResponses = new HashMap<>();
@Override
public void onMessage(WebSocket webSocket, ByteString bytes) {
super.onMessage(webSocket, bytes);
LOG.debug("Binary Message: {}", bytes.hex());
}
});
return websocket;
}
private void sendMessages(Message... messages) {
void sendMessages(Message... messages) {
JSONArray msgs = new JSONArray();
for (Message msg : messages) {
msgs.put(msg.toString());
@ -212,15 +219,6 @@ public class MVLiveClient {
ws.send(msgs.toString());
}
@Override
public void onMessage(WebSocket webSocket, ByteString bytes) {
super.onMessage(webSocket, bytes);
LOG.debug("Binary Message: {}", bytes.hex());
}
});
return websocket;
}
public StreamLocation getStreamLocation(MVLiveModel model) throws IOException, InterruptedException {
start(model);
while (running) {
@ -229,9 +227,9 @@ public class MVLiveClient {
break;
}
}
// if (ws != null) {
// stop();
// }
if (ws != null) {
stop();
}
return new StreamLocation(roomId, roomNumber, masterPlaylist);
}
}

31
common/src/main/java/ctbrec/sites/manyvids/MVLiveModel.java
View File

@ -1,5 +1,6 @@
package ctbrec.sites.manyvids;
import static ctbrec.Model.State.*;
import static ctbrec.io.HttpConstants.*;
import static java.nio.charset.StandardCharsets.*;
@ -28,6 +29,8 @@ import com.iheartradio.m3u8.data.PlaylistData;
import ctbrec.AbstractModel;
import ctbrec.Config;
import ctbrec.Model;
import ctbrec.StringUtil;
import ctbrec.io.HttpException;
import ctbrec.recorder.download.Download;
import ctbrec.recorder.download.StreamSource;
@ -45,8 +48,19 @@ public class MVLiveModel extends AbstractModel {
@Override
public boolean isOnline(boolean ignoreCache) throws IOException, ExecutionException, InterruptedException {
JSONObject json = getRoomLocation();
return json.optBoolean("success");
if (ignoreCache) {
MVLive site = (MVLive) getSite();
for (Model model : site.getModels()) {
if (model.getName().equalsIgnoreCase(getName()) || model.getDisplayName().equalsIgnoreCase(getName())) {
this.onlineState = model.getOnlineState(true);
setName(model.getName());
setDisplayName(model.getDisplayName());
setUrl(model.getUrl());
break;
}
}
}
return this.onlineState == ONLINE;
}
@Override
@ -91,7 +105,6 @@ public class MVLiveModel extends AbstractModel {
try (Response response = getHttpClient().execute(req)) {
if (response.isSuccessful()) {
String body = response.body().string();
LOG.debug(body);
InputStream inputStream = new ByteArrayInputStream(body.getBytes(UTF_8));
PlaylistParser parser = new PlaylistParser(inputStream, Format.EXT_M3U, Encoding.UTF_8, ParsingMode.LENIENT);
Playlist playlist = parser.parse();
@ -104,7 +117,7 @@ public class MVLiveModel extends AbstractModel {
}
public void updateCloudFlareCookies() throws IOException, InterruptedException {
String url = MVLive.WS_ORIGIN + "/api/" + getRoomNumber() + "/player-settings/" + getName();
String url = MVLive.WS_ORIGIN + "/api/" + getRoomNumber() + "/player-settings/" + getDisplayName();
LOG.trace("Getting CF cookies: {}", url);
Request req = new Request.Builder()
.url(url)
@ -112,17 +125,19 @@ public class MVLiveModel extends AbstractModel {
.build();
try (Response response = getHttpClient().execute(req)) {
if (!response.isSuccessful()) {
LOG.debug(response.body().string());
throw new HttpException(response.code(), response.message());
}
}
}
public String getRoomNumber() throws IOException, InterruptedException {
if (roomNumber == null) {
if (StringUtil.isBlank(roomNumber)) {
JSONObject json = getRoomLocation();
if (json.optBoolean("success")) {
roomNumber = json.getString("floorId");
} else {
LOG.debug("Room number response: {}", json.toString(2));
throw new RuntimeException(getName() + " is offline");
}
}
@ -144,7 +159,8 @@ public class MVLiveModel extends AbstractModel {
public JSONObject getRoomLocation() throws IOException {
fetchGeneralCookies();
httpClient.fetchAuthenticationCookies();
String url = MVLive.WS_ORIGIN + "/api/roomlocation/" + getName() + "?private=false";
String url = MVLive.WS_ORIGIN + "/api/roomlocation/" + getDisplayName() + "?private=false";
LOG.trace("Fetching room location from {}", url);
Request req = new Request.Builder()
.url(url)
.header(ACCEPT, MIMETYPE_APPLICATION_JSON)
@ -155,6 +171,7 @@ public class MVLiveModel extends AbstractModel {
try (Response response = getHttpClient().execute(req)) {
if (response.isSuccessful()) {
JSONObject json = new JSONObject(response.body().string());
LOG.trace("Room location response: {}", json.toString(2));
return json;
} else {
throw new HttpException(response.code(), response.message());
@ -180,7 +197,7 @@ public class MVLiveModel extends AbstractModel {
@Override
public int[] getStreamResolution(boolean failFast) throws ExecutionException {
return new int[2];
return new int[] {1280, 720};
}
@Override