davebluke
/
ctbrec
forked from j62/ctbrec
1
0
Fork 0
Code Pull Requests Activity

Use JSON again to parse the models

This commit is contained in:
0xboobface 2019-12-21 21:04:49 +01:00
parent 17cc4a63ad
commit fde24e80ea
1 changed files with 63 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

109
client/src/main/java/ctbrec/ui/sites/flirt4free/Flirt4FreeUpdateService.java
View File

@ -7,8 +7,9 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import org.json.JSONArray;
import org.json.JSONObject;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -25,7 +26,7 @@ import okhttp3.Response;
public class Flirt4FreeUpdateService extends PaginatedScheduledService {
private static final transient Logger LOG = LoggerFactory.getLogger(Flirt4FreeUpdateService.class);
private static final Logger LOG = LoggerFactory.getLogger(Flirt4FreeUpdateService.class);
private static final int MODELS_PER_PAGE = 40;
private String url;
private Flirt4Free flirt4Free;
@ -41,52 +42,10 @@ public class Flirt4FreeUpdateService extends PaginatedScheduledService {
@Override
public List<Model> call() throws IOException {
LOG.debug("Fetching page {}", url);
Request request = new Request.Builder()
.url(url)
.addHeader("User-Agent", Config.getInstance().getSettings().httpUserAgent)
.build();
Request request = new Request.Builder().url(url).addHeader("User-Agent", Config.getInstance().getSettings().httpUserAgent).build();
try (Response response = flirt4Free.getHttpClient().execute(request)) {
if (response.isSuccessful()) {
List<Model> models = new ArrayList<>();
String body = response.body().string();
Elements tags = HtmlParser.getTags(body, "div#live_models div[class*=modelNumber]");
for (Element tag : tags) {
tag.setBaseUri(url);
String modelHtml = tag.html();
Element modelLink = HtmlParser.getTag(modelHtml, "a[class*=modelLink]");
modelLink.setBaseUri(url);
String href = modelLink.attr("href");
String name = href.substring(0, href.length()-1);
name = name.substring(name.indexOf('/', 1) + 1);
Flirt4FreeModel model = (Flirt4FreeModel) flirt4Free.createModel(name);
Element img = HtmlParser.getTag(modelHtml, "a[class*=modelLink] img");
img.setBaseUri(url);
if(img.hasAttr("data-image-url")) {
model.setPreview(img.absUrl("data-image-url"));
} else {
// background-image: url('https://cdn1.vscdns.com/images/models/samples-640x480/3241715.jpg')
Matcher m = Pattern.compile("background-image: url\\('(.*?)'\\)").matcher(img.attr("style"));
if(m.find()) {
model.setPreview(m.group(1));
}
}
Element link = HtmlParser.getTag(modelHtml, "a.name");
model.setDisplayName(link.attr("title"));
model.setUrl(modelLink.absUrl("href"));
model.setDescription("");
String videoHost = modelLink.attr("data-video-host");
String modelId = modelLink.attr("data-model-id").substring(5);
model.setId(modelId);
String streamUrl = "https://manifest.vscdns.com/manifest.m3u8.m3u8?key=nil&provider=level3&secure=true&host=" + videoHost + "&model_id=" + modelId;
model.setStreamUrl(streamUrl);
model.setOnlineState(ctbrec.Model.State.ONLINE);
model.setOnline(true);
models.add(model);
}
return models.stream()
.skip((page-1) * MODELS_PER_PAGE)
.limit(MODELS_PER_PAGE)
.collect(Collectors.toList());
return parseResponse(response);
} else {
throw new HttpException(response.code(), response.message());
}
@ -94,4 +53,62 @@ public class Flirt4FreeUpdateService extends PaginatedScheduledService {
}
};
}
private List<Model> parseResponse(Response response) throws IOException {
List<Model> models = new ArrayList<>();
String body = response.body().string();
Matcher m = Pattern.compile("window\\.homepageData = (\\{.*\\})", Pattern.DOTALL).matcher(body);
if (m.find()) {
JSONObject data = new JSONObject(m.group(1));
JSONArray modelData = data.getJSONArray("liveModels");
LOG.debug("Found {} models", modelData.length());
for (int i = 0; i < modelData.length(); i++) {
Model model = parseModel(modelData.getString(i));
if (model != null) {
models.add(model);
}
}
return models.stream()
.skip((page - 1) * MODELS_PER_PAGE)
.limit(MODELS_PER_PAGE)
.collect(Collectors.toList());
} else {
throw new IOException("Pattern didn't match model JSON data");
}
}
private Model parseModel(String modelHtml) {
if (modelHtml.trim().isEmpty() || !modelHtml.contains("modelLink")) {
return null;
}
Element modelDiv = HtmlParser.getTag(modelHtml, "div[class~=homepageModel]");
String name = modelDiv.attr("data-model-seo-name");
Flirt4FreeModel model = (Flirt4FreeModel) flirt4Free.createModel(name);
model.setDisplayName(modelDiv.attr("data-model-name"));
Element img = HtmlParser.getTag(modelHtml, "a[class*=modelLink] img");
img.setBaseUri(url);
if (img.hasAttr("data-image-url")) {
model.setPreview(img.absUrl("data-image-url"));
} else {
// background-image: url('https://cdn1.vscdns.com/images/models/samples-640x480/3241715.jpg')
Matcher m = Pattern.compile("background-image: url\\('(.*?)'\\)").matcher(img.attr("style"));
if (m.find()) {
model.setPreview(m.group(1));
}
}
Element link = HtmlParser.getTag(modelHtml, "a.name");
model.setDisplayName(link.attr("title"));
Element modelLink = HtmlParser.getTag(modelHtml, "a[class~=modelLink]");
model.setUrl(Flirt4Free.BASE_URI + "/rooms/" + modelDiv.attr("data-model-seo-name") + '/');
model.setDescription("");
String videoHost = modelLink.attr("data-video-host");
String modelId = modelLink.attr("data-model-id").substring(5);
model.setId(modelId);
String streamUrl = "https://manifest.vscdns.com/manifest.m3u8.m3u8?key=nil&provider=level3&secure=true&host=" + videoHost
+ "&model_id=" + modelId;
model.setStreamUrl(streamUrl);
model.setOnlineState(ctbrec.Model.State.ONLINE);
model.setOnline(true);
return model;
}
}