davebluke
/
ctbrec
forked from j62/ctbrec
1
0
Fork 0
Code Pull Requests Activity

Revert switch to JSON parsing 

The switch to parse the JSON array at the end of the page is not
reliable because the page does not always contain the array.
This commit is contained in:
0xboobface 2019-06-29 18:08:45 +02:00
parent 798f950a95
commit 2bedb6497a
1 changed files with 40 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

88
client/src/main/java/ctbrec/ui/sites/flirt4free/Flirt4FreeUpdateService.java
View File

@ -7,9 +7,8 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import org.json.JSONArray;
import org.json.JSONObject;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -42,59 +41,52 @@ public class Flirt4FreeUpdateService extends PaginatedScheduledService {
@Override
public List<Model> call() throws IOException {
LOG.debug("Fetching page {}", url);
Request request = new Request.Builder().url(url).addHeader("User-Agent", Config.getInstance().getSettings().httpUserAgent).build();
Request request = new Request.Builder()
.url(url)
.addHeader("User-Agent", Config.getInstance().getSettings().httpUserAgent)
.build();
try (Response response = flirt4Free.getHttpClient().execute(request)) {
if (response.isSuccessful()) {
List<Model> models = new ArrayList<>();
String body = response.body().string();
Matcher m = Pattern.compile("window\\.homepageData = (\\{.*\\})", Pattern.DOTALL).matcher(body);
if (m.find()) {
JSONObject data = new JSONObject(m.group(1));
JSONArray modelData = data.getJSONArray("liveModels");
LOG.debug("Found {} models", modelData.length());
for (int i = 0; i < modelData.length(); i++) {
String modelHtml = modelData.getString(i);
if (modelHtml.trim().isEmpty() || !modelHtml.contains("modelLink")) {
continue;
Elements tags = HtmlParser.getTags(body, "div#live_models div[class*=modelNumber]");
for (Element tag : tags) {
tag.setBaseUri(url);
String modelHtml = tag.html();
Element modelLink = HtmlParser.getTag(modelHtml, "a[class*=modelLink]");
modelLink.setBaseUri(url);
String href = modelLink.attr("href");
String name = href.substring(0, href.length()-1);
name = name.substring(name.indexOf('/', 1) + 1);
Flirt4FreeModel model = (Flirt4FreeModel) flirt4Free.createModel(name);
Element img = HtmlParser.getTag(modelHtml, "a[class*=modelLink] img");
img.setBaseUri(url);
if(img.hasAttr("data-image-url")) {
model.setPreview(img.absUrl("data-image-url"));
} else {
// background-image: url('https://cdn1.vscdns.com/images/models/samples-640x480/3241715.jpg')
Matcher m = Pattern.compile("background-image: url\\('(.*?)'\\)").matcher(img.attr("style"));
if(m.find()) {
model.setPreview(m.group(1));
}
Element modelDiv = HtmlParser.getTag(modelHtml, "div[class~=homepageModel]");
String name = modelDiv.attr("data-model-seo-name");
Flirt4FreeModel model = (Flirt4FreeModel) flirt4Free.createModel(name);
model.setDisplayName(modelDiv.attr("data-model-name"));
Element img = HtmlParser.getTag(modelHtml, "a[class*=modelLink] img");
img.setBaseUri(url);
if (img.hasAttr("data-image-url")) {
model.setPreview(img.absUrl("data-image-url"));
} else {
// background-image: url('https://cdn1.vscdns.com/images/models/samples-640x480/3241715.jpg')
m = Pattern.compile("background-image: url\\('(.*?)'\\)").matcher(img.attr("style"));
if (m.find()) {
model.setPreview(m.group(1));
}
}
Element link = HtmlParser.getTag(modelHtml, "a.name");
model.setDisplayName(link.attr("title"));
Element modelLink = HtmlParser.getTag(modelHtml, "a[class~=modelLink]");
model.setUrl(Flirt4Free.BASE_URI + "/rooms/" + modelDiv.attr("data-model-seo-name") + '/');
model.setDescription("");
String videoHost = modelLink.attr("data-video-host");
String modelId = modelLink.attr("data-model-id").substring(5);
model.setId(modelId);
String streamUrl = "https://manifest.vscdns.com/manifest.m3u8.m3u8?key=nil&provider=level3&secure=true&host=" + videoHost
+ "&model_id=" + modelId;
model.setStreamUrl(streamUrl);
model.setOnlineState(ctbrec.Model.State.ONLINE);
model.setOnline(true);
//System.err.println("--" + model);
models.add(model);
}
return models.stream()
.skip((page - 1) * MODELS_PER_PAGE)
.limit(MODELS_PER_PAGE)
.collect(Collectors.toList());
} else {
throw new RuntimeException("Pattern didn't match model JSON data");
Element link = HtmlParser.getTag(modelHtml, "a.name");
model.setDisplayName(link.attr("title"));
model.setUrl(modelLink.absUrl("href"));
model.setDescription("");
String videoHost = modelLink.attr("data-video-host");
String modelId = modelLink.attr("data-model-id").substring(5);
model.setId(modelId);
String streamUrl = "https://manifest.vscdns.com/manifest.m3u8.m3u8?key=nil&provider=level3&secure=true&host=" + videoHost + "&model_id=" + modelId;
model.setStreamUrl(streamUrl);
model.setOnlineState(ctbrec.Model.State.ONLINE);
model.setOnline(true);
models.add(model);
}
return models.stream()
.skip((page-1) * MODELS_PER_PAGE)
.limit(MODELS_PER_PAGE)
.collect(Collectors.toList());
} else {
throw new HttpException(response.code(), response.message());
}