davebluke
/
ctbrec
forked from j62/ctbrec
1
0
Fork 0
Code Pull Requests Activity

Revert switch to JSON parsing 

The switch to parse the JSON array at the end of the page is not
reliable because the page does not always contain the array.
This commit is contained in:
0xboobface 2019-06-29 18:08:45 +02:00
parent 798f950a95
commit 2bedb6497a
1 changed files with 40 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

88
client/src/main/java/ctbrec/ui/sites/flirt4free/Flirt4FreeUpdateService.java
View File

@ -7,9 +7,8 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import org.json.JSONArray;
import org.json.JSONObject;
import org.jsoup.nodes.Element; import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@ -42,59 +41,52 @@ public class Flirt4FreeUpdateService extends PaginatedScheduledService {
@Override @Override
public List<Model> call() throws IOException { public List<Model> call() throws IOException {
LOG.debug("Fetching page {}", url); LOG.debug("Fetching page {}", url);
Request request = new Request.Builder().url(url).addHeader("User-Agent", Config.getInstance().getSettings().httpUserAgent).build(); Request request = new Request.Builder()
.url(url)
.addHeader("User-Agent", Config.getInstance().getSettings().httpUserAgent)
.build();
try (Response response = flirt4Free.getHttpClient().execute(request)) { try (Response response = flirt4Free.getHttpClient().execute(request)) {
if (response.isSuccessful()) { if (response.isSuccessful()) {
List<Model> models = new ArrayList<>(); List<Model> models = new ArrayList<>();
String body = response.body().string(); String body = response.body().string();
Matcher m = Pattern.compile("window\\.homepageData = (\\{.*\\})", Pattern.DOTALL).matcher(body); Elements tags = HtmlParser.getTags(body, "div#live_models div[class*=modelNumber]");
if (m.find()) { for (Element tag : tags) {
JSONObject data = new JSONObject(m.group(1)); tag.setBaseUri(url);
JSONArray modelData = data.getJSONArray("liveModels"); String modelHtml = tag.html();
LOG.debug("Found {} models", modelData.length()); Element modelLink = HtmlParser.getTag(modelHtml, "a[class*=modelLink]");
for (int i = 0; i < modelData.length(); i++) { modelLink.setBaseUri(url);
String modelHtml = modelData.getString(i); String href = modelLink.attr("href");
if (modelHtml.trim().isEmpty() || !modelHtml.contains("modelLink")) { String name = href.substring(0, href.length()-1);
continue; name = name.substring(name.indexOf('/', 1) + 1);
Flirt4FreeModel model = (Flirt4FreeModel) flirt4Free.createModel(name);
Element img = HtmlParser.getTag(modelHtml, "a[class*=modelLink] img");
img.setBaseUri(url);
if(img.hasAttr("data-image-url")) {
model.setPreview(img.absUrl("data-image-url"));
} else {
// background-image: url('https://cdn1.vscdns.com/images/models/samples-640x480/3241715.jpg')
Matcher m = Pattern.compile("background-image: url\\('(.*?)'\\)").matcher(img.attr("style"));
if(m.find()) {
model.setPreview(m.group(1));
} }
Element modelDiv = HtmlParser.getTag(modelHtml, "div[class~=homepageModel]");
String name = modelDiv.attr("data-model-seo-name");
Flirt4FreeModel model = (Flirt4FreeModel) flirt4Free.createModel(name);
model.setDisplayName(modelDiv.attr("data-model-name"));
Element img = HtmlParser.getTag(modelHtml, "a[class*=modelLink] img");
img.setBaseUri(url);
if (img.hasAttr("data-image-url")) {
model.setPreview(img.absUrl("data-image-url"));
} else {
// background-image: url('https://cdn1.vscdns.com/images/models/samples-640x480/3241715.jpg')
m = Pattern.compile("background-image: url\\('(.*?)'\\)").matcher(img.attr("style"));
if (m.find()) {
model.setPreview(m.group(1));
}
}
Element link = HtmlParser.getTag(modelHtml, "a.name");
model.setDisplayName(link.attr("title"));
Element modelLink = HtmlParser.getTag(modelHtml, "a[class~=modelLink]");
model.setUrl(Flirt4Free.BASE_URI + "/rooms/" + modelDiv.attr("data-model-seo-name") + '/');
model.setDescription("");
String videoHost = modelLink.attr("data-video-host");
String modelId = modelLink.attr("data-model-id").substring(5);
model.setId(modelId);
String streamUrl = "https://manifest.vscdns.com/manifest.m3u8.m3u8?key=nil&provider=level3&secure=true&host=" + videoHost
+ "&model_id=" + modelId;
model.setStreamUrl(streamUrl);
model.setOnlineState(ctbrec.Model.State.ONLINE);
model.setOnline(true);
//System.err.println("--" + model);
models.add(model);
} }
return models.stream() Element link = HtmlParser.getTag(modelHtml, "a.name");
.skip((page - 1) * MODELS_PER_PAGE) model.setDisplayName(link.attr("title"));
.limit(MODELS_PER_PAGE) model.setUrl(modelLink.absUrl("href"));
.collect(Collectors.toList()); model.setDescription("");
} else { String videoHost = modelLink.attr("data-video-host");
throw new RuntimeException("Pattern didn't match model JSON data"); String modelId = modelLink.attr("data-model-id").substring(5);
model.setId(modelId);
String streamUrl = "https://manifest.vscdns.com/manifest.m3u8.m3u8?key=nil&provider=level3&secure=true&host=" + videoHost + "&model_id=" + modelId;
model.setStreamUrl(streamUrl);
model.setOnlineState(ctbrec.Model.State.ONLINE);
model.setOnline(true);
models.add(model);
} }
return models.stream()
.skip((page-1) * MODELS_PER_PAGE)
.limit(MODELS_PER_PAGE)
.collect(Collectors.toList());
} else { } else {
throw new HttpException(response.code(), response.message()); throw new HttpException(response.code(), response.message());
} }