jafea7
/
ctbrec-5.3.2-experimental
mirror of https://github.com/Jafea7/ctbrec-5.3.2-original.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Move request throttle for Chaturbate to ChaturbateHttpClient

This commit is contained in:
0xb00bface 2021-01-01 13:23:32 +01:00
parent 602c81d18d
commit dbb44e1374
2 changed files with 63 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
common/src/main/java/ctbrec/sites/chaturbate/ChaturbateHttpClient.java
View File

@ -5,6 +5,7 @@ import static ctbrec.io.HttpConstants.*;
import java.io.IOException; import java.io.IOException;
import java.util.NoSuchElementException; import java.util.NoSuchElementException;
import java.util.Objects; import java.util.Objects;
import java.util.concurrent.Semaphore;
import org.jsoup.nodes.Element; import org.jsoup.nodes.Element;
import org.slf4j.Logger; import org.slf4j.Logger;
@ -24,6 +25,9 @@ public class ChaturbateHttpClient extends HttpClient {
private static final Logger LOG = LoggerFactory.getLogger(ChaturbateHttpClient.class); private static final Logger LOG = LoggerFactory.getLogger(ChaturbateHttpClient.class);
protected String token; protected String token;
private static Semaphore requestThrottle = new Semaphore(2, true);
private static long lastRequest = 0;
public ChaturbateHttpClient() { public ChaturbateHttpClient() {
super("chaturbate"); super("chaturbate");
} }
@ -125,8 +129,40 @@ public class ChaturbateHttpClient extends HttpClient {
@Override @Override
public Response execute(Request req) throws IOException { public Response execute(Request req) throws IOException {
Response resp = super.execute(req); boolean throttled = req.url().host().contains("chaturbate.com");
extractCsrfToken(req); return executeThrottled(req, throttled);
return resp; }
private Response executeThrottled(Request req, boolean throttle) throws IOException {
try {
if (throttle) {
acquireSlot();
}
Response resp = super.execute(req);
extractCsrfToken(req);
return resp;
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new IOException("INterrupted during request", e);
} finally {
if (throttle) {
releaseSlot();
}
}
}
private static void acquireSlot() throws InterruptedException {
long pauseBetweenRequests = 500;
requestThrottle.acquire();
long now = System.currentTimeMillis();
long millisSinceLastRequest = now - lastRequest;
if (millisSinceLastRequest < pauseBetweenRequests) {
Thread.sleep(pauseBetweenRequests - millisSinceLastRequest);
}
}
private static void releaseSlot() {
lastRequest = System.currentTimeMillis();
requestThrottle.release();
} }
} }

73
common/src/main/java/ctbrec/sites/chaturbate/ChaturbateModel.java
View File

@ -13,7 +13,6 @@ import java.util.List;
import java.util.Objects; import java.util.Objects;
import java.util.Optional; import java.util.Optional;
import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutionException;
import java.util.concurrent.Semaphore;
import org.json.JSONObject; import org.json.JSONObject;
import org.slf4j.Logger; import org.slf4j.Logger;
@ -47,8 +46,6 @@ public class ChaturbateModel extends AbstractModel { // NOSONAR
private int[] resolution = new int[2]; private int[] resolution = new int[2];
private transient StreamInfo streamInfo; private transient StreamInfo streamInfo;
private long streamInfoTimestamp = 0; private long streamInfoTimestamp = 0;
private static Semaphore requestThrottle = new Semaphore(2, true);
private static long lastRequest = 0;
/** /**
* This constructor exists only for deserialization. Please don't call it directly * This constructor exists only for deserialization. Please don't call it directly
@ -266,39 +263,34 @@ public class ChaturbateModel extends AbstractModel { // NOSONAR
private StreamInfo loadStreamInfo() throws IOException, InterruptedException { private StreamInfo loadStreamInfo() throws IOException, InterruptedException {
long now = System.currentTimeMillis(); long now = System.currentTimeMillis();
long streamInfoAge = now - streamInfoTimestamp; long streamInfoAge = now - streamInfoTimestamp;
if(streamInfo != null && streamInfoAge < 5000) { if (streamInfo != null && streamInfoAge < 5000) {
return streamInfo; return streamInfo;
} }
acquireSlot(); RequestBody body = new FormBody.Builder()
try { .add("room_slug", getName())
RequestBody body = new FormBody.Builder() .add("bandwidth", "high")
.add("room_slug", getName()) .build();
.add("bandwidth", "high") Request req = new Request.Builder()
.build(); .url(getSite().getBaseUrl() + "/get_edge_hls_url_ajax/")
Request req = new Request.Builder() .post(body)
.url(getSite().getBaseUrl() + "/get_edge_hls_url_ajax/") .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
.post(body) .header(X_REQUESTED_WITH, XML_HTTP_REQUEST)
.header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) .build();
.header(X_REQUESTED_WITH, XML_HTTP_REQUEST) try (Response response = getSite().getHttpClient().execute(req)) {
.build(); if (response.isSuccessful()) {
try(Response response = getSite().getHttpClient().execute(req)) { String content = response.body().string();
if(response.isSuccessful()) { LOG.trace("Raw stream info: {}", content);
String content = response.body().string(); Moshi moshi = new Moshi.Builder().build();
LOG.trace("Raw stream info: {}", content); JsonAdapter<StreamInfo> adapter = moshi.adapter(StreamInfo.class);
Moshi moshi = new Moshi.Builder().build(); streamInfo = adapter.fromJson(content);
JsonAdapter<StreamInfo> adapter = moshi.adapter(StreamInfo.class); streamInfoTimestamp = System.currentTimeMillis();
streamInfo = adapter.fromJson(content); return streamInfo;
streamInfoTimestamp = System.currentTimeMillis(); } else {
return streamInfo; int code = response.code();
} else { String message = response.message();
int code = response.code(); throw new HttpException(code, message);
String message = response.message();
throw new HttpException(code, message);
}
} }
} finally {
releaseSlot();
} }
} }
@ -362,21 +354,4 @@ public class ChaturbateModel extends AbstractModel { // NOSONAR
} }
} }
} }
private void acquireSlot() throws InterruptedException {
//LOG.debug("Acquire: {}", requestThrottle.availablePermits());
requestThrottle.acquire();
long now = System.currentTimeMillis();
long millisSinceLastRequest = now - lastRequest;
if(millisSinceLastRequest < 500) {
//LOG.debug("Sleeping: {}", (500-millisSinceLastRequest));
Thread.sleep(500 - millisSinceLastRequest);
}
}
private void releaseSlot() {
lastRequest = System.currentTimeMillis();
requestThrottle.release();
//LOG.debug("Release: {}", requestThrottle.availablePermits());
}
} }