davebluke
/
ctbrec
forked from j62/ctbrec
1
0
Fork 0
Code Pull Requests Activity

Move request throttle for Chaturbate to ChaturbateHttpClient

This commit is contained in:
0xb00bface 2021-01-01 13:23:32 +01:00
parent 602c81d18d
commit dbb44e1374
2 changed files with 63 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
common/src/main/java/ctbrec/sites/chaturbate/ChaturbateHttpClient.java
View File

@ -5,6 +5,7 @@ import static ctbrec.io.HttpConstants.*;
import java.io.IOException;
import java.util.NoSuchElementException;
import java.util.Objects;
import java.util.concurrent.Semaphore;
import org.jsoup.nodes.Element;
import org.slf4j.Logger;
@ -24,6 +25,9 @@ public class ChaturbateHttpClient extends HttpClient {
private static final Logger LOG = LoggerFactory.getLogger(ChaturbateHttpClient.class);
protected String token;
private static Semaphore requestThrottle = new Semaphore(2, true);
private static long lastRequest = 0;
public ChaturbateHttpClient() {
super("chaturbate");
}
@ -125,8 +129,40 @@ public class ChaturbateHttpClient extends HttpClient {
@Override
public Response execute(Request req) throws IOException {
Response resp = super.execute(req);
extractCsrfToken(req);
return resp;
boolean throttled = req.url().host().contains("chaturbate.com");
return executeThrottled(req, throttled);
}
private Response executeThrottled(Request req, boolean throttle) throws IOException {
try {
if (throttle) {
acquireSlot();
}
Response resp = super.execute(req);
extractCsrfToken(req);
return resp;
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new IOException("INterrupted during request", e);
} finally {
if (throttle) {
releaseSlot();
}
}
}
private static void acquireSlot() throws InterruptedException {
long pauseBetweenRequests = 500;
requestThrottle.acquire();
long now = System.currentTimeMillis();
long millisSinceLastRequest = now - lastRequest;
if (millisSinceLastRequest < pauseBetweenRequests) {
Thread.sleep(pauseBetweenRequests - millisSinceLastRequest);
}
}
private static void releaseSlot() {
lastRequest = System.currentTimeMillis();
requestThrottle.release();
}
}

73
common/src/main/java/ctbrec/sites/chaturbate/ChaturbateModel.java
View File

@ -13,7 +13,6 @@ import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Semaphore;
import org.json.JSONObject;
import org.slf4j.Logger;
@ -47,8 +46,6 @@ public class ChaturbateModel extends AbstractModel { // NOSONAR
private int[] resolution = new int[2];
private transient StreamInfo streamInfo;
private long streamInfoTimestamp = 0;
private static Semaphore requestThrottle = new Semaphore(2, true);
private static long lastRequest = 0;
/**
* This constructor exists only for deserialization. Please don't call it directly
@ -266,39 +263,34 @@ public class ChaturbateModel extends AbstractModel { // NOSONAR
private StreamInfo loadStreamInfo() throws IOException, InterruptedException {
long now = System.currentTimeMillis();
long streamInfoAge = now - streamInfoTimestamp;
if(streamInfo != null && streamInfoAge < 5000) {
if (streamInfo != null && streamInfoAge < 5000) {
return streamInfo;
}
acquireSlot();
try {
RequestBody body = new FormBody.Builder()
.add("room_slug", getName())
.add("bandwidth", "high")
.build();
Request req = new Request.Builder()
.url(getSite().getBaseUrl() + "/get_edge_hls_url_ajax/")
.post(body)
.header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
.header(X_REQUESTED_WITH, XML_HTTP_REQUEST)
.build();
try(Response response = getSite().getHttpClient().execute(req)) {
if(response.isSuccessful()) {
String content = response.body().string();
LOG.trace("Raw stream info: {}", content);
Moshi moshi = new Moshi.Builder().build();
JsonAdapter<StreamInfo> adapter = moshi.adapter(StreamInfo.class);
streamInfo = adapter.fromJson(content);
streamInfoTimestamp = System.currentTimeMillis();
return streamInfo;
} else {
int code = response.code();
String message = response.message();
throw new HttpException(code, message);
}
RequestBody body = new FormBody.Builder()
.add("room_slug", getName())
.add("bandwidth", "high")
.build();
Request req = new Request.Builder()
.url(getSite().getBaseUrl() + "/get_edge_hls_url_ajax/")
.post(body)
.header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
.header(X_REQUESTED_WITH, XML_HTTP_REQUEST)
.build();
try (Response response = getSite().getHttpClient().execute(req)) {
if (response.isSuccessful()) {
String content = response.body().string();
LOG.trace("Raw stream info: {}", content);
Moshi moshi = new Moshi.Builder().build();
JsonAdapter<StreamInfo> adapter = moshi.adapter(StreamInfo.class);
streamInfo = adapter.fromJson(content);
streamInfoTimestamp = System.currentTimeMillis();
return streamInfo;
} else {
int code = response.code();
String message = response.message();
throw new HttpException(code, message);
}
} finally {
releaseSlot();
}
}
@ -362,21 +354,4 @@ public class ChaturbateModel extends AbstractModel { // NOSONAR
}
}
}
private void acquireSlot() throws InterruptedException {
//LOG.debug("Acquire: {}", requestThrottle.availablePermits());
requestThrottle.acquire();
long now = System.currentTimeMillis();
long millisSinceLastRequest = now - lastRequest;
if(millisSinceLastRequest < 500) {
//LOG.debug("Sleeping: {}", (500-millisSinceLastRequest));
Thread.sleep(500 - millisSinceLastRequest);
}
}
private void releaseSlot() {
lastRequest = System.currentTimeMillis();
requestThrottle.release();
//LOG.debug("Release: {}", requestThrottle.availablePermits());
}
}