From ca8e0a269e9143ab0e6f3394aad790fe18ae2f62 Mon Sep 17 00:00:00 2001 From: 0xboobface <0xboobface@gmail.com> Date: Sat, 11 May 2019 15:10:42 +0200 Subject: [PATCH] Remove caches from Chaturbate code - Remove caches from the Chaturbate class - Move all model related code from Chaturbate to ChaturbateModel - Use a User-Agent string in all HTTP requests --- .../ctbrec/sites/chaturbate/Chaturbate.java | 184 +------------- .../chaturbate/ChaturbateHttpClient.java | 7 +- .../sites/chaturbate/ChaturbateModel.java | 236 +++++++++++++++--- 3 files changed, 211 insertions(+), 216 deletions(-) diff --git a/common/src/main/java/ctbrec/sites/chaturbate/Chaturbate.java b/common/src/main/java/ctbrec/sites/chaturbate/Chaturbate.java index 708def45..0ddde50f 100644 --- a/common/src/main/java/ctbrec/sites/chaturbate/Chaturbate.java +++ b/common/src/main/java/ctbrec/sites/chaturbate/Chaturbate.java @@ -1,50 +1,23 @@ package ctbrec.sites.chaturbate; -import java.io.EOFException; import java.io.IOException; -import java.io.InputStream; import java.net.URLEncoder; import java.time.Instant; import java.util.ArrayList; import java.util.List; -import java.util.Objects; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; import java.util.regex.Matcher; import java.util.regex.Pattern; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.CacheLoader; -import com.google.common.cache.LoadingCache; -import com.iheartradio.m3u8.Encoding; -import com.iheartradio.m3u8.Format; -import com.iheartradio.m3u8.ParseException; -import com.iheartradio.m3u8.ParsingMode; -import com.iheartradio.m3u8.PlaylistException; -import com.iheartradio.m3u8.PlaylistParser; -import com.iheartradio.m3u8.data.MasterPlaylist; -import com.iheartradio.m3u8.data.Playlist; -import com.iheartradio.m3u8.data.PlaylistData; -import com.squareup.moshi.JsonAdapter; -import com.squareup.moshi.Moshi; - import ctbrec.Config; import ctbrec.Model; import ctbrec.io.HtmlParser; import ctbrec.io.HttpClient; -import ctbrec.io.HttpException; import ctbrec.sites.AbstractSite; -import okhttp3.FormBody; import okhttp3.Request; -import okhttp3.RequestBody; import okhttp3.Response; public class Chaturbate extends AbstractSite { - private static final transient Logger LOG = LoggerFactory.getLogger(Chaturbate.class); static String baseUrl = "https://chaturbate.com"; public static final String AFFILIATE_LINK = "https://chaturbate.com/in/?track=default&tour=grq0&campaign=55vTi"; public static final String REGISTRATION_LINK = "https://chaturbate.com/in/?track=default&tour=g4pe&campaign=55vTi"; @@ -87,7 +60,10 @@ public class Chaturbate extends AbstractSite { } String url = "https://chaturbate.com/p/" + username + "/"; - Request req = new Request.Builder().url(url).build(); + Request req = new Request.Builder() + .url(url) + .header("User-Agent", Config.getInstance().getSettings().httpUserAgent) + .build(); Response resp = getHttpClient().execute(req); if (resp.isSuccessful()) { String profilePage = resp.body().string(); @@ -145,7 +121,7 @@ public class Chaturbate extends AbstractSite { // search online models Request req = new Request.Builder() .url(url) - .addHeader("User-Agent", Config.getInstance().getSettings().httpUserAgent) + .header("User-Agent", Config.getInstance().getSettings().httpUserAgent) .build(); try(Response resp = getHttpClient().execute(req)) { if(resp.isSuccessful()) { @@ -158,7 +134,7 @@ public class Chaturbate extends AbstractSite { url = baseUrl + '/' + q; req = new Request.Builder() .url(url) - .addHeader("User-Agent", Config.getInstance().getSettings().httpUserAgent) + .header("User-Agent", Config.getInstance().getSettings().httpUserAgent) .build(); try(Response resp = getHttpClient().execute(req)) { if(resp.isSuccessful()) { @@ -175,154 +151,6 @@ public class Chaturbate extends AbstractSite { return m instanceof ChaturbateModel; } - // ####################### - private long lastRequest = System.currentTimeMillis(); - - LoadingCache streamInfoCache = CacheBuilder.newBuilder() - .initialCapacity(10_000) - .maximumSize(10_000) - .expireAfterWrite(5, TimeUnit.MINUTES) - .build(new CacheLoader () { - @Override - public StreamInfo load(String model) throws Exception { - return loadStreamInfo(model); - } - }); - - public void sendTip(String name, int tokens) throws IOException { - if (!Objects.equals(System.getenv("CTBREC_DEV"), "1")) { - RequestBody body = new FormBody.Builder() - .add("csrfmiddlewaretoken", ((ChaturbateHttpClient)getHttpClient()).getToken()) - .add("tip_amount", Integer.toString(tokens)) - .add("tip_room_type", "public") - .build(); - Request req = new Request.Builder() - .url("https://chaturbate.com/tipping/send_tip/"+name+"/") - .post(body) - .addHeader("Referer", "https://chaturbate.com/"+name+"/") - .addHeader("X-Requested-With", "XMLHttpRequest") - .build(); - try(Response response = getHttpClient().execute(req)) { - if(!response.isSuccessful()) { - throw new IOException(response.code() + " " + response.message()); - } - } - } - } - - StreamInfo getStreamInfo(String modelName) throws IOException, ExecutionException { - return getStreamInfo(modelName, false); - } - - StreamInfo getStreamInfo(String modelName, boolean failFast) throws IOException, ExecutionException { - if(failFast) { - return streamInfoCache.getIfPresent(modelName); - } else { - return streamInfoCache.get(modelName); - } - } - - StreamInfo loadStreamInfo(String modelName) throws HttpException, IOException, InterruptedException { - throttleRequests(); - RequestBody body = new FormBody.Builder() - .add("room_slug", modelName) - .add("bandwidth", "high") - .build(); - Request req = new Request.Builder() - .url(getBaseUrl() + "/get_edge_hls_url_ajax/") - .post(body) - .addHeader("X-Requested-With", "XMLHttpRequest") - .build(); - Response response = getHttpClient().execute(req); - try { - if(response.isSuccessful()) { - String content = response.body().string(); - LOG.trace("Raw stream info: {}", content); - Moshi moshi = new Moshi.Builder().build(); - JsonAdapter adapter = moshi.adapter(StreamInfo.class); - StreamInfo streamInfo = adapter.fromJson(content); - streamInfoCache.put(modelName, streamInfo); - return streamInfo; - } else { - int code = response.code(); - String message = response.message(); - throw new HttpException(code, message); - } - } finally { - response.close(); - } - } - - public int[] getResolution(String modelName) throws ExecutionException, IOException, ParseException, PlaylistException, InterruptedException { - throttleRequests(); - - int[] res = new int[2]; - StreamInfo streamInfo = getStreamInfo(modelName); - if(!streamInfo.url.startsWith("http")) { - return res; - } - - EOFException ex = null; - for(int i=0; i<2; i++) { - try { - MasterPlaylist master = getMasterPlaylist(modelName); - for (PlaylistData playlistData : master.getPlaylists()) { - if(playlistData.hasStreamInfo() && playlistData.getStreamInfo().hasResolution()) { - int h = playlistData.getStreamInfo().getResolution().height; - int w = playlistData.getStreamInfo().getResolution().width; - if(w > res[1]) { - res[0] = w; - res[1] = h; - } - } - } - ex = null; - break; // this attempt worked, exit loop - } catch(EOFException e) { - // the cause might be, that the playlist url in streaminfo is outdated, - // so let's remove it from cache and retry in the next iteration - streamInfoCache.invalidate(modelName); - ex = e; - } - } - - if(ex != null) { - throw ex; - } - - return res; - } - - private void throttleRequests() throws InterruptedException { - long now = System.currentTimeMillis(); - long diff = now - lastRequest; - if(diff < 500) { - Thread.sleep(diff); - } - lastRequest = now; - } - - public MasterPlaylist getMasterPlaylist(String modelName) throws IOException, ParseException, PlaylistException, ExecutionException { - StreamInfo streamInfo = getStreamInfo(modelName); - return getMasterPlaylist(streamInfo); - } - - public MasterPlaylist getMasterPlaylist(StreamInfo streamInfo) throws IOException, ParseException, PlaylistException { - LOG.trace("Loading master playlist {}", streamInfo.url); - Request req = new Request.Builder().url(streamInfo.url).build(); - try (Response response = getHttpClient().execute(req)) { - if(response.isSuccessful()) { - InputStream inputStream = response.body().byteStream(); - PlaylistParser parser = new PlaylistParser(inputStream, Format.EXT_M3U, Encoding.UTF_8, ParsingMode.LENIENT); - Playlist playlist = parser.parse(); - MasterPlaylist master = playlist.getMasterPlaylist(); - return master; - } else { - throw new HttpException(response.code(), response.message()); - } - } - } - @Override public boolean credentialsAvailable() { String username = Config.getInstance().getSettings().username; diff --git a/common/src/main/java/ctbrec/sites/chaturbate/ChaturbateHttpClient.java b/common/src/main/java/ctbrec/sites/chaturbate/ChaturbateHttpClient.java index fbfffa70..33472446 100644 --- a/common/src/main/java/ctbrec/sites/chaturbate/ChaturbateHttpClient.java +++ b/common/src/main/java/ctbrec/sites/chaturbate/ChaturbateHttpClient.java @@ -55,6 +55,7 @@ public class ChaturbateHttpClient extends HttpClient { try { Request login = new Request.Builder() .url(Chaturbate.baseUrl + "/auth/login/") + .header("User-Agent", Config.getInstance().getSettings().httpUserAgent) .build(); Response response = client.newCall(login).execute(); String content = response.body().string(); @@ -70,6 +71,7 @@ public class ChaturbateHttpClient extends HttpClient { login = new Request.Builder() .url(Chaturbate.baseUrl + "/auth/login/") .header("Referer", Chaturbate.baseUrl + "/auth/login/") + .header("User-Agent", Config.getInstance().getSettings().httpUserAgent) .post(body) .build(); @@ -98,7 +100,10 @@ public class ChaturbateHttpClient extends HttpClient { private boolean checkLogin() throws IOException { String url = "https://chaturbate.com/p/" + Config.getInstance().getSettings().username + "/"; - Request req = new Request.Builder().url(url).build(); + Request req = new Request.Builder() + .url(url) + .header("User-Agent", Config.getInstance().getSettings().httpUserAgent) + .build(); Response resp = execute(req); if (resp.isSuccessful()) { String profilePage = resp.body().string(); diff --git a/common/src/main/java/ctbrec/sites/chaturbate/ChaturbateModel.java b/common/src/main/java/ctbrec/sites/chaturbate/ChaturbateModel.java index 1428c3fd..338571ea 100644 --- a/common/src/main/java/ctbrec/sites/chaturbate/ChaturbateModel.java +++ b/common/src/main/java/ctbrec/sites/chaturbate/ChaturbateModel.java @@ -2,24 +2,36 @@ package ctbrec.sites.chaturbate; import static ctbrec.Model.State.*; +import java.io.EOFException; import java.io.IOException; +import java.io.InputStream; import java.util.ArrayList; import java.util.List; import java.util.Objects; import java.util.Optional; import java.util.concurrent.ExecutionException; +import java.util.concurrent.Semaphore; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.iheartradio.m3u8.Encoding; +import com.iheartradio.m3u8.Format; import com.iheartradio.m3u8.ParseException; +import com.iheartradio.m3u8.ParsingMode; import com.iheartradio.m3u8.PlaylistException; +import com.iheartradio.m3u8.PlaylistParser; import com.iheartradio.m3u8.data.MasterPlaylist; +import com.iheartradio.m3u8.data.Playlist; import com.iheartradio.m3u8.data.PlaylistData; +import com.squareup.moshi.JsonAdapter; +import com.squareup.moshi.Moshi; import ctbrec.AbstractModel; import ctbrec.Config; +import ctbrec.io.HttpException; import ctbrec.recorder.download.StreamSource; +import okhttp3.FormBody; import okhttp3.Request; import okhttp3.RequestBody; import okhttp3.Response; @@ -28,6 +40,10 @@ public class ChaturbateModel extends AbstractModel { private static final transient Logger LOG = LoggerFactory.getLogger(ChaturbateModel.class); private int[] resolution = new int[2]; + private StreamInfo streamInfo; + private long streamInfoTimestamp = 0; + private static Semaphore requestThrottle = new Semaphore(2, true); + private static long lastRequest = 0; /** * This constructor exists only for deserialization. Please don't call it directly @@ -43,11 +59,11 @@ public class ChaturbateModel extends AbstractModel { public boolean isOnline(boolean ignoreCache) throws IOException, ExecutionException, InterruptedException { String roomStatus; if(ignoreCache) { - StreamInfo info = getChaturbate().loadStreamInfo(getName()); + StreamInfo info = loadStreamInfo(); roomStatus = Optional.ofNullable(info).map(i -> i.room_status).orElse(""); LOG.trace("Model {} room status: {}", getName(), info.room_status); } else { - StreamInfo info = getChaturbate().getStreamInfo(getName(), true); + StreamInfo info = getStreamInfo(true); roomStatus = Optional.ofNullable(info).map(i -> i.room_status).orElse(""); } return Objects.equals("public", roomStatus); @@ -60,7 +76,7 @@ public class ChaturbateModel extends AbstractModel { } try { - resolution = getChaturbate().getResolution(getName()); + resolution = getResolution(); } catch(Exception e) { throw new ExecutionException(e); } @@ -73,7 +89,7 @@ public class ChaturbateModel extends AbstractModel { */ @Override public void invalidateCacheEntries() { - getChaturbate().streamInfoCache.invalidate(getName()); + streamInfo = null; } public State getOnlineState() throws IOException, ExecutionException { @@ -83,11 +99,14 @@ public class ChaturbateModel extends AbstractModel { @Override public State getOnlineState(boolean failFast) throws IOException, ExecutionException { if(failFast) { - StreamInfo info = getChaturbate().streamInfoCache.getIfPresent(getName()); - setOnlineStateByRoomStatus(info.room_status); + setOnlineStateByRoomStatus(Optional.ofNullable(streamInfo).map(si -> si.room_status).orElse("Unknown")); } else { - StreamInfo info = getChaturbate().streamInfoCache.get(getName()); - setOnlineStateByRoomStatus(info.room_status); + try { + streamInfo = loadStreamInfo(); + setOnlineStateByRoomStatus(streamInfo.room_status); + } catch (InterruptedException e) { + throw new ExecutionException(e); + } } return onlineState; } @@ -119,41 +138,55 @@ public class ChaturbateModel extends AbstractModel { } } - public StreamInfo getStreamInfo() throws IOException, ExecutionException { - return getChaturbate().getStreamInfo(getName()); - } - public MasterPlaylist getMasterPlaylist() throws IOException, ParseException, PlaylistException, ExecutionException { - return getChaturbate().getMasterPlaylist(getName()); - } - @Override public void receiveTip(Double tokens) throws IOException { - getChaturbate().sendTip(getName(), tokens.intValue()); + if (!Objects.equals(System.getenv("CTBREC_DEV"), "1")) { + RequestBody body = new FormBody.Builder() + .add("csrfmiddlewaretoken", ((ChaturbateHttpClient)getSite().getHttpClient()).getToken()) + .add("tip_amount", Integer.toString(tokens.intValue())) + .add("tip_room_type", "public") + .build(); + Request req = new Request.Builder() + .url("https://chaturbate.com/tipping/send_tip/"+getName()+"/") + .post(body) + .header("Referer", "https://chaturbate.com/"+getName()+"/") + .header("X-Requested-With", "XMLHttpRequest") + .header("User-Agent", Config.getInstance().getSettings().httpUserAgent) + .build(); + try (Response response = getSite().getHttpClient().execute(req)) { + if (!response.isSuccessful()) { + throw new IOException(response.code() + " " + response.message()); + } + } + } } @Override public List getStreamSources() throws IOException, ExecutionException, ParseException, PlaylistException { - invalidateCacheEntries(); - StreamInfo streamInfo = getStreamInfo(); - MasterPlaylist masterPlaylist = getMasterPlaylist(); - List sources = new ArrayList<>(); - for (PlaylistData playlist : masterPlaylist.getPlaylists()) { - if (playlist.hasStreamInfo()) { - StreamSource src = new StreamSource(); - src.bandwidth = playlist.getStreamInfo().getBandwidth(); - src.height = playlist.getStreamInfo().getResolution().height; - String masterUrl = streamInfo.url; - String baseUrl = masterUrl.substring(0, masterUrl.lastIndexOf('/') + 1); - String segmentUri = baseUrl + playlist.getUri(); - src.mediaPlaylistUrl = segmentUri; - if(src.mediaPlaylistUrl.contains("?")) { - src.mediaPlaylistUrl = src.mediaPlaylistUrl.substring(0, src.mediaPlaylistUrl.lastIndexOf('?')); + try { + streamInfo = loadStreamInfo(); + MasterPlaylist masterPlaylist = getMasterPlaylist(); + List sources = new ArrayList<>(); + for (PlaylistData playlist : masterPlaylist.getPlaylists()) { + if (playlist.hasStreamInfo()) { + StreamSource src = new StreamSource(); + src.bandwidth = playlist.getStreamInfo().getBandwidth(); + src.height = playlist.getStreamInfo().getResolution().height; + String masterUrl = streamInfo.url; + String baseUrl = masterUrl.substring(0, masterUrl.lastIndexOf('/') + 1); + String segmentUri = baseUrl + playlist.getUri(); + src.mediaPlaylistUrl = segmentUri; + if(src.mediaPlaylistUrl.contains("?")) { + src.mediaPlaylistUrl = src.mediaPlaylistUrl.substring(0, src.mediaPlaylistUrl.lastIndexOf('?')); + } + LOG.trace("Media playlist {}", src.mediaPlaylistUrl); + sources.add(src); } - LOG.trace("Media playlist {}", src.mediaPlaylistUrl); - sources.add(src); } + return sources; + } catch (InterruptedException e) { + throw new ExecutionException(e); } - return sources; } @Override @@ -167,7 +200,10 @@ public class ChaturbateModel extends AbstractModel { } private boolean follow(boolean follow) throws IOException { - Request req = new Request.Builder().url(getUrl()).build(); + Request req = new Request.Builder() + .url(getUrl()) + .header("User-Agent", Config.getInstance().getSettings().httpUserAgent) + .build(); Response resp = site.getHttpClient().execute(req); resp.close(); @@ -205,7 +241,133 @@ public class ChaturbateModel extends AbstractModel { } } - private Chaturbate getChaturbate() { - return (Chaturbate) site; + private StreamInfo getStreamInfo() throws IOException, ExecutionException, InterruptedException { + return getStreamInfo(false); + } + + private StreamInfo getStreamInfo(boolean failFast) throws IOException, ExecutionException, InterruptedException { + if(failFast) { + return streamInfo; + } else { + return Optional.ofNullable(streamInfo).orElse(loadStreamInfo()); + } + } + + private StreamInfo loadStreamInfo() throws HttpException, IOException, InterruptedException { + long now = System.currentTimeMillis(); + long streamInfoAge = now - streamInfoTimestamp; + if(streamInfo != null && streamInfoAge < 5000) { + return streamInfo; + } + + acquireSlot(); + try { + RequestBody body = new FormBody.Builder() + .add("room_slug", getName()) + .add("bandwidth", "high") + .build(); + Request req = new Request.Builder() + .url(getSite().getBaseUrl() + "/get_edge_hls_url_ajax/") + .post(body) + .header("User-Agent", Config.getInstance().getSettings().httpUserAgent) + .header("X-Requested-With", "XMLHttpRequest") + .build(); + try(Response response = getSite().getHttpClient().execute(req)) { + if(response.isSuccessful()) { + String content = response.body().string(); + LOG.trace("Raw stream info: {}", content); + Moshi moshi = new Moshi.Builder().build(); + JsonAdapter adapter = moshi.adapter(StreamInfo.class); + streamInfo = adapter.fromJson(content); + streamInfoTimestamp = System.currentTimeMillis(); + return streamInfo; + } else { + int code = response.code(); + String message = response.message(); + throw new HttpException(code, message); + } + } + } finally { + releaseSlot(); + } + } + + private int[] getResolution() throws ExecutionException, IOException, ParseException, PlaylistException, InterruptedException { + int[] res = new int[2]; + StreamInfo streamInfo = getStreamInfo(); + if(!streamInfo.url.startsWith("http")) { + return res; + } + + EOFException ex = null; + for(int i=0; i<2; i++) { + try { + MasterPlaylist master = getMasterPlaylist(); + for (PlaylistData playlistData : master.getPlaylists()) { + if(playlistData.hasStreamInfo() && playlistData.getStreamInfo().hasResolution()) { + int h = playlistData.getStreamInfo().getResolution().height; + int w = playlistData.getStreamInfo().getResolution().width; + if(w > res[1]) { + res[0] = w; + res[1] = h; + } + } + } + ex = null; + break; // this attempt worked, exit loop + } catch(EOFException e) { + // the cause might be, that the playlist url in streaminfo is outdated, + // so let's remove it from cache and retry in the next iteration + streamInfo = null; + ex = e; + } + } + + if(ex != null) { + throw ex; + } + + return res; + } + + public MasterPlaylist getMasterPlaylist() throws IOException, ParseException, PlaylistException, ExecutionException, InterruptedException { + StreamInfo streamInfo = getStreamInfo(); + return getMasterPlaylist(streamInfo); + } + + public MasterPlaylist getMasterPlaylist(StreamInfo streamInfo) throws IOException, ParseException, PlaylistException, InterruptedException { + LOG.trace("Loading master playlist {}", streamInfo.url); + Request req = new Request.Builder() + .url(streamInfo.url) + .header("User-Agent", Config.getInstance().getSettings().httpUserAgent) + .build(); + try (Response response = getSite().getHttpClient().execute(req)) { + if (response.isSuccessful()) { + InputStream inputStream = response.body().byteStream(); + PlaylistParser parser = new PlaylistParser(inputStream, Format.EXT_M3U, Encoding.UTF_8, ParsingMode.LENIENT); + Playlist playlist = parser.parse(); + MasterPlaylist master = playlist.getMasterPlaylist(); + return master; + } else { + throw new HttpException(response.code(), response.message()); + } + } + } + + private void acquireSlot() throws InterruptedException { + LOG.debug("Acquire: {}", requestThrottle.availablePermits()); + requestThrottle.acquire(); + long now = System.currentTimeMillis(); + long millisSinceLastRequest = now - lastRequest; + if(millisSinceLastRequest < 500) { + LOG.debug("Sleeping: {}", (500-millisSinceLastRequest)); + Thread.sleep(500 - millisSinceLastRequest); + } + } + + private void releaseSlot() { + lastRequest = System.currentTimeMillis(); + requestThrottle.release(); + LOG.debug("Release: {}", requestThrottle.availablePermits()); } }