Remove caches from Chaturbate code

- Remove caches from the Chaturbate class
- Move all model related code from Chaturbate to ChaturbateModel
- Use a User-Agent string in all HTTP requests

common/src/main/java/ctbrec/sites/chaturbate/Chaturbate.java

@@ -1,50 +1,23 @@
 package ctbrec.sites.chaturbate;
 
-import java.io.EOFException;
 import java.io.IOException;
-import java.io.InputStream;
 import java.net.URLEncoder;
 import java.time.Instant;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.Objects;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.TimeUnit;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.CacheLoader;
-import com.google.common.cache.LoadingCache;
-import com.iheartradio.m3u8.Encoding;
-import com.iheartradio.m3u8.Format;
-import com.iheartradio.m3u8.ParseException;
-import com.iheartradio.m3u8.ParsingMode;
-import com.iheartradio.m3u8.PlaylistException;
-import com.iheartradio.m3u8.PlaylistParser;
-import com.iheartradio.m3u8.data.MasterPlaylist;
-import com.iheartradio.m3u8.data.Playlist;
-import com.iheartradio.m3u8.data.PlaylistData;
-import com.squareup.moshi.JsonAdapter;
-import com.squareup.moshi.Moshi;
-
 import ctbrec.Config;
 import ctbrec.Model;
 import ctbrec.io.HtmlParser;
 import ctbrec.io.HttpClient;
-import ctbrec.io.HttpException;
 import ctbrec.sites.AbstractSite;
-import okhttp3.FormBody;
 import okhttp3.Request;
-import okhttp3.RequestBody;
 import okhttp3.Response;
 
 public class Chaturbate extends AbstractSite {
 
-    private static final transient Logger LOG = LoggerFactory.getLogger(Chaturbate.class);
     static String baseUrl = "https://chaturbate.com";
     public static final String AFFILIATE_LINK = "https://chaturbate.com/in/?track=default&tour=grq0&campaign=55vTi";
     public static final String REGISTRATION_LINK = "https://chaturbate.com/in/?track=default&tour=g4pe&campaign=55vTi";
@@ -87,7 +60,10 @@ public class Chaturbate extends AbstractSite {
         }
 
         String url = "https://chaturbate.com/p/" + username + "/";
-        Request req = new Request.Builder().url(url).build();
+        Request req = new Request.Builder()
+                .url(url)
+                .header("User-Agent", Config.getInstance().getSettings().httpUserAgent)
+                .build();
         Response resp = getHttpClient().execute(req);
         if (resp.isSuccessful()) {
             String profilePage = resp.body().string();
@@ -145,7 +121,7 @@ public class Chaturbate extends AbstractSite {
         // search online models
         Request req = new Request.Builder()
                 .url(url)
-                .addHeader("User-Agent", Config.getInstance().getSettings().httpUserAgent)
+                .header("User-Agent", Config.getInstance().getSettings().httpUserAgent)
                 .build();
         try(Response resp = getHttpClient().execute(req)) {
             if(resp.isSuccessful()) {
@@ -158,7 +134,7 @@ public class Chaturbate extends AbstractSite {
         url = baseUrl + '/' + q;
         req = new Request.Builder()
                 .url(url)
-                .addHeader("User-Agent", Config.getInstance().getSettings().httpUserAgent)
+                .header("User-Agent", Config.getInstance().getSettings().httpUserAgent)
                 .build();
         try(Response resp = getHttpClient().execute(req)) {
             if(resp.isSuccessful()) {
@@ -175,154 +151,6 @@ public class Chaturbate extends AbstractSite {
         return m instanceof ChaturbateModel;
     }
 
-    // #######################
-    private long lastRequest = System.currentTimeMillis();
-
-    LoadingCache<String, StreamInfo> streamInfoCache = CacheBuilder.newBuilder()
-            .initialCapacity(10_000)
-            .maximumSize(10_000)
-            .expireAfterWrite(5, TimeUnit.MINUTES)
-            .build(new CacheLoader<String, StreamInfo> () {
-                @Override
-                public StreamInfo load(String model) throws Exception {
-                    return loadStreamInfo(model);
-                }
-            });
-
-    public void sendTip(String name, int tokens) throws IOException {
-        if (!Objects.equals(System.getenv("CTBREC_DEV"), "1")) {
-            RequestBody body = new FormBody.Builder()
-                    .add("csrfmiddlewaretoken", ((ChaturbateHttpClient)getHttpClient()).getToken())
-                    .add("tip_amount", Integer.toString(tokens))
-                    .add("tip_room_type", "public")
-                    .build();
-            Request req = new Request.Builder()
-                    .url("https://chaturbate.com/tipping/send_tip/"+name+"/")
-                    .post(body)
-                    .addHeader("Referer", "https://chaturbate.com/"+name+"/")
-                    .addHeader("X-Requested-With", "XMLHttpRequest")
-                    .build();
-            try(Response response = getHttpClient().execute(req)) {
-                if(!response.isSuccessful()) {
-                    throw new IOException(response.code() + " " + response.message());
-                }
-            }
-        }
-    }
-
-    StreamInfo getStreamInfo(String modelName) throws IOException, ExecutionException {
-        return getStreamInfo(modelName, false);
-    }
-
-    StreamInfo getStreamInfo(String modelName, boolean failFast) throws IOException, ExecutionException {
-        if(failFast) {
-            return streamInfoCache.getIfPresent(modelName);
-        } else {
-            return streamInfoCache.get(modelName);
-        }
-    }
-
-    StreamInfo loadStreamInfo(String modelName) throws HttpException, IOException, InterruptedException {
-        throttleRequests();
-        RequestBody body = new FormBody.Builder()
-                .add("room_slug", modelName)
-                .add("bandwidth", "high")
-                .build();
-        Request req = new Request.Builder()
-                .url(getBaseUrl() + "/get_edge_hls_url_ajax/")
-                .post(body)
-                .addHeader("X-Requested-With", "XMLHttpRequest")
-                .build();
-        Response response = getHttpClient().execute(req);
-        try {
-            if(response.isSuccessful()) {
-                String content = response.body().string();
-                LOG.trace("Raw stream info: {}", content);
-                Moshi moshi = new Moshi.Builder().build();
-                JsonAdapter<StreamInfo> adapter = moshi.adapter(StreamInfo.class);
-                StreamInfo streamInfo = adapter.fromJson(content);
-                streamInfoCache.put(modelName, streamInfo);
-                return streamInfo;
-            } else {
-                int code = response.code();
-                String message = response.message();
-                throw new HttpException(code, message);
-            }
-        } finally {
-            response.close();
-        }
-    }
-
-    public int[] getResolution(String modelName) throws ExecutionException, IOException, ParseException, PlaylistException, InterruptedException {
-        throttleRequests();
-
-        int[] res = new int[2];
-        StreamInfo streamInfo = getStreamInfo(modelName);
-        if(!streamInfo.url.startsWith("http")) {
-            return res;
-        }
-
-        EOFException ex = null;
-        for(int i=0; i<2; i++) {
-            try {
-                MasterPlaylist master = getMasterPlaylist(modelName);
-                for (PlaylistData playlistData : master.getPlaylists()) {
-                    if(playlistData.hasStreamInfo() && playlistData.getStreamInfo().hasResolution()) {
-                        int h = playlistData.getStreamInfo().getResolution().height;
-                        int w = playlistData.getStreamInfo().getResolution().width;
-                        if(w > res[1]) {
-                            res[0] = w;
-                            res[1] = h;
-                        }
-                    }
-                }
-                ex = null;
-                break; // this attempt worked, exit loop
-            } catch(EOFException e) {
-                // the cause might be, that the playlist url in streaminfo is outdated,
-                // so let's remove it from cache and retry in the next iteration
-                streamInfoCache.invalidate(modelName);
-                ex = e;
-            }
-        }
-
-        if(ex != null) {
-            throw ex;
-        }
-
-        return res;
-    }
-
-    private void throttleRequests() throws InterruptedException {
-        long now = System.currentTimeMillis();
-        long diff = now - lastRequest;
-        if(diff < 500) {
-            Thread.sleep(diff);
-        }
-        lastRequest = now;
-    }
-
-    public MasterPlaylist getMasterPlaylist(String modelName) throws IOException, ParseException, PlaylistException, ExecutionException {
-        StreamInfo streamInfo = getStreamInfo(modelName);
-        return getMasterPlaylist(streamInfo);
-    }
-
-    public MasterPlaylist getMasterPlaylist(StreamInfo streamInfo) throws IOException, ParseException, PlaylistException {
-        LOG.trace("Loading master playlist {}", streamInfo.url);
-        Request req = new Request.Builder().url(streamInfo.url).build();
-        try (Response response = getHttpClient().execute(req)) {
-            if(response.isSuccessful()) {
-                InputStream inputStream = response.body().byteStream();
-                PlaylistParser parser = new PlaylistParser(inputStream, Format.EXT_M3U, Encoding.UTF_8, ParsingMode.LENIENT);
-                Playlist playlist = parser.parse();
-                MasterPlaylist master = playlist.getMasterPlaylist();
-                return master;
-            } else {
-                throw new HttpException(response.code(), response.message());
-            }
-        }
-    }
-
     @Override
     public boolean credentialsAvailable() {
         String username = Config.getInstance().getSettings().username;

common/src/main/java/ctbrec/sites/chaturbate/ChaturbateHttpClient.java

@@ -55,6 +55,7 @@ public class ChaturbateHttpClient extends HttpClient {
         try {
             Request login = new Request.Builder()
                     .url(Chaturbate.baseUrl + "/auth/login/")
+                    .header("User-Agent", Config.getInstance().getSettings().httpUserAgent)
                     .build();
             Response response = client.newCall(login).execute();
             String content = response.body().string();
@@ -70,6 +71,7 @@ public class ChaturbateHttpClient extends HttpClient {
             login = new Request.Builder()
                     .url(Chaturbate.baseUrl + "/auth/login/")
                     .header("Referer", Chaturbate.baseUrl + "/auth/login/")
+                    .header("User-Agent", Config.getInstance().getSettings().httpUserAgent)
                     .post(body)
                     .build();
 
@@ -98,7 +100,10 @@ public class ChaturbateHttpClient extends HttpClient {
 
     private boolean checkLogin() throws IOException {
         String url = "https://chaturbate.com/p/" + Config.getInstance().getSettings().username + "/";
-        Request req = new Request.Builder().url(url).build();
+        Request req = new Request.Builder()
+                .url(url)
+                .header("User-Agent", Config.getInstance().getSettings().httpUserAgent)
+                .build();
         Response resp = execute(req);
         if (resp.isSuccessful()) {
             String profilePage = resp.body().string();

common/src/main/java/ctbrec/sites/chaturbate/ChaturbateModel.java

@@ -2,24 +2,36 @@ package ctbrec.sites.chaturbate;
 
 import static ctbrec.Model.State.*;
 
+import java.io.EOFException;
 import java.io.IOException;
+import java.io.InputStream;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Semaphore;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.iheartradio.m3u8.Encoding;
+import com.iheartradio.m3u8.Format;
 import com.iheartradio.m3u8.ParseException;
+import com.iheartradio.m3u8.ParsingMode;
 import com.iheartradio.m3u8.PlaylistException;
+import com.iheartradio.m3u8.PlaylistParser;
 import com.iheartradio.m3u8.data.MasterPlaylist;
+import com.iheartradio.m3u8.data.Playlist;
 import com.iheartradio.m3u8.data.PlaylistData;
+import com.squareup.moshi.JsonAdapter;
+import com.squareup.moshi.Moshi;
 
 import ctbrec.AbstractModel;
 import ctbrec.Config;
+import ctbrec.io.HttpException;
 import ctbrec.recorder.download.StreamSource;
+import okhttp3.FormBody;
 import okhttp3.Request;
 import okhttp3.RequestBody;
 import okhttp3.Response;
@@ -28,6 +40,10 @@ public class ChaturbateModel extends AbstractModel {
 
     private static final transient Logger LOG = LoggerFactory.getLogger(ChaturbateModel.class);
     private int[] resolution = new int[2];
+    private StreamInfo streamInfo;
+    private long streamInfoTimestamp = 0;
+    private static Semaphore requestThrottle = new Semaphore(2, true);
+    private static long lastRequest = 0;
 
     /**
      * This constructor exists only for deserialization. Please don't call it directly
@@ -43,11 +59,11 @@ public class ChaturbateModel extends AbstractModel {
     public boolean isOnline(boolean ignoreCache) throws IOException, ExecutionException, InterruptedException {
         String roomStatus;
         if(ignoreCache) {
-            StreamInfo info = getChaturbate().loadStreamInfo(getName());
+            StreamInfo info = loadStreamInfo();
             roomStatus = Optional.ofNullable(info).map(i -> i.room_status).orElse("");
             LOG.trace("Model {} room status: {}", getName(), info.room_status);
         } else {
-            StreamInfo info = getChaturbate().getStreamInfo(getName(), true);
+            StreamInfo info = getStreamInfo(true);
             roomStatus = Optional.ofNullable(info).map(i -> i.room_status).orElse("");
         }
         return Objects.equals("public", roomStatus);
@@ -60,7 +76,7 @@ public class ChaturbateModel extends AbstractModel {
         }
 
         try {
-            resolution = getChaturbate().getResolution(getName());
+            resolution = getResolution();
         } catch(Exception e) {
             throw new ExecutionException(e);
         }
@@ -73,7 +89,7 @@ public class ChaturbateModel extends AbstractModel {
      */
     @Override
     public void invalidateCacheEntries() {
-        getChaturbate().streamInfoCache.invalidate(getName());
+        streamInfo = null;
     }
 
     public State getOnlineState() throws IOException, ExecutionException {
@@ -83,11 +99,14 @@ public class ChaturbateModel extends AbstractModel {
     @Override
     public State getOnlineState(boolean failFast) throws IOException, ExecutionException {
         if(failFast) {
-            StreamInfo info = getChaturbate().streamInfoCache.getIfPresent(getName());
+            setOnlineStateByRoomStatus(Optional.ofNullable(streamInfo).map(si -> si.room_status).orElse("Unknown"));
-            setOnlineStateByRoomStatus(info.room_status);
         } else {
-            StreamInfo info = getChaturbate().streamInfoCache.get(getName());
+            try {
-            setOnlineStateByRoomStatus(info.room_status);
+                streamInfo = loadStreamInfo();
+                setOnlineStateByRoomStatus(streamInfo.room_status);
+            } catch (InterruptedException e) {
+                throw new ExecutionException(e);
+            }
         }
         return onlineState;
     }
@@ -119,41 +138,55 @@ public class ChaturbateModel extends AbstractModel {
         }
     }
 
-    public StreamInfo getStreamInfo() throws IOException, ExecutionException {
-        return getChaturbate().getStreamInfo(getName());
-    }
-    public MasterPlaylist getMasterPlaylist() throws IOException, ParseException, PlaylistException, ExecutionException {
-        return getChaturbate().getMasterPlaylist(getName());
-    }
-
     @Override
     public void receiveTip(Double tokens) throws IOException {
-        getChaturbate().sendTip(getName(), tokens.intValue());
+        if (!Objects.equals(System.getenv("CTBREC_DEV"), "1")) {
+            RequestBody body = new FormBody.Builder()
+                    .add("csrfmiddlewaretoken", ((ChaturbateHttpClient)getSite().getHttpClient()).getToken())
+                    .add("tip_amount", Integer.toString(tokens.intValue()))
+                    .add("tip_room_type", "public")
+                    .build();
+            Request req = new Request.Builder()
+                    .url("https://chaturbate.com/tipping/send_tip/"+getName()+"/")
+                    .post(body)
+                    .header("Referer", "https://chaturbate.com/"+getName()+"/")
+                    .header("X-Requested-With", "XMLHttpRequest")
+                    .header("User-Agent", Config.getInstance().getSettings().httpUserAgent)
+                    .build();
+            try (Response response = getSite().getHttpClient().execute(req)) {
+                if (!response.isSuccessful()) {
+                    throw new IOException(response.code() + " " + response.message());
+                }
+            }
+        }
     }
 
     @Override
     public List<StreamSource> getStreamSources() throws IOException, ExecutionException, ParseException, PlaylistException {
-        invalidateCacheEntries();
+        try {
-        StreamInfo streamInfo = getStreamInfo();
+            streamInfo = loadStreamInfo();
-        MasterPlaylist masterPlaylist = getMasterPlaylist();
+            MasterPlaylist masterPlaylist = getMasterPlaylist();
-        List<StreamSource> sources = new ArrayList<>();
+            List<StreamSource> sources = new ArrayList<>();
-        for (PlaylistData playlist : masterPlaylist.getPlaylists()) {
+            for (PlaylistData playlist : masterPlaylist.getPlaylists()) {
-            if (playlist.hasStreamInfo()) {
+                if (playlist.hasStreamInfo()) {
-                StreamSource src = new StreamSource();
+                    StreamSource src = new StreamSource();
-                src.bandwidth = playlist.getStreamInfo().getBandwidth();
+                    src.bandwidth = playlist.getStreamInfo().getBandwidth();
-                src.height = playlist.getStreamInfo().getResolution().height;
+                    src.height = playlist.getStreamInfo().getResolution().height;
-                String masterUrl = streamInfo.url;
+                    String masterUrl = streamInfo.url;
-                String baseUrl = masterUrl.substring(0, masterUrl.lastIndexOf('/') + 1);
+                    String baseUrl = masterUrl.substring(0, masterUrl.lastIndexOf('/') + 1);
-                String segmentUri = baseUrl + playlist.getUri();
+                    String segmentUri = baseUrl + playlist.getUri();
-                src.mediaPlaylistUrl = segmentUri;
+                    src.mediaPlaylistUrl = segmentUri;
-                if(src.mediaPlaylistUrl.contains("?")) {
+                    if(src.mediaPlaylistUrl.contains("?")) {
-                    src.mediaPlaylistUrl = src.mediaPlaylistUrl.substring(0, src.mediaPlaylistUrl.lastIndexOf('?'));
+                        src.mediaPlaylistUrl = src.mediaPlaylistUrl.substring(0, src.mediaPlaylistUrl.lastIndexOf('?'));
+                    }
+                    LOG.trace("Media playlist {}", src.mediaPlaylistUrl);
+                    sources.add(src);
                 }
-                LOG.trace("Media playlist {}", src.mediaPlaylistUrl);
-                sources.add(src);
             }
+            return sources;
+        } catch (InterruptedException e) {
+            throw new ExecutionException(e);
         }
-        return sources;
     }
 
     @Override
@@ -167,7 +200,10 @@ public class ChaturbateModel extends AbstractModel {
     }
 
     private boolean follow(boolean follow) throws IOException {
-        Request req = new Request.Builder().url(getUrl()).build();
+        Request req = new Request.Builder()
+                .url(getUrl())
+                .header("User-Agent", Config.getInstance().getSettings().httpUserAgent)
+                .build();
         Response resp = site.getHttpClient().execute(req);
         resp.close();
 
@@ -205,7 +241,133 @@ public class ChaturbateModel extends AbstractModel {
         }
     }
 
-    private Chaturbate getChaturbate() {
+    private StreamInfo getStreamInfo() throws IOException, ExecutionException, InterruptedException {
-        return (Chaturbate) site;
+        return getStreamInfo(false);
+    }
+
+    private StreamInfo getStreamInfo(boolean failFast) throws IOException, ExecutionException, InterruptedException {
+        if(failFast) {
+            return streamInfo;
+        } else {
+            return Optional.ofNullable(streamInfo).orElse(loadStreamInfo());
+        }
+    }
+
+    private StreamInfo loadStreamInfo() throws HttpException, IOException, InterruptedException {
+        long now = System.currentTimeMillis();
+        long streamInfoAge = now - streamInfoTimestamp;
+        if(streamInfo != null && streamInfoAge < 5000) {
+            return streamInfo;
+        }
+
+        acquireSlot();
+        try {
+            RequestBody body = new FormBody.Builder()
+                    .add("room_slug", getName())
+                    .add("bandwidth", "high")
+                    .build();
+            Request req = new Request.Builder()
+                    .url(getSite().getBaseUrl() + "/get_edge_hls_url_ajax/")
+                    .post(body)
+                    .header("User-Agent", Config.getInstance().getSettings().httpUserAgent)
+                    .header("X-Requested-With", "XMLHttpRequest")
+                    .build();
+            try(Response response = getSite().getHttpClient().execute(req)) {
+                if(response.isSuccessful()) {
+                    String content = response.body().string();
+                    LOG.trace("Raw stream info: {}", content);
+                    Moshi moshi = new Moshi.Builder().build();
+                    JsonAdapter<StreamInfo> adapter = moshi.adapter(StreamInfo.class);
+                    streamInfo = adapter.fromJson(content);
+                    streamInfoTimestamp = System.currentTimeMillis();
+                    return streamInfo;
+                } else {
+                    int code = response.code();
+                    String message = response.message();
+                    throw new HttpException(code, message);
+                }
+            }
+        } finally {
+            releaseSlot();
+        }
+    }
+
+    private int[] getResolution() throws ExecutionException, IOException, ParseException, PlaylistException, InterruptedException {
+        int[] res = new int[2];
+        StreamInfo streamInfo = getStreamInfo();
+        if(!streamInfo.url.startsWith("http")) {
+            return res;
+        }
+
+        EOFException ex = null;
+        for(int i=0; i<2; i++) {
+            try {
+                MasterPlaylist master = getMasterPlaylist();
+                for (PlaylistData playlistData : master.getPlaylists()) {
+                    if(playlistData.hasStreamInfo() && playlistData.getStreamInfo().hasResolution()) {
+                        int h = playlistData.getStreamInfo().getResolution().height;
+                        int w = playlistData.getStreamInfo().getResolution().width;
+                        if(w > res[1]) {
+                            res[0] = w;
+                            res[1] = h;
+                        }
+                    }
+                }
+                ex = null;
+                break; // this attempt worked, exit loop
+            } catch(EOFException e) {
+                // the cause might be, that the playlist url in streaminfo is outdated,
+                // so let's remove it from cache and retry in the next iteration
+                streamInfo = null;
+                ex = e;
+            }
+        }
+
+        if(ex != null) {
+            throw ex;
+        }
+
+        return res;
+    }
+
+    public MasterPlaylist getMasterPlaylist() throws IOException, ParseException, PlaylistException, ExecutionException, InterruptedException {
+        StreamInfo streamInfo = getStreamInfo();
+        return getMasterPlaylist(streamInfo);
+    }
+
+    public MasterPlaylist getMasterPlaylist(StreamInfo streamInfo) throws IOException, ParseException, PlaylistException, InterruptedException {
+        LOG.trace("Loading master playlist {}", streamInfo.url);
+        Request req = new Request.Builder()
+                .url(streamInfo.url)
+                .header("User-Agent", Config.getInstance().getSettings().httpUserAgent)
+                .build();
+        try (Response response = getSite().getHttpClient().execute(req)) {
+            if (response.isSuccessful()) {
+                InputStream inputStream = response.body().byteStream();
+                PlaylistParser parser = new PlaylistParser(inputStream, Format.EXT_M3U, Encoding.UTF_8, ParsingMode.LENIENT);
+                Playlist playlist = parser.parse();
+                MasterPlaylist master = playlist.getMasterPlaylist();
+                return master;
+            } else {
+                throw new HttpException(response.code(), response.message());
+            }
+        }
+    }
+
+    private void acquireSlot() throws InterruptedException {
+        LOG.debug("Acquire: {}", requestThrottle.availablePermits());
+        requestThrottle.acquire();
+        long now = System.currentTimeMillis();
+        long millisSinceLastRequest = now - lastRequest;
+        if(millisSinceLastRequest < 500) {
+            LOG.debug("Sleeping: {}", (500-millisSinceLastRequest));
+            Thread.sleep(500 - millisSinceLastRequest);
+        }
+    }
+
+    private void releaseSlot() {
+        lastRequest = System.currentTimeMillis();
+        requestThrottle.release();
+        LOG.debug("Release: {}", requestThrottle.availablePermits());
     }
 }