j62
/
ctbrec
2
2
Fork 2
Code Issues 5 Pull Requests Packages Projects Releases 3 Wiki Activity

Fix thumbnail caching

This commit is contained in:
0xb00bface 2023-12-30 17:13:57 +01:00
parent cc9a2c640e
commit 959b41e3b9
7 changed files with 131 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGELOG.md
View File

@ -3,6 +3,7 @@
* Added menu entry to force recording of models without changing the prio * Added menu entry to force recording of models without changing the prio
* Added blacklist and whitelist settings to automatically filter out models * Added blacklist and whitelist settings to automatically filter out models
* Added setting to delete orphaned recording metadata (switched off by default) * Added setting to delete orphaned recording metadata (switched off by default)
* Fixed thumbnail caching
5.2.3 5.2.3
======================== ========================

8
client/src/main/java/ctbrec/ui/CamrecApplication.java
View File

@ -14,10 +14,7 @@ import ctbrec.event.EventHandlerConfiguration;
import ctbrec.image.LocalPortraitStore; import ctbrec.image.LocalPortraitStore;
import ctbrec.image.PortraitStore; import ctbrec.image.PortraitStore;
import ctbrec.image.RemotePortraitStore; import ctbrec.image.RemotePortraitStore;
import ctbrec.io.BandwidthMeter; import ctbrec.io.*;
import ctbrec.io.ByteUnitFormatter;
import ctbrec.io.HttpClient;
import ctbrec.io.HttpException;
import ctbrec.io.json.ObjectMapperFactory; import ctbrec.io.json.ObjectMapperFactory;
import ctbrec.notes.LocalModelNotesService; import ctbrec.notes.LocalModelNotesService;
import ctbrec.notes.ModelNotesService; import ctbrec.notes.ModelNotesService;
@ -434,6 +431,9 @@ public class CamrecApplication extends Application {
} }
try { try {
ExternalBrowser.getInstance().close(); ExternalBrowser.getInstance().close();
HttpClientCacheProvider.getCache(config).evictAll();
HttpClientCacheProvider.getCache(config).close();
IoUtils.deleteDirectory(new File(config.getConfigDir(), "cache"));
} catch (IOException e12) { } catch (IOException e12) {
// noop // noop
} }

2
client/src/main/java/ctbrec/ui/settings/SettingsTab.java
View File

@ -234,7 +234,7 @@ public class SettingsTab extends Tab implements TabSelectionListener {
Setting.of("Update overview interval (seconds)", overviewUpdateIntervalInSecs, "Update the thumbnail overviews every x seconds").needsRestart(), Setting.of("Update overview interval (seconds)", overviewUpdateIntervalInSecs, "Update the thumbnail overviews every x seconds").needsRestart(),
Setting.of("Update thumbnails", updateThumbnails, Setting.of("Update thumbnails", updateThumbnails,
"The overviews will still be updated, but the thumbnails won't be changed. This is useful for less powerful systems."), "The overviews will still be updated, but the thumbnails won't be changed. This is useful for less powerful systems."),
Setting.of("Cache size", new CacheSettingsPane(this, config)).needsRestart(), Setting.of("Thumbnails cache size", new CacheSettingsPane(this, config)).needsRestart(),
Setting.of("Manually select stream quality", chooseStreamQuality, "Opens a dialog to select the video resolution before recording"), Setting.of("Manually select stream quality", chooseStreamQuality, "Opens a dialog to select the video resolution before recording"),
Setting.of("Enable live previews (experimental)", livePreviews), Setting.of("Enable live previews (experimental)", livePreviews),
Setting.of("Enable recently watched tab", recentlyWatched).needsRestart(), Setting.of("Enable recently watched tab", recentlyWatched).needsRestart(),

3
client/src/main/java/ctbrec/ui/sites/jasmin/LiveJasminUpdateService.java
View File

@ -182,7 +182,8 @@ public class LiveJasminUpdateService extends PaginatedScheduledService {
model.setOnlineState(LiveJasminModel.mapStatus(m.optInt("status"))); model.setOnlineState(LiveJasminModel.mapStatus(m.optInt("status")));
model.setDisplayName(m.optString("display_name", null)); model.setDisplayName(m.optString("display_name", null));
models.add(model); models.add(model);
}} // if content }
} // if content
} // if data } // if data
} }
} }

2
client/src/main/java/ctbrec/ui/tabs/ThumbCell.java
View File

@ -429,7 +429,7 @@ public class ThumbCell extends StackPane {
.header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent) .header(USER_AGENT, Config.getInstance().getSettings().httpUserAgent)
.header(REFERER, getModel().getSite().getBaseUrl()) .header(REFERER, getModel().getSite().getBaseUrl())
.build(); .build();
try (Response resp = model.getSite().getHttpClient().execute(req)) { try (Response resp = model.getSite().getHttpClient().executeWithCache(req)) {
if (resp.isSuccessful()) { if (resp.isSuccessful()) {
double width = 480; double width = 480;
double height = width * imgAspectRatio; double height = width * imgAspectRatio;

101
common/src/main/java/ctbrec/io/HttpClient.java
View File

@ -8,6 +8,7 @@ import ctbrec.io.json.ObjectMapperFactory;
import ctbrec.io.json.dto.CookieDto; import ctbrec.io.json.dto.CookieDto;
import ctbrec.io.json.mapper.CookieMapper; import ctbrec.io.json.mapper.CookieMapper;
import lombok.Data; import lombok.Data;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import okhttp3.*; import okhttp3.*;
import okhttp3.OkHttpClient.Builder; import okhttp3.OkHttpClient.Builder;
@ -38,11 +39,14 @@ import static java.nio.charset.StandardCharsets.UTF_8;
public abstract class HttpClient { public abstract class HttpClient {
private static final ConnectionPool GLOBAL_HTTP_CONN_POOL = new ConnectionPool(10, 2, TimeUnit.MINUTES); private static final ConnectionPool GLOBAL_HTTP_CONN_POOL = new ConnectionPool(10, 2, TimeUnit.MINUTES);
@Getter
protected CookieJarImpl cookieJar;
protected OkHttpClient client; protected OkHttpClient client;
protected Cache cache; protected Cache cache;
protected CookieJarImpl cookieJar;
protected Config config; protected Config config;
protected boolean loggedIn = false; protected boolean loggedIn = false;
protected long cacheSize;
protected int cacheLifeTime = 600;
private final String name; private final String name;
protected HttpClient(String name, Config config) { protected HttpClient(String name, Config config) {
@ -60,26 +64,26 @@ public abstract class HttpClient {
ProxyType proxyType = config.getSettings().proxyType; ProxyType proxyType = config.getSettings().proxyType;
switch (proxyType) { switch (proxyType) {
case HTTP: case HTTP:
System.setProperty("http.proxyHost", config.getSettings().proxyHost); System.setProperty(ProxyConstants.HTTP_PROXY_HOST, config.getSettings().proxyHost);
System.setProperty("http.proxyPort", config.getSettings().proxyPort); System.setProperty(ProxyConstants.HTTP_PROXY_PORT, config.getSettings().proxyPort);
System.setProperty("https.proxyHost", config.getSettings().proxyHost); System.setProperty(ProxyConstants.HTTPS_PROXY_HOST, config.getSettings().proxyHost);
System.setProperty("https.proxyPort", config.getSettings().proxyPort); System.setProperty(ProxyConstants.HTTPS_PROXY_PORT, config.getSettings().proxyPort);
if (config.getSettings().proxyUser != null && !config.getSettings().proxyUser.isEmpty()) { if (config.getSettings().proxyUser != null && !config.getSettings().proxyUser.isEmpty()) {
String username = config.getSettings().proxyUser; String username = config.getSettings().proxyUser;
String password = config.getSettings().proxyPassword; String password = config.getSettings().proxyPassword;
System.setProperty("http.proxyUser", username); System.setProperty(ProxyConstants.HTTP_PROXY_USER, username);
System.setProperty("http.proxyPassword", password); System.setProperty(ProxyConstants.HTTP_PROXY_PASSWORD, password);
} }
break; break;
case SOCKS4: case SOCKS4:
System.setProperty("socksProxyVersion", "4"); System.setProperty(ProxyConstants.SOCKS_PROXY_VERSION, "4");
System.setProperty("socksProxyHost", config.getSettings().proxyHost); System.setProperty(ProxyConstants.SOCKS_PROXY_HOST, config.getSettings().proxyHost);
System.setProperty("socksProxyPort", config.getSettings().proxyPort); System.setProperty(ProxyConstants.SOCKS_PROXY_PORT, config.getSettings().proxyPort);
break; break;
case SOCKS5: case SOCKS5:
System.setProperty("socksProxyVersion", "5"); System.setProperty(ProxyConstants.SOCKS_PROXY_VERSION, "5");
System.setProperty("socksProxyHost", config.getSettings().proxyHost); System.setProperty(ProxyConstants.SOCKS_PROXY_HOST, config.getSettings().proxyHost);
System.setProperty("socksProxyPort", config.getSettings().proxyPort); System.setProperty(ProxyConstants.SOCKS_PROXY_PORT, config.getSettings().proxyPort);
if (config.getSettings().proxyUser != null && !config.getSettings().proxyUser.isEmpty()) { if (config.getSettings().proxyUser != null && !config.getSettings().proxyUser.isEmpty()) {
String username = config.getSettings().proxyUser; String username = config.getSettings().proxyUser;
String password = config.getSettings().proxyPassword; String password = config.getSettings().proxyPassword;
@ -88,44 +92,55 @@ public abstract class HttpClient {
break; break;
case DIRECT: case DIRECT:
default: default:
System.clearProperty("http.proxyHost"); System.clearProperty(ProxyConstants.HTTP_PROXY_HOST);
System.clearProperty("http.proxyPort"); System.clearProperty(ProxyConstants.HTTP_PROXY_PORT);
System.clearProperty("https.proxyHost"); System.clearProperty(ProxyConstants.HTTPS_PROXY_HOST);
System.clearProperty("https.proxyPort"); System.clearProperty(ProxyConstants.HTTPS_PROXY_PORT);
System.clearProperty("socksProxyVersion"); System.clearProperty(ProxyConstants.SOCKS_PROXY_VERSION);
System.clearProperty("socksProxyHost"); System.clearProperty(ProxyConstants.SOCKS_PROXY_HOST);
System.clearProperty("socksProxyPort"); System.clearProperty(ProxyConstants.SOCKS_PROXY_PORT);
System.clearProperty("java.net.socks.username"); System.clearProperty(ProxyConstants.JAVA_NET_SOCKS_USERNAME);
System.clearProperty("java.net.socks.password"); System.clearProperty(ProxyConstants.JAVA_NET_SOCKS_PASSWORD);
System.clearProperty("http.proxyUser"); System.clearProperty(ProxyConstants.HTTP_PROXY_USER);
System.clearProperty("http.proxyPassword"); System.clearProperty(ProxyConstants.HTTP_PROXY_PASSWORD);
break; break;
} }
} }
public Response execute(Request req) throws IOException { public Response execute(Request req) throws IOException {
if (cache != null) {
log.trace("Cache hit ratio {}/{} = {}", cache.hitCount(), cache.requestCount(), NumberFormat.getPercentInstance().format(cache.hitCount() / (double) cache.requestCount()));
}
Response resp = client.newCall(req).execute(); Response resp = client.newCall(req).execute();
return resp; return resp;
} }
public Response execute(Request request, int timeoutInMillis) throws IOException { public Response execute(Request request, int timeoutInMillis) throws IOException {
if (cache != null) {
log.trace("Cache hit ratio {}/{} = {}", cache.hitCount(), cache.requestCount(), NumberFormat.getPercentInstance().format(cache.hitCount() / (double) cache.requestCount()));
}
return client.newBuilder() // return client.newBuilder() //
.connectTimeout(timeoutInMillis, TimeUnit.MILLISECONDS) // .connectTimeout(timeoutInMillis, TimeUnit.MILLISECONDS) //
.readTimeout(timeoutInMillis, TimeUnit.MILLISECONDS).build() // .readTimeout(timeoutInMillis, TimeUnit.MILLISECONDS).build() //
.newCall(request).execute(); .newCall(request).execute();
} }
public Response executeWithCache(Request req) throws IOException {
log.trace("Cached request for {}", req.url());
if (Objects.nonNull(cache)) {
log.trace("Cache hit ratio {}/{} = {}", cache.hitCount(), cache.requestCount(), NumberFormat.getPercentInstance().format(cache.hitCount() / (double) cache.requestCount()));
}
if (cacheSize > 0 && Objects.nonNull(cache)) {
Request r = req.newBuilder()
.cacheControl(new CacheControl.Builder().maxAge(cacheLifeTime, TimeUnit.SECONDS).build())
.build();
return execute(r);
} else {
return execute(req);
}
}
public abstract boolean login() throws IOException; public abstract boolean login() throws IOException;
public void reconfigure() { public void reconfigure() {
loadProxySettings(); loadProxySettings();
loadCookies(); loadCookies();
cacheSize = (long) config.getSettings().thumbCacheSize * 1024 * 1024;
Builder builder = new OkHttpClient.Builder() Builder builder = new OkHttpClient.Builder()
.cookieJar(cookieJar) .cookieJar(cookieJar)
.connectionPool(GLOBAL_HTTP_CONN_POOL) .connectionPool(GLOBAL_HTTP_CONN_POOL)
@ -133,13 +148,12 @@ public abstract class HttpClient {
.readTimeout(config.getSettings().httpTimeout, TimeUnit.MILLISECONDS) .readTimeout(config.getSettings().httpTimeout, TimeUnit.MILLISECONDS)
.addNetworkInterceptor(new LoggingInterceptor()); .addNetworkInterceptor(new LoggingInterceptor());
long cacheSize = (long) config.getSettings().thumbCacheSize * 1024 * 1024;
if (cacheSize > 0) { if (cacheSize > 0) {
File configDir = config.getConfigDir(); cache = HttpClientCacheProvider.getCache(config);
File cacheDir = new File(configDir, "cache"); if (cache != null) {
cache = new Cache(cacheDir, cacheSize);
builder.cache(cache); builder.cache(cache);
} }
}
ProxyType proxyType = config.getSettings().proxyType; ProxyType proxyType = config.getSettings().proxyType;
if (proxyType == ProxyType.HTTP) { if (proxyType == ProxyType.HTTP) {
@ -265,10 +279,6 @@ public abstract class HttpClient {
} }
} }
public CookieJarImpl getCookieJar() {
return cookieJar;
}
public void logout() { public void logout() {
getCookieJar().clear(); getCookieJar().clear();
loggedIn = false; loggedIn = false;
@ -332,4 +342,19 @@ public abstract class HttpClient {
public void clearCookies() { public void clearCookies() {
logout(); logout();
} }
private static class ProxyConstants {
public static final String HTTP_PROXY_HOST = "http.proxyHost";
public static final String HTTP_PROXY_PORT = "http.proxyPort";
public static final String HTTPS_PROXY_HOST = "https.proxyHost";
public static final String HTTPS_PROXY_PORT = "https.proxyPort";
public static final String HTTP_PROXY_USER = "https.proxyUser";
public static final String HTTP_PROXY_PASSWORD = "https.proxyPassword";
public static final String SOCKS_PROXY_HOST = "socksProxyHost";
public static final String SOCKS_PROXY_PORT = "socksProxyPort";
public static final String SOCKS_PROXY_VERSION = "socksProxyVersion";
public static final String JAVA_NET_SOCKS_USERNAME = "java.net.socks.username";
public static final String JAVA_NET_SOCKS_PASSWORD = "java.net.socks.password";
}
} }

46
common/src/main/java/ctbrec/io/HttpClientCacheProvider.java Normal file
View File

@ -0,0 +1,46 @@
package ctbrec.io;
import ctbrec.Config;
import lombok.extern.slf4j.Slf4j;
import okhttp3.Cache;
import java.io.File;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
@Slf4j
public class HttpClientCacheProvider {
private static final Lock lock = new ReentrantLock();
private static HttpClientCacheProvider provider;
private Cache cache;
private HttpClientCacheProvider(Config config) {
File configDir = config.getConfigDir();
File cacheDir = new File(configDir, "cache");
long cacheSize = (long) config.getSettings().thumbCacheSize * 1024 * 1024;
try {
cache = new Cache(cacheDir, cacheSize);
} catch (Exception ex) {
log.error("Could not create HTTP client cache", ex);
}
}
private Cache getCache() {
return cache;
}
public static Cache getCache(Config config) {
lock.lock();
try {
if (provider == null) {
provider = new HttpClientCacheProvider(config);
}
return provider.getCache();
} finally {
lock.unlock();
}
}
}