diff --git a/client/src/main/java/ctbrec/ui/sites/showup/ShowupUpdateService.java b/client/src/main/java/ctbrec/ui/sites/showup/ShowupUpdateService.java index 02030888..0f38a492 100644 --- a/client/src/main/java/ctbrec/ui/sites/showup/ShowupUpdateService.java +++ b/client/src/main/java/ctbrec/ui/sites/showup/ShowupUpdateService.java @@ -11,8 +11,8 @@ import javafx.concurrent.Task; public class ShowupUpdateService extends PaginatedScheduledService { - private Showup showup; - private String category; + private final Showup showup; + private final String category; public ShowupUpdateService(Showup showup, String category) { this.showup = showup; @@ -21,12 +21,12 @@ public class ShowupUpdateService extends PaginatedScheduledService { @Override protected Task> createTask() { - return new Task>() { + return new Task<>() { @Override public List call() throws IOException { ShowupHttpClient httpClient = (ShowupHttpClient) showup.getHttpClient(); httpClient.setCookie("category", category); - return showup.getModelList(); + return showup.getModelList(true); } }; } diff --git a/common/src/main/java/ctbrec/LoggingInterceptor.java b/common/src/main/java/ctbrec/LoggingInterceptor.java index 8a8a52dc..0960e1e8 100644 --- a/common/src/main/java/ctbrec/LoggingInterceptor.java +++ b/common/src/main/java/ctbrec/LoggingInterceptor.java @@ -1,13 +1,13 @@ package ctbrec; -import java.io.IOException; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - +import kotlin.Pair; import okhttp3.Interceptor; import okhttp3.Request; import okhttp3.Response; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; public class LoggingInterceptor implements Interceptor { @@ -17,13 +17,19 @@ public class LoggingInterceptor implements Interceptor { public Response intercept(Chain chain) throws IOException { long t1 = System.nanoTime(); Request request = chain.request(); - LOG.debug("OkHttp Sending request {} on {}\n{}", request.url(), chain.connection(), request.headers()); - if(request.method().equalsIgnoreCase("POST")) { + LOG.debug("OkHttp Sending request {} on {}", request.url(), chain.connection()); + for (Pair header : request.headers()) { + LOG.debug("{}: {}", header.getFirst(), header.getSecond()); + } + if (request.method().equalsIgnoreCase("POST")) { LOG.debug("Body: {}", request.body()); } Response response = chain.proceed(request); long t2 = System.nanoTime(); - LOG.debug("OkHttp Received {} response for {} in {}ms\n{}", response.code(), response.request().url(), (t2 - t1) / 1e6d, response.headers()); + LOG.debug("OkHttp Received {} response for {} in {}ms", response.code(), response.request().url(), (t2 - t1) / 1e6d); + for (Pair header : response.headers()) { + LOG.debug("{}: {}", header.getFirst(), header.getSecond()); + } return response; } } diff --git a/common/src/main/java/ctbrec/sites/showup/Showup.java b/common/src/main/java/ctbrec/sites/showup/Showup.java index b35cd2da..dda019ef 100644 --- a/common/src/main/java/ctbrec/sites/showup/Showup.java +++ b/common/src/main/java/ctbrec/sites/showup/Showup.java @@ -1,30 +1,30 @@ package ctbrec.sites.showup; -import static ctbrec.Model.State.*; -import static ctbrec.io.HttpConstants.*; - -import java.io.IOException; -import java.io.UnsupportedEncodingException; -import java.net.URLEncoder; -import java.nio.charset.StandardCharsets; -import java.time.Duration; -import java.time.Instant; -import java.util.ArrayList; -import java.util.List; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import org.json.JSONArray; -import org.json.JSONObject; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import ctbrec.Model; import ctbrec.io.HttpClient; import ctbrec.io.HttpException; import ctbrec.sites.AbstractSite; import okhttp3.Request; import okhttp3.Response; +import org.json.JSONArray; +import org.json.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.net.URLEncoder; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static ctbrec.ErrorMessages.HTTP_RESPONSE_BODY_IS_NULL; +import static ctbrec.Model.State.ONLINE; +import static ctbrec.io.HttpConstants.USER_AGENT; +import static java.nio.charset.StandardCharsets.UTF_8; public class Showup extends AbstractSite { @@ -51,7 +51,7 @@ public class Showup extends AbstractSite { } @Override - public Model createModel(String name) { + public ShowupModel createModel(String name) { ShowupModel model = new ShowupModel(); model.setSite(this); model.setName(name); @@ -60,15 +60,15 @@ public class Showup extends AbstractSite { } private String safeUrlEncode(String s) { - try { - return URLEncoder.encode(s, StandardCharsets.UTF_8.name()); - } catch (UnsupportedEncodingException e) { - return s; - } + return URLEncoder.encode(s, UTF_8); } public List getModelList() throws IOException { - if(Duration.between(lastModelListUpdate, Instant.now()).getSeconds() > 10) { + return getModelList(false); + } + + public List getModelList(boolean forceUpdate) throws IOException { + if (Duration.between(lastModelListUpdate, Instant.now()).getSeconds() > 10 || forceUpdate) { lastModelListUpdate = Instant.now(); String url = getBaseUrl() + "/site/get_stream_list/big"; Request req = new Request.Builder() @@ -78,14 +78,15 @@ public class Showup extends AbstractSite { try (Response response = getHttpClient().execute(req)) { if (response.isSuccessful()) { - String body = response.body().string(); + String body = Objects.requireNonNull(response.body(), HTTP_RESPONSE_BODY_IS_NULL).string(); LOG.trace(body); JSONObject json = new JSONObject(body); models = new ArrayList<>(); JSONArray list = json.getJSONArray("list"); for (int i = 0; i < list.length(); i++) { JSONObject entry = list.getJSONObject(i); - ShowupModel model = new ShowupModel(); + ShowupModel model = createModel(entry.getString("username")); + model.setUid(entry.optString("uid", null)); model.setPreview(getBaseUrl() + "/files/" + entry.optString("small_img") + ".jpg"); model.setDescription(entry.optString("description")); model.setName(entry.optString("username")); diff --git a/common/src/main/java/ctbrec/sites/showup/ShowupHttpClient.java b/common/src/main/java/ctbrec/sites/showup/ShowupHttpClient.java index 486bded9..52861cec 100644 --- a/common/src/main/java/ctbrec/sites/showup/ShowupHttpClient.java +++ b/common/src/main/java/ctbrec/sites/showup/ShowupHttpClient.java @@ -1,17 +1,5 @@ package ctbrec.sites.showup; -import static ctbrec.io.HttpConstants.*; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import com.google.common.base.Objects; - import ctbrec.Config; import ctbrec.io.HttpClient; import ctbrec.io.HttpException; @@ -19,9 +7,24 @@ import okhttp3.Cookie; import okhttp3.Request; import okhttp3.Response; +import java.io.IOException; +import java.io.InterruptedIOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.Semaphore; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static ctbrec.ErrorMessages.HTTP_RESPONSE_BODY_IS_NULL; +import static ctbrec.io.HttpConstants.ACCEPT_LANGUAGE; +import static ctbrec.io.HttpConstants.USER_AGENT; + public class ShowupHttpClient extends HttpClient { private String csrfToken; + private final Semaphore requestThrottle = new Semaphore(8, true); protected ShowupHttpClient(Config config) { super("showup", config); @@ -37,16 +40,12 @@ public class ShowupHttpClient extends HttpClient { .build(); Map> cookies = cookieJar.getCookies(); - List cookiesForDomain = cookies.computeIfAbsent(cookie.domain(), k -> new ArrayList()); - for (Iterator iterator = cookiesForDomain.iterator(); iterator.hasNext();) { - Cookie existingCookie = iterator.next(); - if (Objects.equal(existingCookie.name(), cookie.name())) { - iterator.remove(); - } - } + List cookiesForDomain = cookies.computeIfAbsent(cookie.domain(), k -> new ArrayList<>()); + cookiesForDomain.removeIf(existingCookie -> Objects.equals(existingCookie.name(), cookie.name())); cookiesForDomain.add(cookie); } + @SuppressWarnings("unused") public String getCsrfToken() throws IOException { if (csrfToken == null) { Request req = new Request.Builder() @@ -56,7 +55,7 @@ public class ShowupHttpClient extends HttpClient { try (Response response = execute(req)) { if (response.isSuccessful()) { - String body = response.body().string(); + String body = Objects.requireNonNull(response.body(), HTTP_RESPONSE_BODY_IS_NULL).string(); Matcher m = Pattern.compile("var csrf = '(.*?)';").matcher(body); if (m.matches()) { csrfToken = m.group(1); @@ -71,6 +70,19 @@ public class ShowupHttpClient extends HttpClient { return csrfToken; } + @Override + public Response execute(Request req) throws IOException { + try { + requestThrottle.acquire(); + return super.execute(req); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new InterruptedIOException("Interrupted during request"); + } finally { + requestThrottle.release(); + } + } + @Override public boolean login() throws IOException { return checkLoginSuccess(); @@ -93,7 +105,7 @@ public class ShowupHttpClient extends HttpClient { // // try (Response response = execute(req)) { // if (response.isSuccessful()) { - // String responseBody = response.body().string(); + // String responseBody = Objects.requireNonNull(response.body(), HTTP_RESPONSE_BODY_IS_NULL).string(); // if (responseBody.startsWith("{")) { // JSONObject json = new JSONObject(responseBody); // return json.optString("status").equalsIgnoreCase("success"); @@ -114,7 +126,7 @@ public class ShowupHttpClient extends HttpClient { .build(); try (Response response = execute(req)) { if (response.isSuccessful()) { - String responseBody = response.body().string(); + String responseBody = Objects.requireNonNull(response.body(), HTTP_RESPONSE_BODY_IS_NULL).string(); loggedIn = responseBody.contains("Wyloguj"); return loggedIn; } else {