diff --git a/client/src/main/java/ctbrec/ui/ExternalBrowser.java b/client/src/main/java/ctbrec/ui/ExternalBrowser.java index dde7ec13..c0424ceb 100644 --- a/client/src/main/java/ctbrec/ui/ExternalBrowser.java +++ b/client/src/main/java/ctbrec/ui/ExternalBrowser.java @@ -10,6 +10,10 @@ import java.io.InputStreamReader; import java.io.OutputStream; import java.net.Socket; import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; import java.util.function.Consumer; @@ -20,7 +24,7 @@ import org.slf4j.LoggerFactory; import ctbrec.Config; import ctbrec.OS; -import ctbrec.io.StreamRedirector; +import ctbrec.io.ProcessOutputLogger; public class ExternalBrowser implements AutoCloseable { private static final Logger LOG = LoggerFactory.getLogger(ExternalBrowser.class); @@ -36,6 +40,8 @@ public class ExternalBrowser implements AutoCloseable { private volatile boolean browserReady = false; private Object browserReadyLock = new Object(); + private Map> responseFutures = new HashMap<>(); + public static ExternalBrowser getInstance() { return INSTANCE; } @@ -52,13 +58,8 @@ public class ExternalBrowser implements AutoCloseable { var configDir = new File(Config.getInstance().getConfigDir(), "ctbrec-minimal-browser"); String[] cmdline = OS.getBrowserCommand(configDir.getCanonicalPath()); Process p = new ProcessBuilder(cmdline).start(); - if (LOG.isTraceEnabled()) { - new Thread(new StreamRedirector(p.getInputStream(), System.out)).start(); // NOSONAR - new Thread(new StreamRedirector(p.getErrorStream(), System.err)).start(); // NOSONAR - } else { - new Thread(new StreamRedirector(p.getInputStream(), OutputStream.nullOutputStream())).start(); - new Thread(new StreamRedirector(p.getErrorStream(), OutputStream.nullOutputStream())).start(); - } + new Thread(new ProcessOutputLogger(p.getInputStream(), "ExternalBrowser stdout")).start(); + new Thread(new ProcessOutputLogger(p.getErrorStream(), "ExternalBrowser stderr")).start(); LOG.debug("Browser started: {}", Arrays.toString(cmdline)); connectToRemoteControlSocket(); @@ -115,15 +116,22 @@ public class ExternalBrowser implements AutoCloseable { } } - public void executeJavaScript(String javaScript) throws IOException { + public CompletableFuture executeJavaScript(String javaScript) throws IOException { + String id = UUID.randomUUID().toString(); + var future = new CompletableFuture(); var script = new JSONObject(); + script.put("msgid", id); script.put("execute", javaScript); - out.write(script.toString().getBytes(UTF_8)); - out.write('\n'); - out.flush(); - if(javaScript.equals("quit")) { + if (out != null) { + out.write(script.toString().getBytes(UTF_8)); + out.write('\n'); + out.flush(); + responseFutures.put(id, future); + } + if (javaScript.equals("quit")) { stopped = true; } + return future; } @Override @@ -145,8 +153,13 @@ public class ExternalBrowser implements AutoCloseable { } while( !Thread.interrupted() && (line = br.readLine()) != null ) { LOG.debug("Browser output: {}", line); - if (line.startsWith("{") && messageListener != null) { - messageListener.accept(line); + if (line.startsWith("{")) { + JSONObject json = new JSONObject(line); + if (json.has("msgid")) { + handleExecuteScriptResponse(json); + } else { + messageListener.accept(line); + } } } } catch (IOException e) { @@ -162,6 +175,24 @@ public class ExternalBrowser implements AutoCloseable { } } + private void handleExecuteScriptResponse(JSONObject json) { + var msgid = json.getString("msgid"); + LOG.debug("Future {}", msgid); + CompletableFuture future = responseFutures.get(msgid); + if (future != null) { + responseFutures.remove(msgid); + if (json.has("result")) { + LOG.debug("Future {} done", msgid); + future.complete(json.getString("result")); + } else if (json.has("error")) { + LOG.debug("Future {} failed", msgid); + future.completeExceptionally(new Exception(json.getJSONObject("error").toString())); + } + } else { + LOG.warn("No future for previous request {}", msgid); + } + } + private void addProxyConfig(JSONObject jsonConfig) { var proxyType = Config.getInstance().getSettings().proxyType; switch (proxyType) { diff --git a/client/src/main/java/ctbrec/ui/sites/cam4/Cam4ElectronLoginDialog.java b/client/src/main/java/ctbrec/ui/sites/cam4/Cam4ElectronLoginDialog.java index 03b11dd0..0551ba38 100644 --- a/client/src/main/java/ctbrec/ui/sites/cam4/Cam4ElectronLoginDialog.java +++ b/client/src/main/java/ctbrec/ui/sites/cam4/Cam4ElectronLoginDialog.java @@ -1,10 +1,8 @@ package ctbrec.ui.sites.cam4; import java.io.IOException; -import java.net.MalformedURLException; -import java.net.URL; import java.util.Collections; -import java.util.Objects; +import java.util.Optional; import java.util.function.Consumer; import org.json.JSONObject; @@ -23,10 +21,14 @@ public class Cam4ElectronLoginDialog { private static final Logger LOG = LoggerFactory.getLogger(Cam4ElectronLoginDialog.class); public static final String DOMAIN = "cam4.com"; - public static final String URL = Cam4.BASE_URI + "/login"; + public static final String URL = Cam4.BASE_URI; private CookieJar cookieJar; private ExternalBrowser browser; + private boolean dialogsClicked = false; + private boolean loginDialogOpened = false; + private Thread loginChecker; + public Cam4ElectronLoginDialog(CookieJar cookieJar) throws IOException { this.cookieJar = cookieJar; browser = ExternalBrowser.getInstance(); @@ -51,56 +53,116 @@ public class Cam4ElectronLoginDialog { LOG.error("Didn't received a JSON object {}", line); } else { var json = new JSONObject(line); - if (json.has("url")) { - var url = json.getString("url"); + safeCookies(json); - if(url.endsWith("/login")) { - try { - String username = Config.getInstance().getSettings().cam4Username; - if (username != null && !username.trim().isEmpty()) { - browser.executeJavaScript("document.querySelector('#loginPageForm input[name=\"username\"]').value = '" + username + "';"); - } - String password = Config.getInstance().getSettings().cam4Password; - if (password != null && !password.trim().isEmpty()) { - password = password.replace("'", "\\'"); - browser.executeJavaScript("document.querySelector('#loginPageForm input[name=\"password\"]').value = '" + password + "';"); - } - browser.executeJavaScript("document.getElementById('footer').setAttribute('style', 'display:none');"); - browser.executeJavaScript("document.getElementById('promptArea').setAttribute('style', 'display:none');"); - browser.executeJavaScript("document.getElementById('content').setAttribute('style', 'padding: 0');"); - browser.executeJavaScript("document.querySelector('div[class~=\"navbar\"]').setAttribute('style', 'display:none');"); - } catch(Exception e) { - LOG.warn("Couldn't auto fill username and password for Cam4", e); - } - } + clickAwayCookieAndAgeAcknowlegde(); + openLoginDialog(); + //fillInCredentials(); + //clickLoginButton(); - if(json.has("cookies")) { - var cookiesFromBrowser = json.getJSONArray("cookies"); - try { - for (var i = 0; i < cookiesFromBrowser.length(); i++) { - var cookie = cookiesFromBrowser.getJSONObject(i); - if(cookie.getString("domain").contains("cam4")) { - var domain = cookie.getString("domain"); - if(domain.startsWith(".")) { - domain = domain.substring(1); - } - var c = createCookie(domain, cookie); - cookieJar.saveFromResponse(HttpUrl.parse(url), Collections.singletonList(c)); - c = createCookie("cam4.com", cookie); - cookieJar.saveFromResponse(HttpUrl.parse(Cam4.BASE_URI), Collections.singletonList(c)); - } + if (loginChecker == null) { + loginChecker = new Thread(() -> { + while(!Thread.currentThread().isInterrupted()) { + try { + checkIfLoggedIn(); + Thread.sleep(500); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); } - if (Objects.equals(new URL(url).getPath(), "/")) { - closeBrowser(); - } - } catch (MalformedURLException e) { - LOG.error("Couldn't parse new url {}", url, e); } - } + }); + loginChecker.setDaemon(true); + loginChecker.setName("Cam4 External browser login check"); + loginChecker.start(); } } }; + private void checkIfLoggedIn() { + try { + browser.executeJavaScript("document.querySelector('a[id*=\"mainHeader_userMenuContent-logout\"]').text").thenAccept(r -> { + LOG.debug("Result from browser is {}", r); + // found the logout button, we can close the browser, the login was successful + closeBrowser(); + }).exceptionally(ex -> null); + } catch (IOException e) { + LOG.error("Check, if logged in failed", e); + } + } + + private void clickAwayCookieAndAgeAcknowlegde() { + if (!dialogsClicked) { + try { + browser.executeJavaScript("let ageButton = document.querySelector('button[id*=\"disclaimerWithAgeVerification_badge-agreeBtn\"]');" + + "if (ageButton) { ageButton.click(); }"); + browser.executeJavaScript("let cookieButton = document.querySelector('button[id*=\"cookieConsent_consentCookieBtn\"]');" + + "if (cookieButton) { cookieButton.click(); }"); + dialogsClicked = true; + } catch (Exception e) { + LOG.warn("Couldn't click on cookie and age acknowlegde buttons for Cam4", e); + } + } + } + + private void safeCookies(JSONObject json) { + if(json.has("cookies") && json.has("url")) { + var url = json.getString("url"); + var cookiesFromBrowser = json.getJSONArray("cookies"); + for (var i = 0; i < cookiesFromBrowser.length(); i++) { + var cookie = cookiesFromBrowser.getJSONObject(i); + if (cookie.getString("domain").contains("cam4")) { + var domain = cookie.getString("domain"); + if (domain.startsWith(".")) { + domain = domain.substring(1); + } + var c = createCookie(domain, cookie); + cookieJar.saveFromResponse(HttpUrl.parse(url), Collections.singletonList(c)); + c = createCookie("cam4.com", cookie); + cookieJar.saveFromResponse(HttpUrl.parse(Cam4.BASE_URI), Collections.singletonList(c)); + } + } + } + } + + private void openLoginDialog() { + if (!loginDialogOpened) { + try { + browser.executeJavaScript("let loginButton = document.querySelector('button[id=\"loginButton\"]');" + + "console.log('loginButton', loginButton);" + + "if (loginButton) { loginButton.click(); }"); + loginDialogOpened = true; + } catch (Exception e) { + LOG.warn("Couldn't open login dialog for Cam4", e); + } + } + } + + @SuppressWarnings("unused") + private void fillInCredentials() { + try { + String username = Config.getInstance().getSettings().cam4Username; + if (username != null && !username.trim().isEmpty()) { + browser.executeJavaScript("document.querySelector('input[id*=\"loginFrom_usernameInput\"]').value = '" + username + "';"); + } + String password = Config.getInstance().getSettings().cam4Password; + if (password != null && !password.trim().isEmpty()) { + password = password.replace("'", "\\'"); + browser.executeJavaScript("document.querySelector('input[id*=\"loginFrom_passwordInput\"]').value = '" + password + "');"); + } + } catch (Exception e) { + LOG.warn("Couldn't auto fill username and password for Cam4", e); + } + } + + @SuppressWarnings("unused") + private void clickLoginButton() { + try { + browser.executeJavaScript("document.querySelector('button[id*=\"loginFrom_submitButton\"]').click();"); + } catch (Exception e) { + LOG.warn("Couldn't click on login button for Cam4", e); + } + } + private Cookie createCookie(String domain, JSONObject cookie) { Builder b = new Cookie.Builder() .path(cookie.getString("path")) @@ -122,6 +184,7 @@ public class Cam4ElectronLoginDialog { private void closeBrowser() { try { + Optional.ofNullable(loginChecker).ifPresent(Thread::interrupt); browser.close(); } catch(IOException e) { LOG.error("Couldn't send close request to browser", e); diff --git a/client/src/main/java/ctbrec/ui/sites/cam4/Cam4FollowedUpdateService.java b/client/src/main/java/ctbrec/ui/sites/cam4/Cam4FollowedUpdateService.java index 7b769999..a81df6cb 100644 --- a/client/src/main/java/ctbrec/ui/sites/cam4/Cam4FollowedUpdateService.java +++ b/client/src/main/java/ctbrec/ui/sites/cam4/Cam4FollowedUpdateService.java @@ -7,6 +7,8 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import org.json.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import ctbrec.Model; import ctbrec.io.HttpException; @@ -19,6 +21,8 @@ import okhttp3.Request; public class Cam4FollowedUpdateService extends PaginatedScheduledService { + private static final Logger LOG = LoggerFactory.getLogger(Cam4FollowedUpdateService.class); + private Cam4 site; private boolean showOnline = true; @@ -47,6 +51,7 @@ public class Cam4FollowedUpdateService extends PaginatedScheduledService { // login first SiteUiFactory.getUi(site).login(); String url = site.getBaseUrl() + "/directoryCams?directoryJson=true&online=" + showOnline + "&url=true&friends=true&favorites=true&resultsPerPage=90"; + LOG.debug("Fetching page {}", url); Request req = new Request.Builder().url(url).build(); try (var response = site.getHttpClient().execute(req)) { if (response.isSuccessful()) { diff --git a/common/src/main/java/ctbrec/io/ProcessOutputLogger.java b/common/src/main/java/ctbrec/io/ProcessOutputLogger.java new file mode 100644 index 00000000..1326af7e --- /dev/null +++ b/common/src/main/java/ctbrec/io/ProcessOutputLogger.java @@ -0,0 +1,32 @@ +package ctbrec.io; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ProcessOutputLogger implements Runnable { + + private final Logger log; + private InputStream in; + + public ProcessOutputLogger(InputStream in, String loggerName) { + this.in = in; + log = LoggerFactory.getLogger(loggerName); + } + + @Override + public void run() { + try (BufferedReader r = new BufferedReader(new InputStreamReader(in))) { + String line; + while ( (line = r.readLine()) != null) { + log.trace(line); + } + log.trace("Stream redirect thread ended"); + } catch(Exception e) { + log.warn("Couldn't redirect stream: {}", e.getLocalizedMessage()); + } + } +} diff --git a/common/src/main/java/ctbrec/sites/cam4/Cam4HttpClient.java b/common/src/main/java/ctbrec/sites/cam4/Cam4HttpClient.java index 459c388d..0ef18c71 100644 --- a/common/src/main/java/ctbrec/sites/cam4/Cam4HttpClient.java +++ b/common/src/main/java/ctbrec/sites/cam4/Cam4HttpClient.java @@ -1,9 +1,10 @@ package ctbrec.sites.cam4; import static ctbrec.io.HttpConstants.*; +import static java.nio.charset.StandardCharsets.*; import java.io.IOException; -import java.util.Objects; +import java.util.Locale; import org.json.JSONObject; import org.slf4j.Logger; @@ -11,7 +12,9 @@ import org.slf4j.LoggerFactory; import ctbrec.Config; import ctbrec.io.HttpClient; +import okhttp3.MediaType; import okhttp3.Request; +import okhttp3.RequestBody; import okhttp3.Response; public class Cam4HttpClient extends HttpClient { @@ -24,18 +27,42 @@ public class Cam4HttpClient extends HttpClient { @Override public synchronized boolean login() throws IOException { - if(loggedIn) { + if (loggedIn) { return true; } boolean cookiesWorked = checkLoginSuccess(); - if(cookiesWorked) { + if (cookiesWorked) { loggedIn = true; LOG.debug("Logged in with cookies"); return true; } - return false; + String url = Cam4.BASE_URI + "/rest/v2.0/login"; + LOG.debug("Logging in {}", url); + JSONObject bodyJson = new JSONObject(); + bodyJson.put("username", config.getSettings().cam4Username); + bodyJson.put("password", config.getSettings().cam4Password); + RequestBody requestBody = RequestBody.create(bodyJson.toString().getBytes(UTF_8), MediaType.parse(MIMETYPE_APPLICATION_JSON)); + Request req = new Request.Builder() + .url(url) + .header(USER_AGENT, config.getSettings().httpUserAgent) + .header(ACCEPT, MIMETYPE_APPLICATION_JSON) + .header(ACCEPT_LANGUAGE, Locale.ENGLISH.getLanguage()) + .header(ORIGIN, Cam4.BASE_URI) + .header(REFERER, Cam4.BASE_URI) + .post(requestBody) + .build(); + try (Response response = execute(req)) { + String body = response.body().string(); + LOG.debug("Response: {} {}", response.code(), body); + if (response.isSuccessful()) { + JSONObject json = new JSONObject(body); + return json.optInt("userId") != 0; + } else { + return false; + } + } } /** @@ -43,18 +70,26 @@ public class Cam4HttpClient extends HttpClient { * @throws IOException */ public boolean checkLoginSuccess() throws IOException { - String mailUrl = Cam4.BASE_URI + "/mail/unreadThreads"; + String url = Cam4.BASE_URI + "/rest/v2.0/login/user"; + //String url = "http://login.cam4.com:1234/rest/v2.0/login/user"; + LOG.debug("Checkin login success by calling {}", url); Request req = new Request.Builder() - .url(mailUrl) - .addHeader(X_REQUESTED_WITH, XML_HTTP_REQUEST) + .url(url) + .header(USER_AGENT, config.getSettings().httpUserAgent) + //.header(X_REQUESTED_WITH, XML_HTTP_REQUEST) + .header(REFERER, Cam4.BASE_URI + "/library") + .header(ACCEPT, MIMETYPE_APPLICATION_JSON) + .header(ACCEPT_LANGUAGE, Locale.ENGLISH.getLanguage()) .build(); - Response response = execute(req); - if(response.isSuccessful() && response.body().contentLength() > 0) { - JSONObject json = new JSONObject(response.body().string()); - return json.has("status") && Objects.equals("success", json.getString("status")); - } else { - response.close(); - return false; + try (Response response = execute(req)) { + String body = response.body().string(); + LOG.debug("Response: {} {}", response.code(), body); + if (response.isSuccessful()) { + JSONObject json = new JSONObject(body); + return json.optInt("userId") != 0; + } else { + return false; + } } }