Fix Cam4 login

client/src/main/java/ctbrec/ui/ExternalBrowser.java

@@ -10,6 +10,10 @@ import java.io.InputStreamReader;
 import java.io.OutputStream;
 import java.net.Socket;
 import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReentrantLock;
 import java.util.function.Consumer;
@@ -20,7 +24,7 @@ import org.slf4j.LoggerFactory;
 
 import ctbrec.Config;
 import ctbrec.OS;
-import ctbrec.io.StreamRedirector;
+import ctbrec.io.ProcessOutputLogger;
 
 public class ExternalBrowser implements AutoCloseable {
     private static final Logger LOG = LoggerFactory.getLogger(ExternalBrowser.class);
@@ -36,6 +40,8 @@ public class ExternalBrowser implements AutoCloseable {
     private volatile boolean browserReady = false;
     private Object browserReadyLock = new Object();
 
+    private Map<String, CompletableFuture<Object>> responseFutures = new HashMap<>();
+
     public static ExternalBrowser getInstance() {
         return INSTANCE;
     }
@@ -52,13 +58,8 @@ public class ExternalBrowser implements AutoCloseable {
             var configDir = new File(Config.getInstance().getConfigDir(), "ctbrec-minimal-browser");
             String[] cmdline = OS.getBrowserCommand(configDir.getCanonicalPath());
             Process p = new ProcessBuilder(cmdline).start();
-            if (LOG.isTraceEnabled()) {
+            new Thread(new ProcessOutputLogger(p.getInputStream(), "ExternalBrowser stdout")).start();
-                new Thread(new StreamRedirector(p.getInputStream(), System.out)).start(); // NOSONAR
+            new Thread(new ProcessOutputLogger(p.getErrorStream(), "ExternalBrowser stderr")).start();
-                new Thread(new StreamRedirector(p.getErrorStream(), System.err)).start(); // NOSONAR
-            } else {
-                new Thread(new StreamRedirector(p.getInputStream(), OutputStream.nullOutputStream())).start();
-                new Thread(new StreamRedirector(p.getErrorStream(), OutputStream.nullOutputStream())).start();
-            }
             LOG.debug("Browser started: {}", Arrays.toString(cmdline));
 
             connectToRemoteControlSocket();
@@ -115,15 +116,22 @@ public class ExternalBrowser implements AutoCloseable {
         }
     }
 
-    public void executeJavaScript(String javaScript) throws IOException {
+    public CompletableFuture<Object> executeJavaScript(String javaScript) throws IOException {
+        String id = UUID.randomUUID().toString();
+        var future = new CompletableFuture<Object>();
         var script = new JSONObject();
+        script.put("msgid", id);
         script.put("execute", javaScript);
+        if (out != null) {
             out.write(script.toString().getBytes(UTF_8));
             out.write('\n');
             out.flush();
+            responseFutures.put(id, future);
+        }
         if (javaScript.equals("quit")) {
             stopped = true;
         }
+        return future;
     }
 
     @Override
@@ -145,10 +153,15 @@ public class ExternalBrowser implements AutoCloseable {
             }
             while( !Thread.interrupted() && (line = br.readLine()) != null ) {
                 LOG.debug("Browser output: {}", line);
-                if (line.startsWith("{") && messageListener != null) {
+                if (line.startsWith("{")) {
+                    JSONObject json = new JSONObject(line);
+                    if (json.has("msgid")) {
+                        handleExecuteScriptResponse(json);
+                    } else {
                         messageListener.accept(line);
                     }
                 }
+            }
         } catch (IOException e) {
             if(!stopped) {
                 LOG.error("Couldn't read browser output", e);
@@ -162,6 +175,24 @@ public class ExternalBrowser implements AutoCloseable {
         }
     }
 
+    private void handleExecuteScriptResponse(JSONObject json) {
+        var msgid = json.getString("msgid");
+        LOG.debug("Future {}", msgid);
+        CompletableFuture<Object> future = responseFutures.get(msgid);
+        if (future != null) {
+            responseFutures.remove(msgid);
+            if (json.has("result")) {
+                LOG.debug("Future {} done", msgid);
+                future.complete(json.getString("result"));
+            } else if (json.has("error")) {
+                LOG.debug("Future {} failed", msgid);
+                future.completeExceptionally(new Exception(json.getJSONObject("error").toString()));
+            }
+        } else {
+            LOG.warn("No future for previous request {}", msgid);
+        }
+    }
+
     private void addProxyConfig(JSONObject jsonConfig) {
         var proxyType = Config.getInstance().getSettings().proxyType;
         switch (proxyType) {

client/src/main/java/ctbrec/ui/sites/cam4/Cam4ElectronLoginDialog.java

@@ -1,10 +1,8 @@
 package ctbrec.ui.sites.cam4;
 
 import java.io.IOException;
-import java.net.MalformedURLException;
-import java.net.URL;
 import java.util.Collections;
-import java.util.Objects;
+import java.util.Optional;
 import java.util.function.Consumer;
 
 import org.json.JSONObject;
@@ -23,10 +21,14 @@ public class Cam4ElectronLoginDialog {
 
     private static final Logger LOG = LoggerFactory.getLogger(Cam4ElectronLoginDialog.class);
     public static final String DOMAIN = "cam4.com";
-    public static final String URL = Cam4.BASE_URI + "/login";
+    public static final String URL = Cam4.BASE_URI;
     private CookieJar cookieJar;
     private ExternalBrowser browser;
 
+    private boolean dialogsClicked = false;
+    private boolean loginDialogOpened = false;
+    private Thread loginChecker;
+
     public Cam4ElectronLoginDialog(CookieJar cookieJar) throws IOException {
         this.cookieJar = cookieJar;
         browser = ExternalBrowser.getInstance();
@@ -51,32 +53,61 @@ public class Cam4ElectronLoginDialog {
             LOG.error("Didn't received a JSON object {}", line);
         } else {
             var json = new JSONObject(line);
-            if (json.has("url")) {
+            safeCookies(json);
-                var url = json.getString("url");
 
-                if(url.endsWith("/login")) {
+            clickAwayCookieAndAgeAcknowlegde();
+            openLoginDialog();
+            //fillInCredentials();
+            //clickLoginButton();
+
+            if (loginChecker == null) {
+                loginChecker = new Thread(() -> {
+                    while(!Thread.currentThread().isInterrupted()) {
                         try {
-                        String username = Config.getInstance().getSettings().cam4Username;
+                            checkIfLoggedIn();
-                        if (username != null && !username.trim().isEmpty()) {
+                            Thread.sleep(500);
-                            browser.executeJavaScript("document.querySelector('#loginPageForm input[name=\"username\"]').value = '" + username + "';");
+                        } catch (InterruptedException e) {
+                            Thread.currentThread().interrupt();
                         }
-                        String password = Config.getInstance().getSettings().cam4Password;
-                        if (password != null && !password.trim().isEmpty()) {
-                            password = password.replace("'", "\\'");
-                            browser.executeJavaScript("document.querySelector('#loginPageForm input[name=\"password\"]').value = '" + password + "';");
                     }
-                        browser.executeJavaScript("document.getElementById('footer').setAttribute('style', 'display:none');");
+                });
-                        browser.executeJavaScript("document.getElementById('promptArea').setAttribute('style', 'display:none');");
+                loginChecker.setDaemon(true);
-                        browser.executeJavaScript("document.getElementById('content').setAttribute('style', 'padding: 0');");
+                loginChecker.setName("Cam4 External browser login check");
-                        browser.executeJavaScript("document.querySelector('div[class~=\"navbar\"]').setAttribute('style', 'display:none');");
+                loginChecker.start();
+            }
+        }
+    };
+
+    private void checkIfLoggedIn() {
+        try {
+            browser.executeJavaScript("document.querySelector('a[id*=\"mainHeader_userMenuContent-logout\"]').text").thenAccept(r -> {
+                LOG.debug("Result from browser is {}", r);
+                // found the logout button, we can close the browser, the login was successful
+                closeBrowser();
+            }).exceptionally(ex -> null);
+        } catch (IOException e) {
+            LOG.error("Check, if logged in failed", e);
+        }
+    }
+
+    private void clickAwayCookieAndAgeAcknowlegde() {
+        if (!dialogsClicked) {
+            try {
+                browser.executeJavaScript("let ageButton = document.querySelector('button[id*=\"disclaimerWithAgeVerification_badge-agreeBtn\"]');"
+                        + "if (ageButton) { ageButton.click(); }");
+                browser.executeJavaScript("let cookieButton = document.querySelector('button[id*=\"cookieConsent_consentCookieBtn\"]');"
+                        + "if (cookieButton) { cookieButton.click(); }");
+                dialogsClicked = true;
             } catch (Exception e) {
-                        LOG.warn("Couldn't auto fill username and password for Cam4", e);
+                LOG.warn("Couldn't click on cookie and age acknowlegde buttons for Cam4", e);
+            }
         }
     }
 
-                if(json.has("cookies")) {
+    private void safeCookies(JSONObject json) {
+        if(json.has("cookies") && json.has("url")) {
+            var url = json.getString("url");
             var cookiesFromBrowser = json.getJSONArray("cookies");
-                    try {
             for (var i = 0; i < cookiesFromBrowser.length(); i++) {
                 var cookie = cookiesFromBrowser.getJSONObject(i);
                 if (cookie.getString("domain").contains("cam4")) {
@@ -90,16 +121,47 @@ public class Cam4ElectronLoginDialog {
                     cookieJar.saveFromResponse(HttpUrl.parse(Cam4.BASE_URI), Collections.singletonList(c));
                 }
             }
-                        if (Objects.equals(new URL(url).getPath(), "/")) {
-                            closeBrowser();
         }
-                    } catch (MalformedURLException e) {
+    }
-                        LOG.error("Couldn't parse new url {}", url, e);
+
+    private void openLoginDialog() {
+        if (!loginDialogOpened) {
+            try {
+                browser.executeJavaScript("let loginButton = document.querySelector('button[id=\"loginButton\"]');"
+                        + "console.log('loginButton', loginButton);"
+                        + "if (loginButton) { loginButton.click(); }");
+                loginDialogOpened = true;
+            } catch (Exception e) {
+                LOG.warn("Couldn't open login dialog for Cam4", e);
             }
         }
     }
+
+    @SuppressWarnings("unused")
+    private void fillInCredentials() {
+        try {
+            String username = Config.getInstance().getSettings().cam4Username;
+            if (username != null && !username.trim().isEmpty()) {
+                browser.executeJavaScript("document.querySelector('input[id*=\"loginFrom_usernameInput\"]').value = '" + username + "';");
+            }
+            String password = Config.getInstance().getSettings().cam4Password;
+            if (password != null && !password.trim().isEmpty()) {
+                password = password.replace("'", "\\'");
+                browser.executeJavaScript("document.querySelector('input[id*=\"loginFrom_passwordInput\"]').value = '" + password + "');");
+            }
+        } catch (Exception e) {
+            LOG.warn("Couldn't auto fill username and password for Cam4", e);
+        }
+    }
+
+    @SuppressWarnings("unused")
+    private void clickLoginButton() {
+        try {
+            browser.executeJavaScript("document.querySelector('button[id*=\"loginFrom_submitButton\"]').click();");
+        } catch (Exception e) {
+            LOG.warn("Couldn't click on login button for Cam4", e);
+        }
     }
-    };
 
     private Cookie createCookie(String domain, JSONObject cookie) {
         Builder b = new Cookie.Builder()
@@ -122,6 +184,7 @@ public class Cam4ElectronLoginDialog {
 
     private void closeBrowser() {
         try {
+            Optional.ofNullable(loginChecker).ifPresent(Thread::interrupt);
             browser.close();
         } catch(IOException e) {
             LOG.error("Couldn't send close request to browser", e);

client/src/main/java/ctbrec/ui/sites/cam4/Cam4FollowedUpdateService.java

@@ -7,6 +7,8 @@ import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 
 import org.json.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import ctbrec.Model;
 import ctbrec.io.HttpException;
@@ -19,6 +21,8 @@ import okhttp3.Request;
 
 public class Cam4FollowedUpdateService extends PaginatedScheduledService {
 
+    private static final Logger LOG = LoggerFactory.getLogger(Cam4FollowedUpdateService.class);
+
     private Cam4 site;
     private boolean showOnline = true;
 
@@ -47,6 +51,7 @@ public class Cam4FollowedUpdateService extends PaginatedScheduledService {
         // login first
         SiteUiFactory.getUi(site).login();
         String url = site.getBaseUrl() + "/directoryCams?directoryJson=true&online=" + showOnline + "&url=true&friends=true&favorites=true&resultsPerPage=90";
+        LOG.debug("Fetching page {}", url);
         Request req = new Request.Builder().url(url).build();
         try (var response = site.getHttpClient().execute(req)) {
             if (response.isSuccessful()) {

common/src/main/java/ctbrec/io/ProcessOutputLogger.java

@@ -0,0 +1,32 @@
+package ctbrec.io;
+
+import java.io.BufferedReader;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class ProcessOutputLogger implements Runnable {
+
+    private final Logger log;
+    private InputStream in;
+
+    public ProcessOutputLogger(InputStream in, String loggerName) {
+        this.in = in;
+        log = LoggerFactory.getLogger(loggerName);
+    }
+
+    @Override
+    public void run() {
+        try (BufferedReader r = new BufferedReader(new InputStreamReader(in))) {
+            String line;
+            while ( (line = r.readLine()) != null) {
+                log.trace(line);
+            }
+            log.trace("Stream redirect thread ended");
+        } catch(Exception e) {
+            log.warn("Couldn't redirect stream: {}", e.getLocalizedMessage());
+        }
+    }
+}

common/src/main/java/ctbrec/sites/cam4/Cam4HttpClient.java

@@ -1,9 +1,10 @@
 package ctbrec.sites.cam4;
 
 import static ctbrec.io.HttpConstants.*;
+import static java.nio.charset.StandardCharsets.*;
 
 import java.io.IOException;
-import java.util.Objects;
+import java.util.Locale;
 
 import org.json.JSONObject;
 import org.slf4j.Logger;
@@ -11,7 +12,9 @@ import org.slf4j.LoggerFactory;
 
 import ctbrec.Config;
 import ctbrec.io.HttpClient;
+import okhttp3.MediaType;
 import okhttp3.Request;
+import okhttp3.RequestBody;
 import okhttp3.Response;
 
 public class Cam4HttpClient extends HttpClient {
@@ -35,28 +38,60 @@ public class Cam4HttpClient extends HttpClient {
             return true;
         }
 
+        String url = Cam4.BASE_URI + "/rest/v2.0/login";
+        LOG.debug("Logging in {}", url);
+        JSONObject bodyJson = new JSONObject();
+        bodyJson.put("username", config.getSettings().cam4Username);
+        bodyJson.put("password", config.getSettings().cam4Password);
+        RequestBody requestBody = RequestBody.create(bodyJson.toString().getBytes(UTF_8), MediaType.parse(MIMETYPE_APPLICATION_JSON));
+        Request req = new Request.Builder()
+                .url(url)
+                .header(USER_AGENT, config.getSettings().httpUserAgent)
+                .header(ACCEPT, MIMETYPE_APPLICATION_JSON)
+                .header(ACCEPT_LANGUAGE, Locale.ENGLISH.getLanguage())
+                .header(ORIGIN, Cam4.BASE_URI)
+                .header(REFERER, Cam4.BASE_URI)
+                .post(requestBody)
+                .build();
+        try (Response response = execute(req)) {
+            String body = response.body().string();
+            LOG.debug("Response: {} {}", response.code(), body);
+            if (response.isSuccessful()) {
+                JSONObject json = new JSONObject(body);
+                return json.optInt("userId") != 0;
+            } else {
                 return false;
             }
+        }
+    }
 
     /**
      *  check, if the login worked by requesting unchecked mail
      * @throws IOException
      */
     public boolean checkLoginSuccess() throws IOException {
-        String mailUrl = Cam4.BASE_URI + "/mail/unreadThreads";
+        String url = Cam4.BASE_URI + "/rest/v2.0/login/user";
+        //String url = "http://login.cam4.com:1234/rest/v2.0/login/user";
+        LOG.debug("Checkin login success by calling {}", url);
         Request req = new Request.Builder()
-                .url(mailUrl)
+                .url(url)
-                .addHeader(X_REQUESTED_WITH, XML_HTTP_REQUEST)
+                .header(USER_AGENT, config.getSettings().httpUserAgent)
+                //.header(X_REQUESTED_WITH, XML_HTTP_REQUEST)
+                .header(REFERER, Cam4.BASE_URI + "/library")
+                .header(ACCEPT, MIMETYPE_APPLICATION_JSON)
+                .header(ACCEPT_LANGUAGE, Locale.ENGLISH.getLanguage())
                 .build();
-        Response response = execute(req);
+        try (Response response = execute(req)) {
-        if(response.isSuccessful() && response.body().contentLength() > 0) {
+            String body = response.body().string();
-            JSONObject json = new JSONObject(response.body().string());
+            LOG.debug("Response: {} {}", response.code(), body);
-            return json.has("status") && Objects.equals("success", json.getString("status"));
+            if (response.isSuccessful()) {
+                JSONObject json = new JSONObject(body);
+                return json.optInt("userId") != 0;
             } else {
-            response.close();
                 return false;
             }
         }
+    }
 
     protected double getTokenBalance() throws IOException {
         if(!loggedIn) {