Merge branch 'dev' into XxInvictus-patch

CHANGELOG.md

@@ -1,6 +1,11 @@
 5.2.3
 ========================
 * Fix one directory per group
+* Add Stripchat tags thx to @winkru
+* Fix follow / unfollow for Stripchat thx to @winkru
+* Fix: Loading the config failed with model URLs, which contained spaces
+* Fix recording size not properly being reported and transferred between
+  server and client
 
 5.2.2
 ========================

client/pom.xml

@@ -8,7 +8,7 @@
     <parent>
         <groupId>ctbrec</groupId>
         <artifactId>master</artifactId>
-        <version>5.2.2</version>
+        <version>5.2.3</version>
         <relativePath>../master</relativePath>
     </parent>
 

common/pom.xml

@@ -8,7 +8,7 @@
     <parent>
         <groupId>ctbrec</groupId>
         <artifactId>master</artifactId>
-        <version>5.2.2</version>
+        <version>5.2.3</version>
         <relativePath>../master</relativePath>
     </parent>
 

common/src/main/java/ctbrec/MigrateModel5_1_2.java

@@ -8,8 +8,12 @@ public class MigrateModel5_1_2 {
 
     private static final String MARKED_FOR_LATER = "markedForLater";
     private static final String ADDED_TIMESTAMP = "addedTimestamp";
+    private static final String URL = "url";
 
     public static void migrate(JSONObject model) {
+        if (model.has(URL)) {
+            model.put(URL, model.getString(URL).replace(" ", "%20"));
+        }
         if (model.has(MARKED_FOR_LATER)) {
             model.put("bookmarked", model.getBoolean(MARKED_FOR_LATER));
             model.remove(MARKED_FOR_LATER);

common/src/main/java/ctbrec/Recording.java

@@ -226,7 +226,7 @@ public class Recording implements Serializable {
         } else {
             long now = System.currentTimeMillis();
             if (now - lastSizeUpdate > 2500) {
-                log.debug("full size update for {}", this);
+                log.trace("full size update for {}", this);
                 sizeInByte = getSize();
                 lastSizeUpdate = now;
             }

common/src/main/java/ctbrec/recorder/RemoteRecorder.java

@@ -402,7 +402,14 @@ public class RemoteRecorder implements Recorder {
                     if (response.isSuccessful()) {
                         RecordingListResponse resp = mapper.readValue(json, RecordingListResponse.class);
                         if (resp.status.equals(SUCCESS)) {
-                            List<Recording> newRecordings = resp.recordings.stream().map(Mappers.getMapper(RecordingMapper.class)::toRecording).collect(Collectors.toList());
+                            List<Recording> newRecordings = resp.recordings.stream()
+                                    .map(dto -> {
+                                        var rec = Mappers.getMapper(RecordingMapper.class).toRecording(dto);
+                                        rec.setSizeInByte(dto.getSizeInByte());
+                                        rec.setLastSizeUpdate(Instant.now().toEpochMilli());
+                                        return rec;
+                                    })
+                                    .collect(Collectors.toList());
                             // fire changed events
                             for (Recording recording : recordings) {
                                 if (newRecordings.contains(recording)) {

common/src/main/java/ctbrec/recorder/download/AbstractDownload.java

@@ -12,12 +12,14 @@ import java.time.Instant;
 import java.util.List;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
+import java.util.concurrent.atomic.AtomicLong;
 
 import static ctbrec.recorder.download.StreamSource.UNKNOWN;
 
 @Slf4j
 public abstract class AbstractDownload implements RecordingProcess {
 
+    protected AtomicLong downloadedBytes = new AtomicLong();
     protected Instant startTime;
     protected Instant rescheduleTime = Instant.now();
     protected Model model = new UnknownModel();
@@ -112,4 +114,9 @@ public abstract class AbstractDownload implements RecordingProcess {
             }
         }
     }
+
+    @Override
+    public AtomicLong getDownloadedBytes() {
+        return downloadedBytes;
+    }
 }

common/src/main/java/ctbrec/recorder/download/RecordingProcess.java

@@ -12,7 +12,6 @@ import java.util.concurrent.ExecutorService;
 import java.util.concurrent.atomic.AtomicLong;
 
 public interface RecordingProcess extends Callable<RecordingProcess> {
-    AtomicLong downloadedBytes = new AtomicLong();
 
     void init(Config config, Model model, Instant startTime, ExecutorService executorService) throws IOException;
 
@@ -62,8 +61,5 @@ public interface RecordingProcess extends Callable<RecordingProcess> {
 
     void awaitEnd();
 
-    default AtomicLong getDownloadedBytes() {
+    AtomicLong getDownloadedBytes();
-        return downloadedBytes;
-    }
-
 }

common/src/main/java/ctbrec/recorder/download/hls/AbstractHlsDownload.java

@@ -44,7 +44,6 @@ import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorCompletionService;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Future;
-import java.util.concurrent.atomic.AtomicLong;
 
 import static ctbrec.io.HttpConstants.*;
 import static java.nio.charset.StandardCharsets.UTF_8;
@@ -70,7 +69,6 @@ public abstract class AbstractHlsDownload extends AbstractDownload {
     private Instant beforeLastPlaylistRequest = Instant.EPOCH;
     private int consecutivePlaylistTimeouts = 0;
     private int consecutivePlaylistErrors = 0;
-    private final AtomicLong sizeInByte = new AtomicLong();
     protected Instant lastSegmentDownload = Instant.MIN;
     private final List<Instant> segmentErrorTimestamps = new LinkedList<>();
 
@@ -90,14 +88,13 @@ public abstract class AbstractHlsDownload extends AbstractDownload {
 
     @Override
     public long getSizeInByte() {
-        return sizeInByte.get();
+        return getDownloadedBytes().get();
     }
 
     protected abstract OutputStream getSegmentOutputStream(Segment segment) throws IOException;
 
     protected void segmentDownloadFinished(SegmentDownload segmentDownload) { // NOSONAR
-        long bytes = getDownloadedBytes().addAndGet(segmentDownload.size());
+        getDownloadedBytes().addAndGet(segmentDownload.size());
-        sizeInByte.addAndGet(bytes);
         lastSegmentDownload = Instant.now();
     }
 

common/src/main/java/ctbrec/sites/camsoda/CamsodaModel.java

@@ -211,7 +211,7 @@ public class CamsodaModel extends AbstractModel {
         } else {
             try {
                 List<StreamSource> sources = getStreamSources();
-                LOG.debug("stream sources {}", sources);
+                LOG.debug("{}:{} stream sources {}", getSite().getName(), getName(), sources);
                 if (sources.isEmpty()) {
                     return new int[]{0, 0};
                 } else {

common/src/main/java/ctbrec/sites/stripchat/StripchatHttpClient.java

@@ -6,16 +6,15 @@ import ctbrec.io.HttpClient;
 import ctbrec.io.HttpException;
 import lombok.Getter;
 import lombok.extern.slf4j.Slf4j;
-import okhttp3.MediaType;
+import okhttp3.*;
-import okhttp3.Request;
-import okhttp3.RequestBody;
-import okhttp3.Response;
 import org.json.JSONException;
 import org.json.JSONObject;
 
 import java.io.IOException;
+import java.net.URLDecoder;
 
 import static ctbrec.io.HttpConstants.*;
+import static java.nio.charset.StandardCharsets.UTF_8;
 
 @Slf4j
 public class StripchatHttpClient extends HttpClient {
@@ -96,7 +95,7 @@ public class StripchatHttpClient extends HttpClient {
     }
 
     private void loadCsrfToken() throws IOException {
-        String url = Stripchat.baseUri + "/api/front/v2/config?requestPath=%2F&timezoneOffset=0";
+        String url = Stripchat.baseUri + "/api/front/v2/config/data?requestPath=%2F&timezoneOffset=0";
         Request request = new Request.Builder()
                 .url(url)
                 .header(ACCEPT, MIMETYPE_APPLICATION_JSON)
@@ -112,6 +111,25 @@ public class StripchatHttpClient extends HttpClient {
                 csrfToken = data.optString("csrfToken");
                 csrfTimestamp = data.optString("csrfTimestamp");
                 csrfNotifyTimestamp = data.optString("csrfNotifyTimestamp");
+            } else {
+                throw new HttpException(response.code(), response.message());
+            }
+        }
+    }
+
+    private void loadJwtToken() throws IOException {
+        String url = Stripchat.baseUri + "/api/front/v2/config?requestPath=%2F&timezoneOffset=0";
+        Request request = new Request.Builder()
+                .url(url)
+                .header(ACCEPT, MIMETYPE_APPLICATION_JSON)
+                .header(USER_AGENT, config.getSettings().httpUserAgent)
+                .header(ORIGIN, Stripchat.baseUri)
+                .header(REFERER, Stripchat.baseUri)
+                .header(CONTENT_TYPE, MIMETYPE_APPLICATION_JSON)
+                .build();
+        try (Response response = execute(request)) {
+            if (response.isSuccessful()) {
+                JSONObject resp = new JSONObject(response.body().string());
                 JSONObject config = resp.getJSONObject("config");
                 jwtToken = config.optString("jwtToken");
             } else {
@@ -127,7 +145,7 @@ public class StripchatHttpClient extends HttpClient {
      */
     public boolean checkLoginSuccess() throws IOException {
         try {
-            loadCsrfToken();
+            loadJwtToken();
         } catch (Exception e) {
             log.info("Login check returned unsuccessful: {}", e.getLocalizedMessage());
             return false;
@@ -158,4 +176,15 @@ public class StripchatHttpClient extends HttpClient {
         }
         return userId;
     }
+
+    public JSONObject getAmpl() {
+        try {
+            Cookie cookie = getCookieJar().getCookie(HttpUrl.parse(Stripchat.baseUri), "baseAmpl");
+            String json = URLDecoder.decode(cookie.value(), UTF_8);
+            JSONObject ampl = new JSONObject(json);
+            return ampl;
+        } catch (Exception ex) {
+            return new JSONObject();
+        }
+    }
 }

common/src/main/java/ctbrec/sites/stripchat/StripchatModel.java

@@ -25,10 +25,7 @@ import java.io.InputStream;
 import java.text.MessageFormat;
 import java.time.Duration;
 import java.time.Instant;
-import java.util.ArrayList;
+import java.util.*;
-import java.util.List;
-import java.util.Locale;
-import java.util.Optional;
 import java.util.concurrent.ExecutionException;
 
 import static ctbrec.Model.State.*;
@@ -42,7 +39,7 @@ public class StripchatModel extends AbstractModel {
     private int[] resolution = new int[]{0, 0};
     private int modelId = 0;
     private boolean isVr = false;
-    private JSONObject modelInfo;
+    private transient JSONObject modelInfo;
 
     private transient Instant lastInfoRequest = Instant.EPOCH;
 
@@ -85,8 +82,8 @@ public class StripchatModel extends AbstractModel {
     }
 
     private JSONObject getModelInfo() throws IOException {
-        if (Duration.between(lastInfoRequest, Instant.now()).getSeconds() < 5) {
+        if (Objects.nonNull(modelInfo) && Duration.between(lastInfoRequest, Instant.now()).getSeconds() < 5) {
-            return Optional.ofNullable(modelInfo).orElse(new JSONObject());
+            return modelInfo;
         }
         lastInfoRequest = Instant.now();
         modelInfo = loadModelInfo();
@@ -251,6 +248,8 @@ public class StripchatModel extends AbstractModel {
         requestParams.put("csrfToken", client.getCsrfToken());
         requestParams.put("csrfTimestamp", client.getCsrfTimestamp());
         requestParams.put("csrfNotifyTimestamp", client.getCsrfNotifyTimestamp());
+        requestParams.put("uniq", getUniq());
+        requestParams.put("ampl", client.getAmpl());
         RequestBody body = RequestBody.Companion.create(requestParams.toString(), JSON);
         Request request = new Request.Builder()
                 .url(url)
@@ -286,6 +285,7 @@ public class StripchatModel extends AbstractModel {
         requestParams.put("csrfToken", client.getCsrfToken());
         requestParams.put("csrfTimestamp", client.getCsrfTimestamp());
         requestParams.put("csrfNotifyTimestamp", client.getCsrfNotifyTimestamp());
+        requestParams.put("uniq", getUniq());
         RequestBody body = RequestBody.Companion.create(requestParams.toString(), JSON);
         Request request = new Request.Builder()
                 .url(url)
@@ -335,4 +335,15 @@ public class StripchatModel extends AbstractModel {
             return new MergedFfmpegHlsDownload(getSite().getHttpClient());
         }
     }
+
+    protected String getUniq() {
+        Random r = new Random();
+        String dict = "0123456789abcdefghijklmnopqarstvwxyz";
+        char[] text = new char[16];
+        for (int i = 0; i < 16; i++) {
+            text[i] = dict.charAt(r.nextInt(dict.length()));
+        }
+        return new String(text);
+    }
+
 }

master/pom.xml

@@ -6,7 +6,7 @@
     <groupId>ctbrec</groupId>
     <artifactId>master</artifactId>
     <packaging>pom</packaging>
-    <version>5.2.2</version>
+    <version>5.2.3</version>
 
     <modules>
         <module>../common</module>

server/pom.xml

@@ -8,7 +8,7 @@
     <parent>
         <groupId>ctbrec</groupId>
         <artifactId>master</artifactId>
-        <version>5.2.2</version>
+        <version>5.2.3</version>
         <relativePath>../master</relativePath>
     </parent>