j62
/
ctbrec
2
2
Fork 2
Code Issues 4 Pull Requests Packages Projects Releases 3 Wiki Activity

First kind of working model groups 

# Conflicts:
#	common/src/main/java/ctbrec/AbstractModel.java
#	common/src/main/java/ctbrec/Model.java
#	common/src/main/java/ctbrec/ModelGroup.java
#	common/src/main/java/ctbrec/ModelGroupEntry.java
#	common/src/main/java/ctbrec/recorder/NextGenLocalRecorder.java
#	common/src/main/java/ctbrec/recorder/Recorder.java
#	common/src/main/java/ctbrec/recorder/RemoteRecorder.java
This commit is contained in:
0xb00bface 2021-02-19 17:06:57 +01:00
parent 3fea55ff17
commit 4fd7b7ddd0
12 changed files with 385 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
common/src/main/java/ctbrec/AbstractModel.java
View File

@ -41,6 +41,7 @@ public abstract class AbstractModel implements Model {
private Instant lastRecorded;
private Instant recordUntil;
private SubsequentAction recordUntilSubsequentAction;
private Optional<ModelGroupEntry> modelGroup = Optional.empty(); // NOSONAR
@Override
public boolean isOnline() throws IOException, ExecutionException, InterruptedException {
@ -309,4 +310,14 @@ public abstract class AbstractModel implements Model {
}
return true;
}
@Override
public Optional<ModelGroupEntry> getModelGroup() {
return modelGroup;
}
@Override
public void setModelGroup(ModelGroupEntry modelGroup) {
this.modelGroup = Optional.ofNullable(modelGroup);
}
}

4
common/src/main/java/ctbrec/Config.java
View File

@ -19,6 +19,7 @@ import java.util.Map.Entry;
import java.util.Objects;
import java.util.Optional;
import java.util.stream.Collectors;
import java.util.UUID;
import org.apache.commons.io.FileUtils;
import org.slf4j.Logger;
@ -31,6 +32,7 @@ import ctbrec.Settings.SplitStrategy;
import ctbrec.io.FileJsonAdapter;
import ctbrec.io.ModelJsonAdapter;
import ctbrec.io.PostProcessorJsonAdapter;
import ctbrec.io.UuidJSonAdapter;
import ctbrec.recorder.postprocessing.DeleteTooShort;
import ctbrec.recorder.postprocessing.PostProcessor;
import ctbrec.recorder.postprocessing.RemoveKeepFile;
@ -74,6 +76,7 @@ public class Config {
.add(Model.class, new ModelJsonAdapter(sites))
.add(PostProcessor.class, new PostProcessorJsonAdapter())
.add(File.class, new FileJsonAdapter())
.add(UUID.class, new UuidJSonAdapter())
.build();
JsonAdapter<Settings> adapter = moshi.adapter(Settings.class).lenient();
File configFile = new File(configDir, filename);
@ -234,6 +237,7 @@ public class Config {
.add(Model.class, new ModelJsonAdapter())
.add(PostProcessor.class, new PostProcessorJsonAdapter())
.add(File.class, new FileJsonAdapter())
.add(UUID.class, new UuidJSonAdapter())
.build();
JsonAdapter<Settings> adapter = moshi.adapter(Settings.class).indent(" ");
String json = adapter.toJson(settings);

4
common/src/main/java/ctbrec/Model.java
View File

@ -4,6 +4,7 @@ import java.io.IOException;
import java.io.Serializable;
import java.time.Instant;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.ExecutionException;
import javax.xml.bind.JAXBException;
@ -148,4 +149,7 @@ public interface Model extends Comparable<Model>, Serializable {
*/
public boolean exists() throws IOException;
public void setModelGroup(ModelGroupEntry modelGroupEntry);
public Optional<ModelGroupEntry> getModelGroup();
}

74
common/src/main/java/ctbrec/ModelGroup.java Normal file
View File

@ -0,0 +1,74 @@
package ctbrec;
import java.io.Serializable;
import java.util.Collections;
import java.util.LinkedList;
import java.util.List;
import java.util.Objects;
import java.util.UUID;
public class ModelGroup implements Serializable {
private static final long serialVersionUID = 1L;
private UUID id;
private String name;
private transient List<Model> models = new LinkedList<>();
public UUID getId() {
return id;
}
public void setId(UUID id) {
this.id = id;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public List<Model> getModels() {
return models;
}
public void setModels(List<Model> models) {
this.models = models;
}
@Override
public int hashCode() {
return Objects.hash(id);
}
@Override
public boolean equals(Object obj) {
if (this == obj) {
return true;
}
if (obj == null) {
return false;
}
if (getClass() != obj.getClass()) {
return false;
}
ModelGroup other = (ModelGroup) obj;
return Objects.equals(id, other.id);
}
@Override
public String toString() {
return "ModelGroup [id=" + id + ", name=" + name + ", models=" + models + "]";
}
public void add(Model model) {
models.add(model);
Collections.sort(models, (m1, m2) -> {
int index1 = m1.getModelGroup().map(ModelGroupEntry::getIndex).orElse(0);
int index2 = m2.getModelGroup().map(ModelGroupEntry::getIndex).orElse(0);
return index1 - index2;
});
}
}

53
common/src/main/java/ctbrec/ModelGroupEntry.java Normal file
View File

@ -0,0 +1,53 @@
package ctbrec;
import java.io.Serializable;
import java.util.Objects;
import java.util.UUID;
public class ModelGroupEntry implements Serializable {
private static final long serialVersionUID = 0L;
private UUID id;
private int index;
public UUID getId() {
return id;
}
public void setId(UUID id) {
this.id = id;
}
public int getIndex() {
return index;
}
public void setIndex(int index) {
this.index = index;
}
@Override
public int hashCode() {
return Objects.hash(id, index);
}
@Override
public boolean equals(Object obj) {
if (this == obj) {
return true;
}
if (obj == null) {
return false;
}
if (getClass() != obj.getClass()) {
return false;
}
ModelGroupEntry other = (ModelGroupEntry) obj;
return Objects.equals(id, other.id) && index == other.index;
}
@Override
public String toString() {
return "ModelGroupEntry [id=" + id + ", index=" + index + "]";
}
}

3
common/src/main/java/ctbrec/Settings.java
View File

@ -3,8 +3,10 @@ package ctbrec;
import java.io.File;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import ctbrec.event.EventHandlerConfiguration;
import ctbrec.recorder.postprocessing.PostProcessor;
@ -108,6 +110,7 @@ public class Settings {
public long minimumSpaceLeftInBytes = 0;
public Map<String, String> modelNotes = new HashMap<>();
public List<Model> models = new ArrayList<>();
public Set<ModelGroup> modelGroups = new HashSet<>();
@Deprecated
public List<Model> modelsIgnored = new ArrayList<>();
public boolean monitorClipboard = false;

16
common/src/main/java/ctbrec/io/ModelJsonAdapter.java
View File

@ -5,6 +5,7 @@ import java.lang.reflect.InvocationTargetException;
import java.time.Instant;
import java.util.List;
import java.util.Optional;
import java.util.UUID;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -15,6 +16,7 @@ import com.squareup.moshi.JsonReader.Token;
import com.squareup.moshi.JsonWriter;
import ctbrec.Model;
import ctbrec.ModelGroupEntry;
import ctbrec.SubsequentAction;
import ctbrec.sites.Site;
import ctbrec.sites.chaturbate.ChaturbateModel;
@ -69,6 +71,18 @@ public class ModelJsonAdapter extends JsonAdapter<Model> {
model.setRecordUntil(Instant.ofEpochMilli(reader.nextLong()));
} else if (key.equals("recordUntilSubsequentAction")) {
model.setRecordUntilSubsequentAction(SubsequentAction.valueOf(reader.nextString()));
} else if (key.equals("groupId")) {
ModelGroupEntry entry = new ModelGroupEntry();
entry.setId(UUID.fromString(reader.nextString()));
model.setModelGroup(entry);
} else if (key.equals("groupIndex")) {
model.getModelGroup().ifPresent(mg -> {
try {
mg.setIndex(reader.nextInt());
} catch (IOException e) {
LOG.error("Error while reading model group index", e);
}
});
} else if (key.equals("siteSpecific")) {
reader.beginObject();
try {
@ -114,6 +128,8 @@ public class ModelJsonAdapter extends JsonAdapter<Model> {
writer.name("lastRecorded").value(model.getLastRecorded().toEpochMilli());
writer.name("recordUntil").value(model.getRecordUntil().toEpochMilli());
writer.name("recordUntilSubsequentAction").value(model.getRecordUntilSubsequentAction().name());
writer.name("groupId").value(model.getModelGroup().map(ModelGroupEntry::getId).map(Object::toString).orElse(null));
writer.name("groupIndex").value(model.getModelGroup().map(ModelGroupEntry::getIndex).orElse(0));
writer.name("siteSpecific");
writer.beginObject();
model.writeSiteSpecificData(writer);

22
common/src/main/java/ctbrec/io/UuidJSonAdapter.java Normal file
View File

@ -0,0 +1,22 @@
package ctbrec.io;
import java.io.IOException;
import java.util.UUID;
import com.squareup.moshi.JsonAdapter;
import com.squareup.moshi.JsonReader;
import com.squareup.moshi.JsonWriter;
public class UuidJSonAdapter extends JsonAdapter<UUID> {
@Override
public UUID fromJson(JsonReader reader) throws IOException {
return UUID.fromString(reader.nextString());
}
@Override
public void toJson(JsonWriter writer, UUID value) throws IOException {
writer.value(value.toString());
}
}

95
common/src/main/java/ctbrec/recorder/NextGenLocalRecorder.java
View File

@ -22,6 +22,7 @@ import java.util.Map;
import java.util.NoSuchElementException;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.CompletableFuture;
@ -45,6 +46,8 @@ import com.google.common.eventbus.Subscribe;
import ctbrec.Config;
import ctbrec.Model;
import ctbrec.ModelGroup;
import ctbrec.ModelGroupEntry;
import ctbrec.Recording;
import ctbrec.Recording.State;
import ctbrec.event.Event;
@ -54,6 +57,7 @@ import ctbrec.event.NoSpaceLeftEvent;
import ctbrec.event.RecordingStateChangedEvent;
import ctbrec.io.HttpClient;
import ctbrec.recorder.download.Download;
import ctbrec.recorder.postprocessing.PostProcessingContext;
import ctbrec.recorder.postprocessing.PostProcessor;
import ctbrec.sites.Site;
@ -88,18 +92,8 @@ public class NextGenLocalRecorder implements Recorder {
downloadPool = Executors.newScheduledThreadPool(5, createThreadFactory("Download", MAX_PRIORITY));
threadPoolScaler = new ThreadPoolScaler((ThreadPoolExecutor) downloadPool, 5);
recordingManager = new RecordingManager(config, sites);
config.getSettings().models.stream().forEach(m -> {
if (m.getSite() != null) {
if (m.getSite().isEnabled()) {
models.add(m);
} else {
LOG.info("{} disabled -> ignoring {}", m.getSite().getName(), m.getName());
}
} else {
LOG.info("Site for model {} is unknown -> ignoring", m.getName());
}
});
loadModels();
createModelGroups();
int ppThreads = config.getSettings().postProcessingThreads;
ppPool = new ThreadPoolExecutor(ppThreads, ppThreads, 5, TimeUnit.MINUTES, ppQueue, createThreadFactory("PP", MIN_PRIORITY));
@ -127,6 +121,30 @@ public class NextGenLocalRecorder implements Recorder {
}, 1, 1, TimeUnit.SECONDS);
}
private void loadModels() {
config.getSettings().models.stream().forEach(m -> {
if (m.getSite() != null) {
if (m.getSite().isEnabled()) {
models.add(m);
} else {
LOG.info("{} disabled -> ignoring {}", m.getSite().getName(), m.getName());
}
} else {
LOG.info("Site for model {} is unknown -> ignoring", m.getName());
}
});
}
private void createModelGroups() {
for (Model model : models) {
if(model.getModelGroup().isPresent()) {
ModelGroupEntry groupEntry = model.getModelGroup().get(); // NOSONAR
ModelGroup group = getModelGroup(groupEntry.getId());
group.add(model);
}
}
}
private void startCompletionHandler() {
downloadCompletionPool.submit(() -> {
while (!Thread.currentThread().isInterrupted()) {
@ -209,9 +227,10 @@ public class NextGenLocalRecorder implements Recorder {
recordingManager.saveRecording(recording);
recording.postprocess();
List<PostProcessor> postProcessors = config.getSettings().postProcessors;
PostProcessingContext ctx = createPostProcessingContext(recording);
for (PostProcessor postProcessor : postProcessors) {
LOG.debug("Running post-processor: {}", postProcessor.getName());
boolean continuePP = postProcessor.postprocess(recording, recordingManager, config);
boolean continuePP = postProcessor.postprocess(ctx);
if (!continuePP) {
break;
}
@ -237,6 +256,15 @@ public class NextGenLocalRecorder implements Recorder {
});
}
private PostProcessingContext createPostProcessingContext(Recording recording) {
PostProcessingContext ctx = new PostProcessingContext();
ctx.setConfig(config);
ctx.setRecorder(this);
ctx.setRecording(recording);
ctx.setRecordingManager(recordingManager);
return ctx;
}
private void setRecordingStatus(Recording recording, State status) {
recording.setStatus(status);
RecordingStateChangedEvent evt = new RecordingStateChangedEvent(recording.getDownload().getTarget(), status, recording.getModel(),
@ -761,4 +789,45 @@ public class NextGenLocalRecorder implements Recorder {
public int getModelCount() {
return (int) models.stream().filter(m -> !m.isMarkedForLaterRecording()).count();
}
@Override
public Set<ModelGroup> getModelGroups() {
return config.getSettings().modelGroups;
}
@Override
public ModelGroup getModelGroup(UUID id) {
for (ModelGroup group : getModelGroups()) {
if (Objects.equals(group.getId(), id)) {
return group;
}
}
throw new NoSuchElementException("ModelGroup with id " + id + " not found");
}
@Override
public ModelGroup createModelGroup(String name) {
ModelGroup group = new ModelGroup();
group.setName(name);
config.getSettings().modelGroups.add(group);
try {
config.save();
} catch (IOException e) {
LOG.error("Couldn't save new model group", e);
}
return group;
}
@Override
public void deleteModelGroup(ModelGroup group) {
for (Model model : group.getModels()) {
model.setModelGroup(null);
}
config.getSettings().modelGroups.remove(group);
try {
config.save();
} catch (IOException e) {
LOG.error("Couldn't delete model group", e);
}
}
}

20
common/src/main/java/ctbrec/recorder/Recorder.java
View File

@ -1,15 +1,18 @@
package ctbrec.recorder;
import ctbrec.Model;
import ctbrec.Recording;
import ctbrec.io.HttpClient;
import java.io.IOException;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import java.util.List;
import java.util.Set;
import java.util.UUID;
import java.util.stream.Collectors;
import ctbrec.Model;
import ctbrec.ModelGroup;
import ctbrec.Recording;
import ctbrec.io.HttpClient;
public interface Recorder {
public void addModel(Model model) throws IOException, InvalidKeyException, NoSuchAlgorithmException;
@ -143,5 +146,14 @@ public interface Recorder {
*/
public void resume() throws InvalidKeyException, NoSuchAlgorithmException, IOException;
/**
* Returns the number of models, which are on the recording list and not marked for later recording
* @return
*/
public int getModelCount();
public Set<ModelGroup> getModelGroups();
public ModelGroup createModelGroup(String name);
public ModelGroup getModelGroup(UUID uuid);
public void deleteModelGroup(ModelGroup group);
}

36
common/src/main/java/ctbrec/recorder/RecordingPreconditions.java
View File

@ -12,6 +12,8 @@ import org.slf4j.LoggerFactory;
import ctbrec.Config;
import ctbrec.Model;
import ctbrec.ModelGroup;
import ctbrec.ModelGroupEntry;
import ctbrec.Recording;
import ctbrec.recorder.download.Download;
@ -36,6 +38,7 @@ public class RecordingPreconditions {
ensureEnoughSpaceForRecording();
ensureDownloadSlotAvailable(model);
ensureModelIsOnline(model);
ensureNoOtherFromModelGroupIsRecording(model);
}
private void ensureModelIsOnline(Model model) {
@ -130,4 +133,37 @@ public class RecordingPreconditions {
int concurrentRecordings = Config.getInstance().getSettings().concurrentRecordings;
return concurrentRecordings == 0 || concurrentRecordings > 0 && recorder.getRecordingProcesses().size() < concurrentRecordings;
}
private void ensureNoOtherFromModelGroupIsRecording(Model model) {
if (model.getModelGroup().isPresent()) {
ModelGroupEntry modelGroupEntry = model.getModelGroup().get(); // NOSONAR
ModelGroup modelGroup = recorder.getModelGroup(modelGroupEntry.getId());
for (Model groupModel : modelGroup.getModels()) {
if (groupModel.equals(model)) {
return; // no other model with lower group index is online, start recording
} else if (otherModelCanBeRecorded(groupModel)) {
throw new PreconditionNotMetException(groupModel + " from the same group is already recorded");
}
}
}
}
private boolean otherModelCanBeRecorded(Model model) {
try {
ensureRecorderIsActive();
ensureModelIsNotSuspended(model);
ensureModelIsNotMarkedForLaterRecording(model);
ensureRecordUntilIsInFuture(model);
ensureModelShouldBeRecorded(model);
ensureEnoughSpaceForRecording();
ensureDownloadSlotAvailable(model);
ensureModelIsOnline(model);
return true;
} catch (PreconditionNotMetException e) {
// precondition for other model not met
} catch (IOException e) {
LOG.warn("Couldn't check if preconditions of other model from group are met. Assuming she's offline", e);
}
return false;
}
}

85
common/src/main/java/ctbrec/recorder/RemoteRecorder.java
View File

@ -1,25 +1,5 @@
package ctbrec.recorder;
import com.squareup.moshi.JsonAdapter;
import com.squareup.moshi.Moshi;
import ctbrec.Config;
import ctbrec.Hmac;
import ctbrec.Model;
import ctbrec.Recording;
import ctbrec.event.EventBusHolder;
import ctbrec.event.NoSpaceLeftEvent;
import ctbrec.event.RecordingStateChangedEvent;
import ctbrec.io.*;
import ctbrec.sites.Site;
import okhttp3.MediaType;
import okhttp3.Request;
import okhttp3.Request.Builder;
import okhttp3.RequestBody;
import okhttp3.Response;
import org.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
@ -27,7 +7,43 @@ import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import java.time.Duration;
import java.time.Instant;
import java.util.*;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.NoSuchElementException;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.UUID;
import org.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.squareup.moshi.JsonAdapter;
import com.squareup.moshi.Moshi;
import ctbrec.Config;
import ctbrec.Hmac;
import ctbrec.Model;
import ctbrec.ModelGroup;
import ctbrec.Recording;
import ctbrec.event.EventBusHolder;
import ctbrec.event.NoSpaceLeftEvent;
import ctbrec.event.RecordingStateChangedEvent;
import ctbrec.io.BandwidthMeter;
import ctbrec.io.FileJsonAdapter;
import ctbrec.io.HttpClient;
import ctbrec.io.HttpException;
import ctbrec.io.InstantJsonAdapter;
import ctbrec.io.ModelJsonAdapter;
import ctbrec.sites.Site;
import okhttp3.MediaType;
import okhttp3.Request;
import okhttp3.Request.Builder;
import okhttp3.RequestBody;
import okhttp3.Response;
public class RemoteRecorder implements Recorder {
@ -592,4 +608,31 @@ public class RemoteRecorder implements Recorder {
public int getModelCount() {
return (int) models.stream().filter(m -> !m.isMarkedForLaterRecording()).count();
}
@Override
public Set<ModelGroup> getModelGroups() {
// TODO Auto-generated method stub
return null;
}
@Override
public ModelGroup createModelGroup(String name) {
// TODO Auto-generated method stub
return null;
}
@Override
public void deleteModelGroup(ModelGroup group) {
// TODO Auto-generated method stub
}
@Override
public ModelGroup getModelGroup(UUID id) {
for (ModelGroup group : getModelGroups()) {
if (Objects.equals(group.getId(), id)) {
return group;
}
}
throw new NoSuchElementException("ModelGroup with id " + id + " not found");
}
}