Add setting to configure the directory structure

In client mode you can now select how the stream is stored. There
are three options:
* all recordings in one directory
* one directory for each model
* one directory for each recording
This commit is contained in:
0xboobface 2018-11-18 18:23:19 +01:00
parent f7c581a517
commit 69cfc8a6ec
11 changed files with 224 additions and 125 deletions

View File

@ -7,6 +7,8 @@ import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.file.Files;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.List;
import java.util.Objects;
@ -105,4 +107,31 @@ public class Config {
public File getConfigDir() {
return configDir;
}
public File getFileForRecording(Model model) {
File dirForRecording = getDirForRecording(model);
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd_HH-mm");
String startTime = sdf.format(new Date());
File targetFile = new File(dirForRecording, model.getName() + '_' + startTime + ".ts");
if(getSettings().splitRecordings > 0) {
LOG.debug("Splitting recordings every {} seconds", getSettings().splitRecordings);
targetFile = new File(targetFile.getAbsolutePath().replaceAll("\\.ts", "-00000.ts"));
}
return targetFile;
}
private File getDirForRecording(Model model) {
switch(getSettings().recordingsDirStructure) {
case ONE_PER_MODEL:
return new File(getSettings().recordingsDir, model.getName());
case ONE_PER_RECORDING:
File modelDir = new File(getSettings().recordingsDir, model.getName());
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd_HH-mm");
String startTime = sdf.format(new Date());
return new File(modelDir, startTime);
case FLAT:
default:
return new File(getSettings().recordingsDir);
}
}
}

View File

@ -1,6 +1,5 @@
package ctbrec;
import java.io.File;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.time.Instant;
@ -94,6 +93,7 @@ public class Recording {
final int prime = 31;
int result = 1;
result = prime * result + ((modelName == null) ? 0 : modelName.hashCode());
result = prime * result + ((path == null) ? 0 : path.hashCode());
result = prime * result + ((startDate == null) ? 0 : startDate.hashCode());
return result;
}
@ -105,35 +105,21 @@ public class Recording {
if (obj == null)
return false;
Recording other = (Recording) obj;
if (getModelName() == null) {
if (modelName == null) {
if (other.getModelName() != null)
return false;
} else if (!getModelName().equals(other.getModelName()))
} else if (!modelName.equals(other.getModelName()))
return false;
if (getStartDate() == null) {
if (path == null) {
if (other.getPath() != null)
return false;
} else if (!path.equals(other.getPath()))
return false;
if (startDate == null) {
if (other.getStartDate() != null)
return false;
} else if (!getStartDate().equals(other.getStartDate()))
} else if (!startDate.equals(other.getStartDate()))
return false;
return true;
}
public static File mergedFileFromDirectory(File recDir) {
String date = recDir.getName();
String model = recDir.getParentFile().getName();
String filename = model + "-" + date + ".ts";
File mergedFile = new File(recDir, filename);
return mergedFile;
}
public static boolean isMergedRecording(File recDir) {
File mergedFile = mergedFileFromDirectory(recDir);
return mergedFile.exists();
}
public static boolean isMergedRecording(Recording recording) {
String recordingsDir = Config.getInstance().getSettings().recordingsDir;
File recDir = new File(recordingsDir, recording.getPath());
return isMergedRecording(recDir);
}
}

View File

@ -13,6 +13,22 @@ public class Settings {
SOCKS5
}
public enum DirectoryStructure {
FLAT("all recordings in one directory"),
ONE_PER_MODEL("one directory for each model"),
ONE_PER_RECORDING("one directory for each recording");
private String description;
DirectoryStructure(String description) {
this.description = description;
}
@Override
public String toString() {
return description;
}
}
public boolean singlePlayer = true;
public boolean localRecording = true;
public int httpPort = 8080;
@ -20,6 +36,7 @@ public class Settings {
public String httpUserAgent = "Mozilla/5.0 Gecko/20100101 Firefox/62.0";
public String httpServer = "localhost";
public String recordingsDir = System.getProperty("user.home") + File.separator + "ctbrec";
public DirectoryStructure recordingsDirStructure = DirectoryStructure.FLAT;
public String mediaPlayer = "/usr/bin/mpv";
public String postProcessing = "";
public String username = ""; // chaturbate username TODO maybe rename this onetime

View File

@ -3,6 +3,7 @@ package ctbrec.recorder;
import static ctbrec.Recording.STATUS.*;
import java.io.File;
import java.io.FilenameFilter;
import java.io.IOException;
import java.nio.file.Files;
import java.security.InvalidKeyException;
@ -15,6 +16,7 @@ import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
@ -33,6 +35,7 @@ import ctbrec.Config;
import ctbrec.Model;
import ctbrec.OS;
import ctbrec.Recording;
import ctbrec.Recording.STATUS;
import ctbrec.io.HttpException;
import ctbrec.io.StreamRedirectThread;
import ctbrec.recorder.PlaylistGenerator.InvalidPlaylistException;
@ -44,8 +47,9 @@ import ctbrec.recorder.server.RecorderHttpClient;
public class LocalRecorder implements Recorder {
private static final transient Logger LOG = LoggerFactory.getLogger(LocalRecorder.class);
private static final boolean IGNORE_CACHE = true;
private static final String DATE_FORMAT = "yyyy-MM-dd_HH-mm";
private List<Model> models = Collections.synchronizedList(new ArrayList<>());
private Map<Model, Download> recordingProcesses = Collections.synchronizedMap(new HashMap<>());
private Map<File, PlaylistGenerator> playlistGenerators = new HashMap<>();
@ -180,8 +184,8 @@ public class LocalRecorder implements Recorder {
MergedHlsDownload d = (MergedHlsDownload) download;
String[] args = new String[] {
postProcessing,
d.getDirectory().getAbsolutePath(),
d.getTargetFile().getAbsolutePath(),
d.getTarget().getParentFile().getAbsolutePath(),
d.getTarget().getAbsolutePath(),
d.getModel().getName(),
d.getModel().getSite().getName(),
Long.toString(download.getStartTime().getEpochSecond())
@ -330,7 +334,7 @@ public class LocalRecorder implements Recorder {
restart.add(m);
if(config.isServerMode()) {
try {
finishRecording(d.getDirectory());
finishRecording(d.getTarget());
} catch(Exception e) {
LOG.error("Error while finishing recording for model {}", m.getName(), e);
}
@ -448,7 +452,7 @@ public class LocalRecorder implements Recorder {
File recordingsDir = new File(config.getSettings().recordingsDir);
File recDir = new File(recordingsDir, rec.getPath());
for (Entry<Model, Download> download : recordingProcesses.entrySet()) {
if (download.getValue().getDirectory().equals(recDir)) {
if (download.getValue().getTarget().equals(recDir)) {
recordingProcessFound = true;
}
}
@ -476,6 +480,74 @@ public class LocalRecorder implements Recorder {
@Override
public List<Recording> getRecordings() {
if(Config.getInstance().isServerMode()) {
return listSegmentedRecordings();
} else {
return listMergedRecordings();
}
}
private List<Recording> listMergedRecordings() {
File recordingsDir = new File(config.getSettings().recordingsDir);
List<File> possibleRecordings = new LinkedList<>();
listRecursively(recordingsDir, possibleRecordings, (dir, name) -> name.matches(".*?_\\d{4}-\\d{2}-\\d{2}_\\d{2}-\\d{2}\\.ts"));
SimpleDateFormat sdf = new SimpleDateFormat(DATE_FORMAT);
List<Recording> recordings = new ArrayList<>();
for (File ts: possibleRecordings) {
try {
String filename = ts.getName();
String dateString = filename.substring(filename.length() - 3 - DATE_FORMAT.length(), filename.length() - 3);
Date startDate = sdf.parse(dateString);
Recording recording = new Recording();
recording.setModelName(filename.substring(0, filename.length() - 4 - DATE_FORMAT.length()));
recording.setStartDate(Instant.ofEpochMilli(startDate.getTime()));
String path = ts.getAbsolutePath().replace(config.getSettings().recordingsDir, "");
if(!path.startsWith("/")) {
path = '/' + path;
}
recording.setPath(path);
recording.setSizeInByte(ts.length());
recording.setStatus(getStatus(recording));
recordings.add(recording);
} catch(Exception e) {
LOG.error("Ignoring {} - {}", ts.getAbsolutePath(), e.getMessage());
}
}
return recordings;
}
private STATUS getStatus(Recording recording) {
File absolutePath = new File(Config.getInstance().getSettings().recordingsDir, recording.getPath());
PlaylistGenerator playlistGenerator = playlistGenerators.get(absolutePath);
if (playlistGenerator != null) {
recording.setProgress(playlistGenerator.getProgress());
return GENERATING_PLAYLIST;
}
if (config.isServerMode()) {
if (recording.hasPlaylist()) {
return FINISHED;
} else {
return RECORDING;
}
} else {
boolean dirUsedByRecordingProcess = false;
for (Download download : recordingProcesses.values()) {
if(absolutePath.equals(download.getTarget())) {
dirUsedByRecordingProcess = true;
break;
}
}
if(dirUsedByRecordingProcess) {
return RECORDING;
} else {
return FINISHED;
}
}
}
private List<Recording> listSegmentedRecordings() {
List<Recording> recordings = new ArrayList<>();
File recordingsDir = new File(config.getSettings().recordingsDir);
File[] subdirs = recordingsDir.listFiles();
@ -484,25 +556,19 @@ public class LocalRecorder implements Recorder {
}
for (File subdir : subdirs) {
// only consider directories
if (!subdir.isDirectory()) {
continue;
}
// ignore empty directories
File[] recordingsDirs = subdir.listFiles();
if(recordingsDirs.length == 0) {
if(recordingsDirs == null || recordingsDirs.length == 0) {
continue;
}
// start going over valid directories
for (File rec : recordingsDirs) {
String pattern = "yyyy-MM-dd_HH-mm";
SimpleDateFormat sdf = new SimpleDateFormat(pattern);
SimpleDateFormat sdf = new SimpleDateFormat(DATE_FORMAT);
if (rec.isDirectory()) {
try {
// ignore directories, which are probably not created by ctbrec
if (rec.getName().length() != pattern.length()) {
if (rec.getName().length() != DATE_FORMAT.length()) {
continue;
}
// ignore empty directories
@ -518,33 +584,7 @@ public class LocalRecorder implements Recorder {
recording.setSizeInByte(getSize(rec));
File playlist = new File(rec, "playlist.m3u8");
recording.setHasPlaylist(playlist.exists());
PlaylistGenerator playlistGenerator = playlistGenerators.get(rec);
if (playlistGenerator != null) {
recording.setStatus(GENERATING_PLAYLIST);
recording.setProgress(playlistGenerator.getProgress());
} else {
if (config.isServerMode()) {
if (recording.hasPlaylist()) {
recording.setStatus(FINISHED);
} else {
recording.setStatus(RECORDING);
}
} else {
boolean dirUsedByRecordingProcess = false;
for (Download download : recordingProcesses.values()) {
if(rec.equals(download.getDirectory())) {
dirUsedByRecordingProcess = true;
break;
}
}
if(dirUsedByRecordingProcess) {
recording.setStatus(RECORDING);
} else {
recording.setStatus(FINISHED);
}
}
}
recording.setStatus(getStatus(recording));
recordings.add(recording);
} catch (Exception e) {
LOG.debug("Ignoring {} - {}", rec.getAbsolutePath(), e.getMessage());
@ -555,6 +595,20 @@ public class LocalRecorder implements Recorder {
return recordings;
}
private void listRecursively(File dir, List<File> result, FilenameFilter filenameFilter) {
File[] files = dir.listFiles();
if(files != null) {
for (File file : files) {
if(file.isDirectory()) {
listRecursively(file, result, filenameFilter);
}
if(filenameFilter.accept(dir, file.getName())) {
result.add(file);
}
}
}
}
private long getSize(File rec) {
long size = 0;
File[] files = rec.listFiles();
@ -567,11 +621,28 @@ public class LocalRecorder implements Recorder {
@Override
public void delete(Recording recording) throws IOException {
File recordingsDir = new File(config.getSettings().recordingsDir);
File directory = new File(recordingsDir, recording.getPath());
delete(directory);
File path = new File(recordingsDir, recording.getPath());
LOG.debug("Deleting {}", path);
if(path.isFile()) {
Files.delete(path.toPath());
deleteEmptyParents(path);
} else {
deleteDirectory(path);
deleteEmptyParents(path);
}
}
private void delete(File directory) throws IOException {
private void deleteEmptyParents(File path) throws IOException {
File parent = path.getParentFile();
while(parent != null && parent.list() != null && parent.list().length == 0) {
LOG.debug("Deleting empty directory {}", parent.getAbsolutePath());
Files.delete(parent.toPath());
parent = parent.getParentFile();
}
}
private void deleteDirectory(File directory) throws IOException {
if (!directory.exists()) {
throw new IOException("Recording does not exist");
}

View File

@ -1,10 +1,8 @@
package ctbrec.recorder.download;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.nio.file.Path;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Collections;
@ -42,7 +40,6 @@ public abstract class AbstractHlsDownload implements Download {
HttpClient client;
volatile boolean running = false;
volatile boolean alive = true;
Path downloadDir;
Instant startTime;
Model model;
@ -117,11 +114,6 @@ public abstract class AbstractHlsDownload implements Download {
return alive;
}
@Override
public File getDirectory() {
return downloadDir.toFile();
}
@Override
public Instant getStartTime() {
return startTime;

View File

@ -11,7 +11,7 @@ public interface Download {
public void start(Model model, Config config) throws IOException;
public void stop();
public boolean isAlive();
public File getDirectory();
public File getTarget();
public Model getModel();
public Instant getStartTime();
}

View File

@ -33,6 +33,8 @@ public class HlsDownload extends AbstractHlsDownload {
private static final transient Logger LOG = LoggerFactory.getLogger(HlsDownload.class);
protected Path downloadDir;
public HlsDownload(HttpClient client) {
super(client);
}
@ -180,4 +182,9 @@ public class HlsDownload extends AbstractHlsDownload {
return false;
}
}
@Override
public File getTarget() {
return downloadDir.toFile();
}
}

View File

@ -10,18 +10,15 @@ import java.io.InputStream;
import java.net.MalformedURLException;
import java.net.URL;
import java.nio.channels.FileChannel;
import java.nio.file.FileSystems;
import java.nio.file.Files;
import java.nio.file.LinkOption;
import java.nio.file.Path;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import java.text.DecimalFormat;
import java.text.SimpleDateFormat;
import java.time.Duration;
import java.time.Instant;
import java.time.ZonedDateTime;
import java.util.Date;
import java.util.concurrent.Callable;
import org.slf4j.Logger;
@ -38,7 +35,6 @@ import com.iheartradio.m3u8.PlaylistException;
import ctbrec.Config;
import ctbrec.Hmac;
import ctbrec.Model;
import ctbrec.Recording;
import ctbrec.io.HttpClient;
import ctbrec.io.HttpException;
import ctbrec.recorder.ProgressListener;
@ -62,7 +58,8 @@ public class MergedHlsDownload extends AbstractHlsDownload {
super(client);
}
public File getTargetFile() {
@Override
public File getTarget() {
return targetFile;
}
@ -70,7 +67,6 @@ public class MergedHlsDownload extends AbstractHlsDownload {
try {
running = true;
super.startTime = Instant.now();
downloadDir = targetFile.getParentFile().toPath();
mergeThread = createMergeThread(targetFile, progressListener, false);
LOG.debug("Merge thread started");
mergeThread.start();
@ -105,28 +101,16 @@ public class MergedHlsDownload extends AbstractHlsDownload {
public void start(Model model, Config config) throws IOException {
this.config = config;
try {
running = true;
super.startTime = Instant.now();
super.model = model;
startTime = ZonedDateTime.now();
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd_HH-mm");
String startTime = sdf.format(new Date());
Path modelDir = FileSystems.getDefault().getPath(config.getSettings().recordingsDir, model.getName());
downloadDir = FileSystems.getDefault().getPath(modelDir.toString(), startTime);
if(!model.isOnline(IGNORE_CACHE)) {
throw new IOException(model.getName() +"'s room is not public");
}
targetFile = Recording.mergedFileFromDirectory(downloadDir.toFile());
File target = targetFile;
if(config.getSettings().splitRecordings > 0) {
LOG.debug("Splitting recordings every {} seconds", config.getSettings().splitRecordings);
target = new File(targetFile.getAbsolutePath().replaceAll("\\.ts", "-00000.ts"));
}
running = true;
super.startTime = Instant.now();
super.model = model;
targetFile = Config.getInstance().getFileForRecording(model);
String segments = getSegmentPlaylistUrl(model);
mergeThread = createMergeThread(target, null, true);
mergeThread = createMergeThread(targetFile, null, true);
mergeThread.start();
if(segments != null) {
downloadSegments(segments, true);
@ -285,6 +269,7 @@ public class MergedHlsDownload extends AbstractHlsDownload {
FileChannel channel = null;
try {
Path downloadDir = targetFile.getParentFile().toPath();
if (!Files.exists(downloadDir, LinkOption.NOFOLLOW_LINKS)) {
Files.createDirectories(downloadDir);
}

View File

@ -113,14 +113,8 @@ public class Player {
Runtime rt = Runtime.getRuntime();
try {
if (Config.getInstance().getSettings().localRecording && rec != null) {
File dir = new File(Config.getInstance().getSettings().recordingsDir, rec.getPath());
File file = null;
if(Recording.isMergedRecording(rec)) {
file = Recording.mergedFileFromDirectory(dir);
} else {
file = new File(dir, "playlist.m3u8");
}
playerProcess = rt.exec(Config.getInstance().getSettings().mediaPlayer + " " + file, OS.getEnvironment(), dir);
File file = new File(Config.getInstance().getSettings().recordingsDir, rec.getPath());
playerProcess = rt.exec(Config.getInstance().getSettings().mediaPlayer + " " + file, OS.getEnvironment(), file.getParentFile());
} else {
if(Config.getInstance().getSettings().requireAuthentication) {
URL u = new URL(url);

View File

@ -295,9 +295,9 @@ public class RecordingsTab extends Tab implements TabSelectionListener {
openDir.setOnAction((e) -> {
String recordingsDir = Config.getInstance().getSettings().recordingsDir;
String path = recording.getPath();
File recdir = new File(recordingsDir, path);
File tsFile = new File(recordingsDir, path);
new Thread(() -> {
DesktopIntegration.open(recdir);
DesktopIntegration.open(tsFile.getParent());
}).start();
});
if(Config.getInstance().getSettings().localRecording) {

View File

@ -1,5 +1,7 @@
package ctbrec.ui;
import static ctbrec.Settings.DirectoryStructure.*;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
@ -12,6 +14,7 @@ import org.slf4j.LoggerFactory;
import ctbrec.Config;
import ctbrec.Hmac;
import ctbrec.Settings;
import ctbrec.Settings.DirectoryStructure;
import ctbrec.sites.ConfigUI;
import ctbrec.sites.Site;
import javafx.beans.value.ChangeListener;
@ -70,6 +73,7 @@ public class SettingsTab extends Tab implements TabSelectionListener {
private ProxySettingsPane proxySettingsPane;
private ComboBox<Integer> maxResolution;
private ComboBox<SplitAfterOption> splitAfter;
private ComboBox<DirectoryStructure> directoryStructure;
private List<Site> sites;
private Label restartLabel;
private Accordion credentialsAccordion = new Accordion();
@ -245,8 +249,9 @@ public class SettingsTab extends Tab implements TabSelectionListener {
}
private Node createLocationsPanel() {
int row = 0;
GridPane layout = createGridLayout();
layout.add(new Label("Recordings Directory"), 0, 0);
layout.add(new Label("Recordings Directory"), 0, row);
recordingsDirectory = new TextField(Config.getInstance().getSettings().recordingsDir);
recordingsDirectory.focusedProperty().addListener(createRecordingsDirectoryFocusListener());
recordingsDirectory.setPrefWidth(400);
@ -254,30 +259,42 @@ public class SettingsTab extends Tab implements TabSelectionListener {
GridPane.setHgrow(recordingsDirectory, Priority.ALWAYS);
GridPane.setColumnSpan(recordingsDirectory, 2);
GridPane.setMargin(recordingsDirectory, new Insets(0, 0, 0, CHECKBOX_MARGIN));
layout.add(recordingsDirectory, 1, 0);
layout.add(recordingsDirectory, 1, row);
recordingsDirectoryButton = createRecordingsBrowseButton();
layout.add(recordingsDirectoryButton, 3, 0);
layout.add(recordingsDirectoryButton, 3, row++);
layout.add(new Label("Player"), 0, 1);
mediaPlayer = new TextField(Config.getInstance().getSettings().mediaPlayer);
mediaPlayer.focusedProperty().addListener(createMpvFocusListener());
GridPane.setFillWidth(mediaPlayer, true);
GridPane.setHgrow(mediaPlayer, Priority.ALWAYS);
GridPane.setColumnSpan(mediaPlayer, 2);
GridPane.setMargin(mediaPlayer, new Insets(0, 0, 0, CHECKBOX_MARGIN));
layout.add(mediaPlayer, 1, 1);
layout.add(createMpvBrowseButton(), 3, 1);
layout.add(new Label("Directory Structure"), 0, row);
List<DirectoryStructure> options = new ArrayList<>();
options.add(FLAT);
options.add(ONE_PER_MODEL);
options.add(ONE_PER_RECORDING);
directoryStructure = new ComboBox<>(FXCollections.observableList(options));
directoryStructure.setValue(Config.getInstance().getSettings().recordingsDirStructure);
directoryStructure.setOnAction((evt) -> Config.getInstance().getSettings().recordingsDirStructure = directoryStructure.getValue());
GridPane.setColumnSpan(directoryStructure, 2);
GridPane.setMargin(directoryStructure, new Insets(0, 0, 0, CHECKBOX_MARGIN));
layout.add(directoryStructure, 1, row++);
layout.add(new Label("Post-Processing"), 0, 2);
layout.add(new Label("Post-Processing"), 0, row);
postProcessing = new TextField(Config.getInstance().getSettings().postProcessing);
postProcessing.focusedProperty().addListener(createPostProcessingFocusListener());
GridPane.setFillWidth(postProcessing, true);
GridPane.setHgrow(postProcessing, Priority.ALWAYS);
GridPane.setColumnSpan(postProcessing, 2);
GridPane.setMargin(postProcessing, new Insets(0, 0, 0, CHECKBOX_MARGIN));
layout.add(postProcessing, 1, 2);
layout.add(postProcessing, 1, row);
postProcessingDirectoryButton = createPostProcessingBrowseButton();
layout.add(postProcessingDirectoryButton, 3, 2);
layout.add(postProcessingDirectoryButton, 3, row++);
layout.add(new Label("Player"), 0, row);
mediaPlayer = new TextField(Config.getInstance().getSettings().mediaPlayer);
mediaPlayer.focusedProperty().addListener(createMpvFocusListener());
GridPane.setFillWidth(mediaPlayer, true);
GridPane.setHgrow(mediaPlayer, Priority.ALWAYS);
GridPane.setColumnSpan(mediaPlayer, 2);
GridPane.setMargin(mediaPlayer, new Insets(0, 0, 0, CHECKBOX_MARGIN));
layout.add(mediaPlayer, 1, row);
layout.add(createMpvBrowseButton(), 3, row++);
TitledPane locations = new TitledPane("Locations", layout);
locations.setCollapsible(false);
@ -394,6 +411,7 @@ public class SettingsTab extends Tab implements TabSelectionListener {
maxResolution.setDisable(!local);
postProcessing.setDisable(!local);
postProcessingDirectoryButton.setDisable(!local);
directoryStructure.setDisable(!local);
}
private ChangeListener<? super Boolean> createRecordingsDirectoryFocusListener() {