Add support for hlsdl

This commit is contained in:
0xb00bface 2021-01-09 22:03:01 +01:00
parent 4421c6f9c3
commit 8e22112603
11 changed files with 415 additions and 25 deletions

View File

@ -9,6 +9,7 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.CompletableFuture;
import java.util.stream.Collectors;
import org.slf4j.Logger;
@ -117,6 +118,7 @@ public class SettingsTab extends Tab implements TabSelectionListener {
private SimpleLongProperty leaveSpaceOnDevice;
private SimpleStringProperty ffmpegParameters;
private SimpleBooleanProperty logFFmpegOutput;
private SimpleBooleanProperty loghlsdlOutput;
private SimpleStringProperty fileExtension;
private SimpleStringProperty server;
private SimpleIntegerProperty port;
@ -126,6 +128,8 @@ public class SettingsTab extends Tab implements TabSelectionListener {
private SimpleBooleanProperty totalModelCountInTitle;
private SimpleBooleanProperty transportLayerSecurity;
private SimpleBooleanProperty fastScrollSpeed;
private SimpleBooleanProperty useHlsdl;
private SimpleFileProperty hlsdlExecutable;
private ExclusiveSelectionProperty recordLocal;
private SimpleIntegerProperty postProcessingThreads;
private IgnoreList ignoreList;
@ -170,6 +174,7 @@ public class SettingsTab extends Tab implements TabSelectionListener {
leaveSpaceOnDevice = new SimpleLongProperty(null, "minimumSpaceLeftInBytes", (long) new GigabytesConverter().convertTo(settings.minimumSpaceLeftInBytes));
ffmpegParameters = new SimpleStringProperty(null, "ffmpegMergedDownloadArgs", settings.ffmpegMergedDownloadArgs);
logFFmpegOutput = new SimpleBooleanProperty(null, "logFFmpegOutput", settings.logFFmpegOutput);
loghlsdlOutput = new SimpleBooleanProperty(null, "loghlsdlOutput", settings.loghlsdlOutput);
fileExtension = new SimpleStringProperty(null, "ffmpegFileSuffix", settings.ffmpegFileSuffix);
server = new SimpleStringProperty(null, "httpServer", settings.httpServer);
port = new SimpleIntegerProperty(null, "httpPort", settings.httpPort);
@ -184,6 +189,8 @@ public class SettingsTab extends Tab implements TabSelectionListener {
onlineCheckSkipsPausedModels = new SimpleBooleanProperty(null, "onlineCheckSkipsPausedModels", settings.onlineCheckSkipsPausedModels);
fastScrollSpeed = new SimpleBooleanProperty(null, "fastScrollSpeed", settings.fastScrollSpeed);
confirmationDialogs = new SimpleBooleanProperty(null, "confirmationForDangerousActions", settings.confirmationForDangerousActions);
useHlsdl = new SimpleBooleanProperty(null, "useHlsdl", settings.useHlsdl);
hlsdlExecutable = new SimpleFileProperty(null, "hlsdlExecutable", settings.hlsdlExecutable);
}
private void createGui() {
@ -269,6 +276,11 @@ public class SettingsTab extends Tab implements TabSelectionListener {
Category.of("Advanced / Devtools",
Group.of("Logging",
Setting.of("Log FFmpeg output", logFFmpegOutput, "Log FFmpeg output to files in the system's temp directory")
),
Group.of("hlsdl (experimental)",
Setting.of("Use hlsdl (if possible)", useHlsdl, "Use hlsdl to record the live streams. Some features might not work correctly."),
Setting.of("hlsdl executable", hlsdlExecutable, "Path to the hlsdl executable"),
Setting.of("Log hlsdl output", loghlsdlOutput, "Log hlsdl output to files in the system's temp directory")
)
)
);
@ -300,7 +312,6 @@ public class SettingsTab extends Tab implements TabSelectionListener {
setContent(stackPane);
prefs.expandTree();
prefs.getSetting("httpServer").ifPresent(s -> bindEnabledProperty(s, recordLocal));
prefs.getSetting("httpPort").ifPresent(s -> bindEnabledProperty(s, recordLocal));
prefs.getSetting("servletContext").ifPresent(s -> bindEnabledProperty(s, recordLocal));
@ -321,10 +332,10 @@ public class SettingsTab extends Tab implements TabSelectionListener {
prefs.getSetting("concurrentRecordings").ifPresent(s -> bindEnabledProperty(s, recordLocal.not()));
prefs.getSetting("concurrentRecordings").ifPresent(s -> bindEnabledProperty(s, recordLocal.not()));
prefs.getSetting("downloadFilename").ifPresent(s -> bindEnabledProperty(s, recordLocal));
prefs.getSetting("hlsdlExecutable").ifPresent(s -> bindEnabledProperty(s, useHlsdl.not()));
prefs.getSetting("loghlsdlOutput").ifPresent(s -> bindEnabledProperty(s, useHlsdl.not()));
postProcessingStepPanel.disableProperty().bind(recordLocal.not());
variablesHelpButton.disableProperty().bind(recordLocal);
}
private void splitValuesChanged(ObservableValue<?> value, Object oldV, Object newV) {
@ -425,11 +436,13 @@ public class SettingsTab extends Tab implements TabSelectionListener {
}
public void saveConfig() {
try {
Config.getInstance().save();
} catch (IOException e) {
LOG.error("Couldn't save config", e);
}
CompletableFuture.runAsync(() -> {
try {
Config.getInstance().save();
} catch (IOException e) {
LOG.error("Couldn't save config", e);
}
});
}
@Override

View File

@ -28,6 +28,8 @@ until a recording is finished. 0 means unlimited.
- **generatePlaylist** (server only) - [`true`,`false`] Generate a playlist once a recording terminates.
- **hlsdlExecutable** - Path to the hlsdl executable, which is used, if `useHlsdl` is set to true
- **httpPort** - [1 - 65536] The TCP port, the server listens on. In the server config, this will tell the server, which port to use. In the application this will set
the port ctbrec tries to connect to, if it is run in remote mode.
@ -43,7 +45,9 @@ the port ctbrec tries to connect to, if it is run in remote mode.
- **livePreviews** (app only) - Enables the live preview feature in the app.
- **logFFmpegOutput** - The output from FFmpeg (from recordings or post-processing steps) will be logged in temporary files.
- **logFFmpegOutput** - [`true`,`false`] The output from FFmpeg (from recordings or post-processing steps) will be logged in temporary files.
- **loghlsdlOutput** - [`true`,`false`] The output from hlsdl will be logged in temporary files. Only in effect, if `useHlsdl` is set to true
- **minimumResolution** - [1 - 2147483647]. Sets the minimum video height for a recording. ctbrec tries to find a stream quality, which is higher than or equal to this value. If the only provided stream quality is below this threshold, ctbrec won't record the stream.
@ -73,5 +77,7 @@ which have the defined length (roughly). Has to be activated with `splitStrategy
- **splitRecordingsBiggerThanBytes** - [0 - 9223372036854775807] in bytes. Split recordings, if the size on disk exceeds this value. The recordings are split up into several individual recordings,
which have the defined size (roughly). Has to be activated with `splitStrategy`.
- **useHlsdl** - [`true`,`false`] Use hlsdl to record the live streams. You also have to set `hlsdlExecutable`, if hlsdl is not globally available on your system. hlsdl won't be used for MV Live, LiveJasmin and Showup.
- **webinterface** (server only) - [`true`,`false`] Enables the webinterface for the server. You can access it with http://host:port/static/index.html Don't activate this on
a machine, which can be accessed from the internet, because this is totally unprotected at the moment.

View File

@ -17,6 +17,7 @@ import ctbrec.recorder.download.Download;
import ctbrec.recorder.download.HttpHeaderFactory;
import ctbrec.recorder.download.HttpHeaderFactoryImpl;
import ctbrec.recorder.download.hls.HlsDownload;
import ctbrec.recorder.download.hls.HlsdlDownload;
import ctbrec.recorder.download.hls.MergedFfmpegHlsDownload;
import ctbrec.sites.Site;
import okhttp3.Request;
@ -275,10 +276,14 @@ public abstract class AbstractModel implements Model {
@Override
public Download createDownload() {
if (Config.isServerMode() && !Config.getInstance().getSettings().recordSingleFile) {
return new HlsDownload(getSite().getHttpClient());
if (Config.getInstance().getSettings().useHlsdl) {
return new HlsdlDownload();
} else {
return new MergedFfmpegHlsDownload(getSite().getHttpClient());
if (Config.isServerMode() && !Config.getInstance().getSettings().recordSingleFile) {
return new HlsDownload(getSite().getHttpClient());
} else {
return new MergedFfmpegHlsDownload(getSite().getHttpClient());
}
}
}

View File

@ -213,7 +213,7 @@ public class Config {
return settings;
}
public void save() throws IOException {
public synchronized void save() throws IOException {
Moshi moshi = new Moshi.Builder()
.add(Model.class, new ModelJsonAdapter())
.add(PostProcessor.class, new PostProcessorJsonAdapter())

View File

@ -69,6 +69,7 @@ public class Settings {
public String flirt4freePassword;
public String flirt4freeUsername;
public boolean generatePlaylist = true;
public String hlsdlExecutable = "hlsdl";
public int httpPort = 8080;
public int httpSecurePort = 8443;
public String httpServer = "localhost";
@ -84,6 +85,7 @@ public class Settings {
public boolean livePreviews = false;
public boolean localRecording = true;
public boolean logFFmpegOutput = false;
public boolean loghlsdlOutput = false;
public int minimumResolution = 0;
public int maximumResolution = 8640;
public int maximumResolutionPlayer = 0;
@ -156,6 +158,7 @@ public class Settings {
public boolean transportLayerSecurity = true;
public int thumbWidth = 180;
public boolean updateThumbnails = true;
public boolean useHlsdl = false;
@Deprecated
public String username = "";
public int windowHeight = 800;

View File

@ -9,15 +9,15 @@ import java.util.concurrent.ScheduledExecutorService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class FfmpegStreamRedirector implements Runnable {
private static final Logger LOG = LoggerFactory.getLogger(FfmpegStreamRedirector.class);
public class ProcessStreamRedirector implements Runnable {
private static final Logger LOG = LoggerFactory.getLogger(ProcessStreamRedirector.class);
private InputStream in;
private OutputStream out;
private boolean keepGoing = true;
private ScheduledExecutorService executor;
public FfmpegStreamRedirector(ScheduledExecutorService executor, InputStream in, OutputStream out) {
public ProcessStreamRedirector(ScheduledExecutorService executor, InputStream in, OutputStream out) {
super();
this.executor = executor;
this.in = in;
@ -37,7 +37,7 @@ public class FfmpegStreamRedirector implements Runnable {
executor.schedule(this, 100, MILLISECONDS);
}
} catch (Exception e) {
LOG.debug("Error while reading from FFmpeg output stream: {}", e.getLocalizedMessage());
LOG.debug("Error while reading from process output stream: {}", e.getLocalizedMessage());
keepGoing = false;
}
}

View File

@ -15,7 +15,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import ctbrec.io.DevNull;
import ctbrec.io.FfmpegStreamRedirector;
import ctbrec.io.ProcessStreamRedirector;
import ctbrec.recorder.download.ProcessExitedUncleanException;
public class FFmpeg {
@ -30,8 +30,8 @@ public class FFmpeg {
private Consumer<Integer> exitCallback;
private File ffmpegLog = null;
private OutputStream ffmpegLogStream;
private FfmpegStreamRedirector stdoutRedirector;
private FfmpegStreamRedirector stderrRedirector;
private ProcessStreamRedirector stdoutRedirector;
private ProcessStreamRedirector stderrRedirector;
private FFmpeg() {}
@ -85,8 +85,8 @@ public class FFmpeg {
} else {
ffmpegLogStream = new DevNull();
}
stdoutRedirector = new FfmpegStreamRedirector(processOutputReader, process.getInputStream(), ffmpegLogStream);
stderrRedirector = new FfmpegStreamRedirector(processOutputReader, process.getErrorStream(), ffmpegLogStream);
stdoutRedirector = new ProcessStreamRedirector(processOutputReader, process.getInputStream(), ffmpegLogStream);
stderrRedirector = new ProcessStreamRedirector(processOutputReader, process.getErrorStream(), ffmpegLogStream);
processOutputReader.submit(stdoutRedirector);
processOutputReader.submit(stderrRedirector);
}

View File

@ -0,0 +1,150 @@
package ctbrec.recorder.download.hls;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.file.Files;
import java.util.Arrays;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ThreadFactory;
import java.util.function.Consumer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import ctbrec.io.DevNull;
import ctbrec.io.ProcessStreamRedirector;
import ctbrec.recorder.download.ProcessExitedUncleanException;
public class Hlsdl {
private static final Logger LOG = LoggerFactory.getLogger(Hlsdl.class);
private static ScheduledExecutorService processOutputReader = Executors.newScheduledThreadPool(2, createThreadFactory("hlsdl output stream reader"));
private Process process;
private boolean logOutput = false;
private Consumer<Process> startCallback;
private Consumer<Integer> exitCallback;
private File processLog = null;
private OutputStream processLogStream;
private ProcessStreamRedirector stdoutRedirector;
private ProcessStreamRedirector stderrRedirector;
private Hlsdl() {}
private static ThreadFactory createThreadFactory(String name) {
return r -> {
Thread t = new Thread(r);
t.setName(name);
t.setDaemon(true);
t.setPriority(Thread.MIN_PRIORITY);
return t;
};
}
public void exec(String[] cmdline, String[] env, File executionDir) throws IOException, InterruptedException {
LOG.debug("hlsdl command line: {}", Arrays.toString(cmdline));
process = Runtime.getRuntime().exec(cmdline, env, executionDir);
afterStart();
int exitCode = process.waitFor();
afterExit(exitCode);
}
private void afterStart() throws IOException {
notifyStartCallback(process);
setupLogging();
}
private void afterExit(int exitCode) throws IOException {
LOG.debug("hlsdl exit code was {}", exitCode);
processLogStream.flush();
processLogStream.close();
stdoutRedirector.setKeepGoing(false);
stderrRedirector.setKeepGoing(false);
notifyExitCallback(exitCode);
if (exitCode != 1) {
if (processLog != null && processLog.exists()) {
Files.delete(processLog.toPath());
}
} else {
throw new ProcessExitedUncleanException("hlsdl exit code was " + exitCode);
}
}
private void setupLogging() throws IOException {
if (logOutput) {
if (processLog == null) {
processLog = File.createTempFile("hlsdl_", ".log");
}
LOG.debug("Logging hlsdl output to {}", processLog);
processLog.deleteOnExit();
processLogStream = new FileOutputStream(processLog);
} else {
processLogStream = new DevNull();
}
stdoutRedirector = new ProcessStreamRedirector(processOutputReader, process.getInputStream(), processLogStream);
stderrRedirector = new ProcessStreamRedirector(processOutputReader, process.getErrorStream(), processLogStream);
processOutputReader.submit(stdoutRedirector);
processOutputReader.submit(stderrRedirector);
}
private void notifyStartCallback(Process process) {
try {
startCallback.accept(process);
} catch(Exception e) {
LOG.error("Exception in onStart callback", e);
}
}
private void notifyExitCallback(int exitCode) {
try {
exitCallback.accept(exitCode);
} catch(Exception e) {
LOG.error("Exception in onExit callback", e);
}
}
public int waitFor() throws InterruptedException {
return process.waitFor();
}
public static class Builder {
private boolean logOutput = false;
private File logFile;
private Consumer<Process> startCallback;
private Consumer<Integer> exitCallback;
public Builder logOutput(boolean logOutput) {
this.logOutput = logOutput;
return this;
}
public Builder logFile(File logFile) {
this.logFile = logFile;
return this;
}
public Builder onStarted(Consumer<Process> callback) {
this.startCallback = callback;
return this;
}
public Builder onExit(Consumer<Integer> callback) {
this.exitCallback = callback;
return this;
}
public Hlsdl build() {
Hlsdl instance = new Hlsdl();
instance.logOutput = logOutput;
instance.startCallback = startCallback != null ? startCallback : p -> {};
instance.exitCallback = exitCallback != null ? exitCallback : exitCode -> {};
instance.processLog = logFile;
return instance;
}
}
}

View File

@ -0,0 +1,182 @@
package ctbrec.recorder.download.hls;
import static ctbrec.recorder.download.StreamSource.*;
import java.io.EOFException;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.time.Instant;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.concurrent.ExecutionException;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import javax.xml.bind.JAXBException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.iheartradio.m3u8.ParseException;
import com.iheartradio.m3u8.PlaylistException;
import ctbrec.Config;
import ctbrec.Model;
import ctbrec.OS;
import ctbrec.Recording;
import ctbrec.recorder.download.AbstractDownload;
import ctbrec.recorder.download.StreamSource;
public class HlsdlDownload extends AbstractDownload {
private static final transient Logger LOG = LoggerFactory.getLogger(HlsdlDownload.class);
protected Model model;
protected File targetFile;
protected transient Config config;
protected transient Process hlsdlProcess;
protected transient boolean running;
@Override
public void init(Config config, Model model, Instant startTime) {
super.startTime = startTime;
this.config = config;
this.model = model;
String fileSuffix = config.getSettings().ffmpegFileSuffix;
targetFile = config.getFileForRecording(model, fileSuffix, startTime);
// TODO splittingStrategy = initSplittingStrategy(config.getSettings());
}
@Override
public void start() throws IOException {
try {
running = true;
Thread.currentThread().setName("Download " + model.getName());
Files.createDirectories(targetFile.getParentFile().toPath());
Hlsdl hlsdl = new Hlsdl.Builder()
.logOutput(config.getSettings().loghlsdlOutput)
.onStarted(p -> hlsdlProcess = p)
.build();
String[] cmdline = createCommandLine();
hlsdl.exec(cmdline, OS.getEnvironment(), targetFile.getParentFile());
} catch (ParseException e) {
throw new IOException("Couldn't parse stream information", e);
} catch (PlaylistException e) {
throw new IOException("Couldn't parse HLS playlist", e);
} catch (EOFException e) {
// end of playlist reached
LOG.debug("Reached end of playlist for model {}", model);
} catch (Exception e) {
throw new IOException("Exception while downloading segments", e);
} finally {
stop();
running = false;
LOG.debug("Download for {} terminated", model);
}
}
private String[] createCommandLine() throws IOException, ExecutionException, ParseException, PlaylistException, JAXBException {
String playlistUrl = getSegmentPlaylistUrl(model);
Map<String, String> headers = model.getHttpHeaderFactory().createSegmentPlaylistHeaders();
String[] cmdline = new String[9 + headers.size() * 2];
int idx = 0;
cmdline[idx++] = config.getSettings().hlsdlExecutable;
cmdline[idx++] = "-c";
cmdline[idx++] = "-r";
cmdline[idx++] = "3";
cmdline[idx++] = "-w";
cmdline[idx++] = "3";
for (Entry<String, String> header : headers.entrySet()) {
cmdline[idx++] = "-h";
cmdline[idx++] = header.getKey() + ": " + header.getValue();
}
cmdline[idx++] = "-o";
cmdline[idx++] = targetFile.getCanonicalPath();
cmdline[idx] = playlistUrl;
return cmdline;
}
protected String getSegmentPlaylistUrl(Model model) throws IOException, ExecutionException, ParseException, PlaylistException, JAXBException {
LOG.debug("{} stream idx: {}", model.getName(), model.getStreamUrlIndex());
List<StreamSource> streamSources = model.getStreamSources();
Collections.sort(streamSources);
for (StreamSource streamSource : streamSources) {
LOG.debug("{} src {}", model.getName(), streamSource);
}
String url = null;
if (model.getStreamUrlIndex() >= 0 && model.getStreamUrlIndex() < streamSources.size()) {
// TODO don't use the index, but the bandwidth. if the bandwidth does not match, take the closest one
LOG.debug("{} selected {}", model.getName(), streamSources.get(model.getStreamUrlIndex()));
url = streamSources.get(model.getStreamUrlIndex()).getMediaPlaylistUrl();
} else {
// filter out stream resolutions, which are out of range of the configured min and max
int minRes = Config.getInstance().getSettings().minimumResolution;
int maxRes = Config.getInstance().getSettings().maximumResolution;
List<StreamSource> filteredStreamSources = streamSources.stream()
.filter(src -> src.height == 0 || src.height == UNKNOWN || minRes <= src.height)
.filter(src -> src.height == 0 || src.height == UNKNOWN || maxRes >= src.height)
.collect(Collectors.toList());
if (filteredStreamSources.isEmpty()) {
throw new ExecutionException(new RuntimeException("No stream left in playlist"));
} else {
LOG.debug("{} selected {}", model.getName(), filteredStreamSources.get(filteredStreamSources.size() - 1));
url = filteredStreamSources.get(filteredStreamSources.size() - 1).getMediaPlaylistUrl();
}
}
LOG.debug("Segment playlist url {}", url);
return url;
}
@Override
public void stop() {
if (running) {
running = false;
if (hlsdlProcess != null) {
hlsdlProcess.destroy();
if (hlsdlProcess.isAlive()) {
LOG.info("hlsdl didn't terminate. Destroying the process with force!");
hlsdlProcess.destroyForcibly();
hlsdlProcess = null;
}
}
}
}
@Override
public void postprocess(Recording recording) {
// nothing to do
}
@Override
public File getTarget() {
return targetFile;
}
@Override
public String getPath(Model model) {
String absolutePath = targetFile.getAbsolutePath();
String recordingsDir = Config.getInstance().getSettings().recordingsDir;
String relativePath = absolutePath.replaceFirst(Pattern.quote(recordingsDir), "");
return relativePath;
}
@Override
public boolean isSingleFile() {
return true;
}
@Override
public long getSizeInByte() {
return getTarget().length();
}
@Override
public Model getModel() {
return model;
}
}

View File

@ -0,0 +1,27 @@
package ctbrec.sites.fc2live;
import java.io.IOException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import ctbrec.recorder.download.hls.HlsdlDownload;
public class Fc2HlsdlDownload extends HlsdlDownload {
private static final Logger LOG = LoggerFactory.getLogger(Fc2HlsdlDownload.class);
@Override
public void start() throws IOException {
Fc2Model fc2Model = (Fc2Model) model;
try {
fc2Model.openWebsocket();
super.start();
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
LOG.error("Couldn't start download for {}", model, e);
} finally {
fc2Model.closeWebsocket();
}
}
}

View File

@ -381,10 +381,14 @@ public class Fc2Model extends AbstractModel {
@Override
public Download createDownload() {
if(Config.isServerMode() && !Config.getInstance().getSettings().recordSingleFile) {
return new Fc2HlsDownload(getSite().getHttpClient());
if (Config.getInstance().getSettings().useHlsdl) {
return new Fc2HlsdlDownload();
} else {
return new Fc2MergedHlsDownload(getSite().getHttpClient());
if (Config.isServerMode() && !Config.getInstance().getSettings().recordSingleFile) {
return new Fc2HlsDownload(getSite().getHttpClient());
} else {
return new Fc2MergedHlsDownload(getSite().getHttpClient());
}
}
}