From a31debcdea607d0f1feef2ad5f600b22859de3b0 Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Sat, 5 Dec 2020 21:30:54 +0100 Subject: [PATCH] Add possibility to split recordings with different strategies --- .../resources/html/docs/ConfigurationFile.md | 13 +- common/src/main/java/ctbrec/Config.java | 6 + common/src/main/java/ctbrec/Recording.java | 41 +--- common/src/main/java/ctbrec/Settings.java | 11 ++ common/src/main/java/ctbrec/io/IoUtils.java | 29 +++ .../ctbrec/recorder/NextGenLocalRecorder.java | 1 + .../ctbrec/recorder/download/Download.java | 2 + .../recorder/download/SplittingStrategy.java | 9 + .../recorder/download/dash/DashDownload.java | 6 + .../download/hls/AbstractHlsDownload.java | 26 +++ .../hls/CombinedSplittingStrategy.java | 31 +++ .../recorder/download/hls/FFmpegDownload.java | 5 + .../recorder/download/hls/HlsDownload.java | 182 ++++++++++-------- .../download/hls/MergedFfmpegHlsDownload.java | 37 ++-- .../download/hls/NoopSplittingStrategy.java | 19 ++ .../download/hls/SizeSplittingStrategy.java | 22 +++ .../download/hls/TimeSplittingStrategy.java | 28 +++ .../sites/showup/ShowupMergedDownload.java | 4 +- 18 files changed, 322 insertions(+), 150 deletions(-) create mode 100644 common/src/main/java/ctbrec/recorder/download/SplittingStrategy.java create mode 100644 common/src/main/java/ctbrec/recorder/download/hls/CombinedSplittingStrategy.java create mode 100644 common/src/main/java/ctbrec/recorder/download/hls/NoopSplittingStrategy.java create mode 100644 common/src/main/java/ctbrec/recorder/download/hls/SizeSplittingStrategy.java create mode 100644 common/src/main/java/ctbrec/recorder/download/hls/TimeSplittingStrategy.java diff --git a/client/src/main/resources/html/docs/ConfigurationFile.md b/client/src/main/resources/html/docs/ConfigurationFile.md index e3ea4bf1..0222f242 100644 --- a/client/src/main/resources/html/docs/ConfigurationFile.md +++ b/client/src/main/resources/html/docs/ConfigurationFile.md @@ -53,16 +53,21 @@ the port ctbrec tries to connect to, if it is run in remote mode. - **onlineCheckSkipsPausedModels** - [`true`,`false`] Skip the online check for paused models. If you have many models in the recording list, this can reduce the delay when a recording starts after a model came online. -- **postProcessing** - Absolute path to a script, which is executed once a recording is finished. See [Post-Processing](/docs/PostProcessing.md). +- **postProcessing** - **Deprecated. See [Post-Processing](/docs/PostProcessing.md)** Absolute path to a script, which is executed once a recording is finished. - **recordingsDir** - Where ctbrec saves the recordings. - **recordingsDirStructure** (server only) - [`FLAT`, `ONE_PER_MODEL`, `ONE_PER_RECORDING`] How recordings are stored in the file system. `FLAT` - all recordings in one directory; `ONE_PER_MODEL` - one directory per model; `ONE_PER_RECORDING` - each recordings ends up in its own directory. Change this only, if you have `recordSingleFile` set to `true` -- **recordSingleFile** (server only) - [`true`,`false`] - How recordings are stored in the file system. `true` means, each recording is saved in one large file. `false` means, ctbrec just downloads the stream segments. +- **recordSingleFile** (server only) - [`true`,`false`] - How recordings are stored in the file system. `true` means, each recording is saved in one large file. `false` means, ctbrec just downloads the stream segments. -- **splitRecordings** - [0 - 2147483647] in seconds. Split recordings after this amount of seconds. The recordings are split up into several individual recordings, -which have the defined length (roughly). 0 means no splitting. The server does not support splitRecordings. +- **splitStrategy** - [`DONT`, `TIME`, `SIZE`, `TIME_OR_SIZE`] Defines if and how to split recordings. Also see `splitRecordingsAfterSecs` and `splitRecordingsBiggerThanBytes` + +- **splitRecordingsAfterSecs** - [0 - 2147483647] in seconds. Split recordings after this amount of seconds. The recordings are split up into several individual recordings, +which have the defined length (roughly). Has to be activated with `splitStrategy`. + +- **splitRecordingsBiggerThanBytes** - [0 - 9223372036854775807] in bytes. Split recordings, if the size on disk exceeds this value. The recordings are split up into several individual recordings, +which have the defined size (roughly). Has to be activated with `splitStrategy`. - **webinterface** (server only) - [`true`,`false`] Enables the webinterface for the server. You can access it with http://host:port/static/index.html Don't activate this on a machine, which can be accessed from the internet, because this is totally unprotected at the moment. diff --git a/common/src/main/java/ctbrec/Config.java b/common/src/main/java/ctbrec/Config.java index d4709ded..c943ab92 100644 --- a/common/src/main/java/ctbrec/Config.java +++ b/common/src/main/java/ctbrec/Config.java @@ -23,6 +23,7 @@ import org.slf4j.LoggerFactory; import com.squareup.moshi.JsonAdapter; import com.squareup.moshi.Moshi; +import ctbrec.Settings.SplitStrategy; import ctbrec.io.FileJsonAdapter; import ctbrec.io.ModelJsonAdapter; import ctbrec.io.PostProcessorJsonAdapter; @@ -139,6 +140,11 @@ public class Config { settings.chaturbatePassword = settings.password; settings.password = null; } + if (settings.splitRecordings > 0) { + settings.splitStrategy = SplitStrategy.TIME; + settings.splitRecordingsAfterSecs = settings.splitRecordings; + settings.splitRecordings = 0; + } } private void makeBackup(File source) { diff --git a/common/src/main/java/ctbrec/Recording.java b/common/src/main/java/ctbrec/Recording.java index ad34b717..f2ccc264 100644 --- a/common/src/main/java/ctbrec/Recording.java +++ b/common/src/main/java/ctbrec/Recording.java @@ -3,35 +3,23 @@ package ctbrec; import static ctbrec.Recording.State.*; import java.io.File; -import java.io.IOException; import java.io.Serializable; -import java.nio.file.FileVisitOption; -import java.nio.file.FileVisitResult; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.SimpleFileVisitor; -import java.nio.file.attribute.BasicFileAttributes; import java.time.Duration; import java.time.Instant; import java.time.LocalDateTime; import java.time.ZoneId; import java.time.format.DateTimeFormatter; -import java.util.EnumSet; import java.util.HashSet; import java.util.Optional; import java.util.Set; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import ctbrec.event.EventBusHolder; import ctbrec.event.RecordingStateChangedEvent; +import ctbrec.io.IoUtils; import ctbrec.recorder.download.Download; import ctbrec.recorder.download.VideoLengthDetector; public class Recording implements Serializable { - private static final transient Logger LOG = LoggerFactory.getLogger(Recording.class); - private String id; private Model model; private transient Download download; @@ -253,11 +241,11 @@ public class Recording implements Serializable { private long getSize() { File rec = getAbsoluteFile(); if (rec.isDirectory()) { - return getDirectorySize(rec); + return IoUtils.getDirectorySize(rec); } else { if (!rec.exists()) { if (rec.getName().endsWith(".m3u8")) { - return getDirectorySize(rec.getParentFile()); + return IoUtils.getDirectorySize(rec.getParentFile()); } else { return -1; } @@ -267,29 +255,6 @@ public class Recording implements Serializable { } } - private long getDirectorySize(File dir) { - final long[] size = { 0 }; - int maxDepth = 1; // Don't expect subdirs, so don't even try - try { - Files.walkFileTree(dir.toPath(), EnumSet.noneOf(FileVisitOption.class), maxDepth, new SimpleFileVisitor() { - @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { - size[0] += attrs.size(); - return FileVisitResult.CONTINUE; - } - - @Override - public FileVisitResult visitFileFailed(Path file, IOException exc) { - // Ignore file access issues - return FileVisitResult.CONTINUE; - } - }); - } catch (IOException e) { - LOG.error("Couldn't determine size of recording {}", this, e); - } - return size[0]; - } - public void refresh() { sizeInByte = getSize(); } diff --git a/common/src/main/java/ctbrec/Settings.java b/common/src/main/java/ctbrec/Settings.java index 355a75c5..ed233308 100644 --- a/common/src/main/java/ctbrec/Settings.java +++ b/common/src/main/java/ctbrec/Settings.java @@ -34,6 +34,13 @@ public class Settings { SOCKS5 } + public enum SplitStrategy { + DONT, + TIME, + SIZE, + TIME_OR_SIZE + } + public String bongacamsBaseUrl = "https://bongacams.com"; public String bongaPassword = ""; public String bongaUsername = ""; @@ -124,7 +131,11 @@ public class Settings { public String showupUsername = ""; public String showupPassword = ""; public boolean singlePlayer = true; + @Deprecated public int splitRecordings = 0; + public SplitStrategy splitStrategy = SplitStrategy.DONT; + public int splitRecordingsAfterSecs = 0; + public long splitRecordingsBiggerThanBytes = 0; public String startTab = "Settings"; public String streamatePassword = ""; public String streamateUsername = ""; diff --git a/common/src/main/java/ctbrec/io/IoUtils.java b/common/src/main/java/ctbrec/io/IoUtils.java index 644e540b..a76d6879 100644 --- a/common/src/main/java/ctbrec/io/IoUtils.java +++ b/common/src/main/java/ctbrec/io/IoUtils.java @@ -2,7 +2,13 @@ package ctbrec.io; import java.io.File; import java.io.IOException; +import java.nio.file.FileVisitOption; +import java.nio.file.FileVisitResult; import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.EnumSet; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -50,4 +56,27 @@ public class IoUtils { throw new IOException("Couldn't delete all files in " + directory); } } + + public static long getDirectorySize(File dir) { + final long[] size = { 0 }; + int maxDepth = 1; // Don't expect subdirs, so don't even try + try { + Files.walkFileTree(dir.toPath(), EnumSet.noneOf(FileVisitOption.class), maxDepth, new SimpleFileVisitor() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { + size[0] += attrs.size(); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFileFailed(Path file, IOException exc) { + // Ignore file access issues + return FileVisitResult.CONTINUE; + } + }); + } catch (IOException e) { + LOG.error("Couldn't determine size of directory {}", dir, e); + } + return size[0]; + } } diff --git a/common/src/main/java/ctbrec/recorder/NextGenLocalRecorder.java b/common/src/main/java/ctbrec/recorder/NextGenLocalRecorder.java index 90a38395..7e1a3c96 100644 --- a/common/src/main/java/ctbrec/recorder/NextGenLocalRecorder.java +++ b/common/src/main/java/ctbrec/recorder/NextGenLocalRecorder.java @@ -159,6 +159,7 @@ public class NextGenLocalRecorder implements Recorder { ppPool.submit(() -> { try { setRecordingStatus(recording, State.POST_PROCESSING); + recording.refresh(); recordingManager.saveRecording(recording); recording.postprocess(); List postProcessors = config.getSettings().postProcessors; diff --git a/common/src/main/java/ctbrec/recorder/download/Download.java b/common/src/main/java/ctbrec/recorder/download/Download.java index 171a8d98..b5497a1d 100644 --- a/common/src/main/java/ctbrec/recorder/download/Download.java +++ b/common/src/main/java/ctbrec/recorder/download/Download.java @@ -38,4 +38,6 @@ public interface Download extends Serializable { * @return true, if the recording is only a single file */ public boolean isSingleFile(); + + public long getSizeInByte(); } diff --git a/common/src/main/java/ctbrec/recorder/download/SplittingStrategy.java b/common/src/main/java/ctbrec/recorder/download/SplittingStrategy.java new file mode 100644 index 00000000..aba7eaab --- /dev/null +++ b/common/src/main/java/ctbrec/recorder/download/SplittingStrategy.java @@ -0,0 +1,9 @@ +package ctbrec.recorder.download; + +import ctbrec.Settings; + +public interface SplittingStrategy { + + void init(Settings settings); + boolean splitNecessary(Download download); +} diff --git a/common/src/main/java/ctbrec/recorder/download/dash/DashDownload.java b/common/src/main/java/ctbrec/recorder/download/dash/DashDownload.java index 9f57d93a..e28a6ba6 100644 --- a/common/src/main/java/ctbrec/recorder/download/dash/DashDownload.java +++ b/common/src/main/java/ctbrec/recorder/download/dash/DashDownload.java @@ -37,6 +37,7 @@ import ctbrec.Recording; import ctbrec.io.BandwidthMeter; import ctbrec.io.HttpClient; import ctbrec.io.HttpException; +import ctbrec.io.IoUtils; import ctbrec.recorder.download.AbstractDownload; import ctbrec.recorder.download.dash.SegmentTimelineType.S; import ctbrec.recorder.download.hls.PostProcessingException; @@ -416,4 +417,9 @@ public class DashDownload extends AbstractDownload { return false; } + @Override + public long getSizeInByte() { + return IoUtils.getDirectorySize(downloadDir.toFile()); + } + } diff --git a/common/src/main/java/ctbrec/recorder/download/hls/AbstractHlsDownload.java b/common/src/main/java/ctbrec/recorder/download/hls/AbstractHlsDownload.java index 4af8d12d..9cd2101e 100644 --- a/common/src/main/java/ctbrec/recorder/download/hls/AbstractHlsDownload.java +++ b/common/src/main/java/ctbrec/recorder/download/hls/AbstractHlsDownload.java @@ -44,6 +44,7 @@ import com.iheartradio.m3u8.data.TrackData; import ctbrec.Config; import ctbrec.Model; import ctbrec.Recording.State; +import ctbrec.Settings; import ctbrec.UnknownModel; import ctbrec.io.BandwidthMeter; import ctbrec.io.HttpClient; @@ -51,6 +52,7 @@ import ctbrec.io.HttpException; import ctbrec.recorder.PlaylistGenerator.InvalidPlaylistException; import ctbrec.recorder.download.AbstractDownload; import ctbrec.recorder.download.HttpHeaderFactory; +import ctbrec.recorder.download.SplittingStrategy; import ctbrec.recorder.download.StreamSource; import ctbrec.sites.Site; import okhttp3.Request; @@ -67,6 +69,7 @@ public abstract class AbstractHlsDownload extends AbstractDownload { protected Model model = new UnknownModel(); protected transient LinkedBlockingQueue downloadQueue = new LinkedBlockingQueue<>(50); protected transient ExecutorService downloadThreadPool = new ThreadPoolExecutor(0, 5, 20, TimeUnit.SECONDS, downloadQueue, createThreadFactory()); + protected transient SplittingStrategy splittingStrategy; protected State state = State.UNKNOWN; private int playlistEmptyCount = 0; @@ -235,4 +238,27 @@ public abstract class AbstractHlsDownload extends AbstractDownload { this.url = url; } } + + protected SplittingStrategy initSplittingStrategy(Settings settings) { + SplittingStrategy strategy; + switch (settings.splitStrategy) { + case TIME: + strategy = new TimeSplittingStrategy(); + break; + case SIZE: + strategy = new SizeSplittingStrategy(); + break; + case TIME_OR_SIZE: + SplittingStrategy timeSplittingStrategy = new TimeSplittingStrategy(); + SplittingStrategy sizeSplittingStrategy = new SizeSplittingStrategy(); + strategy = new CombinedSplittingStrategy(timeSplittingStrategy, sizeSplittingStrategy); + break; + case DONT: + default: + strategy = new NoopSplittingStrategy(); + break; + } + strategy.init(settings); + return strategy; + } } diff --git a/common/src/main/java/ctbrec/recorder/download/hls/CombinedSplittingStrategy.java b/common/src/main/java/ctbrec/recorder/download/hls/CombinedSplittingStrategy.java new file mode 100644 index 00000000..a4f9284c --- /dev/null +++ b/common/src/main/java/ctbrec/recorder/download/hls/CombinedSplittingStrategy.java @@ -0,0 +1,31 @@ +package ctbrec.recorder.download.hls; + +import ctbrec.Settings; +import ctbrec.recorder.download.Download; +import ctbrec.recorder.download.SplittingStrategy; + +public class CombinedSplittingStrategy implements SplittingStrategy { + + private SplittingStrategy[] splittingStrategies; + + public CombinedSplittingStrategy(SplittingStrategy... splittingStrategies) { + this.splittingStrategies = splittingStrategies; + } + + @Override + public void init(Settings settings) { + for (SplittingStrategy splittingStrategy : splittingStrategies) { + splittingStrategy.init(settings); + } + } + + @Override + public boolean splitNecessary(Download download) { + for (SplittingStrategy splittingStrategy : splittingStrategies) { + if (splittingStrategy.splitNecessary(download)) { + return true; + } + } + return false; + } +} diff --git a/common/src/main/java/ctbrec/recorder/download/hls/FFmpegDownload.java b/common/src/main/java/ctbrec/recorder/download/hls/FFmpegDownload.java index 6ef507a2..c58239d6 100644 --- a/common/src/main/java/ctbrec/recorder/download/hls/FFmpegDownload.java +++ b/common/src/main/java/ctbrec/recorder/download/hls/FFmpegDownload.java @@ -145,4 +145,9 @@ public class FFmpegDownload extends AbstractHlsDownload { return true; } + @Override + public long getSizeInByte() { + return getTarget().length(); + } + } diff --git a/common/src/main/java/ctbrec/recorder/download/hls/HlsDownload.java b/common/src/main/java/ctbrec/recorder/download/hls/HlsDownload.java index 8491971a..8130178b 100644 --- a/common/src/main/java/ctbrec/recorder/download/hls/HlsDownload.java +++ b/common/src/main/java/ctbrec/recorder/download/hls/HlsDownload.java @@ -13,7 +13,6 @@ import java.nio.file.Files; import java.nio.file.Path; import java.text.DecimalFormat; import java.text.NumberFormat; -import java.time.Duration; import java.time.Instant; import java.time.ZoneId; import java.time.ZonedDateTime; @@ -40,6 +39,7 @@ import ctbrec.Recording.State; import ctbrec.io.BandwidthMeter; import ctbrec.io.HttpClient; import ctbrec.io.HttpException; +import ctbrec.io.IoUtils; import ctbrec.recorder.PlaylistGenerator; import ctbrec.recorder.download.HttpHeaderFactory; import okhttp3.Request; @@ -48,6 +48,8 @@ import okhttp3.Response; public class HlsDownload extends AbstractHlsDownload { + private static final int TEN_SECONDS = 10_000; + private static final Logger LOG = LoggerFactory.getLogger(HlsDownload.class); protected transient Path downloadDir; @@ -55,8 +57,8 @@ public class HlsDownload extends AbstractHlsDownload { private int segmentCounter = 1; private NumberFormat nf = new DecimalFormat("000000"); private transient AtomicBoolean downloadFinished = new AtomicBoolean(false); - private ZonedDateTime splitRecStartTime; protected transient Config config; + private transient int waitFactor = 1; public HlsDownload(HttpClient client) { super(client); @@ -71,6 +73,7 @@ public class HlsDownload extends AbstractHlsDownload { String formattedStartTime = formatter.format(ZonedDateTime.ofInstant(this.startTime, ZoneId.systemDefault())); Path modelDir = FileSystems.getDefault().getPath(config.getSettings().recordingsDir, model.getSanitizedNamed()); downloadDir = FileSystems.getDefault().getPath(modelDir.toString(), formattedStartTime); + splittingStrategy = initSplittingStrategy(config.getSettings()); } @Override @@ -78,7 +81,6 @@ public class HlsDownload extends AbstractHlsDownload { try { running = true; Thread.currentThread().setName("Download " + model.getName()); - splitRecStartTime = ZonedDateTime.now(); String segments = getSegmentPlaylistUrl(model); if (segments != null) { if (!downloadDir.toFile().exists()) { @@ -86,45 +88,13 @@ public class HlsDownload extends AbstractHlsDownload { } int lastSegmentNumber = 0; int nextSegmentNumber = 0; - int waitFactor = 1; while (running) { SegmentPlaylist playlist = getNextSegments(segments); emptyPlaylistCheck(playlist); - if (nextSegmentNumber > 0 && playlist.seq > nextSegmentNumber) { - waitFactor *= 2; - LOG.warn("Missed segments {} < {} in download for {} - setting wait factor to 1/{}", nextSegmentNumber, playlist.seq, model, - waitFactor); - } - int skip = nextSegmentNumber - playlist.seq; - for (String segment : playlist.segments) { - if (skip > 0) { - skip--; - } else { - URL segmentUrl = new URL(segment); - String prefix = nf.format(segmentCounter++); - SegmentDownload segmentDownload = new SegmentDownload(playlist, segmentUrl, downloadDir, client, prefix); - enqueueDownload(segmentDownload, prefix, segmentUrl); - } - } - - // split recordings - boolean split = splitRecording(); - if (split) { - break; - } - - long waitForMillis = 0; - if (lastSegmentNumber == playlist.seq) { - // playlist didn't change -> wait for at least half the target duration - waitForMillis = (long) playlist.targetDuration * 1000 / waitFactor; - LOG.trace("Playlist didn't change... waiting for {}ms", waitForMillis); - } else { - // playlist did change -> wait for at least last segment duration - waitForMillis = 1; - LOG.trace("Playlist changed... waiting for {}ms", waitForMillis); - } - - waitSomeTime(waitForMillis); + logMissedSegments(playlist, nextSegmentNumber); + enqueueNewSegments(playlist, nextSegmentNumber); + splitRecordingIfNecessary(); + waitSomeTime(playlist, lastSegmentNumber, waitFactor); // this if check makes sure, that we don't decrease nextSegment. for some reason // streamate playlists sometimes jump back. e.g. max sequence = 79 -> 80 -> 79 @@ -144,45 +114,96 @@ public class HlsDownload extends AbstractHlsDownload { // end of playlist reached LOG.debug("Reached end of playlist for model {}", model); } catch (HttpException e) { - if (e.getResponseCode() == 404) { - ctbrec.Model.State modelState; - try { - modelState = model.getOnlineState(false); - } catch (ExecutionException e1) { - modelState = ctbrec.Model.State.UNKNOWN; - } - LOG.info("Playlist not found (404). Model {} probably went offline. Model state: {}", model, modelState); - waitSomeTime(10_000); - } else if (e.getResponseCode() == 403) { - ctbrec.Model.State modelState; - try { - modelState = model.getOnlineState(false); - } catch (ExecutionException e1) { - modelState = ctbrec.Model.State.UNKNOWN; - } - LOG.info("Playlist access forbidden (403). Model {} probably went private or offline. Model state: {}", model, modelState); - waitSomeTime(10_000); - } else { - throw e; - } + handleHttpException(e); } catch (Exception e) { throw new IOException("Couldn't download segment", e); } finally { - downloadThreadPool.shutdown(); - try { - LOG.debug("Waiting for last segments for {}", model); - downloadThreadPool.awaitTermination(60, TimeUnit.SECONDS); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - downloadFinished.set(true); - synchronized (downloadFinished) { - downloadFinished.notifyAll(); - } - LOG.debug("Download for {} terminated", model); + finalizeDownload(); } } + private void finalizeDownload() { + downloadThreadPool.shutdown(); + try { + LOG.debug("Waiting for last segments for {}", model); + downloadThreadPool.awaitTermination(60, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + downloadFinished.set(true); + synchronized (downloadFinished) { + downloadFinished.notifyAll(); + } + LOG.debug("Download for {} terminated", model); + } + + private void handleHttpException(HttpException e) throws IOException { + if (e.getResponseCode() == 404) { + ctbrec.Model.State modelState; + try { + modelState = model.getOnlineState(false); + } catch (ExecutionException e1) { + modelState = ctbrec.Model.State.UNKNOWN; + } + LOG.info("Playlist not found (404). Model {} probably went offline. Model state: {}", model, modelState); + waitSomeTime(TEN_SECONDS); + } else if (e.getResponseCode() == 403) { + ctbrec.Model.State modelState; + try { + modelState = model.getOnlineState(false); + } catch (ExecutionException e1) { + modelState = ctbrec.Model.State.UNKNOWN; + } + LOG.info("Playlist access forbidden (403). Model {} probably went private or offline. Model state: {}", model, modelState); + waitSomeTime(TEN_SECONDS); + } else { + throw e; + } + } + + private void splitRecordingIfNecessary() { + if (splittingStrategy.splitNecessary(this)) { + internalStop(); + } + } + + private void enqueueNewSegments(SegmentPlaylist playlist, int nextSegmentNumber) throws IOException, ExecutionException, InterruptedException { + int skip = nextSegmentNumber - playlist.seq; + for (String segment : playlist.segments) { + if (skip > 0) { + skip--; + } else { + URL segmentUrl = new URL(segment); + String prefix = nf.format(segmentCounter++); + SegmentDownload segmentDownload = new SegmentDownload(playlist, segmentUrl, downloadDir, client, prefix); + enqueueDownload(segmentDownload, prefix, segmentUrl); + } + } + } + + private void logMissedSegments(SegmentPlaylist playlist, int nextSegmentNumber) { + if (nextSegmentNumber > 0 && playlist.seq > nextSegmentNumber) { + waitFactor *= 2; + LOG.warn("Missed segments {} < {} in download for {} - setting wait factor to 1/{}", nextSegmentNumber, playlist.seq, model, + waitFactor); + } + } + + private void waitSomeTime(SegmentPlaylist playlist, int lastSegmentNumber, int waitFactor) { + long waitForMillis = 0; + if (lastSegmentNumber == playlist.seq) { + // playlist didn't change -> wait for at least half the target duration + waitForMillis = (long) playlist.targetDuration * 1000 / waitFactor; + LOG.trace("Playlist didn't change... waiting for {}ms", waitForMillis); + } else { + // playlist did change -> wait for at least last segment duration + waitForMillis = 1; + LOG.trace("Playlist changed... waiting for {}ms", waitForMillis); + } + + waitSomeTime(waitForMillis); + } + private void enqueueDownload(SegmentDownload segmentDownload, String prefix, URL segmentUrl) throws IOException, ExecutionException, InterruptedException { try { downloadThreadPool.submit(segmentDownload); @@ -228,18 +249,6 @@ public class HlsDownload extends AbstractHlsDownload { } - private boolean splitRecording() { - if (config.getSettings().splitRecordings > 0) { - Duration recordingDuration = Duration.between(splitRecStartTime, ZonedDateTime.now()); - long seconds = recordingDuration.getSeconds(); - if (seconds >= config.getSettings().splitRecordings) { - internalStop(); - return true; - } - } - return false; - } - @Override public void stop() { if (running) { @@ -334,4 +343,9 @@ public class HlsDownload extends AbstractHlsDownload { public boolean isSingleFile() { return false; } + + @Override + public long getSizeInByte() { + return IoUtils.getDirectorySize(getTarget()); + } } diff --git a/common/src/main/java/ctbrec/recorder/download/hls/MergedFfmpegHlsDownload.java b/common/src/main/java/ctbrec/recorder/download/hls/MergedFfmpegHlsDownload.java index d5a0767d..136a159f 100644 --- a/common/src/main/java/ctbrec/recorder/download/hls/MergedFfmpegHlsDownload.java +++ b/common/src/main/java/ctbrec/recorder/download/hls/MergedFfmpegHlsDownload.java @@ -11,9 +11,7 @@ import java.io.OutputStream; import java.net.MalformedURLException; import java.net.URL; import java.nio.file.Files; -import java.time.Duration; import java.time.Instant; -import java.time.ZonedDateTime; import java.util.Arrays; import java.util.HashMap; import java.util.LinkedList; @@ -53,14 +51,13 @@ public class MergedFfmpegHlsDownload extends AbstractHlsDownload { private static final long serialVersionUID = 1L; private static final Logger LOG = LoggerFactory.getLogger(MergedFfmpegHlsDownload.class); private static final boolean IGNORE_CACHE = true; - private ZonedDateTime splitRecStartTime; private File targetFile; private transient Config config; private transient Process ffmpeg; private transient OutputStream ffmpegStdIn; protected transient Thread ffmpegThread; private transient Object ffmpegStartMonitor = new Object(); - private Queue> downloads = new LinkedList<>(); + private transient Queue> downloads = new LinkedList<>(); public MergedFfmpegHlsDownload(HttpClient client) { super(client); @@ -73,6 +70,7 @@ public class MergedFfmpegHlsDownload extends AbstractHlsDownload { this.model = model; String fileSuffix = config.getSettings().ffmpegFileSuffix; targetFile = config.getFileForRecording(model, fileSuffix, startTime); + splittingStrategy = initSplittingStrategy(config.getSettings()); } @Override @@ -86,7 +84,6 @@ public class MergedFfmpegHlsDownload extends AbstractHlsDownload { running = true; Thread.currentThread().setName("Download " + model.getName()); super.startTime = Instant.now(); - splitRecStartTime = ZonedDateTime.now(); String segments = getSegmentPlaylistUrl(model); @@ -211,11 +208,7 @@ public class MergedFfmpegHlsDownload extends AbstractHlsDownload { } if (livestreamDownload) { - // split up the recording, if configured - boolean split = splitRecording(); - if (split) { - break; - } + splitRecordingIfNecessary(); // wait some time until requesting the segment playlist again to not hammer the server waitForNewSegments(lsp, lastSegment, downloadTookMillis); @@ -245,6 +238,12 @@ public class MergedFfmpegHlsDownload extends AbstractHlsDownload { ffmpegThread.interrupt(); } + protected void splitRecordingIfNecessary() { + if (splittingStrategy.splitNecessary(this)) { + internalStop(); + } + } + private void downloadRecording(SegmentPlaylist lsp) throws IOException { for (String segment : lsp.segments) { URL segmentUrl = new URL(segment); @@ -337,18 +336,6 @@ public class MergedFfmpegHlsDownload extends AbstractHlsDownload { writeSegment(segmentData, 0, segmentData.length); } - protected boolean splitRecording() { - if (config.getSettings().splitRecordings > 0) { - Duration recordingDuration = Duration.between(splitRecStartTime, ZonedDateTime.now()); - long seconds = recordingDuration.getSeconds(); - if (seconds >= config.getSettings().splitRecordings) { - internalStop(); - return true; - } - } - return false; - } - private void waitForNewSegments(SegmentPlaylist lsp, int lastSegment, long downloadTookMillis) { try { long wait = 0; @@ -493,6 +480,7 @@ public class MergedFfmpegHlsDownload extends AbstractHlsDownload { @Override public void postprocess(Recording recording) { + // nothing to do } public void downloadFinishedRecording(String segmentPlaylistUri, File target, ProgressListener progressListener, long sizeInBytes) throws Exception { @@ -549,4 +537,9 @@ public class MergedFfmpegHlsDownload extends AbstractHlsDownload { public boolean isSingleFile() { return true; } + + @Override + public long getSizeInByte() { + return getTarget().length(); + } } diff --git a/common/src/main/java/ctbrec/recorder/download/hls/NoopSplittingStrategy.java b/common/src/main/java/ctbrec/recorder/download/hls/NoopSplittingStrategy.java new file mode 100644 index 00000000..6af9404b --- /dev/null +++ b/common/src/main/java/ctbrec/recorder/download/hls/NoopSplittingStrategy.java @@ -0,0 +1,19 @@ +package ctbrec.recorder.download.hls; + +import ctbrec.Settings; +import ctbrec.recorder.download.Download; +import ctbrec.recorder.download.SplittingStrategy; + +public class NoopSplittingStrategy implements SplittingStrategy { + + @Override + public void init(Settings settings) { + // settings not needed + } + + @Override + public boolean splitNecessary(Download download) { + return false; + } + +} diff --git a/common/src/main/java/ctbrec/recorder/download/hls/SizeSplittingStrategy.java b/common/src/main/java/ctbrec/recorder/download/hls/SizeSplittingStrategy.java new file mode 100644 index 00000000..e37eadb0 --- /dev/null +++ b/common/src/main/java/ctbrec/recorder/download/hls/SizeSplittingStrategy.java @@ -0,0 +1,22 @@ +package ctbrec.recorder.download.hls; + +import ctbrec.Settings; +import ctbrec.recorder.download.Download; +import ctbrec.recorder.download.SplittingStrategy; + +public class SizeSplittingStrategy implements SplittingStrategy { + + private Settings settings; + + @Override + public void init(Settings settings) { + this.settings = settings; + } + + @Override + public boolean splitNecessary(Download download) { + long sizeInByte = download.getSizeInByte(); + return sizeInByte >= settings.splitRecordingsBiggerThanBytes; + } + +} diff --git a/common/src/main/java/ctbrec/recorder/download/hls/TimeSplittingStrategy.java b/common/src/main/java/ctbrec/recorder/download/hls/TimeSplittingStrategy.java new file mode 100644 index 00000000..8fb5a119 --- /dev/null +++ b/common/src/main/java/ctbrec/recorder/download/hls/TimeSplittingStrategy.java @@ -0,0 +1,28 @@ +package ctbrec.recorder.download.hls; + +import java.time.Duration; +import java.time.ZoneId; +import java.time.ZonedDateTime; + +import ctbrec.Settings; +import ctbrec.recorder.download.Download; +import ctbrec.recorder.download.SplittingStrategy; + +public class TimeSplittingStrategy implements SplittingStrategy { + + private Settings settings; + + @Override + public void init(Settings settings) { + this.settings = settings; + } + + @Override + public boolean splitNecessary(Download download) { + ZonedDateTime startTime = download.getStartTime().atZone(ZoneId.systemDefault()); + Duration recordingDuration = Duration.between(startTime, ZonedDateTime.now()); + long seconds = recordingDuration.getSeconds(); + return seconds >= settings.splitRecordingsAfterSecs; + } + +} diff --git a/common/src/main/java/ctbrec/sites/showup/ShowupMergedDownload.java b/common/src/main/java/ctbrec/sites/showup/ShowupMergedDownload.java index b9516ad0..78a33af3 100644 --- a/common/src/main/java/ctbrec/sites/showup/ShowupMergedDownload.java +++ b/common/src/main/java/ctbrec/sites/showup/ShowupMergedDownload.java @@ -48,8 +48,8 @@ public class ShowupMergedDownload extends MergedFfmpegHlsDownload { BandwidthMeter.add(length); writeSegment(buffer, 0, length); keepGoing = running && !Thread.interrupted() && model.isOnline(true); - if (livestreamDownload && splitRecording()) { - break; + if (livestreamDownload) { + splitRecordingIfNecessary(); } } } else {