From b959c57b8f9dbf05742f1383a154523cdd1c6dd9 Mon Sep 17 00:00:00 2001 From: 0xb00bface <0xboobface@gmail.com> Date: Fri, 10 Sep 2021 21:37:11 +0200 Subject: [PATCH] Generate / update playlist while recording --- .../main/java/ctbrec/RecordingDownload.java | 5 +- .../resources/html/docs/ConfigurationFile.md | 4 - common/src/main/java/ctbrec/Settings.java | 2 - .../recorder/AccuratePlaylistGenerator.java | 93 ------------ .../recorder/FastPlaylistGenerator.java | 69 --------- .../recorder/InvalidPlaylistException.java | 7 + .../recorder/InvalidTrackLengthException.java | 7 + .../ctbrec/recorder/PlaylistGenerator.java | 129 ---------------- .../recorder/download/AbstractDownload.java | 6 +- .../download/hls/AbstractHlsDownload.java | 55 ++++--- .../recorder/download/hls/FFmpegDownload.java | 3 +- .../recorder/download/hls/HlsDownload.java | 142 ++++++++++++++---- .../download/hls/MergedFfmpegHlsDownload.java | 13 +- .../download/hls/SegmentDownload.java | 12 +- .../download/hls/SegmentPlaylist.java | 15 +- .../ctbrec/recorder/server/ConfigServlet.java | 2 - 16 files changed, 197 insertions(+), 367 deletions(-) delete mode 100644 common/src/main/java/ctbrec/recorder/AccuratePlaylistGenerator.java delete mode 100644 common/src/main/java/ctbrec/recorder/FastPlaylistGenerator.java create mode 100644 common/src/main/java/ctbrec/recorder/InvalidPlaylistException.java create mode 100644 common/src/main/java/ctbrec/recorder/InvalidTrackLengthException.java delete mode 100644 common/src/main/java/ctbrec/recorder/PlaylistGenerator.java diff --git a/client/src/main/java/ctbrec/RecordingDownload.java b/client/src/main/java/ctbrec/RecordingDownload.java index 23e2cedb..7c437e69 100644 --- a/client/src/main/java/ctbrec/RecordingDownload.java +++ b/client/src/main/java/ctbrec/RecordingDownload.java @@ -17,6 +17,7 @@ import ctbrec.io.HttpException; import ctbrec.recorder.ProgressListener; import ctbrec.recorder.download.hls.MergedFfmpegHlsDownload; import ctbrec.recorder.download.hls.SegmentPlaylist; +import ctbrec.recorder.download.hls.SegmentPlaylist.Segment; import okhttp3.Request; import okhttp3.Response; @@ -56,8 +57,8 @@ public class RecordingDownload extends MergedFfmpegHlsDownload { SegmentPlaylist segmentPlaylist = getNextSegments(segmentPlaylistUri); long loadedBytes = 0; - for (String segmentUrl : segmentPlaylist.segments) { - loadedBytes += downloadFile(segmentUrl, loadedBytes, sizeInBytes, progressListener); + for (Segment segment : segmentPlaylist.segments) { + loadedBytes += downloadFile(segment.url, loadedBytes, sizeInBytes, progressListener); int progress = (int) (loadedBytes / (double) sizeInBytes * 100); progressListener.update(progress); } diff --git a/client/src/main/resources/html/docs/ConfigurationFile.md b/client/src/main/resources/html/docs/ConfigurationFile.md index bc6f5f38..8d659d49 100644 --- a/client/src/main/resources/html/docs/ConfigurationFile.md +++ b/client/src/main/resources/html/docs/ConfigurationFile.md @@ -24,10 +24,6 @@ until a recording is finished. 0 means unlimited. - **determineResolution** (app only) - [`true`,`false`] Display the stream resolution on the thumbnails. -- **fastPlaylistGenerator** (server only) - Use a fast playlist generator, which is not as accurate. This might lead to inaccurate skipping and timelines in media players. Useful for weak devices. - -- **generatePlaylist** (server only) - [`true`,`false`] Generate a playlist once a recording terminates. - - **hlsdlExecutable** - Path to the hlsdl executable, which is used, if `useHlsdl` is set to true - **httpPort** - [1 - 65536] The TCP port, the server listens on. In the server config, this will tell the server, which port to use. In the application this will set diff --git a/common/src/main/java/ctbrec/Settings.java b/common/src/main/java/ctbrec/Settings.java index ef4c05c9..38bb0cf7 100644 --- a/common/src/main/java/ctbrec/Settings.java +++ b/common/src/main/java/ctbrec/Settings.java @@ -69,7 +69,6 @@ public class Settings { public List disabledSites = new ArrayList<>(); public String downloadFilename = "${modelSanitizedName}-${localDateTime}"; public List eventHandlers = new ArrayList<>(); - public boolean fastPlaylistGenerator = false; public boolean fastScrollSpeed = true; public String fc2livePassword = ""; public String fc2liveUsername = ""; @@ -79,7 +78,6 @@ public class Settings { public String flirt4freeUsername; public String fontFamily = "Sans-Serif"; public int fontSize = 14; - public boolean generatePlaylist = true; public String hlsdlExecutable = "hlsdl"; public int httpPort = 8080; public int httpSecurePort = 8443; diff --git a/common/src/main/java/ctbrec/recorder/AccuratePlaylistGenerator.java b/common/src/main/java/ctbrec/recorder/AccuratePlaylistGenerator.java deleted file mode 100644 index 73abfed2..00000000 --- a/common/src/main/java/ctbrec/recorder/AccuratePlaylistGenerator.java +++ /dev/null @@ -1,93 +0,0 @@ -package ctbrec.recorder; - -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.FilenameFilter; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.common.io.Files; -import com.iheartradio.m3u8.Encoding; -import com.iheartradio.m3u8.Format; -import com.iheartradio.m3u8.ParseException; -import com.iheartradio.m3u8.ParsingMode; -import com.iheartradio.m3u8.PlaylistException; -import com.iheartradio.m3u8.PlaylistParser; -import com.iheartradio.m3u8.data.MediaPlaylist; -import com.iheartradio.m3u8.data.Playlist; -import com.iheartradio.m3u8.data.TrackData; -import com.iheartradio.m3u8.data.TrackInfo; - -import ctbrec.MpegUtil; - - -public class AccuratePlaylistGenerator extends PlaylistGenerator { - private static final Logger LOG = LoggerFactory.getLogger(AccuratePlaylistGenerator.class); - - AccuratePlaylistGenerator() { - } - - @Override - public File generate(File directory, String fileSuffix) throws IOException, ParseException, PlaylistException { - LOG.info("Starting playlist generation for {}", directory); - File[] files = scanDirectoryForSegments(directory, fileSuffix); - - // create a track containing all files - List track = new ArrayList<>(); - int total = files.length; - int done = 0; - for (File file : files) { - try { - float duration = 0; - if (file.getName().toLowerCase().endsWith(".ts")) { - duration = (float) MpegUtil.getFileDurationInSecs(file); - if (duration <= 0) { - throw new InvalidTrackLengthException("Track has negative duration: " + file.getName()); - } - } - - track.add(new TrackData.Builder() - .withUri(file.getName()) - .withTrackInfo(new TrackInfo(duration, file.getName())) - .build()); - } catch (Exception | AssertionError e) { - LOG.warn("Couldn't determine duration for {}. Skipping this file.", file.getName()); - File corruptedFile = new File(directory, file.getName() + ".corrupt"); - Files.move(file, corruptedFile); - } - done++; - double percentage = (double) done / (double) total; - updateProgressListeners(percentage); - } - - return writePlaylistFile(directory, track); - } - - @Override - public void validate(File recDir) throws IOException, ParseException, PlaylistException { - File playlist = new File(recDir, "playlist.m3u8"); - if (playlist.exists()) { - try (FileInputStream fin = new FileInputStream(playlist)) { - PlaylistParser playlistParser = new PlaylistParser(fin, Format.EXT_M3U, Encoding.UTF_8, ParsingMode.LENIENT); - Playlist m3u = playlistParser.parse(); - MediaPlaylist mediaPlaylist = m3u.getMediaPlaylist(); - int playlistSize = mediaPlaylist.getTracks().size(); - File[] segments = recDir.listFiles((FilenameFilter) (dir, name) -> name.endsWith(".ts")); - if (segments.length == 0) { - throw new InvalidPlaylistException("No segments found. Playlist is empty"); - } else if (segments.length != playlistSize) { - throw new InvalidPlaylistException("Playlist size and amount of segments differ (" + segments.length + " != " + playlistSize + ")"); - } else { - LOG.debug("Generated playlist looks good"); - } - } - } else { - throw new FileNotFoundException(playlist.getAbsolutePath() + " does not exist"); - } - } -} diff --git a/common/src/main/java/ctbrec/recorder/FastPlaylistGenerator.java b/common/src/main/java/ctbrec/recorder/FastPlaylistGenerator.java deleted file mode 100644 index 5667309c..00000000 --- a/common/src/main/java/ctbrec/recorder/FastPlaylistGenerator.java +++ /dev/null @@ -1,69 +0,0 @@ -package ctbrec.recorder; - -import java.io.File; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.iheartradio.m3u8.ParseException; -import com.iheartradio.m3u8.PlaylistException; -import com.iheartradio.m3u8.data.TrackData; -import com.iheartradio.m3u8.data.TrackInfo; - -import ctbrec.MpegUtil; - - -public class FastPlaylistGenerator extends PlaylistGenerator { - static final Logger LOG = LoggerFactory.getLogger(FastPlaylistGenerator.class); - - FastPlaylistGenerator() { - } - - @Override - public File generate(File directory, String fileSuffix) throws IOException, ParseException, PlaylistException { - LOG.info("Starting playlist generation for {}", directory); - File[] files = scanDirectoryForSegments(directory, fileSuffix); - - // create a track containing all files - List track = new ArrayList<>(); - int total = files.length; - int done = 0; - float duration = getFileDuration(files); - for (File file : files) { - track.add(new TrackData.Builder() - .withUri(file.getName()) - .withTrackInfo(new TrackInfo(duration, file.getName())) - .build()); - done++; - double percentage = (double) done / (double) total; - updateProgressListeners(percentage); - } - - return writePlaylistFile(directory, track); - } - - private float getFileDuration(File[] files) { - for (File file : files) { - try { - float duration = 0; - if (file.getName().toLowerCase().endsWith(".ts")) { - duration = (float) MpegUtil.getFileDurationInSecs(file); - if (duration > 0) { - return duration; - } - } - } catch (Exception e) { - LOG.warn("Couldn't determine duration for {}. Skipping this file.", file.getName()); - } - } - throw new RuntimeException("Couldn't determine playlist segment length"); - } - - @Override - public void validate(File recDir) throws IOException, ParseException, PlaylistException { - // don't validate anything - } -} diff --git a/common/src/main/java/ctbrec/recorder/InvalidPlaylistException.java b/common/src/main/java/ctbrec/recorder/InvalidPlaylistException.java new file mode 100644 index 00000000..4631cd87 --- /dev/null +++ b/common/src/main/java/ctbrec/recorder/InvalidPlaylistException.java @@ -0,0 +1,7 @@ +package ctbrec.recorder; + +public class InvalidPlaylistException extends RuntimeException { + public InvalidPlaylistException(String msg) { + super(msg); + } +} \ No newline at end of file diff --git a/common/src/main/java/ctbrec/recorder/InvalidTrackLengthException.java b/common/src/main/java/ctbrec/recorder/InvalidTrackLengthException.java new file mode 100644 index 00000000..1548f586 --- /dev/null +++ b/common/src/main/java/ctbrec/recorder/InvalidTrackLengthException.java @@ -0,0 +1,7 @@ +package ctbrec.recorder; + +public class InvalidTrackLengthException extends RuntimeException { + public InvalidTrackLengthException(String msg) { + super(msg); + } +} \ No newline at end of file diff --git a/common/src/main/java/ctbrec/recorder/PlaylistGenerator.java b/common/src/main/java/ctbrec/recorder/PlaylistGenerator.java deleted file mode 100644 index e6c33906..00000000 --- a/common/src/main/java/ctbrec/recorder/PlaylistGenerator.java +++ /dev/null @@ -1,129 +0,0 @@ -package ctbrec.recorder; - -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.iheartradio.m3u8.Encoding; -import com.iheartradio.m3u8.Format; -import com.iheartradio.m3u8.ParseException; -import com.iheartradio.m3u8.PlaylistException; -import com.iheartradio.m3u8.PlaylistWriter; -import com.iheartradio.m3u8.data.MediaPlaylist; -import com.iheartradio.m3u8.data.Playlist; -import com.iheartradio.m3u8.data.PlaylistType; -import com.iheartradio.m3u8.data.TrackData; - -public abstract class PlaylistGenerator { - - private static final Logger LOG = LoggerFactory.getLogger(PlaylistGenerator.class); - private int lastPercentage; - private List listeners = new ArrayList<>(); - - public static PlaylistGenerator newInstance(boolean fast) { - if(fast) { - return new FastPlaylistGenerator(); - } else { - return new AccuratePlaylistGenerator(); - } - } - - public File generate(File directory) throws IOException, ParseException, PlaylistException { - return generate(directory, "ts"); - } - - public abstract File generate(File directory, String fileSuffix) throws IOException, ParseException, PlaylistException; - - public void addProgressListener(ProgressListener l) { - listeners.add(l); - } - - public int getProgress() { - return lastPercentage; - } - - protected void updateProgressListeners(double percentage) { - int p = (int) (percentage * 100); - if (p > lastPercentage) { - for (ProgressListener progressListener : listeners) { - progressListener.update(p); - } - lastPercentage = p; - } - } - - public static class InvalidPlaylistException extends RuntimeException { - public InvalidPlaylistException(String msg) { - super(msg); - } - } - - public static class InvalidTrackLengthException extends RuntimeException { - public InvalidTrackLengthException(String msg) { - super(msg); - } - } - - public abstract void validate(File recDir) throws IOException, ParseException, PlaylistException; - - protected File[] scanDirectoryForSegments(File directory, String fileSuffix) { - // get a list of all ts files and sort them by sequence - File[] files = directory.listFiles(f -> f.getName().endsWith('.' + fileSuffix)); - if (files == null || files.length == 0) { - LOG.debug("{} is empty. Not going to generate a playlist", directory); - throw new InvalidPlaylistException("Directory is empty"); - } - - Arrays.sort(files, (f1, f2) -> { - String n1 = f1.getName(); - String n2 = f2.getName(); - return n1.compareTo(n2); - }); - return files; - } - - protected File writePlaylistFile(File directory, List track) throws IOException, ParseException, PlaylistException { - // create a media playlist - float targetDuration = getAvgDuration(track); - MediaPlaylist playlist = new MediaPlaylist.Builder() - .withPlaylistType(PlaylistType.VOD) - .withMediaSequenceNumber(0) - .withTargetDuration((int) targetDuration) - .withTracks(track).build(); - - // create a master playlist containing the media playlist - Playlist master = new Playlist.Builder() - .withCompatibilityVersion(4) - .withExtended(true) - .withMediaPlaylist(playlist) - .build(); - - // write the playlist to a file - File output = new File(directory, "playlist.m3u8"); - try (FileOutputStream fos = new FileOutputStream(output)) { - PlaylistWriter writer = new PlaylistWriter.Builder() - .withFormat(Format.EXT_M3U) - .withEncoding(Encoding.UTF_8) - .withOutputStream(fos) - .build(); - writer.write(master); - LOG.debug("Finished playlist generation for {}", directory); - } - return output; - } - - private float getAvgDuration(List track) { - float targetDuration = 0; - for (TrackData trackData : track) { - targetDuration += trackData.getTrackInfo().duration; - } - targetDuration /= track.size(); - return targetDuration; - } -} diff --git a/common/src/main/java/ctbrec/recorder/download/AbstractDownload.java b/common/src/main/java/ctbrec/recorder/download/AbstractDownload.java index d66393c4..c260fb7f 100644 --- a/common/src/main/java/ctbrec/recorder/download/AbstractDownload.java +++ b/common/src/main/java/ctbrec/recorder/download/AbstractDownload.java @@ -19,9 +19,9 @@ public abstract class AbstractDownload implements Download { protected Instant rescheduleTime = Instant.now(); protected Model model = new UnknownModel(); - protected transient Config config; - protected transient SplittingStrategy splittingStrategy; - protected transient ExecutorService downloadExecutor; + protected Config config; + protected SplittingStrategy splittingStrategy; + protected ExecutorService downloadExecutor; @Override public void init(Config config, Model model, Instant startTime, ExecutorService executorService) throws IOException { diff --git a/common/src/main/java/ctbrec/recorder/download/hls/AbstractHlsDownload.java b/common/src/main/java/ctbrec/recorder/download/hls/AbstractHlsDownload.java index 8f79f160..d0499308 100644 --- a/common/src/main/java/ctbrec/recorder/download/hls/AbstractHlsDownload.java +++ b/common/src/main/java/ctbrec/recorder/download/hls/AbstractHlsDownload.java @@ -31,8 +31,10 @@ import java.util.Locale; import java.util.Map; import java.util.Map.Entry; import java.util.Optional; -import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorCompletionService; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; import java.util.stream.Collectors; import javax.xml.bind.JAXBException; @@ -59,10 +61,11 @@ import ctbrec.io.BandwidthMeter; import ctbrec.io.HttpClient; import ctbrec.io.HttpConstants; import ctbrec.io.HttpException; -import ctbrec.recorder.PlaylistGenerator.InvalidPlaylistException; +import ctbrec.recorder.InvalidPlaylistException; import ctbrec.recorder.download.AbstractDownload; import ctbrec.recorder.download.HttpHeaderFactory; import ctbrec.recorder.download.StreamSource; +import ctbrec.recorder.download.hls.SegmentPlaylist.Segment; import ctbrec.sites.Site; import okhttp3.Request; import okhttp3.Request.Builder; @@ -92,12 +95,19 @@ public abstract class AbstractHlsDownload extends AbstractDownload { private int selectedResolution = UNKNOWN; private List recordingEvents = new LinkedList<>(); + protected ExecutorCompletionService segmentDownloadService; protected AbstractHlsDownload(HttpClient client) { this.client = client; } - protected abstract OutputStream getSegmentOutputStream(String prefix, String fileName) throws IOException; + @Override + public void init(Config config, Model model, Instant startTime, ExecutorService executorService) throws IOException { + super.init(config, model, startTime, executorService); + segmentDownloadService = new ExecutorCompletionService<>(downloadExecutor); + } + + protected abstract OutputStream getSegmentOutputStream(Segment segment) throws IOException; protected void segmentDownloadFinished(SegmentDownload segmentDownload) { // NOSONAR if (Duration.between(lastSegmentDownload, Instant.now()).getSeconds() > MAX_SECONDS_WITHOUT_TRANSFER) { @@ -123,8 +133,9 @@ public abstract class AbstractHlsDownload extends AbstractDownload { enqueueNewSegments(segmentPlaylist, nextSegmentNumber); splitRecordingIfNecessary(); calculateRescheduleTime(); + processFinishedSegments(); - // this if check makes sure, that we don't decrease nextSegment. for some reason + // this if-check makes sure, that we don't decrease nextSegment. for some reason // streamate playlists sometimes jump back. e.g. max sequence = 79 -> 80 -> 79 lastSegmentNumber = segmentPlaylist.seq; if (lastSegmentNumber + segmentPlaylist.segments.size() > nextSegmentNumber) { @@ -134,6 +145,7 @@ public abstract class AbstractHlsDownload extends AbstractDownload { while (recordingEvents.size() > 30) { recordingEvents.remove(0); } + } catch (ParseException e) { LOG.error("Couldn't parse HLS playlist for model {}\n{}", model, e.getInput(), e); stop(); @@ -163,20 +175,24 @@ public abstract class AbstractHlsDownload extends AbstractDownload { return this; } - protected void execute(SegmentDownload segmentDownload) { - CompletableFuture.supplyAsync(() -> downloadExecutor.submit(segmentDownload), downloadExecutor) - .whenComplete((result, executor) -> { - try { - segmentDownloadFinished(result.get()); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - LOG.error("Error in segmentDownloadFinished", e); - } catch (ExecutionException e) { - LOG.error("Error in segmentDownloadFinished", e); + protected void processFinishedSegments() { + downloadExecutor.submit((Runnable)() -> { + Future future; + while ((future = segmentDownloadService.poll()) != null) { + try { + segmentDownloadFinished(future.get()); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + LOG.error("Error in segmentDownloadFinished", e); + } catch (ExecutionException e) { + LOG.error("Error in segmentDownloadFinished", e); + } } }); } + protected abstract void execute(SegmentDownload segmentDownload); + protected void handleHttpException(HttpException e) throws IOException { if (e.getResponseCode() == 404) { checkIfModelIsStillOnline("Playlist not found (404). Model {} probably went offline. Model state: {}"); @@ -307,7 +323,7 @@ public abstract class AbstractHlsDownload extends AbstractDownload { uri = new URL(context, uri).toExternalForm(); } lsp.totalDuration += trackData.getTrackInfo().duration; - lsp.segments.add(uri); + lsp.segments.add(new Segment(uri, trackData.getTrackInfo().duration)); if (trackData.hasEncryptionData()) { lsp.encrypted = true; EncryptionData data = trackData.getEncryptionData(); @@ -367,15 +383,14 @@ public abstract class AbstractHlsDownload extends AbstractDownload { protected void enqueueNewSegments(SegmentPlaylist playlist, int nextSegmentNumber) throws IOException { int skip = nextSegmentNumber - playlist.seq; - for (String segment : playlist.segments) { + for (Segment segment : playlist.segments) { if (skip > 0) { skip--; } else { - URL segmentUrl = new URL(segment); String prefix = nf.format(segmentCounter++); - File tmp = new File(segmentUrl.getFile()); - OutputStream targetStream = getSegmentOutputStream(prefix, tmp.getName()); - SegmentDownload segmentDownload = new SegmentDownload(model, playlist, segmentUrl, client, targetStream); + segment.prefix = prefix; + OutputStream targetStream = getSegmentOutputStream(segment); + SegmentDownload segmentDownload = new SegmentDownload(model, playlist, segment, client, targetStream); execute(segmentDownload); } } diff --git a/common/src/main/java/ctbrec/recorder/download/hls/FFmpegDownload.java b/common/src/main/java/ctbrec/recorder/download/hls/FFmpegDownload.java index 0ad3ee11..90a0083d 100644 --- a/common/src/main/java/ctbrec/recorder/download/hls/FFmpegDownload.java +++ b/common/src/main/java/ctbrec/recorder/download/hls/FFmpegDownload.java @@ -27,6 +27,7 @@ import ctbrec.Recording; import ctbrec.io.HttpClient; import ctbrec.io.StreamRedirector; import ctbrec.recorder.download.ProcessExitedUncleanException; +import ctbrec.recorder.download.hls.SegmentPlaylist.Segment; /** * Does the whole HLS download with FFmpeg. Not used at the moment, because FFMpeg can't @@ -162,7 +163,7 @@ public class FFmpegDownload extends AbstractHlsDownload { } @Override - protected OutputStream getSegmentOutputStream(String prefix, String fileName) throws IOException { + protected OutputStream getSegmentOutputStream(Segment segment) throws IOException { // TODO Auto-generated method stub return null; } diff --git a/common/src/main/java/ctbrec/recorder/download/hls/HlsDownload.java b/common/src/main/java/ctbrec/recorder/download/hls/HlsDownload.java index cc1d4a8c..f4da7927 100644 --- a/common/src/main/java/ctbrec/recorder/download/hls/HlsDownload.java +++ b/common/src/main/java/ctbrec/recorder/download/hls/HlsDownload.java @@ -5,6 +5,8 @@ import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStream; +import java.net.MalformedURLException; +import java.net.URL; import java.nio.file.FileSystems; import java.nio.file.Files; import java.nio.file.Path; @@ -12,28 +14,45 @@ import java.time.Instant; import java.time.ZoneId; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; +import java.util.LinkedList; +import java.util.List; +import java.util.Queue; import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; import java.util.regex.Pattern; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.iheartradio.m3u8.Encoding; +import com.iheartradio.m3u8.Format; import com.iheartradio.m3u8.ParseException; import com.iheartradio.m3u8.PlaylistException; +import com.iheartradio.m3u8.PlaylistWriter; +import com.iheartradio.m3u8.data.MediaPlaylist; +import com.iheartradio.m3u8.data.Playlist; +import com.iheartradio.m3u8.data.PlaylistType; +import com.iheartradio.m3u8.data.TrackData; +import com.iheartradio.m3u8.data.TrackInfo; import ctbrec.Config; import ctbrec.Model; import ctbrec.Recording; -import ctbrec.Recording.State; import ctbrec.io.HttpClient; import ctbrec.io.IoUtils; -import ctbrec.recorder.PlaylistGenerator; +import ctbrec.recorder.download.hls.SegmentPlaylist.Segment; public class HlsDownload extends AbstractHlsDownload { private static final Logger LOG = LoggerFactory.getLogger(HlsDownload.class); - protected transient Path downloadDir; + protected Path downloadDir; + + private Queue> segmentDownloads = new LinkedList<>(); + + private List segments = new LinkedList<>(); + + private float targetDuration; public HlsDownload(HttpClient client) { super(client); @@ -49,6 +68,83 @@ public class HlsDownload extends AbstractHlsDownload { createTargetDirectory(); } + @Override + public AbstractHlsDownload call() throws Exception { + super.call(); + updatePlaylist(); + return this; + } + + @Override + protected SegmentPlaylist getNextSegments(String segmentPlaylistUrl) throws IOException, ParseException, PlaylistException { + SegmentPlaylist segmentPlaylist = super.getNextSegments(segmentPlaylistUrl); + targetDuration = segmentPlaylist.targetDuration; + return segmentPlaylist; + } + + private void updatePlaylist() { + downloadExecutor.submit(() -> { + addNewSegmentsToPlaylist(); + try { + MediaPlaylist playlist = new MediaPlaylist.Builder() + .withPlaylistType(PlaylistType.VOD) + .withMediaSequenceNumber(0) + .withTargetDuration(Math.round(targetDuration)) + .withTracks(segments) + .build(); + + // create a master playlist containing the media playlist + Playlist master = new Playlist.Builder() + .withCompatibilityVersion(4) + .withExtended(true) + .withMediaPlaylist(playlist) + .build(); + + // write the playlist to a file + File output = new File(getTarget(), "playlist.m3u8"); + try (FileOutputStream fos = new FileOutputStream(output)) { + PlaylistWriter writer = new PlaylistWriter.Builder() + .withFormat(Format.EXT_M3U) + .withEncoding(Encoding.UTF_8) + .withOutputStream(fos) + .build(); + writer.write(master); + } + } catch (IOException | ParseException | PlaylistException e) { + LOG.error("Updating segment playlist failed", e); + } + LOG.trace("Segment queue size for {}: {}", model, segmentDownloads.size()); + }); + } + + private void addNewSegmentsToPlaylist() { + Future future; + while ((future = segmentDownloads.peek()) != null && !Thread.currentThread().isInterrupted()) { + try { + if (running && future.isDone()) { + segmentDownloads.poll(); // future is done remove from queue + SegmentDownload segmentDownload = future.get(); + segments.add(toTrack(segmentDownload.getSegment())); + } else { + // first download in queue not finished, let's continue with other stuff + break; + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } catch (Exception e) { + LOG.error("Segment download failed for model {}", model, e); + } + } + } + + private TrackData toTrack(Segment segment) { + String filename = segment.targetFile.getName(); + return new TrackData.Builder() + .withUri(filename) + .withTrackInfo(new TrackInfo(segment.duration, filename)) + .build(); + } + protected void createTargetDirectory() throws IOException { if (!downloadDir.toFile().exists()) { Files.createDirectories(downloadDir); @@ -62,30 +158,7 @@ public class HlsDownload extends AbstractHlsDownload { @Override public void postprocess(Recording recording) { - Thread.currentThread().setName("PP " + model.getName()); - recording.setStatusWithEvent(State.GENERATING_PLAYLIST); - try { - generatePlaylist(recording); - recording.setStatusWithEvent(State.POST_PROCESSING); - } catch (Exception e) { - throw new PostProcessingException(e); - } - } - - protected File generatePlaylist(Recording recording) throws IOException, ParseException, PlaylistException { - File recDir = recording.getAbsoluteFile(); - if (!config.getSettings().generatePlaylist) { - return null; - } - PlaylistGenerator playlistGenerator = PlaylistGenerator.newInstance(config.getSettings().fastPlaylistGenerator); - playlistGenerator.addProgressListener(recording::setProgress); - File playlist = playlistGenerator.generate(recDir); - if (playlist != null) { - playlistGenerator.validate(recDir); - } - recording.setProgress(-1); - return playlist; - + // nothing to do } @Override @@ -123,8 +196,10 @@ public class HlsDownload extends AbstractHlsDownload { } @Override - protected OutputStream getSegmentOutputStream(String prefix, String fileName) throws FileNotFoundException { - String prefixedFileName = prefix + '_' + fileName; + protected OutputStream getSegmentOutputStream(Segment segment) throws FileNotFoundException, MalformedURLException { + URL segmentUrl = new URL(segment.url); + File tmp = new File(segmentUrl.getFile()); + String prefixedFileName = segment.prefix + '_' + tmp.getName(); int questionMarkPosition = prefixedFileName.indexOf('?'); if (questionMarkPosition > 0) { prefixedFileName = prefixedFileName.substring(0, questionMarkPosition); @@ -132,8 +207,8 @@ public class HlsDownload extends AbstractHlsDownload { if (!prefixedFileName.endsWith(".ts")) { prefixedFileName += ".ts"; } - File file = FileSystems.getDefault().getPath(downloadDir.toAbsolutePath().toString(), prefixedFileName).toFile(); - return new FileOutputStream(file); + segment.targetFile = FileSystems.getDefault().getPath(downloadDir.toAbsolutePath().toString(), prefixedFileName).toFile(); + return new FileOutputStream(segment.targetFile); } @Override @@ -141,4 +216,9 @@ public class HlsDownload extends AbstractHlsDownload { super.segmentDownloadFinished(segmentDownload); IoUtils.close(segmentDownload.getOutputStream(), "Couldn't close segment file"); } + + @Override + protected void execute(SegmentDownload segmentDownload) { + segmentDownloads.add(segmentDownloadService.submit(segmentDownload)); + } } diff --git a/common/src/main/java/ctbrec/recorder/download/hls/MergedFfmpegHlsDownload.java b/common/src/main/java/ctbrec/recorder/download/hls/MergedFfmpegHlsDownload.java index 725a5fe7..1aa3c314 100644 --- a/common/src/main/java/ctbrec/recorder/download/hls/MergedFfmpegHlsDownload.java +++ b/common/src/main/java/ctbrec/recorder/download/hls/MergedFfmpegHlsDownload.java @@ -25,17 +25,18 @@ import ctbrec.Recording; import ctbrec.io.HttpClient; import ctbrec.recorder.FFmpeg; import ctbrec.recorder.download.ProcessExitedUncleanException; +import ctbrec.recorder.download.hls.SegmentPlaylist.Segment; public class MergedFfmpegHlsDownload extends AbstractHlsDownload { private static final Logger LOG = LoggerFactory.getLogger(MergedFfmpegHlsDownload.class); protected File targetFile; - protected transient FFmpeg ffmpeg; - protected transient Process ffmpegProcess; - protected transient OutputStream ffmpegStdIn; - protected transient BlockingQueue> queue = new LinkedBlockingQueue<>(); - protected transient Lock ffmpegStreamLock = new ReentrantLock(); + protected FFmpeg ffmpeg; + protected Process ffmpegProcess; + protected OutputStream ffmpegStdIn; + protected BlockingQueue> queue = new LinkedBlockingQueue<>(); + protected Lock ffmpegStreamLock = new ReentrantLock(); public MergedFfmpegHlsDownload(HttpClient client) { super(client); @@ -212,7 +213,7 @@ public class MergedFfmpegHlsDownload extends AbstractHlsDownload { } @Override - protected OutputStream getSegmentOutputStream(String prefix, String fileName) throws IOException { + protected OutputStream getSegmentOutputStream(Segment segment) throws IOException { return new ByteArrayOutputStream(); } } diff --git a/common/src/main/java/ctbrec/recorder/download/hls/SegmentDownload.java b/common/src/main/java/ctbrec/recorder/download/hls/SegmentDownload.java index 5709130b..578245e0 100644 --- a/common/src/main/java/ctbrec/recorder/download/hls/SegmentDownload.java +++ b/common/src/main/java/ctbrec/recorder/download/hls/SegmentDownload.java @@ -7,6 +7,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.InterruptedIOException; import java.io.OutputStream; +import java.net.MalformedURLException; import java.net.URL; import java.security.InvalidAlgorithmParameterException; import java.security.InvalidKeyException; @@ -25,6 +26,7 @@ import ctbrec.io.BandwidthMeter; import ctbrec.io.HttpClient; import ctbrec.io.HttpException; import ctbrec.recorder.download.HttpHeaderFactory; +import ctbrec.recorder.download.hls.SegmentPlaylist.Segment; import okhttp3.Request; import okhttp3.Request.Builder; import okhttp3.Response; @@ -35,15 +37,17 @@ public class SegmentDownload implements Callable { private URL url; private HttpClient client; private SegmentPlaylist playlist; + private Segment segment; private Model model; private OutputStream out; - public SegmentDownload(Model model, SegmentPlaylist playlist, URL url, HttpClient client, OutputStream out) { + public SegmentDownload(Model model, SegmentPlaylist playlist, Segment segment, HttpClient client, OutputStream out) throws MalformedURLException { this.model = model; this.playlist = playlist; - this.url = url; + this.segment = segment; this.client = client; this.out = out; + this.url = new URL(segment.url); } @Override @@ -98,7 +102,7 @@ public class SegmentDownload implements Callable { return out; } - public URL getUrl() { - return url; + public Segment getSegment() { + return segment; } } \ No newline at end of file diff --git a/common/src/main/java/ctbrec/recorder/download/hls/SegmentPlaylist.java b/common/src/main/java/ctbrec/recorder/download/hls/SegmentPlaylist.java index 5cb11ffc..2ba20ba9 100644 --- a/common/src/main/java/ctbrec/recorder/download/hls/SegmentPlaylist.java +++ b/common/src/main/java/ctbrec/recorder/download/hls/SegmentPlaylist.java @@ -1,5 +1,6 @@ package ctbrec.recorder.download.hls; +import java.io.File; import java.util.ArrayList; import java.util.List; @@ -9,7 +10,7 @@ public class SegmentPlaylist { public float totalDuration = 0; public float avgSegDuration = 0; public float targetDuration = 0; - public List segments = new ArrayList<>(); + public List segments = new ArrayList<>(); public boolean encrypted = false; public String encryptionMethod = "AES-128"; public String encryptionKeyUrl; @@ -17,4 +18,16 @@ public class SegmentPlaylist { public SegmentPlaylist(String url) { this.url = url; } + + public static class Segment { + public String url; + public String prefix; + public File targetFile; + public float duration; + + public Segment(String url, float duration) { + this.url = url; + this.duration = duration; + } + } } diff --git a/server/src/main/java/ctbrec/recorder/server/ConfigServlet.java b/server/src/main/java/ctbrec/recorder/server/ConfigServlet.java index e7792665..5d8ee0ed 100644 --- a/server/src/main/java/ctbrec/recorder/server/ConfigServlet.java +++ b/server/src/main/java/ctbrec/recorder/server/ConfigServlet.java @@ -57,8 +57,6 @@ public class ConfigServlet extends AbstractCtbrecServlet { addParameter("httpSecurePort", "HTTPS port", DataType.INTEGER, settings.httpSecurePort, json); addParameter("httpUserAgent", "User-Agent", DataType.STRING, settings.httpUserAgent, json); addParameter("httpUserAgentMobile", "Mobile User-Agent", DataType.STRING, settings.httpUserAgentMobile, json); - addParameter("generatePlaylist", "Generate Playlist", DataType.BOOLEAN, settings.generatePlaylist, json); - addParameter("fastPlaylistGenerator", "Use Fast Playlist Generator", DataType.BOOLEAN, settings.fastPlaylistGenerator, json); addParameter("minimumResolution", "Minimum Resolution", DataType.INTEGER, settings.minimumResolution, json); addParameter("maximumResolution", "Maximum Resolution", DataType.INTEGER, settings.maximumResolution, json); addParameter("minimumSpaceLeftInBytes", "Leave Space On Device (bytes)", DataType.LONG, settings.minimumSpaceLeftInBytes, json);